diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml new file mode 100644 index 00000000..eec6bff9 --- /dev/null +++ b/.github/workflows/docker_build_tpls.yml @@ -0,0 +1,118 @@ +name: Docker Build TPLs + +on: pull_request + +# Cancels in-progress workflows for a PR when updated +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_images: + name: ${{ matrix.name }} + + runs-on: ubuntu-22.04 + + strategy: + + # In-progress jobs will not be cancelled if there is a failure + fail-fast : false + + matrix: + include: + - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) + DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 + INSTALL_DIR_ROOT: /workrd/SCR/GEOSX/install/gcc8/GEOSX_TPL + - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) + DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake + INSTALL_DIR_ROOT: /data/gpfs/Users/j0436735/travis-deployments/GPU/GEOSX_TPL + - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) + DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake + INSTALL_DIR_ROOT: /data/gpfs/Users/j0436735/travis-deployments/CPU/GEOSX_TPL + - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + DOCKERFILE: docker/Stanford/Dockerfile + DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 + HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake + INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/CPU + - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10, zlib 1.2.11) + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11 + DOCKERFILE: docker/Stanford/Dockerfile + DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 + HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake + INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/GPU + - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3) + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + DOCKERFILE: docker/gcc-ubuntu/Dockerfile + DOCKER_ROOT_IMAGE: ubuntu:20.04 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" + INSTALL_DIR_ROOT: /opt/GEOSX + - name: Ubuntu (20.04, gcc 10.3.0, open-mpi 4.0.3) - github codespaces + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + DOCKERFILE: docker/gcc-ubuntu/Dockerfile + DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" + INSTALL_DIR_ROOT: /opt/GEOSX + - name: Ubuntu (22.04, gcc 11.2.0, open-mpi 4.1.2) + DOCKER_ROOT_IMAGE: ubuntu:22.04 + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + DOCKERFILE: docker/gcc-ubuntu/Dockerfile + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" + INSTALL_DIR_ROOT: /opt/GEOSX + - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.2.152) + DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.2.152 + DOCKERFILE: docker/clang-cuda/Dockerfile + INSTALL_DIR_ROOT: /opt/GEOSX + - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.0) + DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.0 + DOCKERFILE: docker/gcc-cuda/Dockerfile + INSTALL_DIR_ROOT: /opt/GEOSX + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + lfs: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Print environment + run: printenv + + - name: Run the docker build and push docker script + env: + DOCKERFILE: ${{ matrix.DOCKERFILE }} + DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} + DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }} + INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT }} + HOST_CONFIG: ${{ matrix.HOST_CONFIG }} + DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE }} + PULL_REQUEST_NUMBER: ${{ github.event.number }} + BUILD_NUMBER: ${{ github.run_number }} + COMMIT: ${{ github.sha }} + BUILD_DIR: ${{ github.workspace }} + run: bash -x ./scripts/docker_build_and_push.sh + + # Convenience job - passes when all docker images are built. + check_that_all_images_built: + runs-on: ubuntu-22.04 + needs: [build_images] + steps: + - name: Success + run: "true" \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c1073f79..00000000 --- a/.travis.yml +++ /dev/null @@ -1,141 +0,0 @@ -language: minimal -vm: - size: large - -geosx_osx_build: &geosx_osx_build - stage: build - os: osx - osx_image: xcode13.4 - install: - - BREW_OPENMPI_VERSION=4.1.1 - - BREW_OPENMPI_TAP=${USER}/local-open-mpi - - brew tap-new ${BREW_OPENMPI_TAP} - - brew extract --version=${BREW_OPENMPI_VERSION} open-mpi ${BREW_OPENMPI_TAP} - - HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_MAKE_JOBS=$(nproc) brew install ${BREW_OPENMPI_TAP}/open-mpi@${BREW_OPENMPI_VERSION} git-lfs - before_script: - - git lfs install - - git lfs pull - script: - - GEOSX_DIR=/usr/local/GEOSX && sudo mkdir -p -m a=rwx ${GEOSX_DIR} - - GEOSX_TPL_DIR=${GEOSX_DIR}/GEOSX_TPL - - python3 scripts/config-build.py -hc ${TRAVIS_BUILD_DIR}/host-configs/darwin-clang.cmake -bt Release -ip ${GEOSX_TPL_DIR} -DNUM_PROC=$(nproc) -DGEOSXTPL_ENABLE_DOXYGEN:BOOL=OFF -DENABLE_VTK:BOOL=OFF -DENABLE_TRILINOS:BOOL=OFF - - cd build-darwin-clang-release - - make - # The deployment phase is done here because the `deploy` stage of travis does not work for PR. - - python3 -m pip install google-cloud-storage - - cd ${TRAVIS_BUILD_DIR} - - openssl aes-256-cbc -K $encrypted_5ac030ea614b_key -iv $encrypted_5ac030ea614b_iv -in geosx-key.json.enc -out geosx-key.json -d - - python3 macosx_TPL_mngt.py ${GEOSX_TPL_DIR} geosx-key.json ${BREW_OPENMPI_VERSION} - -geosx_linux_build: &geosx_linux_build - stage: build - os: linux - dist: focal - services: docker - script: - # We save memory for the docker context - - echo .git > .dockerignore - # This script will build and push a DOCKER_REPOSITORY:DOCKER_TAG image build from DOCKERFILE - # with (optional) DOCKER_COMPILER_BUILD_ARG build arguments. - # A specific host-config file can be defined through variable HOST_CONFIG. - # For the case of Total cluster only, DOCKER_ROOT_IMAGE is used to define docker base image. - # Where the TPL are installed in the docker can be specified by parameter INSTALL_DIR. - # Unlike DOCKER_TAG, these variables shall be defined by the "yaml derived classes" in a stage prior to `script` stage. - - DOCKER_TAG=${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER} - - echo "${DOCKER_USERNAME}" - - echo "${DOCKER_REPOSITORY}:${DOCKER_TAG}" - - docker build ${DOCKER_COMPILER_BUILD_ARG} - --build-arg HOST_CONFIG=${HOST_CONFIG:-host-configs/environment.cmake} - --build-arg DOCKER_ROOT_IMAGE=${DOCKER_ROOT_IMAGE:-undefined} - --build-arg INSTALL_DIR=${INSTALL_DIR:-/opt/GEOSX_TPL} - --tag ${DOCKER_REPOSITORY}:${DOCKER_TAG} - --file ${DOCKERFILE} - --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" - --label "org.opencontainers.image.source=https://github.com/GEOSX/thirdPartyLibs" - --label "org.opencontainers.image.revision=${TRAVIS_COMMIT}" - --label "org.opencontainers.image.title=Building environment for GEOSX" - . - # The deployment phase is done here because the `deploy` stage of travis does not work for PR. - # And `after_success` does not fail the build in case of error. - - echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin - - docker push ${DOCKER_REPOSITORY}:${DOCKER_TAG} - -jobs: - include: - - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 - - DOCKERFILE=docker/TotalEnergies/Dockerfile - - DOCKER_ROOT_IMAGE=totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 - - INSTALL_DIR=/workrd/SCR/GEOSX/install/gcc8/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 - - DOCKERFILE=docker/TotalEnergies/Dockerfile - - DOCKER_ROOT_IMAGE=totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - - HOST_CONFIG=docker/TotalEnergies/pecan-GPU.cmake - - INSTALL_DIR=/data/gpfs/Users/j0436735/travis-deployments/GPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 - - DOCKERFILE=docker/TotalEnergies/Dockerfile - - DOCKER_ROOT_IMAGE=totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - - HOST_CONFIG=docker/TotalEnergies/pecan-CPU.cmake - - INSTALL_DIR=/data/gpfs/Users/j0436735/travis-deployments/CPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - - DOCKERFILE=docker/Stanford/Dockerfile - - DOCKER_ROOT_IMAGE=jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 - - HOST_CONFIG=docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake - - INSTALL_DIR=/home/groups/tchelepi/geosx-sherlock/CPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10, zlib 1.2.11) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11 - - DOCKERFILE=docker/Stanford/Dockerfile - - DOCKER_ROOT_IMAGE=jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 - - HOST_CONFIG=docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake - - INSTALL_DIR=/home/groups/tchelepi/geosx-sherlock/GPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Mac OSX - <<: *geosx_osx_build - - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3) - <<: *geosx_linux_build - before_script: - - GCC_MAJOR_VERSION=9 - - DOCKER_REPOSITORY=geosx/ubuntu20.04-gcc${GCC_MAJOR_VERSION} - - DOCKERFILE=docker/gcc-ubuntu/Dockerfile - - DOCKER_ROOT_IMAGE=ubuntu:20.04 - - DOCKER_COMPILER_BUILD_ARG="--build-arg GCC_MAJOR_VERSION=${GCC_MAJOR_VERSION}" - - name: Ubuntu (20.04, gcc 10.3.0, open-mpi 4.0.3) - github codespaces - <<: *geosx_linux_build - before_script: - # This docker root image brings the components to use github's codespaces feature. - - DOCKER_ROOT_IMAGE=mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 - - GCC_MAJOR_VERSION=10 - - DOCKER_REPOSITORY=geosx/ubuntu20.04-gcc${GCC_MAJOR_VERSION} - - DOCKERFILE=docker/gcc-ubuntu/Dockerfile - - DOCKER_COMPILER_BUILD_ARG="--build-arg GCC_MAJOR_VERSION=${GCC_MAJOR_VERSION}" - - name: Ubuntu (22.04, gcc 11.2.0, open-mpi 4.1.2) - <<: *geosx_linux_build - before_script: - - DOCKER_ROOT_IMAGE=ubuntu:22.04 - - GCC_MAJOR_VERSION=11 - - DOCKER_REPOSITORY=geosx/ubuntu22.04-gcc${GCC_MAJOR_VERSION} - - DOCKERFILE=docker/gcc-ubuntu/Dockerfile - - DOCKER_COMPILER_BUILD_ARG="--build-arg GCC_MAJOR_VERSION=${GCC_MAJOR_VERSION}" - - INSTALL_DIR=/opt/GEOSX/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.2.152) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/ubuntu20.04-clang10.0.0-cuda11.2.152 - - DOCKERFILE=docker/clang-cuda/Dockerfile - - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.0) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/centos7.7-gcc8.3.1-cuda11.8.0 - - DOCKERFILE=docker/gcc-cuda/Dockerfile diff --git a/CMakeLists.txt b/CMakeLists.txt index ae4d15f6..3df13303 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.23.1) project(geosx_tpl LANGUAGES C CXX Fortran) option( GEOSXTPL_ENABLE_DOXYGEN "" ON ) -if ( ENABLE_DOXYGEN EQUAL OFF ) +if ( NOT ENABLE_DOXYGEN ) set( GEOSXTPL_ENABLE_DOXYGEN "OFF" ) endif() set( ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE ) diff --git a/cmake/blt b/cmake/blt index 6a73c2cc..09f7a4be 160000 --- a/cmake/blt +++ b/cmake/blt @@ -1 +1 @@ -Subproject commit 6a73c2cc3ad8147d2686c4eaa79beb43e77e1724 +Subproject commit 09f7a4be48886e1175885e10ed6af1225ae12a90 diff --git a/geosx-key.json.enc b/geosx-key.json.enc deleted file mode 100644 index 6541f68b..00000000 Binary files a/geosx-key.json.enc and /dev/null differ diff --git a/macosx_TPL_mngt.py b/macosx_TPL_mngt.py deleted file mode 100644 index 3d0206d3..00000000 --- a/macosx_TPL_mngt.py +++ /dev/null @@ -1,155 +0,0 @@ -import os -import sys -import logging -import argparse -from io import BytesIO -import tarfile - -from google.oauth2 import service_account -from google.cloud import storage -from google.auth.transport.requests import AuthorizedSession - -from requests.utils import quote - - -TPL_BUCKET_NAME = "geosx" - - -def parse_args(arguments): - """ - Parse the command line arguments provided as input. - We do not let argparse fetch them itself. - Returns an arparse structure. - """ - parser = argparse.ArgumentParser(description="Uploading the TPL for MACOSX so they can be reused for GEOSX.") - parser.add_argument("tpl_dir", metavar="TPL_DIR", help="Path to the TPL folder") - parser.add_argument("service_account_file", metavar="CONFIG_JSON", help="Path to the service accoubt json file.") - parser.add_argument("brew_openmpi_version", metavar="BREW_OPENMPI_VERSION", help="Current version of openmpi in brew.") - return parser.parse_args(arguments) - - -def tpl_name_builder(): - """ - Builds and returns the GCP blob name (mostly from environment variables). - Before modifying this function, keep in mind that GEOSX uses this naming convention to download the tarball. - Consider modifying GEOSX accordingly. - """ - return "TPL/%s-%s-%s.tar" % (os.environ['TRAVIS_OS_NAME'], os.environ['TRAVIS_PULL_REQUEST'], os.environ['TRAVIS_BUILD_NUMBER']) - - -def old_tpl_in_pr_predicate(blob): - """ - A predicate to check if the given CGP blob may be deleted from its bucket. - For the moment we only delete the blobs coming from the same pull request - because we can assume they cannot be used by client code. - """ - try: - return blob.metadata['TRAVIS_PULL_REQUEST'] == os.environ["TRAVIS_PULL_REQUEST"] - except Exception: - logging.warning('Could not retrieve metainformation for blob "%s" in bucket "%s".' % (blob.name, blob.bucket.name)) - return False - - -def build_credentials(service_account_file): - """ - Builds and returns the GCP credentials from the JSON config file (decyphered by travis). - """ - return service_account.Credentials.from_service_account_file( - service_account_file, scopes=("https://www.googleapis.com/auth/devstorage.full_control",) - ) - - -def build_storage_client(credentials): - """ - Builds and returns the GCP storage client. - This functions requires GCP credentials. - """ - return storage.Client(project=credentials.project_id, credentials=credentials) - - -def upload_metadata(blob, brew_openmpi_version, tpl_dir, credentials): - """ - Uploads the metadata of the blob. These metadata can be used to delete the old blobs - (instead of relying on an implicit convention for the blob name). - """ - metadata = {"metadata":{"TRAVIS_PULL_REQUEST": os.environ["TRAVIS_PULL_REQUEST"], - "TRAVIS_BUILD_NUMBER": os.environ["TRAVIS_BUILD_NUMBER"], - "TRAVIS_COMMIT": os.environ["TRAVIS_COMMIT"], - "BREW_OPENMPI_VERSION": brew_openmpi_version, - "GEOSX_TPL_DIR": tpl_dir}} - authed_session = AuthorizedSession(credentials) - url = "https://www.googleapis.com/storage/v1/b/%s/o/%s" % (quote(blob.bucket.name, safe=""), quote(blob.name, safe="")) - req = authed_session.patch(url, json=metadata) - if not req.ok: - raise ValueError(req.reason) - - -def remove_old_blobs(storage_client, blob_filter, bucket_name=TPL_BUCKET_NAME): - """ - Removes the olb GCP blobs from the GCP bucket `bucket_name`. - This functions reliese on the predicate `blob_filter` to select the blocs to delete. - """ - bucket = storage_client.get_bucket(bucket_name) - for b in filter(blob_filter, bucket.list_blobs()): - b.delete() - logging.info('Removed blob "%s" from bucket "%s"' % (b.name, bucket.name)) - - -def upload_tpl(fp, fp_size, destination_blob_name, storage_client, bucket_name=TPL_BUCKET_NAME): - """ - Uploads the content of the file-like instance `fp` of size `fp_size (in bytes)` - to the blob `destination_blob_name` into the `bucket_name` GCP bucket. - No assumptiom on the `fp` position is done and it will me rewinded anyhow - and `fp_size` bytes will be read. - - Returns the created blob instance. - """ - bucket = storage_client.get_bucket(bucket_name) - blob = bucket.blob(destination_blob_name) - blob.upload_from_file(fp, size=fp_size, rewind=True) - blob.make_public() - return blob - - -def compress_tpl_dir(tpl_dir): - """ - Tar (no compression for binaries) the `tpl_dir` file or direcory name. - The resulting archive is returned in a 2-tuple as a file-like object alonside its size (in bytes). - """ - fp = BytesIO() - with tarfile.open(mode="w|", fileobj=fp) as tar: - # We don't want the archive to copy the whole path to the root folder - archive_name = os.path.basename(os.path.normpath(tpl_dir)) - tar.add(tpl_dir, arcname=archive_name) - size = fp.tell() - return fp, size - - -def main(arguments): - """ - Uploads the bucket and its metainformation, detetes some old blobs (but not all). - The `arguments` are the command line arguments (excluding the program itself). - Returns 0 in case of succes of throws. - """ - logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', - level=logging.INFO) - args = parse_args(arguments) - tpl_buff, tpl_size = compress_tpl_dir(args.tpl_dir) - - credentials = build_credentials(args.service_account_file) - storage_client = build_storage_client(credentials) - - blob_name = tpl_name_builder() - remove_old_blobs(storage_client, old_tpl_in_pr_predicate) - blob = upload_tpl(tpl_buff, tpl_size, blob_name, storage_client) - upload_metadata(blob, args.brew_openmpi_version, args.tpl_dir, credentials) - logging.info('Uploaded blob "%s" to bucket "%s"' % (blob.name, blob.bucket.name)) - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main(sys.argv[1:])) - except Exception as e: - logging.error(repr(e)) - sys.exit(1) diff --git a/scripts/docker_build_and_push.sh b/scripts/docker_build_and_push.sh new file mode 100644 index 00000000..588e7c31 --- /dev/null +++ b/scripts/docker_build_and_push.sh @@ -0,0 +1,30 @@ +#!/bin/bash +env + +# We save memory for the docker context +echo .git > .dockerignore +# This script will build and push a DOCKER_REPOSITORY:DOCKER_TAG image build from DOCKERFILE +# with (optional) DOCKER_COMPILER_BUILD_ARG build arguments. +# A specific host-config file can be defined through variable HOST_CONFIG. +# For the case of Total cluster only, DOCKER_ROOT_IMAGE is used to define docker base image. +# Where the TPL are installed in the docker can be specified by parameter INSTALL_DIR. +# Unlike DOCKER_TAG, these variables shall be defined by the "yaml derived classes" in a stage prior to `script` stage. +DOCKER_TAG=${PULL_REQUEST_NUMBER}-${BUILD_NUMBER} +echo "Docker tag is ${DOCKER_REPOSITORY}:${DOCKER_TAG}" + +INSTALL_DIR=${INSTALL_DIR_ROOT}/GEOSX_TPL-${PULL_REQUEST_NUMBER}-${BUILD_NUMBER}-${COMMIT:0:7} +echo "Installation directory is ${INSTALL_DIR}" + +docker build ${DOCKER_COMPILER_BUILD_ARG} \ +--build-arg HOST_CONFIG=${HOST_CONFIG:-host-configs/environment.cmake} \ +--build-arg DOCKER_ROOT_IMAGE=${DOCKER_ROOT_IMAGE:-undefined} \ +--build-arg INSTALL_DIR=${INSTALL_DIR} \ +--tag ${DOCKER_REPOSITORY}:${DOCKER_TAG} \ +--file ${DOCKERFILE} \ +--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \ +--label "org.opencontainers.image.source=https://github.com/GEOSX/thirdPartyLibs" \ +--label "org.opencontainers.image.revision=${COMMIT}" \ +--label "org.opencontainers.image.title=Building environment for GEOSX" \ +. + +docker push ${DOCKER_REPOSITORY}:${DOCKER_TAG}