From b483572f80934341dd551fe3a115e43921b58afa Mon Sep 17 00:00:00 2001 From: tbeltzun <129868353+tbeltzun@users.noreply.github.com> Date: Mon, 10 Jul 2023 18:21:05 +0200 Subject: [PATCH 1/5] fix boolean comparison --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ae4d15f6..ceb12cb9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.23.1) project(geosx_tpl LANGUAGES C CXX Fortran) option( GEOSXTPL_ENABLE_DOXYGEN "" ON ) -if ( ENABLE_DOXYGEN EQUAL OFF ) +if ( ENABLE_DOXYGEN ) set( GEOSXTPL_ENABLE_DOXYGEN "OFF" ) endif() set( ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE ) From 90a1792da4caec8fdc2040c291fa0355df882623 Mon Sep 17 00:00:00 2001 From: tbeltzun <129868353+tbeltzun@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:16:28 +0200 Subject: [PATCH 2/5] fix logic --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ceb12cb9..3df13303 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.23.1) project(geosx_tpl LANGUAGES C CXX Fortran) option( GEOSXTPL_ENABLE_DOXYGEN "" ON ) -if ( ENABLE_DOXYGEN ) +if ( NOT ENABLE_DOXYGEN ) set( GEOSXTPL_ENABLE_DOXYGEN "OFF" ) endif() set( ENABLE_DOXYGEN OFF CACHE BOOL "" FORCE ) From 75d56a018d68c34f51e5d4eba1a8c1e42581ac30 Mon Sep 17 00:00:00 2001 From: jfranc Date: Fri, 28 Jul 2023 17:13:32 -0700 Subject: [PATCH 3/5] changing path for automated deployment --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c1073f79..a61ebb47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,7 +92,7 @@ jobs: - DOCKERFILE=docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE=jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 - HOST_CONFIG=docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake - - INSTALL_DIR=/home/groups/tchelepi/geosx-sherlock/CPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} + - INSTALL_DIR=/oak/stanford/schools/ees/COLLABORATIONS/geosx/CPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10, zlib 1.2.11) <<: *geosx_linux_build env: @@ -100,7 +100,7 @@ jobs: - DOCKERFILE=docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE=jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 - HOST_CONFIG=docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake - - INSTALL_DIR=/home/groups/tchelepi/geosx-sherlock/GPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} + - INSTALL_DIR=/oak/stanford/schools/ees/COLLABORATIONS/geosx/GPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - name: Mac OSX <<: *geosx_osx_build - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3) From 00e085e1459a3d2a6c6ca8d6ca8a0657fe7df4d7 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Mon, 14 Aug 2023 11:54:23 -0700 Subject: [PATCH 4/5] Github Actions Pipeline (#225) * Test with testing PR tag * Update paths * Trigger on pull request * Use PR number * Remove Mac OSX builds * Shorten github sha for installation directory name * Adjust Sherlock paths for gha --- .github/workflows/docker_build_tpls.yml | 118 ++++++++++++++++++ .travis.yml | 27 ----- geosx-key.json.enc | Bin 2320 -> 0 bytes macosx_TPL_mngt.py | 155 ------------------------ scripts/docker_build_and_push.sh | 30 +++++ 5 files changed, 148 insertions(+), 182 deletions(-) create mode 100644 .github/workflows/docker_build_tpls.yml delete mode 100644 geosx-key.json.enc delete mode 100644 macosx_TPL_mngt.py create mode 100644 scripts/docker_build_and_push.sh diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml new file mode 100644 index 00000000..eec6bff9 --- /dev/null +++ b/.github/workflows/docker_build_tpls.yml @@ -0,0 +1,118 @@ +name: Docker Build TPLs + +on: pull_request + +# Cancels in-progress workflows for a PR when updated +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_images: + name: ${{ matrix.name }} + + runs-on: ubuntu-22.04 + + strategy: + + # In-progress jobs will not be cancelled if there is a failure + fail-fast : false + + matrix: + include: + - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) + DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 + INSTALL_DIR_ROOT: /workrd/SCR/GEOSX/install/gcc8/GEOSX_TPL + - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) + DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake + INSTALL_DIR_ROOT: /data/gpfs/Users/j0436735/travis-deployments/GPU/GEOSX_TPL + - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) + DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake + INSTALL_DIR_ROOT: /data/gpfs/Users/j0436735/travis-deployments/CPU/GEOSX_TPL + - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + DOCKERFILE: docker/Stanford/Dockerfile + DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 + HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake + INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/CPU + - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10, zlib 1.2.11) + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11 + DOCKERFILE: docker/Stanford/Dockerfile + DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 + HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake + INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/GPU + - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3) + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + DOCKERFILE: docker/gcc-ubuntu/Dockerfile + DOCKER_ROOT_IMAGE: ubuntu:20.04 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" + INSTALL_DIR_ROOT: /opt/GEOSX + - name: Ubuntu (20.04, gcc 10.3.0, open-mpi 4.0.3) - github codespaces + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + DOCKERFILE: docker/gcc-ubuntu/Dockerfile + DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" + INSTALL_DIR_ROOT: /opt/GEOSX + - name: Ubuntu (22.04, gcc 11.2.0, open-mpi 4.1.2) + DOCKER_ROOT_IMAGE: ubuntu:22.04 + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + DOCKERFILE: docker/gcc-ubuntu/Dockerfile + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" + INSTALL_DIR_ROOT: /opt/GEOSX + - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.2.152) + DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.2.152 + DOCKERFILE: docker/clang-cuda/Dockerfile + INSTALL_DIR_ROOT: /opt/GEOSX + - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.0) + DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.0 + DOCKERFILE: docker/gcc-cuda/Dockerfile + INSTALL_DIR_ROOT: /opt/GEOSX + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + lfs: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Print environment + run: printenv + + - name: Run the docker build and push docker script + env: + DOCKERFILE: ${{ matrix.DOCKERFILE }} + DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} + DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }} + INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT }} + HOST_CONFIG: ${{ matrix.HOST_CONFIG }} + DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE }} + PULL_REQUEST_NUMBER: ${{ github.event.number }} + BUILD_NUMBER: ${{ github.run_number }} + COMMIT: ${{ github.sha }} + BUILD_DIR: ${{ github.workspace }} + run: bash -x ./scripts/docker_build_and_push.sh + + # Convenience job - passes when all docker images are built. + check_that_all_images_built: + runs-on: ubuntu-22.04 + needs: [build_images] + steps: + - name: Success + run: "true" \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index a61ebb47..9d18766c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,31 +2,6 @@ language: minimal vm: size: large -geosx_osx_build: &geosx_osx_build - stage: build - os: osx - osx_image: xcode13.4 - install: - - BREW_OPENMPI_VERSION=4.1.1 - - BREW_OPENMPI_TAP=${USER}/local-open-mpi - - brew tap-new ${BREW_OPENMPI_TAP} - - brew extract --version=${BREW_OPENMPI_VERSION} open-mpi ${BREW_OPENMPI_TAP} - - HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_MAKE_JOBS=$(nproc) brew install ${BREW_OPENMPI_TAP}/open-mpi@${BREW_OPENMPI_VERSION} git-lfs - before_script: - - git lfs install - - git lfs pull - script: - - GEOSX_DIR=/usr/local/GEOSX && sudo mkdir -p -m a=rwx ${GEOSX_DIR} - - GEOSX_TPL_DIR=${GEOSX_DIR}/GEOSX_TPL - - python3 scripts/config-build.py -hc ${TRAVIS_BUILD_DIR}/host-configs/darwin-clang.cmake -bt Release -ip ${GEOSX_TPL_DIR} -DNUM_PROC=$(nproc) -DGEOSXTPL_ENABLE_DOXYGEN:BOOL=OFF -DENABLE_VTK:BOOL=OFF -DENABLE_TRILINOS:BOOL=OFF - - cd build-darwin-clang-release - - make - # The deployment phase is done here because the `deploy` stage of travis does not work for PR. - - python3 -m pip install google-cloud-storage - - cd ${TRAVIS_BUILD_DIR} - - openssl aes-256-cbc -K $encrypted_5ac030ea614b_key -iv $encrypted_5ac030ea614b_iv -in geosx-key.json.enc -out geosx-key.json -d - - python3 macosx_TPL_mngt.py ${GEOSX_TPL_DIR} geosx-key.json ${BREW_OPENMPI_VERSION} - geosx_linux_build: &geosx_linux_build stage: build os: linux @@ -101,8 +76,6 @@ jobs: - DOCKER_ROOT_IMAGE=jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 - HOST_CONFIG=docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake - INSTALL_DIR=/oak/stanford/schools/ees/COLLABORATIONS/geosx/GPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Mac OSX - <<: *geosx_osx_build - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3) <<: *geosx_linux_build before_script: diff --git a/geosx-key.json.enc b/geosx-key.json.enc deleted file mode 100644 index 6541f68b9b6121a37daff47ce3c6e372667ea4e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2320 zcmV+r3Gepg%=hT8(9-R&3F4V)m1i!uZ%Q{)KYnWEy)SzaoT0Ccks=Y{n@ZRDI(C%R zX7NX0d(_^geSX&(9qzaOOFipB=5A5$#5x@@R5kP_dnUPixGGyAD0NPdNN#Q`wfTo( z0eZ-*_W|wtdZ&0(3m0TSB%lzehX-UdU}+mSl~uT-Sv=xpe)|g{%k^z-MVLTb{$D{2 z?$EB5>3PHVWD|vQlyWfz&Pcet${*Rm%MSuBZa3BLPg4e5mb#1#;Q@{^9g=KoTM=T! zl=Q&6vTMF3ij<9B2mR+6FDy+Dj{~utofis(6n&`5zB~fzFB!0uX$Yar?w)j@&LUdM zhm`}0iq|a|FDIM>X%gZe+a4CPYa|IRV{+ z%oVjGGfAb|(2{ac%(OymzO8VKULUvb5L>4E->A7?6t{~uzRB+t#~(0PE!Y)Zg@H(k zs5`$w3L|gl9Nd0KL|sk)MgX+R-`t6_D_io$UIh}^Qn{%lVIp0}+=;%)R4UQcWlKbT z+yQ0Kg71<&OQNJ+AnF*05M{Lqe+?P-P%lrG{|-A*jn!hR*9quiE>u=f_uR|gR9;Pp z-30^(#01W?UMWBDX6=ZRv%HYx!vGS)zm*t_!;rjUAs|zwWmlJNntzE12get~Jk)Dk zD^;Idw-|{};Iu#1cB8(@`=bh=b(q=5hwqZ3>EhuD)jBL98oNfI>%l<_lx;T&j;H?O zE?@Os6`h(?P2Mv7pP(hz806uhDfm6zo;zIgW5Z@&%6iKtIB*K408_~XtC^lUSv|t% z@g?b}fJeG&Ruk{}{U9dP>_#g$mqF|helwtH-Xozlkwh)9C67y#_s5gKV@%N)^DRN! z3=mdpra};SOI=3Zm#z%CXwh6A^J9Q=b_>4Jeob`7=NJYr&3WgR^;zC}3|RACLPxj7*sSsSGuneX*0P=Rc`R`{xh z`-J)y>ji?{CVLitfD@_#(!(k)i2CfHejI%kAq?l;pc-g>2wKk2wHj=h`L~B-MV=QO zgD{_6N;`G(qu)rdD6GFCw4x1*Y42frPgkdPY5g0y&yoDSZLR)f4kRPe8zj@wnL5Z2 zp2i{#ARTI5qGVH0K32A@#wTe3f`D5Mwd(Aa zI$_S4JcfimMv6pDDa&~O?D2Bi?#|;0G?SH$ zVOj#ZN{=-sD~|pted;FJj%~Y-Weh(g+?*Dn!==dp34oWFx_L`z@@<^>|4A@!6;_*F znfE`V@V3k&7{Qw4)LP-sG*&9IYr2mWzzyd=!)AxhK?r0MYt%Am`w6VUyQ53F_+yQu z6E5V>T%hG2UgsdOjmBMM5?$c7A)$-mCQm(8OI{6NBW^%SFOb~*@YJthdH!R(Bc2y> z>Oq@f<657|hS8io=h>2BGm!Ln(Fvcnls)=-yEnCMtA~M4Zoof`#S}%2K8x~vTs=aj zN-DZ^f>@Bgq%f03|%XIad)2Q|%LGm+CRx5}dcRMKE8^GorZMU@wl`B&9L9`_-LYg_O;f!>w93X$# z+544qcX)k~8A-}EPLsev1h5ewWb-PixRR0|TV={}>6kfQb5@X4zG(1oJ<_WPT$xND zLrR1pfYq8y>gRNl!oie1H-vg97YXJg>zDX*yO1UV(fF7AoK71Pu}_lj)&KqUzGyV+ zbSA-w^x4hEJzL0c!?kN^w#eaI6&qO56*pg;39#jwqid?)k?32^cQf}9V3G?>yNHU1 z@a9J6Q}3PEL;4OG^f?>wDXk}e1=ddj{Bi_nVhZ&$`$boW%FEIrbdKr%tEr_Fbv&ii z7m*?R`JuEl4ZXSmHg9M>=}~f?pIiU0L(;LV=<4M!!fCmAn{ND>+j7ksBH`0R@D5~) z0UzGQp!D^sQ)vG%Qk6Ff)4wfmFu5>A6OT40{p*WnPS{p& z-{)o;M+%vut>;eKF>xp}*BTjNdO(Pr7=hBSo?y4s^p1$gg2<}FCBp6vUA=~^#n}vZ z)Sln{?ZbVV2g`-VI`|Nf^XG~lL7Xa%lg@$G!O2!bxj4{1j#b#uj?Fo2)XVgZmMaRi z{gEx0*LhGhx)`Iz=K(7o`k(!B{$Z>*SzK+4eUL!BaX$Q7v|GpVwLtaw0x~%QsP>Q>in|mdT3N!!qy?h735Ph$RkYm~-xM q^~V^!oQkXO*p*>%gchHwX0rYC*b#8n(r$uM?#6KzadtN+k4g|et&64r diff --git a/macosx_TPL_mngt.py b/macosx_TPL_mngt.py deleted file mode 100644 index 3d0206d3..00000000 --- a/macosx_TPL_mngt.py +++ /dev/null @@ -1,155 +0,0 @@ -import os -import sys -import logging -import argparse -from io import BytesIO -import tarfile - -from google.oauth2 import service_account -from google.cloud import storage -from google.auth.transport.requests import AuthorizedSession - -from requests.utils import quote - - -TPL_BUCKET_NAME = "geosx" - - -def parse_args(arguments): - """ - Parse the command line arguments provided as input. - We do not let argparse fetch them itself. - Returns an arparse structure. - """ - parser = argparse.ArgumentParser(description="Uploading the TPL for MACOSX so they can be reused for GEOSX.") - parser.add_argument("tpl_dir", metavar="TPL_DIR", help="Path to the TPL folder") - parser.add_argument("service_account_file", metavar="CONFIG_JSON", help="Path to the service accoubt json file.") - parser.add_argument("brew_openmpi_version", metavar="BREW_OPENMPI_VERSION", help="Current version of openmpi in brew.") - return parser.parse_args(arguments) - - -def tpl_name_builder(): - """ - Builds and returns the GCP blob name (mostly from environment variables). - Before modifying this function, keep in mind that GEOSX uses this naming convention to download the tarball. - Consider modifying GEOSX accordingly. - """ - return "TPL/%s-%s-%s.tar" % (os.environ['TRAVIS_OS_NAME'], os.environ['TRAVIS_PULL_REQUEST'], os.environ['TRAVIS_BUILD_NUMBER']) - - -def old_tpl_in_pr_predicate(blob): - """ - A predicate to check if the given CGP blob may be deleted from its bucket. - For the moment we only delete the blobs coming from the same pull request - because we can assume they cannot be used by client code. - """ - try: - return blob.metadata['TRAVIS_PULL_REQUEST'] == os.environ["TRAVIS_PULL_REQUEST"] - except Exception: - logging.warning('Could not retrieve metainformation for blob "%s" in bucket "%s".' % (blob.name, blob.bucket.name)) - return False - - -def build_credentials(service_account_file): - """ - Builds and returns the GCP credentials from the JSON config file (decyphered by travis). - """ - return service_account.Credentials.from_service_account_file( - service_account_file, scopes=("https://www.googleapis.com/auth/devstorage.full_control",) - ) - - -def build_storage_client(credentials): - """ - Builds and returns the GCP storage client. - This functions requires GCP credentials. - """ - return storage.Client(project=credentials.project_id, credentials=credentials) - - -def upload_metadata(blob, brew_openmpi_version, tpl_dir, credentials): - """ - Uploads the metadata of the blob. These metadata can be used to delete the old blobs - (instead of relying on an implicit convention for the blob name). - """ - metadata = {"metadata":{"TRAVIS_PULL_REQUEST": os.environ["TRAVIS_PULL_REQUEST"], - "TRAVIS_BUILD_NUMBER": os.environ["TRAVIS_BUILD_NUMBER"], - "TRAVIS_COMMIT": os.environ["TRAVIS_COMMIT"], - "BREW_OPENMPI_VERSION": brew_openmpi_version, - "GEOSX_TPL_DIR": tpl_dir}} - authed_session = AuthorizedSession(credentials) - url = "https://www.googleapis.com/storage/v1/b/%s/o/%s" % (quote(blob.bucket.name, safe=""), quote(blob.name, safe="")) - req = authed_session.patch(url, json=metadata) - if not req.ok: - raise ValueError(req.reason) - - -def remove_old_blobs(storage_client, blob_filter, bucket_name=TPL_BUCKET_NAME): - """ - Removes the olb GCP blobs from the GCP bucket `bucket_name`. - This functions reliese on the predicate `blob_filter` to select the blocs to delete. - """ - bucket = storage_client.get_bucket(bucket_name) - for b in filter(blob_filter, bucket.list_blobs()): - b.delete() - logging.info('Removed blob "%s" from bucket "%s"' % (b.name, bucket.name)) - - -def upload_tpl(fp, fp_size, destination_blob_name, storage_client, bucket_name=TPL_BUCKET_NAME): - """ - Uploads the content of the file-like instance `fp` of size `fp_size (in bytes)` - to the blob `destination_blob_name` into the `bucket_name` GCP bucket. - No assumptiom on the `fp` position is done and it will me rewinded anyhow - and `fp_size` bytes will be read. - - Returns the created blob instance. - """ - bucket = storage_client.get_bucket(bucket_name) - blob = bucket.blob(destination_blob_name) - blob.upload_from_file(fp, size=fp_size, rewind=True) - blob.make_public() - return blob - - -def compress_tpl_dir(tpl_dir): - """ - Tar (no compression for binaries) the `tpl_dir` file or direcory name. - The resulting archive is returned in a 2-tuple as a file-like object alonside its size (in bytes). - """ - fp = BytesIO() - with tarfile.open(mode="w|", fileobj=fp) as tar: - # We don't want the archive to copy the whole path to the root folder - archive_name = os.path.basename(os.path.normpath(tpl_dir)) - tar.add(tpl_dir, arcname=archive_name) - size = fp.tell() - return fp, size - - -def main(arguments): - """ - Uploads the bucket and its metainformation, detetes some old blobs (but not all). - The `arguments` are the command line arguments (excluding the program itself). - Returns 0 in case of succes of throws. - """ - logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', - level=logging.INFO) - args = parse_args(arguments) - tpl_buff, tpl_size = compress_tpl_dir(args.tpl_dir) - - credentials = build_credentials(args.service_account_file) - storage_client = build_storage_client(credentials) - - blob_name = tpl_name_builder() - remove_old_blobs(storage_client, old_tpl_in_pr_predicate) - blob = upload_tpl(tpl_buff, tpl_size, blob_name, storage_client) - upload_metadata(blob, args.brew_openmpi_version, args.tpl_dir, credentials) - logging.info('Uploaded blob "%s" to bucket "%s"' % (blob.name, blob.bucket.name)) - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main(sys.argv[1:])) - except Exception as e: - logging.error(repr(e)) - sys.exit(1) diff --git a/scripts/docker_build_and_push.sh b/scripts/docker_build_and_push.sh new file mode 100644 index 00000000..588e7c31 --- /dev/null +++ b/scripts/docker_build_and_push.sh @@ -0,0 +1,30 @@ +#!/bin/bash +env + +# We save memory for the docker context +echo .git > .dockerignore +# This script will build and push a DOCKER_REPOSITORY:DOCKER_TAG image build from DOCKERFILE +# with (optional) DOCKER_COMPILER_BUILD_ARG build arguments. +# A specific host-config file can be defined through variable HOST_CONFIG. +# For the case of Total cluster only, DOCKER_ROOT_IMAGE is used to define docker base image. +# Where the TPL are installed in the docker can be specified by parameter INSTALL_DIR. +# Unlike DOCKER_TAG, these variables shall be defined by the "yaml derived classes" in a stage prior to `script` stage. +DOCKER_TAG=${PULL_REQUEST_NUMBER}-${BUILD_NUMBER} +echo "Docker tag is ${DOCKER_REPOSITORY}:${DOCKER_TAG}" + +INSTALL_DIR=${INSTALL_DIR_ROOT}/GEOSX_TPL-${PULL_REQUEST_NUMBER}-${BUILD_NUMBER}-${COMMIT:0:7} +echo "Installation directory is ${INSTALL_DIR}" + +docker build ${DOCKER_COMPILER_BUILD_ARG} \ +--build-arg HOST_CONFIG=${HOST_CONFIG:-host-configs/environment.cmake} \ +--build-arg DOCKER_ROOT_IMAGE=${DOCKER_ROOT_IMAGE:-undefined} \ +--build-arg INSTALL_DIR=${INSTALL_DIR} \ +--tag ${DOCKER_REPOSITORY}:${DOCKER_TAG} \ +--file ${DOCKERFILE} \ +--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \ +--label "org.opencontainers.image.source=https://github.com/GEOSX/thirdPartyLibs" \ +--label "org.opencontainers.image.revision=${COMMIT}" \ +--label "org.opencontainers.image.title=Building environment for GEOSX" \ +. + +docker push ${DOCKER_REPOSITORY}:${DOCKER_TAG} From 18b356b7e5e8a13bc3eac55b375f795c4f56806f Mon Sep 17 00:00:00 2001 From: Randolph Settgast Date: Tue, 15 Aug 2023 09:09:25 -0700 Subject: [PATCH 5/5] remove .travis.yml file (#235) --- .travis.yml | 114 ---------------------------------------------------- cmake/blt | 2 +- 2 files changed, 1 insertion(+), 115 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9d18766c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,114 +0,0 @@ -language: minimal -vm: - size: large - -geosx_linux_build: &geosx_linux_build - stage: build - os: linux - dist: focal - services: docker - script: - # We save memory for the docker context - - echo .git > .dockerignore - # This script will build and push a DOCKER_REPOSITORY:DOCKER_TAG image build from DOCKERFILE - # with (optional) DOCKER_COMPILER_BUILD_ARG build arguments. - # A specific host-config file can be defined through variable HOST_CONFIG. - # For the case of Total cluster only, DOCKER_ROOT_IMAGE is used to define docker base image. - # Where the TPL are installed in the docker can be specified by parameter INSTALL_DIR. - # Unlike DOCKER_TAG, these variables shall be defined by the "yaml derived classes" in a stage prior to `script` stage. - - DOCKER_TAG=${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER} - - echo "${DOCKER_USERNAME}" - - echo "${DOCKER_REPOSITORY}:${DOCKER_TAG}" - - docker build ${DOCKER_COMPILER_BUILD_ARG} - --build-arg HOST_CONFIG=${HOST_CONFIG:-host-configs/environment.cmake} - --build-arg DOCKER_ROOT_IMAGE=${DOCKER_ROOT_IMAGE:-undefined} - --build-arg INSTALL_DIR=${INSTALL_DIR:-/opt/GEOSX_TPL} - --tag ${DOCKER_REPOSITORY}:${DOCKER_TAG} - --file ${DOCKERFILE} - --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" - --label "org.opencontainers.image.source=https://github.com/GEOSX/thirdPartyLibs" - --label "org.opencontainers.image.revision=${TRAVIS_COMMIT}" - --label "org.opencontainers.image.title=Building environment for GEOSX" - . - # The deployment phase is done here because the `deploy` stage of travis does not work for PR. - # And `after_success` does not fail the build in case of error. - - echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin - - docker push ${DOCKER_REPOSITORY}:${DOCKER_TAG} - -jobs: - include: - - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 - - DOCKERFILE=docker/TotalEnergies/Dockerfile - - DOCKER_ROOT_IMAGE=totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 - - INSTALL_DIR=/workrd/SCR/GEOSX/install/gcc8/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 - - DOCKERFILE=docker/TotalEnergies/Dockerfile - - DOCKER_ROOT_IMAGE=totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - - HOST_CONFIG=docker/TotalEnergies/pecan-GPU.cmake - - INSTALL_DIR=/data/gpfs/Users/j0436735/travis-deployments/GPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 - - DOCKERFILE=docker/TotalEnergies/Dockerfile - - DOCKER_ROOT_IMAGE=totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - - HOST_CONFIG=docker/TotalEnergies/pecan-CPU.cmake - - INSTALL_DIR=/data/gpfs/Users/j0436735/travis-deployments/CPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - - DOCKERFILE=docker/Stanford/Dockerfile - - DOCKER_ROOT_IMAGE=jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 - - HOST_CONFIG=docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake - - INSTALL_DIR=/oak/stanford/schools/ees/COLLABORATIONS/geosx/CPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10, zlib 1.2.11) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11 - - DOCKERFILE=docker/Stanford/Dockerfile - - DOCKER_ROOT_IMAGE=jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 - - HOST_CONFIG=docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake - - INSTALL_DIR=/oak/stanford/schools/ees/COLLABORATIONS/geosx/GPU/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3) - <<: *geosx_linux_build - before_script: - - GCC_MAJOR_VERSION=9 - - DOCKER_REPOSITORY=geosx/ubuntu20.04-gcc${GCC_MAJOR_VERSION} - - DOCKERFILE=docker/gcc-ubuntu/Dockerfile - - DOCKER_ROOT_IMAGE=ubuntu:20.04 - - DOCKER_COMPILER_BUILD_ARG="--build-arg GCC_MAJOR_VERSION=${GCC_MAJOR_VERSION}" - - name: Ubuntu (20.04, gcc 10.3.0, open-mpi 4.0.3) - github codespaces - <<: *geosx_linux_build - before_script: - # This docker root image brings the components to use github's codespaces feature. - - DOCKER_ROOT_IMAGE=mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 - - GCC_MAJOR_VERSION=10 - - DOCKER_REPOSITORY=geosx/ubuntu20.04-gcc${GCC_MAJOR_VERSION} - - DOCKERFILE=docker/gcc-ubuntu/Dockerfile - - DOCKER_COMPILER_BUILD_ARG="--build-arg GCC_MAJOR_VERSION=${GCC_MAJOR_VERSION}" - - name: Ubuntu (22.04, gcc 11.2.0, open-mpi 4.1.2) - <<: *geosx_linux_build - before_script: - - DOCKER_ROOT_IMAGE=ubuntu:22.04 - - GCC_MAJOR_VERSION=11 - - DOCKER_REPOSITORY=geosx/ubuntu22.04-gcc${GCC_MAJOR_VERSION} - - DOCKERFILE=docker/gcc-ubuntu/Dockerfile - - DOCKER_COMPILER_BUILD_ARG="--build-arg GCC_MAJOR_VERSION=${GCC_MAJOR_VERSION}" - - INSTALL_DIR=/opt/GEOSX/GEOSX_TPL-${TRAVIS_PULL_REQUEST}-${TRAVIS_BUILD_NUMBER}-${TRAVIS_COMMIT:0:7} - - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.2.152) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/ubuntu20.04-clang10.0.0-cuda11.2.152 - - DOCKERFILE=docker/clang-cuda/Dockerfile - - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.0) - <<: *geosx_linux_build - env: - - DOCKER_REPOSITORY=geosx/centos7.7-gcc8.3.1-cuda11.8.0 - - DOCKERFILE=docker/gcc-cuda/Dockerfile diff --git a/cmake/blt b/cmake/blt index 6a73c2cc..09f7a4be 160000 --- a/cmake/blt +++ b/cmake/blt @@ -1 +1 @@ -Subproject commit 6a73c2cc3ad8147d2686c4eaa79beb43e77e1724 +Subproject commit 09f7a4be48886e1175885e10ed6af1225ae12a90