diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..40b0478 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,85 @@ +# Any commit to master branch re-builds images, re-runs tests, and pushes SHA tags to DockerHub +name: Build +on: + pull_request: + push: + branches: + - main + paths-ignore: + - 'LICENSE' + - 'README.md' + workflow_dispatch: + +env: + DOCKER_REGISTRY: ghcr.io + DOCKER_ORG: ${{ github.repository_owner }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_REF: ${{ github.ref }} + +jobs: + build-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + IMAGE: [base] + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Get date tag + id: get_date + run: | + DATE_TAG="$( date -u '+%Y.%m.%d' )" + echo "date_tag=$DATE_TAG" >> $GITHUB_OUTPUT + + - name: Get registry and org + id: registry_org + run: | + ORG=$(echo "${{ env.DOCKER_ORG }}" | tr '[:upper:]' '[:lower:]') + echo "image_base=${{ env.DOCKER_REGISTRY }}/${ORG}" >> $GITHUB_OUTPUT + + # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + - name: Free up disk space + run: | + df -h + docker image ls + sudo apt clean + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc + df -h + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ steps.registry_org.outputs.image_base }}/${{ matrix.IMAGE }} + tags: | + # set latest tag for default branch + type=raw,value=latest + type=raw,value=${{ steps.get_date.outputs.date_tag }} + + - name: Log in to registry + uses: docker/login-action@v2 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: ${{ matrix.IMAGE }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + load: true + + - name: Inspect Image + run: | + docker run ${{ steps.registry_org.outputs.image_base }}/${{ matrix.IMAGE }}:latest micromamba list + docker images ls diff --git a/.gitignore b/.gitignore index 68bc17f..89273aa 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,9 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Docker image tar files +*.tar.gz + +# MacOS +.DS_Store diff --git a/base/Dockerfile b/base/Dockerfile new file mode 100644 index 0000000..9670441 --- /dev/null +++ b/base/Dockerfile @@ -0,0 +1,46 @@ +FROM mambaorg/micromamba:focal-cuda-11.8.0 + +ARG CAUSTICS_BRANCH=dev + +# Tell apt-get to not block installs by asking for interactive human input +ENV DEBIAN_FRONTEND=noninteractive \ + # Setup locale to be UTF-8, avoiding gnarly hard to debug encoding errors + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +# Switch over to root user to install apt-get packages +USER root + +# Install basic apt packages +RUN echo "Installing Apt-get packages..." \ + && apt-get update --fix-missing > /dev/null \ + && apt-get install -y apt-utils \ + && apt-get install -y \ + git \ + wget \ + zip \ + tzdata \ + python3-venv \ + graphviz > /dev/null \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Switch back to mamba user after all apt-get packages are installed +USER $MAMBA_USER + +# Copy over the conda lock file from host machine to docker build engine +COPY --chown=$MAMBA_USER:$MAMBA_USER conda-linux-64.lock /tmp/conda-linux-64.lock + +# Install the packages listed in the conda lock file +RUN micromamba install --name base --yes --file /tmp/conda-linux-64.lock \ + && micromamba clean --all --yes + +# Set to activate mamba environment, otherwise python will not be found +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +# Install caustics from a branch for now +RUN echo "Caustics branch: ${CAUSTICS_BRANCH}" +ENV CAUSTICS_REPO="git+https://github.com/Ciela-Institute/caustics.git@${CAUSTICS_BRANCH}" + +RUN echo "Installing caustics" \ + && pip install --no-cache ${CAUSTICS_REPO} diff --git a/base/conda-linux-64.lock b/base/conda-linux-64.lock new file mode 100644 index 0000000..8b8a7ba --- /dev/null +++ b/base/conda-linux-64.lock @@ -0,0 +1,78 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: cbce00e63ec521f5d4e80da0d632c032a937b85a6ee4373b90eca120bc17dea3 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 +https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda#2f4327a1cbe7f022401b236e915a5fef +https://conda.anaconda.org/nvidia/linux-64/cuda-cudart-11.8.89-0.tar.bz2#b68c7ef3eda01e95d5903fb508c5e440 +https://conda.anaconda.org/nvidia/linux-64/cuda-cupti-11.8.87-0.tar.bz2#2f4b4933285400137cf029fef9a7daa6 +https://conda.anaconda.org/nvidia/linux-64/cuda-nvrtc-11.8.89-0.tar.bz2#f4af75ee32661708c979630cdb8f4987 +https://conda.anaconda.org/nvidia/linux-64/cuda-nvtx-11.8.86-0.tar.bz2#1825ffc3feb608f2752073935e90bb49 +https://conda.anaconda.org/conda-forge/noarch/cuda-version-12.3-h32bc705_2.conda#2d19844038fc677d667ef91ab8164192 +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda#7aca3059a1729aa76c597603f10b0dd3 +https://conda.anaconda.org/nvidia/linux-64/libcublas-11.11.3.6-0.tar.bz2#7700a48c99151d2b77e7838aa0852da9 +https://conda.anaconda.org/nvidia/linux-64/libcufft-10.9.0.58-0.tar.bz2#dbb21687334ce5f8e6a233cb18ee406b +https://conda.anaconda.org/nvidia/linux-64/libcusolver-11.4.1.48-0.tar.bz2#a497123295be4e0bd221da5bf215f8b8 +https://conda.anaconda.org/nvidia/linux-64/libcusparse-11.7.5.86-0.tar.bz2#853c37fabd07b5b91d3007afc82c3ed4 +https://conda.anaconda.org/nvidia/linux-64/libnpp-11.8.0.86-0.tar.bz2#03822c4b5dae5988ba9dcb7eae837345 +https://conda.anaconda.org/nvidia/linux-64/libnvjpeg-11.9.0.86-0.tar.bz2#e42d6f0f20feb0cba0165d5cae33362f +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h7e041cc_5.conda#f6f6600d18a4047b54f803cf708b868a +https://conda.anaconda.org/conda-forge/linux-64/mkl-include-2022.1.0-h84fe81f_915.tar.bz2#2dcd1acca05c11410d4494d7fc7dfa2a +https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-4_cp311.conda#d786502c97404c94d7d58d258a445a65 +https://conda.anaconda.org/pytorch/noarch/pytorch-mutex-1.0-cuda.tar.bz2#a948316e36fb5b11223b3fcfa93f8358 +https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda#161081fc7cec0bfda0d86d7cb595f8d8 +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_kmp_llvm.tar.bz2#562b26ba2e19059551a811e72ab7f793 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda#d4ff227c46917d3b4565302a2bbb276b +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda#69b8b6202a07720f448be700e300ccf4 +https://conda.anaconda.org/conda-forge/linux-64/gmp-6.3.0-h59595ed_0.conda#0e33ef437202db431aa5a928248cf2e8 +https://conda.anaconda.org/conda-forge/linux-64/icu-73.2-h59595ed_0.conda#cc47e1facc155f91abd89b11e48e72ff +https://conda.anaconda.org/conda-forge/linux-64/libcufile-1.8.1.2-hd3aeb46_0.conda#86641b13f98d24e22b6673e456dd7d45 +https://conda.anaconda.org/conda-forge/linux-64/libcurand-10.3.4.107-hd3aeb46_0.conda#11619fe5021476180219c6a02f0a7c14 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.5.0-hcb278e6_1.conda#6305a3dd2752c76335295da4e581f2fd +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-ha4646dd_5.conda#7a6bd7a12a4bd359e2afe6c0fa1acace +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-hd590300_2.conda#d66573916ffcf376178462f1b61c941e +https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7 +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b +https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda#5aa797f8787fe7a17d1b0821485b5adc +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda#f36c115f1ee199da648e0597ec2047ad +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-h59595ed_2.conda#7dbaa197d7ba6032caf7ae7f32c1efa0 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_0.conda#51a753e64a3027bd7e23a189b1f6e91e +https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0 +https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae +https://conda.anaconda.org/nvidia/linux-64/cuda-libraries-11.8.0-0.tar.bz2#3a43d100104e52ac8209a834c82ab231 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_5.conda#e73e9cfd1191783392131e6238bdb3e9 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.1-h2797004_0.conda#fc4ccadfbf6d4784de88c41704792562 +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.12.5-h232c23b_0.conda#c442ebfda7a475f5e78f1c8e45f1e919 +https://conda.anaconda.org/conda-forge/linux-64/llvm-openmp-15.0.7-h0cdce71_0.conda#589c9a3575a050b583241c3d688ad9aa +https://conda.anaconda.org/conda-forge/linux-64/mpfr-4.2.1-h9458935_0.conda#4c28f3210b30250037a4a627eeee9e0f +https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda#d453b98d9c83e71da0741bb0ff4d76bc +https://conda.anaconda.org/nvidia/linux-64/cuda-runtime-11.8.0-0.tar.bz2#3ca379d762f8d7bd727df9e2c9b30664 +https://conda.anaconda.org/conda-forge/linux-64/libhwloc-2.9.3-default_h554bfaf_1009.conda#f36ddc11ca46958197a45effdd286e45 +https://conda.anaconda.org/conda-forge/linux-64/mpc-1.3.1-hfe3b2da_0.conda#289c71e83dc0daa7d4c81f04180778ca +https://conda.anaconda.org/conda-forge/linux-64/python-3.11.7-hab00c5b_1_cpython.conda#27cf681282c11dba7b0b1fd266e8f289 +https://conda.anaconda.org/conda-forge/noarch/filelock-3.13.1-pyhd8ed1ab_0.conda#0c1729b74a8152fde6a38ba0a2ab9f45 +https://conda.anaconda.org/conda-forge/linux-64/gmpy2-2.1.2-py311h6a5fa03_1.tar.bz2#3515bd4a3d92bbd3cc2d25aac335e34d +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.5-py311h459d7ec_0.conda#a322b4185121935c871d201ae00ac143 +https://conda.anaconda.org/conda-forge/noarch/mpmath-1.3.0-pyhd8ed1ab_0.conda#dbf6e2d89137da32fa6670f3bffc024e +https://conda.anaconda.org/conda-forge/noarch/networkx-3.2.1-pyhd8ed1ab_0.conda#425fce3b531bed6ec3c74fab3e5f0a1c +https://conda.anaconda.org/pytorch/linux-64/pytorch-cuda-11.8-h7e8668a_5.tar.bz2#48e990086eb245cce92f09b45a34651e +https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py311h459d7ec_1.conda#52719a74ad130de8fb5d047dc91f247a +https://conda.anaconda.org/conda-forge/noarch/setuptools-69.0.3-pyhd8ed1ab_0.conda#40695fdfd15a92121ed2922900d0308b +https://conda.anaconda.org/conda-forge/linux-64/tbb-2021.11.0-h00ab1b0_1.conda#4531d2927578e7e254ff3bcf6457518c +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.9.0-pyha770c72_0.conda#a92a6440c3fe7052d63244f3aba2a4a7 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.42.0-pyhd8ed1ab_0.conda#1cdea58981c5cbc17b51973bcaddcea7 +https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.3-pyhd8ed1ab_0.conda#e7d8df6509ba635247ff9aea31134262 +https://conda.anaconda.org/conda-forge/linux-64/mkl-2022.1.0-h84fe81f_915.tar.bz2#b9c8f925797a93dbff45e1626b025a6b +https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda#f586ac1e56c8638b64f9c8122a7b8a67 +https://conda.anaconda.org/conda-forge/noarch/sympy-1.12-pypyh9d50eac_103.conda#2f7d6347d7acf6edf1ac7f2189f44c8f +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-16_linux64_mkl.tar.bz2#85f61af03fd291dae33150ffe89dc09a +https://conda.anaconda.org/conda-forge/linux-64/mkl-devel-2022.1.0-ha770c72_916.tar.bz2#69ba49e445f87aea2cba343a71a35ca2 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_mkl.tar.bz2#361bf757b95488de76c4f123805742d3 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_mkl.tar.bz2#a2f166748917d6d6e4707841ca1f519e +https://conda.anaconda.org/conda-forge/linux-64/liblapacke-3.9.0-16_linux64_mkl.tar.bz2#44ccc4d4dca6a8d57fa17442bc64b5a1 +https://conda.anaconda.org/conda-forge/linux-64/blas-devel-3.9.0-16_linux64_mkl.tar.bz2#3f92c1c9e1c0e183462c5071aa02cae1 +https://conda.anaconda.org/conda-forge/linux-64/blas-2.116-mkl.tar.bz2#c196a26abf6b4f132c88828ab7c2231c +https://conda.anaconda.org/pytorch/linux-64/pytorch-2.2.0-py3.11_cuda11.8_cudnn8.7.0_0.tar.bz2#ff212242316c68bbb32dbd8c3d2ed6a2 +https://conda.anaconda.org/pytorch/linux-64/torchtriton-2.2.0-py311.tar.bz2#da613ad3f914b4b335de5baa5d184da3 diff --git a/base/env.yaml b/base/env.yaml new file mode 100644 index 0000000..f46169b --- /dev/null +++ b/base/env.yaml @@ -0,0 +1,10 @@ +name: base +channels: +- conda-forge +- nvidia +- nodefaults +dependencies: +- python=3.11 +- pytorch::pytorch>=2.0.0 +- pytorch::pytorch-cuda==11.8 +- pip