Build conda package and upload to anaconda (#515)

* add a file to build wheel * build wheel action * build wheel only for py3.8 * restore unnecessary changes * conda build and upload in workflow * wheel location changed * source location in meta.yaml * add cibuildwheel and change openblas option * conda build * conda build workflow * install conda-build * test pip * d * upload * conda path * build upload * addd && * rename step name * add multiple py versions * correct format * change py version to string * restore * refactor * py versions add * Jinja2 py version * export py version * add tag * matrix and force upload * add obspy * add cpp build * add gsl * add boost * libboost * add main label * add py 3.12 * arm build * micromamba * new env * anaconda * change * copy * add pkg * make work * add cmake * add doc for conda and coiled * Update deploy_mspass_with_conda_and_coiled.rst Revised the "Get MsPASS Conda package" section based on what worked for me when I tried it. Key thing was it was not optional add the conda channels - previous suggested something that didn't work. * update doc add connect to mongoatlas * add cartopy * try old version cmake=3.10 * camke 3.13 * cmake version * use an old base container * no update * mannual download cmake 3.13 * gdebi-core * older base container * change to miniconda * install conda mannually * conda clean fix * install cmake * debug build conda * change workdir, add py version * push to mspass org * remove MANIFEST.in; use version from project; reduce pkg size * use build script to prep source * go back to use exclude files * add build script * fix problem finding version number * try removing files from source * remove recipe from dist * fix wrong build script;remove pandas version * change doc to mspass * standardize versioning * add git into container for version * remove .git from container at the end --------- Co-authored-by: Gary Pavlis <[email protected]> Co-authored-by: wangyinz <[email protected]>
mspass-team · May 15, 2024 · 57afad5 · 57afad5
1 parent cdf07be
commit 57afad5
Show file tree

Hide file tree

Showing 13 changed files with 345 additions and 10 deletions.
diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml
@@ -0,0 +1,61 @@
+name: publish_conda
+
+on:
+  push:
+    branches: [ master ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+
+jobs:
+  conda-build-arm:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Setup for Multi-platform image
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push Docker image arm64
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: Dockerfile_arm_conda
+          platforms: linux/arm64
+          build-args: |
+            PYTHON_VERSION=${{ matrix.python-version }}
+            ANACONDA_API_TOKEN=${{ secrets.CONDA_TOKEN }}
+
+  conda-build-amd64:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: s-weigand/setup-conda@v1
+      - run: |
+          export PYTHON_VERSION=${{ matrix.python-version }}
+          conda install conda-build anaconda-client
+          conda config --set anaconda_upload no
+          conda config --add channels conda-forge
+          export ANACONDA_API_TOKEN=${{ secrets.CONDA_TOKEN }}
+          PACKAGE_PATH=$(conda build . --python ${{ matrix.python-version }} --output)
+          conda build . --python ${{ matrix.python-version }}
+          if [ "${{ matrix.python-version }}" == "3.11" ]; then
+            anaconda -t $ANACONDA_API_TOKEN upload $PACKAGE_PATH --label main --label py3.11 --force
+          else
+            anaconda -t $ANACONDA_API_TOKEN upload $PACKAGE_PATH --label py${{ matrix.python-version }} --force
+          fi
+
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -80,3 +80,4 @@ jobs:
         BRANCH: gh-pages 
         FOLDER: docs/build/html 
         CLEAN: false
+
diff --git a/Dockerfile b/Dockerfile
@@ -190,8 +190,9 @@ ADD setup.py /mspass/setup.py
 ADD pyproject.toml /mspass/pyproject.toml
 ADD requirements.txt /mspass/requirements.txt
 ADD python /mspass/python
+ADD .git /mspass/.git
 RUN pip3 install /mspass -v \
-	&& rm -rf /mspass/build && docker-clean
+	&& rm -rf /mspass/build /mspass/.git && docker-clean
 
 # Install jedi
 RUN pip3 --no-cache-dir install jedi==0.17.2 && docker-clean

diff --git a/Dockerfile_arm_conda b/Dockerfile_arm_conda
@@ -0,0 +1,33 @@
+FROM --platform=linux/arm64/v8 ubuntu:22.04
+COPY . /mspass
+RUN mv /mspass /mspasspy_build
+WORKDIR /mspasspy_build
+RUN ls
+RUN pwd
+ARG PYTHON_VERSION
+ENV PATH /opt/conda/bin:$PATH
+RUN apt-get update --fix-missing && \
+    apt-get install -y wget bzip2 ca-certificates curl git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    /opt/conda/bin/conda clean -a -y && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
+    echo "conda activate base" >> ~/.bashrc
+
+ARG ANACONDA_API_TOKEN
+
+RUN apt-get update && apt-get install -y cmake
+RUN cd /mspasspy_build && sed -i '38d' meta.yaml \
+    && conda install -y conda-build anaconda-client \
+    && conda config --set anaconda_upload no \
+    && conda config --add channels conda-forge \
+    && export CONDA_BUILD_DEBUG=1 \
+    && PACKAGE_PATH=$(conda build . --python ${PYTHON_VERSION} --output) \
+    && conda build . --python ${PYTHON_VERSION} --debug \
+    && anaconda -t $ANACONDA_API_TOKEN upload $PACKAGE_PATH --label py${PYTHON_VERSION} --force
+
diff --git a/Dockerfile_dev b/Dockerfile_dev
@@ -190,8 +190,9 @@ ADD setup.py /mspass/setup.py
 ADD python /mspass/python
 ADD pyproject.toml /mspass/pyproject.toml
 ADD requirements.txt /mspass/requirements.txt
+ADD .git /mspass/.git
 RUN pip3 install -C--global-option=build -C--global-option=--debug /mspass -v \
-	&& rm -rf /mspass/build && docker-clean
+	&& rm -rf /mspass/build /mspass/.git && docker-clean
 
 # Add docs and dependencies to build docs
 ADD docs /mspass/docs

diff --git a/Dockerfile_mpi b/Dockerfile_mpi
@@ -186,8 +186,9 @@ ADD setup.py /mspass/setup.py
 ADD python /mspass/python
 ADD pyproject.toml /mspass/pyproject.toml
 ADD requirements.txt /mspass/requirements.txt
+ADD .git /mspass/.git
 RUN pip3 install /mspass -v \
-	&& rm -rf /mspass/build && docker-clean
+	&& rm -rf /mspass/build /mspass/.git && docker-clean
 
 # Install jedi
 RUN pip3 --no-cache-dir install jedi==0.17.2 && docker-clean

diff --git a/cxx/cmake/openblas-download.cmake b/cxx/cmake/openblas-download.cmake
@@ -8,12 +8,12 @@ ExternalProject_Add(
   openblas
   SOURCE_DIR "@OPENBLAS_DOWNLOAD_ROOT@/openblas-src"
   GIT_REPOSITORY
-    https://github.com/xianyi/OpenBLAS.git 
+    https://github.com/OpenMathLib/OpenBLAS.git 
   GIT_TAG
-    v0.3.7
+    v0.3.27
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
-  BUILD_COMMAND make libs -j 8 PREFIX=${PROJECT_BINARY_DIR} NO_SHARED=1 USE_THREAD=0 USE_OPENMP=0
+  BUILD_COMMAND make libs -j 8 PREFIX=${PROJECT_BINARY_DIR} NO_SHARED=1 USE_THREAD=0 USE_OPENMP=0 DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 TARGET=GENERIC
   BUILD_IN_SOURCE 1
   INSTALL_COMMAND make install PREFIX=${PROJECT_BINARY_DIR} NO_SHARED=1
   TEST_COMMAND ""

diff --git a/docs/source/getting_started/deploy_mspass_with_conda_and_coiled.rst b/docs/source/getting_started/deploy_mspass_with_conda_and_coiled.rst
@@ -0,0 +1,145 @@
+.. _deploy_mspass_with_conda_and_coiled:
+
+Deploy MsPASS with Conda and Coiled
+===============================
+
+Overview
+-------------
+This section provides a concise summary of the steps required to run 
+MsPASS using Conda and Coiled. The instructions assume you are working 
+in a cloud environment (AWS, GCP, Azure). You can also only use conda 
+to install MsPASS locally without Coiled (step 3).
+
+1. Install Coiled
+---------------------
+Fetch and install Coiled following instructions on the
+`Coiled web site <https://docs.coiled.io/user_guide/setup/index.html>`__.
+
+Install the Coiled client Python library with pip or conda.
+
+.. code-block::
+
+    pip install coiled "dask[complete]"
+    coiled login
+
+This will redirect you to the Coiled website to authenticate your computer. 
+
+
+2. Connect to your cloud
+---------------------------------------------
+Next grant Coiled permission to run in your cloud account(AWS, GCP, Azure). 
+Coiled creates the IAM policies and network configuration for your account, 
+asking you for permission at each step. 
+
+.. code-block::
+
+    coiled setup aws
+    coiled setup gcp
+
+You can configure Coiled with custom network configuration in the 
+`user portal <https://cloud.coiled.io/settings/setup/infrastructure>`__.
+
+3. Get MsPASS Conda package
+-------------------------------------------
+If you have not run MsPASS before you will need to get the 
+`conda package <https://anaconda.org/mspass/mspasspy>`__.
+from our standard repository.  Alternatively if you want to get the most
+recent updates you may also need to do this step. 
+
+We strongly advise you create a separate environment for mspass
+to avoiding breaking any existing python packages you may have 
+installed previous.  Make sure you are on the `base` environment 
+ant enter
+
+.. code-block::
+
+    conda create --name mspass
+
+Noting the name "mspass" is not special and it can be changed if you 
+prefer something else.  You chould then make the new 
+environment current with the standand conda command:
+
+.. code-block::
+
+    conda activate mspass
+
+You will almost certainly need to add key "channels" as follows:
+
+.. code-block::
+
+    conda config --add channels mspass
+    conda config --add channels conda-forge 
+
+Then install mspass in this environment with
+
+.. code-block::
+
+    conda install -y mspasspy
+
+4. Run MsPASS
+-------------------------
+After installing, Coiled will then pick up all those things installed locally, 
+and install them on your cluster. For running things on Coiled, you could 
+try `coiled run your_code.py` (or follow one of the examples in 
+`coiled docs <https://docs.coiled.io/user_guide/usage/examples.html>`__)!
+
+
+For example, to connect to MongoDB using Atlas:
+
+.. code-block::
+
+    from pymongo.mongo_client import MongoClient
+    from pymongo.server_api import ServerApi
+    from urllib.parse import quote_plus
+    username = "your username"
+    password = "your password"
+
+    # URL-encode the username and password
+    uri_username = quote_plus(username)
+    uri_password = quote_plus(password)
+    uri = "mongodb+srv://username:[email protected]/"
+
+    # Create a new client and connect to the server
+    client = MongoClient(uri, server_api=ServerApi('1'))
+
+    # Send a ping to confirm a successful connection
+    try:
+        client.admin.command('ping')
+        print("Pinged your deployment. You successfully connected to MongoDB!")
+    except Exception as e:
+        print(e)
+
+If the ping success, it means we successfully connect to your remote mongo database.
+Let's build a MsPASS client and start to use MsPASS:
+
+.. code-block::
+    
+    from mspasspy.db.client import DBClient
+    dbclient=DBClient(uri)
+    dbclient.list_database_names() # view all the databases
+    db = dbclient['mspass']        # choose a database
+    db.index_mseed_file('CIGSC__BHZ___2017180.ms', some_path) # index mseed files
+
+
+To use Dask:
+
+.. code-block::
+
+    from coiled import Cluster
+
+    cluster = Cluster(n_workers=20)
+    client = cluster.get_client()
+
+Once you have a Dask cluster you can then run Python code on that cluster. 
+Here is a simple code you could run:
+
+.. code-block::
+
+    def inc(x):
+        return x + 1
+
+    future = client.submit(inc, 10)
+    future.result() # returns 11
+
+You can find more useful examples in Coiled documentation and reach out to 
+Coiled team ([email protected]) for any usage questions.
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -31,6 +31,7 @@ package.
    getting_started/deploy_mspass_with_docker_compose
    getting_started/deploy_mspass_on_HPC
    getting_started/getting_started_overview
+   getting_started/deploy_mspass_with_conda_and_coiled
 
 .. toctree::
    :maxdepth: 1

diff --git a/meta.yaml b/meta.yaml
@@ -0,0 +1,85 @@
+#{% set version_match = load_file_regex(load_file='pyproject.toml', regex_pattern='version = "(.*)"') %}
+#{% set version = version_match[1] %}
+
+package:
+  name: mspasspy
+  version: {{ environ.get('GIT_DESCRIBE_TAG', '') }}
+  #  version: {{ version }}
+
+source:
+  path: .
+
+build:
+  script: scripts/conda_build.sh
+  include_recipe: False
+  number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }}
+  string: {{ environ.get('GIT_BUILD_STR', '') }}
+
+requirements:
+  build:
+    - {{ compiler('c') }}
+    - {{ compiler('cxx') }}
+    - {{ compiler('gfortran') }}
+    - gsl
+
+
+  host:
+    - python {{ PYTHON_VERSION }}
+    - pip
+    - gsl
+    - ipympl
+    - boost
+    - boost-cpp
+    - lapack
+    - libblas
+    - yaml-cpp
+
+  run:
+    - python {{ PYTHON_VERSION }}
+    - boost
+    - libboost-devel
+    - libzlib
+    - gsl
+    - pyyaml
+    - yaml-cpp
+    - libblas
+    - pymongo
+    - numpy
+    - obspy
+    - dill
+    - click
+    - cartopy
+    - dask
+    - distributed
+    - cloudpickle
+    - schema
+    - decorator
+    - boto3
+    - botocore
+    - moto
+    - requests 
+    - setuptools
+    - xarray
+    - zarr
+    - pandas
+    - numba
+    - multitaper
+    - ipympl
+
+
+about:
+  home: https://github.com/mspass-team/mspass
+  license: BSD 3-Clause
+  license_file: LICENSE
+  license_family: BSD
+  license_url: https://github.com/mspass-team/mspass/blob/master/LICENSE
+  summary: Massive Parallel Analysis System for Seismologists
+  description: |
+    The Massive Parallel Analysis System for Seismologists is an open source framework 
+    for seismic data processing and management. It has three core components. 
+    A scalable parallel processing framework based on a dataflow computation model. 
+    A NoSQL database system centered on document store. 
+    A container-based virtualization environment.
+  dev_url: https://github.com/mspass-team/mspass
+  doc_url: https://www.mspass.org
+  doc_source_url: https://github.com/mspass-team/mspass/blob/master/docs/source/index.rst
Original file line number	Diff line number	Diff line change
Expand Up		@@ -80,3 +80,4 @@ jobs:
		BRANCH: gh-pages
		FOLDER: docs/build/html
		CLEAN: false