diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml new file mode 100644 index 000000000..d88a15443 --- /dev/null +++ b/.github/workflows/conda-build.yml @@ -0,0 +1,61 @@ +name: publish_conda + +on: + push: + branches: [ master ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + +jobs: + conda-build-arm: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11'] + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Setup for Multi-platform image + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push Docker image arm64 + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile_arm_conda + platforms: linux/arm64 + build-args: | + PYTHON_VERSION=${{ matrix.python-version }} + ANACONDA_API_TOKEN=${{ secrets.CONDA_TOKEN }} + + conda-build-amd64: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11'] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: s-weigand/setup-conda@v1 + - run: | + export PYTHON_VERSION=${{ matrix.python-version }} + conda install conda-build anaconda-client + conda config --set anaconda_upload no + conda config --add channels conda-forge + export ANACONDA_API_TOKEN=${{ secrets.CONDA_TOKEN }} + PACKAGE_PATH=$(conda build . --python ${{ matrix.python-version }} --output) + conda build . --python ${{ matrix.python-version }} + if [ "${{ matrix.python-version }}" == "3.11" ]; then + anaconda -t $ANACONDA_API_TOKEN upload $PACKAGE_PATH --label main --label py3.11 --force + else + anaconda -t $ANACONDA_API_TOKEN upload $PACKAGE_PATH --label py${{ matrix.python-version }} --force + fi + diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0c0d6b39e..0bc855294 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -80,3 +80,4 @@ jobs: BRANCH: gh-pages FOLDER: docs/build/html CLEAN: false + diff --git a/Dockerfile b/Dockerfile index fd2c92337..48f57ece2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -190,8 +190,9 @@ ADD setup.py /mspass/setup.py ADD pyproject.toml /mspass/pyproject.toml ADD requirements.txt /mspass/requirements.txt ADD python /mspass/python +ADD .git /mspass/.git RUN pip3 install /mspass -v \ - && rm -rf /mspass/build && docker-clean + && rm -rf /mspass/build /mspass/.git && docker-clean # Install jedi RUN pip3 --no-cache-dir install jedi==0.17.2 && docker-clean diff --git a/Dockerfile_arm_conda b/Dockerfile_arm_conda new file mode 100644 index 000000000..30e0fdcd0 --- /dev/null +++ b/Dockerfile_arm_conda @@ -0,0 +1,33 @@ +FROM --platform=linux/arm64/v8 ubuntu:22.04 +COPY . /mspass +RUN mv /mspass /mspasspy_build +WORKDIR /mspasspy_build +RUN ls +RUN pwd +ARG PYTHON_VERSION +ENV PATH /opt/conda/bin:$PATH +RUN apt-get update --fix-missing && \ + apt-get install -y wget bzip2 ca-certificates curl git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda clean -a -y && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc + +ARG ANACONDA_API_TOKEN + +RUN apt-get update && apt-get install -y cmake +RUN cd /mspasspy_build && sed -i '38d' meta.yaml \ + && conda install -y conda-build anaconda-client \ + && conda config --set anaconda_upload no \ + && conda config --add channels conda-forge \ + && export CONDA_BUILD_DEBUG=1 \ + && PACKAGE_PATH=$(conda build . --python ${PYTHON_VERSION} --output) \ + && conda build . --python ${PYTHON_VERSION} --debug \ + && anaconda -t $ANACONDA_API_TOKEN upload $PACKAGE_PATH --label py${PYTHON_VERSION} --force + diff --git a/Dockerfile_dev b/Dockerfile_dev index 26d9e48a8..c2db236ad 100644 --- a/Dockerfile_dev +++ b/Dockerfile_dev @@ -190,8 +190,9 @@ ADD setup.py /mspass/setup.py ADD python /mspass/python ADD pyproject.toml /mspass/pyproject.toml ADD requirements.txt /mspass/requirements.txt +ADD .git /mspass/.git RUN pip3 install -C--global-option=build -C--global-option=--debug /mspass -v \ - && rm -rf /mspass/build && docker-clean + && rm -rf /mspass/build /mspass/.git && docker-clean # Add docs and dependencies to build docs ADD docs /mspass/docs diff --git a/Dockerfile_mpi b/Dockerfile_mpi index 8e9f07bfb..a6848e607 100644 --- a/Dockerfile_mpi +++ b/Dockerfile_mpi @@ -186,8 +186,9 @@ ADD setup.py /mspass/setup.py ADD python /mspass/python ADD pyproject.toml /mspass/pyproject.toml ADD requirements.txt /mspass/requirements.txt +ADD .git /mspass/.git RUN pip3 install /mspass -v \ - && rm -rf /mspass/build && docker-clean + && rm -rf /mspass/build /mspass/.git && docker-clean # Install jedi RUN pip3 --no-cache-dir install jedi==0.17.2 && docker-clean diff --git a/cxx/cmake/openblas-download.cmake b/cxx/cmake/openblas-download.cmake index ebc04e854..cd0a75839 100644 --- a/cxx/cmake/openblas-download.cmake +++ b/cxx/cmake/openblas-download.cmake @@ -8,12 +8,12 @@ ExternalProject_Add( openblas SOURCE_DIR "@OPENBLAS_DOWNLOAD_ROOT@/openblas-src" GIT_REPOSITORY - https://github.com/xianyi/OpenBLAS.git + https://github.com/OpenMathLib/OpenBLAS.git GIT_TAG - v0.3.7 + v0.3.27 UPDATE_COMMAND "" CONFIGURE_COMMAND "" - BUILD_COMMAND make libs -j 8 PREFIX=${PROJECT_BINARY_DIR} NO_SHARED=1 USE_THREAD=0 USE_OPENMP=0 + BUILD_COMMAND make libs -j 8 PREFIX=${PROJECT_BINARY_DIR} NO_SHARED=1 USE_THREAD=0 USE_OPENMP=0 DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 TARGET=GENERIC BUILD_IN_SOURCE 1 INSTALL_COMMAND make install PREFIX=${PROJECT_BINARY_DIR} NO_SHARED=1 TEST_COMMAND "" diff --git a/docs/source/getting_started/deploy_mspass_with_conda_and_coiled.rst b/docs/source/getting_started/deploy_mspass_with_conda_and_coiled.rst new file mode 100644 index 000000000..028db3f14 --- /dev/null +++ b/docs/source/getting_started/deploy_mspass_with_conda_and_coiled.rst @@ -0,0 +1,145 @@ +.. _deploy_mspass_with_conda_and_coiled: + +Deploy MsPASS with Conda and Coiled +=============================== + +Overview +------------- +This section provides a concise summary of the steps required to run +MsPASS using Conda and Coiled. The instructions assume you are working +in a cloud environment (AWS, GCP, Azure). You can also only use conda +to install MsPASS locally without Coiled (step 3). + +1. Install Coiled +--------------------- +Fetch and install Coiled following instructions on the +`Coiled web site `__. + +Install the Coiled client Python library with pip or conda. + +.. code-block:: + + pip install coiled "dask[complete]" + coiled login + +This will redirect you to the Coiled website to authenticate your computer. + + +2. Connect to your cloud +--------------------------------------------- +Next grant Coiled permission to run in your cloud account(AWS, GCP, Azure). +Coiled creates the IAM policies and network configuration for your account, +asking you for permission at each step. + +.. code-block:: + + coiled setup aws + coiled setup gcp + +You can configure Coiled with custom network configuration in the +`user portal `__. + +3. Get MsPASS Conda package +------------------------------------------- +If you have not run MsPASS before you will need to get the +`conda package `__. +from our standard repository. Alternatively if you want to get the most +recent updates you may also need to do this step. + +We strongly advise you create a separate environment for mspass +to avoiding breaking any existing python packages you may have +installed previous. Make sure you are on the `base` environment +ant enter + +.. code-block:: + + conda create --name mspass + +Noting the name "mspass" is not special and it can be changed if you +prefer something else. You chould then make the new +environment current with the standand conda command: + +.. code-block:: + + conda activate mspass + +You will almost certainly need to add key "channels" as follows: + +.. code-block:: + + conda config --add channels mspass + conda config --add channels conda-forge + +Then install mspass in this environment with + +.. code-block:: + + conda install -y mspasspy + +4. Run MsPASS +------------------------- +After installing, Coiled will then pick up all those things installed locally, +and install them on your cluster. For running things on Coiled, you could +try `coiled run your_code.py` (or follow one of the examples in +`coiled docs `__)! + + +For example, to connect to MongoDB using Atlas: + +.. code-block:: + + from pymongo.mongo_client import MongoClient + from pymongo.server_api import ServerApi + from urllib.parse import quote_plus + username = "your username" + password = "your password" + + # URL-encode the username and password + uri_username = quote_plus(username) + uri_password = quote_plus(password) + uri = "mongodb+srv://username:somestring@cluster0.domain.mongodb.net/" + + # Create a new client and connect to the server + client = MongoClient(uri, server_api=ServerApi('1')) + + # Send a ping to confirm a successful connection + try: + client.admin.command('ping') + print("Pinged your deployment. You successfully connected to MongoDB!") + except Exception as e: + print(e) + +If the ping success, it means we successfully connect to your remote mongo database. +Let's build a MsPASS client and start to use MsPASS: + +.. code-block:: + + from mspasspy.db.client import DBClient + dbclient=DBClient(uri) + dbclient.list_database_names() # view all the databases + db = dbclient['mspass'] # choose a database + db.index_mseed_file('CIGSC__BHZ___2017180.ms', some_path) # index mseed files + + +To use Dask: + +.. code-block:: + + from coiled import Cluster + + cluster = Cluster(n_workers=20) + client = cluster.get_client() + +Once you have a Dask cluster you can then run Python code on that cluster. +Here is a simple code you could run: + +.. code-block:: + + def inc(x): + return x + 1 + + future = client.submit(inc, 10) + future.result() # returns 11 + +You can find more useful examples in Coiled documentation and reach out to +Coiled team (support@coiled.io) for any usage questions. diff --git a/docs/source/index.rst b/docs/source/index.rst index d88d854b3..7a987d2a8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -31,6 +31,7 @@ package. getting_started/deploy_mspass_with_docker_compose getting_started/deploy_mspass_on_HPC getting_started/getting_started_overview + getting_started/deploy_mspass_with_conda_and_coiled .. toctree:: :maxdepth: 1 diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 000000000..2d9ea3719 --- /dev/null +++ b/meta.yaml @@ -0,0 +1,85 @@ +#{% set version_match = load_file_regex(load_file='pyproject.toml', regex_pattern='version = "(.*)"') %} +#{% set version = version_match[1] %} + +package: + name: mspasspy + version: {{ environ.get('GIT_DESCRIBE_TAG', '') }} + # version: {{ version }} + +source: + path: . + +build: + script: scripts/conda_build.sh + include_recipe: False + number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} + string: {{ environ.get('GIT_BUILD_STR', '') }} + +requirements: + build: + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - {{ compiler('gfortran') }} + - gsl + + + host: + - python {{ PYTHON_VERSION }} + - pip + - gsl + - ipympl + - boost + - boost-cpp + - lapack + - libblas + - yaml-cpp + + run: + - python {{ PYTHON_VERSION }} + - boost + - libboost-devel + - libzlib + - gsl + - pyyaml + - yaml-cpp + - libblas + - pymongo + - numpy + - obspy + - dill + - click + - cartopy + - dask + - distributed + - cloudpickle + - schema + - decorator + - boto3 + - botocore + - moto + - requests + - setuptools + - xarray + - zarr + - pandas + - numba + - multitaper + - ipympl + + +about: + home: https://github.com/mspass-team/mspass + license: BSD 3-Clause + license_file: LICENSE + license_family: BSD + license_url: https://github.com/mspass-team/mspass/blob/master/LICENSE + summary: Massive Parallel Analysis System for Seismologists + description: | + The Massive Parallel Analysis System for Seismologists is an open source framework + for seismic data processing and management. It has three core components. + A scalable parallel processing framework based on a dataflow computation model. + A NoSQL database system centered on document store. + A container-based virtualization environment. + dev_url: https://github.com/mspass-team/mspass + doc_url: https://www.mspass.org + doc_source_url: https://github.com/mspass-team/mspass/blob/master/docs/source/index.rst diff --git a/pyproject.toml b/pyproject.toml index 9703537fa..4a1e32900 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,14 @@ [project] name = "mspasspy" -version = "1.0.0" description = "Massive Parallel Analysis System for Seismologists" authors = [ {name = "Ian Wang", email = "yinzhi.wang.cug@gmail.com"}, {name = "Gary Pavlis", email = "pavlis@indiana.edu"}, ] -dynamic = ["dependencies", "scripts"] +dynamic = ["dependencies", "scripts", "version"] [build-system] -requires = ["setuptools", "wheel"] +requires = ["setuptools>=64", "setuptools_scm>=8", "wheel"] build-backend = "setuptools.build_meta" [tool.setuptools.dynamic] @@ -30,3 +29,5 @@ mspass-dbclean = "mspasspy.db.script.dbclean:main" mspass-dbverify = "mspasspy.db.script.dbverify:main" mspass-normalize_mseed = "mspasspy.db.script.normalize_mseed:main" +[tool.setuptools_scm] + diff --git a/requirements.txt b/requirements.txt index 11ccab825..e98b9cd21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ requests setuptools xarray zarr -pandas==1.5.3 +pandas numba multitaper ipympl diff --git a/scripts/conda_build.sh b/scripts/conda_build.sh new file mode 100755 index 000000000..df1cc8323 --- /dev/null +++ b/scripts/conda_build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Install the package +${PYTHON} -m pip install . -vv +