diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index c61fa3f..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: CI - -on: - pull_request: [] - -jobs: - formatting: - runs-on: ubuntu-latest - steps: - - name: Check out the code - uses: actions/checkout@v4 - - - name: Install poetry - run: pipx install poetry - - - name: Determine dependencies - run: poetry lock - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - cache: poetry - - - name: Install Dependencies using Poetry - run: poetry install - - - name: Check formatting - run: poetry run black --check . - - linting: - runs-on: ubuntu-latest - steps: - - name: Check out the code - uses: actions/checkout@v3 - - - name: Install poetry - run: pipx install poetry - - - name: Determine dependencies - run: poetry lock - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - cache: poetry - - - name: Install Dependencies using Poetry - run: poetry install - - - name: Check code - run: poetry run flake8 - -# testing: -# runs-on: ubuntu-latest -# steps: -# - name: Check out the code -# uses: actions/checkout@v3 - -# - name: Install poetry -# run: pipx install poetry - -# - name: Determine dependencies -# run: poetry lock - -# - uses: actions/setup-python@v4 -# with: -# python-version: "3.9" -# cache: poetry - -# - name: Install Dependencies using Poetry -# run: poetry install - -# - name: Run pytest -# run: poetry run coverage run -m pytest tests/tests.py - -# - name: Run Coverage -# run: poetry run coverage report -m - - testing: - runs-on: ubuntu-latest - permissions: - packages: read - strategy: - fail-fast: false - matrix: - container: ["fluxrm/flux-sched:jammy"] - - container: - image: ${{ matrix.container }} - options: "--platform=linux/amd64 --user root -it" - - name: ${{ matrix.container }} - steps: - - name: Update environment for Flux - run: | - echo "/opt/conda/bin" >> $GITHUB_PATH - echo "/root/.local/bin" >> $GITHUB_PATH - echo "FLUX_MODULE_PATH=${FLUX_MODULE_PATH}:/usr/lib/x86_64-linux-gnu/flux/modules" >> ${GITHUB_ENV} - - - name: Make Space - run: | - rm -rf /usr/share/dotnet - rm -rf /opt/ghc - - - uses: actions/checkout@v4 - - name: Install poetry - run: | - sudo apt-get install -y python3-pip - ln -s $(which python3) /bin/python - - - name: Install dependencies - run: | - python3 -m pip install git+https://github.com/vsoch/snakemake@update-lazy-property-import - python3 -m pip install . - - - name: Start Flux and Test Workflow - run: | - # We must have python3->python accessible for this to work - su fluxuser - echo "ROOT=$PWD" >> $GITHUB_ENV - cd example - which snakemake - python3 -m pip freeze | grep snakemake - pip freeze | grep snakemake - flux start snakemake --show-failed-logs --verbose --executor flux --jobs=1 --no-shared-fs - - - name: Test Flux with Conda - run: | - # conda python needs to be after system python - wget https://repo.anaconda.com/archive/Anaconda3-2023.09-0-Linux-x86_64.sh - sudo bash Anaconda3-2023.09-0-Linux-x86_64.sh -p /opt/conda -b - export PATH=$PATH:/opt/conda/bin - # Now install again - python3 -m pip install git+https://github.com/vsoch/snakemake@update-lazy-property-import - cd $ROOT - python3 -m pip install . - which python - which conda - cd $ROOT/example/conda - flux start snakemake --show-failed-logs --verbose --executor flux --jobs=1 --use-conda --conda-frontend=conda \ No newline at end of file diff --git a/.github/workflows/ci_true_api.yml b/.github/workflows/ci_true_api.yml new file mode 100644 index 0000000..dc67d4f --- /dev/null +++ b/.github/workflows/ci_true_api.yml @@ -0,0 +1,23 @@ +name: test-flux + +on: + workflow_dispatch: + +jobs: + testing-true-api: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install poetry + run: pip install poetry + + - name: Determine dependencies + run: poetry lock + + - name: Install dependencies + run: poetry install diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..ce809e6 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,18 @@ +name: test snakemake-executor-flux + +on: + pull_request: [] + +jobs: + formatting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup black linter + run: conda create --quiet --name black pytest + + - name: Check Spelling + uses: crate-ci/typos@7ad296c72fa8265059cc03d1eda562fbdfcd6df2 # v1.9.0 + with: + files: ./README.md diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index f381283..3582920 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -26,7 +26,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.9" + python-version: "3.11" - name: Install poetry run: pipx install poetry @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" cache: poetry - name: Install Dependencies using Poetry diff --git a/README.md b/README.md index 08657cb..9d2de1c 100644 --- a/README.md +++ b/README.md @@ -101,5 +101,11 @@ plugin for use or development (that doesn't need to be added to upstream snakema ## Developer +To do the same run but bind the local plugin directory: + +```bash +docker run -it -v $PWD/:/home/fluxuser/plugin flux-snake bash +``` + The instructions for creating and scaffolding this plugin are [here](https://github.com/snakemake/poetry-snakemake-plugin#scaffolding-an-executor-plugin). -Instructions for writing your plugin with examples are provided via the [snakemake-executor-plugin-interface](https://github.com/snakemake/snakemake-executor-plugin-interface). \ No newline at end of file +Instructions for writing your plugin with examples are provided via the [snakemake-executor-plugin-interface](https://github.com/snakemake/snakemake-executor-plugin-interface). diff --git a/docs/further.md b/docs/further.md new file mode 100644 index 0000000..9d2de1c --- /dev/null +++ b/docs/further.md @@ -0,0 +1,111 @@ +# Snakemake Executor Flux + +This is an example implementation for an external snakemake plugin. +Since we already have one for Flux (and it can run in a container) the example +is for Flux. You can use this repository as a basis to design your own executor to work +with snakemake! + +## Usage + +### Tutorial + +For this tutorial you will need Docker installer. + +[Flux-framework](https://flux-framework.org/) is a flexible resource scheduler that can work on both high performance computing systems and cloud (e.g., Kubernetes). +Since it is more modern (e.g., has an official Python API) we define it under a cloud resource. For this example, we will show you how to set up a "single node" local +Flux container to interact with snakemake using the plugin here. You can use the [Dockerfile](examples/Dockerfile) that will provide a container with Flux and snakemake +Note that we install from source and bind to `/home/fluxuser/snakemake` with the intention of being able to develop (if desired). + +First, build the container: + +```bash +$ docker build -f example/Dockerfile -t flux-snake . +``` + +We will add the plugin here to `/home/fluxuser/plugin`, install it, and shell in as the fluxuser to optimally interact with flux. +After the container builds, shell in: + +```bash +$ docker run -it flux-snake bash +``` + +And start a flux instance: + +```bash +$ flux start --test-size=4 +``` + +Go into the examples directory (where the Snakefile is) and run snakemake, targeting your executor plugin. + +```bash +$ cd ./example + +# This says "use the custom executor module named snakemake_executor_plugin_flux" +$ snakemake --jobs 1 --executor flux +``` +```console +Building DAG of jobs... +Using shell: /bin/bash +Job stats: +job count min threads max threads +------------------------ ------- ------------- ------------- +all 1 1 1 +multilingual_hello_world 2 1 1 +total 3 1 1 + +Select jobs to execute... + +[Fri Jun 16 19:24:22 2023] +rule multilingual_hello_world: + output: hola/world.txt + jobid: 2 + reason: Missing output files: hola/world.txt + wildcards: greeting=hola + resources: tmpdir=/tmp + +Job 2 has been submitted with flux jobid ƒcjn4t3R (log: .snakemake/flux_logs/multilingual_hello_world/greeting_hola.log). +[Fri Jun 16 19:24:32 2023] +Finished job 2. +1 of 3 steps (33%) done +Select jobs to execute... + +[Fri Jun 16 19:24:32 2023] +rule multilingual_hello_world: + output: hello/world.txt + jobid: 1 + reason: Missing output files: hello/world.txt + wildcards: greeting=hello + resources: tmpdir=/tmp + +Job 1 has been submitted with flux jobid ƒhAPLa79 (log: .snakemake/flux_logs/multilingual_hello_world/greeting_hello.log). +[Fri Jun 16 19:24:42 2023] +Finished job 1. +2 of 3 steps (67%) done +Select jobs to execute... + +[Fri Jun 16 19:24:42 2023] +localrule all: + input: hello/world.txt, hola/world.txt + jobid: 0 + reason: Input files updated by another job: hello/world.txt, hola/world.txt + resources: tmpdir=/tmp + +[Fri Jun 16 19:24:42 2023] +Finished job 0. +3 of 3 steps (100%) done +Complete log: .snakemake/log/2023-06-16T192422.186675.snakemake.log +``` + +And that's it! Continue reading to learn more about plugin design, and how you can also design your own executor +plugin for use or development (that doesn't need to be added to upstream snakemake). + +## Developer + +To do the same run but bind the local plugin directory: + +```bash +docker run -it -v $PWD/:/home/fluxuser/plugin flux-snake bash +``` + +The instructions for creating and scaffolding this plugin are [here](https://github.com/snakemake/poetry-snakemake-plugin#scaffolding-an-executor-plugin). +Instructions for writing your plugin with examples are provided via the [snakemake-executor-plugin-interface](https://github.com/snakemake/snakemake-executor-plugin-interface). diff --git a/docs/intro.md b/docs/intro.md new file mode 100644 index 0000000..e964995 --- /dev/null +++ b/docs/intro.md @@ -0,0 +1 @@ +This is the [Flux Framework](https://flux-framework.org) external executor plugin for snakemake. diff --git a/example/Dockerfile b/example/Dockerfile index 79ea57a..49a3522 100644 --- a/example/Dockerfile +++ b/example/Dockerfile @@ -1,29 +1,209 @@ -FROM fluxrm/flux-sched:jammy -# This is run from the repository root -# docker build -t flux-snake . -# docker run -it flux-snake -USER root -ENV PATH=/opt/conda/bin:/root/.local/bin:$PATH -ENV FLUX_MODULE_PATH=${FLUX_MODULE_PATH}:/usr/lib/x86_64-linux-gnu/flux/modules +FROM ubuntu:jammy -# Note this is a custom branch that currently has the functionality for plugins -RUN apt-get update && apt-get install -y python3-pip python3-venv +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends git \ + libfftw3-dev libfftw3-bin pdsh libfabric-dev libfabric1 \ + dnsutils telnet strace cmake git g++ openmpi-bin \ + openmpi-common wget curl unzip libopenmpi-dev \ + software-properties-common gpg-agent build-essential + +RUN add-apt-repository ppa:deadsnakes/ppa && apt-get update && \ + apt install -y python3.12 python3.12-dev + +RUN update-alternatives --install /usr/bin/python3 python /usr/bin/python3.12 1 && \ + update-alternatives --install /usr/bin/python python3 /usr/bin/python3.12 1 + +RUN wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py + +RUN pip3 install -U git+https://github.com/snakemake/snakemake-interface-common@main && \ + pip3 install -U git+https://github.com/snakemake/snakemake-interface-executor-plugins && \ + pip3 install -U git+https://github.com/snakemake/snakemake-interface-storage-plugins@main && \ + pip3 install -U git+https://github.com/snakemake/snakemake-storage-plugin-s3@main && \ + pip3 install -U git+https://github.com/snakemake/snakemake-storage-plugin-gcs@main && \ + pip3 install -U git+https://github.com/snakemake/snakemake@main + +# This first section from src/test/docker/bionic/Dockerfile in flux-core +# https://github.com/flux-framework/flux-core/blob/master/src/test/docker/bionic/Dockerfile +RUN apt-get update && \ + apt-get -qq install -y --no-install-recommends \ + apt-utils && \ + rm -rf /var/lib/apt/lists/* + +# Utilities +RUN apt-get update && \ + apt-get -qq install -y --no-install-recommends \ + locales \ + ca-certificates \ + curl \ + wget \ + man \ + git \ + flex \ + ssh \ + sudo \ + vim \ + luarocks \ + munge \ + lcov \ + ccache \ + lua5.2 \ + mpich \ + valgrind \ + jq && \ + rm -rf /var/lib/apt/lists/* + +# Compilers, autotools +RUN apt-get update && \ + apt-get -qq install -y --no-install-recommends \ + build-essential \ + pkg-config \ + autotools-dev \ + libtool \ + libffi-dev \ + autoconf \ + automake \ + make \ + cmake \ + clang \ + clang-tidy \ + gcc \ + g++ && \ + rm -rf /var/lib/apt/lists/* + +# Other deps +RUN apt-get update && \ + apt-get -qq install -y --no-install-recommends \ + libsodium-dev \ + libzmq3-dev \ + libczmq-dev \ + libjansson-dev \ + libmunge-dev \ + libncursesw5-dev \ + liblua5.2-dev \ + liblz4-dev \ + libsqlite3-dev \ + uuid-dev \ + libhwloc-dev \ + libmpich-dev \ + libs3-dev \ + libevent-dev \ + libarchive-dev \ + libpam-dev && \ + rm -rf /var/lib/apt/lists/* + +# Testing utils and libs +RUN apt-get update && \ + apt-get -qq install -y --no-install-recommends \ + faketime \ + libfaketime \ + pylint \ + cppcheck \ + enchant-2 \ + aspell \ + aspell-en && \ + rm -rf /var/lib/apt/lists/* + +RUN locale-gen en_US.UTF-8 + +# NOTE: luaposix installed by rocks due to Ubuntu bug: #1752082 https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 +RUN luarocks install luaposix + + +# Install openpmix, prrte +WORKDIR /opt/prrte +RUN git clone https://github.com/openpmix/openpmix.git && \ + git clone https://github.com/openpmix/prrte.git && \ + ls -l && \ + set -x && \ + cd openpmix && \ + git checkout fefaed568f33bf86f28afb6e45237f1ec5e4de93 && \ + ./autogen.pl && \ + PYTHON=/usr/bin/python3 ./configure --prefix=/usr --disable-static && make -j 4 install && \ + ldconfig && \ + cd .. && \ + cd prrte && \ + git checkout 477894f4720d822b15cab56eee7665107832921c && \ + ./autogen.pl && \ + PYTHON=/usr/bin/python3 ./configure --prefix=/usr && make -j 4 install && \ + cd ../.. && \ + rm -rf prrte + +ENV LANG=C.UTF-8 + +# This is from the docker check script (run interactively during a test) +# https://github.com/flux-framework/flux-core/blob/master/src/test/docker/checks/Dockerfile +ARG USER=fluxuser +ARG UID=1000 +ARG GID=1000 + +# Install flux-security by hand for now: +# +WORKDIR /opt +RUN git clone --depth 1 https://github.com/flux-framework/flux-security && \ + cd flux-security && \ + ./autogen.sh && \ + PYTHON=/usr/bin/python3 ./configure --prefix=/usr --sysconfdir=/etc && \ + make && \ + make install && \ + cd .. && \ + rm -rf flux-security + +# Add configured user to image with sudo access: +# +RUN set -x && groupadd -g $UID $USER && \ + useradd -g $USER -u $UID -d /home/$USER -m $USER && \ + printf "$USER ALL= NOPASSWD: ALL\\n" >> /etc/sudoers + +# Setup MUNGE directories & key +RUN mkdir -p /var/run/munge && \ + dd if=/dev/urandom bs=1 count=1024 > /etc/munge/munge.key && \ + chown -R munge /etc/munge/munge.key /var/run/munge && \ + chmod 600 /etc/munge/munge.key + +# Build flux core +RUN python3 -m pip install cffi ply +RUN git clone https://github.com/flux-framework/flux-core && \ + cd flux-core && \ + ./autogen.sh && \ + PYTHON=/usr/bin/python3 PYTHON_PREFIX=PYTHON_EXEC_PREFIX=/usr/lib/python3.12/site-packages ./configure \ + --prefix=/usr \ + --sysconfdir=/etc \ + --with-systemdsystemunitdir=/etc/systemd/system \ + --localstatedir=/var \ + --with-flux-security && \ + make clean && \ + make && \ + sudo make install + +# This is from the same src/test/docker/bionic/Dockerfile but in flux-sched +# Flux-sched deps +RUN sudo apt-get update +RUN sudo apt-get -qq install -y --no-install-recommends \ + libboost-graph-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libboost-regex-dev \ + libyaml-cpp-dev \ + libedit-dev + +ENV LD_LIBRARY_PATH=/opt/conda/lib:$LD_LIBRARY_PATH + +# Build Flux Sched +# https://github.com/flux-framework/flux-sched/blob/master/src/test/docker/docker-run-checks.sh#L152-L158 +RUN git clone https://github.com/flux-framework/flux-sched && \ + cd flux-sched && \ + git fetch && \ + ./autogen.sh && \ + PYTHON=/usr/bin/python3 ./configure --prefix=/usr --sysconfdir=/etc \ + --with-systemdsystemunitdir=/etc/systemd/system \ + --localstatedir=/var && \ + make && \ + sudo make install && \ + ldconfig WORKDIR /home/fluxuser/plugin COPY . /home/fluxuser/plugin -RUN chown -R fluxuser /home/fluxuser - -ENV PATH=/opt/conda/bin:/home/fluxuser/.local/bin:$PATH -USER fluxuser - -# Run/install poetry as fluxuser -RUN python3 -m pip install --user pipx && \ - python3 -m pipx ensurepath && \ - pipx install poetry && \ - sudo ln -s /bin/python3 /bin/python - -RUN poetry lock && \ - poetry install && \ - pip install git+https://github.com/vsoch/snakemake@update-lazy-property-import && \ - pip install -e . && \ - sudo chown -R fluxuser /home/fluxuser + +RUN pip install . diff --git a/pyproject.toml b/pyproject.toml index dabc38d..639d468 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,21 +5,27 @@ description = "A snakemake executor plugin for the Flux scheduler" authors = ["vsoch "] readme = "README.md" packages = [{include = "snakemake_executor_plugin_flux"}] +license = "MIT" +repository = "https://github.com/snakemake/snakemake-executor-plugin-flux" +documentation = "https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/flux.html" +keywords = ["snakemake", "plugin", "executor", "flux", "flux-framework"] [tool.poetry.dependencies] -python = "^3.9" -snakemake-interface-common = "^1.10.0" -# snakemake-interface-executor-plugins = "^5.0.2" -snakemake-interface-executor-plugins = { git = "https://github.com/snakemake/snakemake-interface-executor-plugins", branch = "main" } +python = "^3.11" +snakemake-interface-common = "^1.14.0" +snakemake-interface-executor-plugins = "^8.1.1" + +[tool.coverage.run] +omit = [".*", "*/site-packages/*", "Snakefile"] [tool.poetry.group.dev.dependencies] black = "^23.9.1" flake8 = "^6.1.0" coverage = "^7.3.1" pytest = "^7.4.2" -# snakemake = "^7.32.4" -snakemake = { git = "https://github.com/vsoch/snakemake", branch = "update-lazy-property-import" } +snakemake = {git = "https://github.com/snakemake/snakemake.git", branch="main"} [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + diff --git a/snakemake_executor_plugin_flux/__init__.py b/snakemake_executor_plugin_flux/__init__.py index f76671c..c4e3ca2 100644 --- a/snakemake_executor_plugin_flux/__init__.py +++ b/snakemake_executor_plugin_flux/__init__.py @@ -10,8 +10,9 @@ # (cluster, cloud, etc.). Only Snakemake's standard execution # plugins (snakemake-executor-plugin-dryrun, snakemake-executor-plugin-local) # are expected to specify False here. + job_deploy_sources=True, non_local_exec=True, # Flux can have a shared filesystem if run in an HPC context, or not if cloud # so we cannot set it one way or the other. - implies_no_shared_fs=True, + implies_no_shared_fs=False, ) diff --git a/snakemake_executor_plugin_flux/executor.py b/snakemake_executor_plugin_flux/executor.py index be3b31e..9d7dc38 100644 --- a/snakemake_executor_plugin_flux/executor.py +++ b/snakemake_executor_plugin_flux/executor.py @@ -26,19 +26,7 @@ def __init__( workflow: WorkflowExecutorInterface, logger: LoggerExecutorInterface, ): - super().__init__( - workflow, - logger, - # configure behavior of RemoteExecutor below - # whether arguments for setting the remote provider shall be passed to jobs - pass_default_remote_provider_args=True, - # whether arguments for setting default resources shall be passed to jobs - pass_default_resources_args=True, - # whether environment variables shall be passed to jobs - pass_envvar_declarations_to_cmd=True, - # specify initial amount of seconds to sleep before checking for job status - init_seconds_before_status_checks=0, - ) + super().__init__(workflow, logger) # Attach variables for easy access self.workdir = os.path.realpath(os.path.dirname(self.workflow.persistence.path)) @@ -58,25 +46,12 @@ def get_envvar_declarations(self): https://github.com/snakemake/snakemake-interface-executor-plugins/pull/31 is able to be merged. """ - if self.pass_envvar_declarations_to_cmd: - return " ".join( - f"{var}={repr(os.environ[var])}" - for var in self.workflow.remote_execution_settings.envvars or {} - ) - else: - return "" + return " ".join( + f"{var}={repr(os.environ[var])}" + for var in self.workflow.remote_execution_settings.envvars or {} + ) def run_job(self, job: JobExecutorInterface): - # Implement here how to run a job. - # You can access the job's resources, etc. - # via the job object. - # After submitting the job, you have to call - # self.report_job_submission(job_info). - # with job_info being of type - # snakemake_interface_executor_plugins.executors.base.SubmittedJobInfo. - # If required, make sure to pass the job's id to the job_info object, as keyword - # argument 'external_job_id'. - flux_logfile = job.logfile_suggestion(os.path.join(".snakemake", "flux_logs")) os.makedirs(os.path.dirname(flux_logfile), exist_ok=True) @@ -119,28 +94,6 @@ def _get_jobname(self, job): async def check_active_jobs( self, active_jobs: List[SubmittedJobInfo] ) -> Generator[SubmittedJobInfo, None, None]: - # Check the status of active jobs. - - # You have to iterate over the given list active_jobs. - # If you provided it above, each will have its external_jobid set according - # to the information you provided at submission time. - # For jobs that have finished successfully, you have to call - # self.report_job_success(active_job). - # For jobs that have errored, you have to call - # self.report_job_error(active_job). - # This will also take care of providing a proper error message. - # Usually there is no need to perform additional logging here. - # Jobs that are still running have to be yielded. - # - # For queries to the remote middleware, please use - # self.status_rate_limiter like this: - # - # async with self.status_rate_limiter: - # # query remote middleware here - # - # To modify the time until the next call of this method, - # you can set self.next_sleep_seconds here. - # Loop through active jobs and act on status for j in active_jobs: jobid = j.external_jobid