Merge pull request #90 from inab/full_circle

Workflow Run RO-Crate import implemented (first stage)
inab · May 23, 2024 · 03cb492 · 03cb492
2 parents 6d003a2 + 92a6e21
commit 03cb492
Show file tree

Hide file tree

Showing 96 changed files with 4,502 additions and 2,001 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,26 @@
 fail_fast: true
 
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      # This one has been added to fail fast on syntax errors
+      # before running expensive pylint
+      - id: check-ast
+        types_or:
+          - python
+          - pyi
+        exclude: "/(site-packages|development-[^/]*|docs|node_modules|__pycache__|\\..*)/$"
+      - id: check-json
+      - id: check-yaml
+        types: [file]
+        files: \.(yaml|yml|wfex\.(stage|export|ctxt))$
+  - repo: https://github.com/koalaman/shellcheck-precommit.git
+    rev: v0.10.0
+    hooks:
+      - id: shellcheck
+        #args: [--format,json]
+        stages: [manual]
   - repo: local
     hooks:
       - id: pylint
@@ -9,27 +29,47 @@ repos:
         require_serial: true
         types:
           - python
-        exclude: "^[^/]*env/|dev-[^/]*/"
-        entry: pylint -j 4 --source-roots .
+        exclude: "^[^/]*env/|dev-[^/]*/|docs/"
+        entry: pylint
+        args: [-j , "4", --source-roots, .]
       - id: mypy
         name: Local MyPy
         language: system
-        types:
+        types_or:
           - python
-#        files: \.py$
-#        exclude: "^[^/]*env/|development-[^/]*/|docs/"
+          - pyi
         exclude: "/(site-packages|development-[^/]*|docs|node_modules|__pycache__|\\..*)/$"
         require_serial: true
-#        entry: ./basic-installer.bash mypy --strict
-        entry: mypy --strict --show-error-codes --no-warn-unused-ignores
-# Main problem: python executable path, used to find the environment is hard
+        entry: mypy
+        args: [--strict, --show-error-codes, --no-warn-unused-ignores]
+
+## Main problem: python executable path, used to find the environment, is hardcoded
 #  - repo: https://github.com/pre-commit/mirrors-mypy.git
-#    rev: v1.2.0
+#    rev: v1.10.0
 #    hooks:
 #      - id: mypy
-#        exclude: "^[^/]*env/|dev-[^/]*/"
+#        alias: mirrormypy
+#        exclude: "/(site-packages|development-[^/]*|docs|node_modules|__pycache__|\\..*)/$"
 #        require_serial: true
-#        args: [--strict, --show-error-codes, --no-warn-unused-ignores, --python-executable, .pyWEenv/bin/python]
+#        entry: env PYTHONPATH=. mypy
+#        args: [--strict, --show-error-codes, --no-warn-unused-ignores, --python-executable, .full/bin/python]
+#        stages: [manual]
+#  - repo: meta
+#    hooks:
+#      - id: identity
+#        types:
+#          - python
+#        exclude: "/(site-packages|development-[^/]*|docs|node_modules|__pycache__|\\..*)/$"
+#        stages: [manual]
+#  - repo: https://github.com/pylint-dev/pylint.git
+#    rev: v3.2.0
+#    hooks:
+#      - id: pylint
+#        alias: mirrorpylint
+#        exclude: "^[^/]*env/|dev-[^/]*/"
+#        entry: env PYTHONPATH=. pylint
+#        args: [-j, "4", --source-roots, .]
+#        stages: [manual]
 #  - repo: meta
   - repo: https://github.com/inab/python-extended-json-schema-validator.git
     rev: v0.15.1

diff --git a/CITATION.cff b/CITATION.cff
@@ -28,4 +28,4 @@ message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/inab/WfExS-backend"
 type: software
 title: "WfExS-backend"
-version: 0.99.0
+version: 0.99.1
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
@@ -56,7 +56,7 @@ representative at an online or offline event.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at <wfexs-backend[email protected]>.
+reported to the community leaders responsible for enforcement at <[email protected]>.
 All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the

diff --git a/README.md b/README.md
@@ -68,13 +68,13 @@ python WfExS-backend.py --full-help
 usage: WfExS-backend.py [-h] [--log-file LOGFILENAME] [-q] [-v] [-d]
                         [-L LOCALCONFIGFILENAME] [--cache-dir CACHEDIR] [-V]
                         [--full-help]
-                        {init,cache,staged-workdir,export,list-fetchers,list-exporters,list-licences,config-validate,stage,re-stage,mount-workdir,export-stage,offline-execute,execute,export-results,export-crate}
+                        {init,cache,staged-workdir,export,list-fetchers,list-exporters,list-container-factories,list-workflow-engines,list-licences,config-validate,stage,re-stage,import,mount-workdir,export-stage,offline-execute,execute,export-results,export-crate}
                         ...
 
-WfExS (workflow execution service) backend 0.10.1-12-g24f2de3
-(24f2de3bb0b0f3f8a59c90ec16fdf07b66ebe641, branch jmfernandez)
+WfExS (workflow execution service) backend 0.99.0-43-g5058e32
+(5058e32bba74aecc3fef81c9b954b80afbbbb146, branch full_circle)
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   --log-file LOGFILENAME
                         Store messages in a file instead of using standard
@@ -86,31 +86,37 @@ optional arguments:
   -L LOCALCONFIGFILENAME, --local-config LOCALCONFIGFILENAME
                         Local installation configuration file (can also be set
                         up through WFEXS_CONFIG_FILE environment variable)
-                        (default: /home/jmfernandez/projects/WfExS-
-                        backend/wfexs_config.yml)
+                        (default: /home/jmfernandez/projects/WfExS/WfExS-
+                        backend_full_circle/wfexs_config.yml)
   --cache-dir CACHEDIR  Caching directory (default: None)
   -V, --version         show program's version number and exit
   --full-help           It returns full help (default: False)
 
 commands:
   Command to run. It must be one of these
 
-  {init,cache,staged-workdir,export,list-fetchers,list-exporters,list-licences,config-validate,stage,re-stage,mount-workdir,export-stage,offline-execute,execute,export-results,export-crate}
+  {init,cache,staged-workdir,export,list-fetchers,list-exporters,list-container-factories,list-workflow-engines,list-licences,config-validate,stage,re-stage,import,mount-workdir,export-stage,offline-execute,execute,export-results,export-crate}
     init                Init local setup
     cache               Cache handling subcommands
     staged-workdir      Staged working directories handling subcommands
     export              Staged working directories export subcommands
     list-fetchers       List the supported fetchers / schemes
     list-exporters      List the supported export plugins
+    list-container-factories
+                        List the supported container factories
+    list-workflow-engines
+                        List the supported workflow engines
     list-licences       List the documented licences, both embedded and
-                        fetched from SPDX release 3.22
+                        fetched from SPDX release 3.23
     config-validate     Validate the configuration files to be used for
                         staging and execution
     stage               Prepare the staging (working) directory for workflow
                         execution, fetching dependencies and contents
     re-stage            Prepare a new staging (working) directory for workflow
                         execution, repeating the fetch of dependencies and
                         contents
+    import              Workflow Run RO-Crate import into a new staged working
+                        directory
     mount-workdir       Mount the encrypted staging directory on secure
                         staging scenarios
     export-stage        Export the staging directory as an RO-Crate
@@ -321,6 +327,28 @@ optional arguments:
 ```
 usage: WfExS-backend.py list-exporters [-h]
 
+optional arguments:
+  -h, --help  show this help message and exit
+
+```
+</details>
+<details>
+<summary>Subparser <code>list-container-factories</code></summary>
+
+```
+usage: WfExS-backend.py list-container-factories [-h]
+
+optional arguments:
+  -h, --help  show this help message and exit
+
+```
+</details>
+<details>
+<summary>Subparser <code>list-workflow-engines</code></summary>
+
+```
+usage: WfExS-backend.py list-workflow-engines [-h]
+
 optional arguments:
   -h, --help  show this help message and exit
 
@@ -403,15 +431,24 @@ secure workdir arguments:
 <summary>Subparser <code>re-stage</code></summary>
 
 ```
-usage: WfExS-backend.py re-stage [-h] [-Z SECURITYCONTEXTSCONFIGFILENAME]
+Subparser 're-stage'
+usage: WfExS-backend.py re-stage [-h] [-W WORKFLOWCONFIGFILENAME] [-s] [-S]
+                                 [-Z SECURITYCONTEXTSCONFIGFILENAME]
                                  [-n NICKNAME_PREFIX] [--orcid ORCIDS]
                                  [--public-key-file PUBLIC_KEY_FILES]
                                  [--private-key-file PRIVATE_KEY_FILE]
                                  [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR]
                                  -J WORKFLOWWORKINGDIRECTORY
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
+  -W WORKFLOWCONFIGFILENAME, --workflow-config WORKFLOWCONFIGFILENAME
+                        Optional configuration file, describing some inputs
+                        which will replace the base, original ones (default:
+                        None)
+  -s, --no-secure       Make unsecured working directory (default: True)
+  -S, --secure          Make secured working directory (default) (default:
+                        False)
   -Z SECURITYCONTEXTSCONFIGFILENAME, --creds-config SECURITYCONTEXTSCONFIGFILENAME
                         Configuration file, describing security contexts,
                         which hold credentials and similar (default: None)
@@ -440,6 +477,61 @@ secure workdir arguments:
                         the private key needed to unlock an encrypted working
                         directory. (default: )
 
+```
+</details>
+<details>
+<summary>Subparser <code>import</code></summary>
+
+```
+Subparser 'import'
+usage: WfExS-backend.py import [-h] -R WORKFLOWROCRATEFILENAMEORURI
+                               [-W WORKFLOWCONFIGFILENAME] [-s] [-S]
+                               [-Z SECURITYCONTEXTSCONFIGFILENAME]
+                               [-n NICKNAME_PREFIX] [--orcid ORCIDS]
+                               [--public-key-file PUBLIC_KEY_FILES]
+                               [--private-key-file PRIVATE_KEY_FILE]
+                               [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR]
+
+options:
+  -h, --help            show this help message and exit
+  -R WORKFLOWROCRATEFILENAMEORURI, --workflow-rocrate WORKFLOWROCRATEFILENAMEORURI
+                        Workflow Run RO-Crate describing a previous workflow
+                        execution. It can be either a local path or an URI
+                        resolvable from WfExS with no authentication (default:
+                        None)
+  -W WORKFLOWCONFIGFILENAME, --workflow-config WORKFLOWCONFIGFILENAME
+                        Optional configuration file, describing some inputs
+                        which will replace the base, original ones (default:
+                        None)
+  -s, --no-secure       Make unsecured working directory (default: True)
+  -S, --secure          Make secured working directory (default) (default:
+                        False)
+  -Z SECURITYCONTEXTSCONFIGFILENAME, --creds-config SECURITYCONTEXTSCONFIGFILENAME
+                        Configuration file, describing security contexts,
+                        which hold credentials and similar (default: None)
+  -n NICKNAME_PREFIX, --nickname-prefix NICKNAME_PREFIX
+                        Nickname prefix to be used on staged workdir creation
+                        (default: None)
+  --orcid ORCIDS        ORCID(s) of the person(s) staging, running or
+                        exporting the workflow scenario (default: [])
+  --public-key-file PUBLIC_KEY_FILES
+                        This parameter switches on secure processing. Path to
+                        the public key(s) to be used to encrypt the working
+                        directory (default: [])
+
+secure workdir arguments:
+  Private key and passphrase to access secured working directories
+
+  --private-key-file PRIVATE_KEY_FILE
+                        This parameter passes the name of the file containing
+                        the private key needed to unlock an encrypted working
+                        directory. (default: None)
+  --private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR
+                        This parameter passes the name of the environment
+                        variable containing the passphrase needed to decrypt
+                        the private key needed to unlock an encrypted working
+                        directory. (default: )
+
 ```
 </details>
 <details>

diff --git a/TODO.md b/TODO.md
@@ -55,15 +55,17 @@ references to the inputs, that we are going to use to instantiate the workflows.
 
   - [x] **Step 7.k**: Upload to Dataverse.
 
-- [ ] **Step 8**: (partially implemented) Create execution provenance, which includes uploading URLs of results and / or DOIs / URIs.
+- [x] **Step 8**: (partially implemented) Create execution provenance, which includes uploading URLs of results and / or DOIs / URIs.
 
 - [x] **Step 9**: Generate RO-Crate from execution provenance and exported results.
 
   - [x] **Step 9.a**: Generated RO-Crate should be consumable by WorkflowHub.
 
-  - [ ] **Step 9.c**: Generated RO-Crate should be consumable by WfExS-backend.
+  - [x] **Step 9.c**: Generated RO-Crate should be consumable by WfExS-backend.
 
-  - [ ] **Step 9.d**: Add full circle capabilities. Re-execute workflow with the very same parameters from previously generated RO-Crate.
+  - [x] **Step 9.d**: Add full circle capabilities. Re-execute workflow with the very same parameters from previously generated RO-Crate (only metadata).
+
+  - [ ] **Step 9.e**: Add full circle capabilities. Re-execute workflow with the very same parameters from previously generated RO-Crate (reusing payloads).
 
 
 ## Other features

diff --git a/WfExS-backend.py b/WfExS-backend.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/apptainer-local-installer.bash b/apptainer-local-installer.bash
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/apptainer12-local-installer.bash b/apptainer12-local-installer.bash
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/basic-installer.bash b/basic-installer.bash
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -15,4 +15,5 @@ mccabe >= 0.7.0
 mypy >= 1.1.1
 json-schema-for-humans >= 0.45.1
 pre-commit >= 2.17.0
+# This is the last version of black supporting Python 3.7
 black == 23.3.0