updates to processes docs and cwl package + update rtd rendering/links (

#61)
crim-ca · Jun 5, 2020 · d3612ac · d3612ac
1 parent d851afa
commit d3612ac
Show file tree

Hide file tree

Showing 13 changed files with 186 additions and 49 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -12,6 +12,7 @@ Changes:
 - Add some sections for documentation (`#61 <https://github.com/crim-ca/weaver/issues/61>`_).
 - Add support of documentation RST file redirection to generated HTML for reference resolution in both Github source
   and Readthedocs served pages.
+- Improve documentation links, ReadTheDocs format and TOC references.
 
 Fixes:
 ------

diff --git a/README.rst b/README.rst
@@ -96,7 +96,7 @@ a few. Because of this, not all features offered in `Weaver` are guaranteed to b
 behaving `ADES` and/or `EMS` instances. The reference specification is tracked to preserve the minimal conformance
 requirements and provide feedback to |ogc|_ in this effect.
 
-Weaver can be launched either as an *EMS* or an *ADES* according to configuration values it is deployed with.
+Weaver can be launched either as an `EMS` or an `ADES` according to configuration values it is deployed with.
 For more details, see `Configuration`_ and `Documentation`_ sections.
 
 .. |ogc| replace:: Open Geospatial Consortium (OGC)

diff --git a/docs/source/api.rst b/docs/source/api.rst
@@ -0,0 +1,6 @@
+.. This creates an auto-link to the OpenAPI specification as 'api.rst' matches generated 'api.html'
+.. The title is the name employed in the TOC, but nothing is actually displayed as the page redirects to 'api.html'
+.. _api:
+
+OpenAPI Specification
+=======================
diff --git a/docs/source/appendix.rst b/docs/source/appendix.rst
@@ -1,6 +1,29 @@
 .. _appendix:
+.. include:: references.rst
+
+************
+Glossary
+************
+
+.. glossary::
+    :sorted:
+
+    ADES: Application Deployment and Execution Service
+    `CWL`_: Common Workflow Language
+    EMS: Execution Management Service
+    `OGC`_: Open Geospatial Consortium
+    WPS: Web Processing Service
 
 ************
 Useful Links
 ************
 
+- |cwl-home|_
+- |cwl-spec|_
+- |cwl-guide|_
+- |cwl-cmdtool|_
+- |cwl-workflow|_
+- |oas|_
+- |ogc|_
+- |ogc-proc-api|_
+- `Weaver Issues`_
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -48,16 +48,18 @@
 # extensions coming with Sphinx (named "sphinx.ext.*") or your custom ones.
 sys.path.append(os.path.join(DOC_DIR_ROOT, "_extensions"))
 extensions = [
-    "doc_redirect",
-    "sphinxcontrib.redoc",
-    "sphinx.ext.autodoc",
-    "sphinx.ext.todo",
-    "sphinx.ext.viewcode",
-    "sphinx.ext.intersphinx",
-    "autoapi.extension",
-    "sphinx_autodoc_typehints",
-    "pywps.ext_autodoc",
-    "sphinx_paramlinks",
+    "doc_redirect",         # redirect literal RST references -> built HTML page
+    "sphinxcontrib.redoc",  # generate live OpenAPI with this doc
+    "sphinx.ext.autodoc",   # document code docstrings
+    "cloud_sptheme.ext.autodoc_sections",   # allow sections in docstrings code
+    "sphinx.ext.githubpages",   # for publishing the doc to github pages
+    "sphinx.ext.todo",          # support directives
+    "sphinx.ext.viewcode",      # add links to highlighted source code
+    "sphinx.ext.intersphinx",   # add links to other projects documentation
+    "autoapi.extension",        # generate source code documentation
+    "sphinx_autodoc_typehints",     # support '# type: (...) -> ...' typing
+    "pywps.ext_autodoc",        # extra autodoc for PyWPS processes
+    "sphinx_paramlinks",        # allow ':paramref:`<>`' references in docstrings
 ]
 
 # note: see custom extension documentation
@@ -144,9 +146,9 @@ def doc_redirect_include(file_path):
 # built documents.
 #
 # The short X.Y version.
-version = "0.3"
+version = __meta__.__version__
 # The full version, including alpha/beta/rc tags.
-release = "0.3"
+release = __meta__.__version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -198,14 +200,17 @@ def doc_redirect_include(file_path):
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 # html_theme = "alabaster"
-html_theme = "nature"
+# html_theme = "nature"
+html_theme = "sphinx_rtd_theme"
 
 # otherwise, readthedocs.org uses their theme by default, so no need to specify it
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-# html_theme_options = {}
+html_theme_options = {
+    "navigation_depth": 2,  # TOC, RTD theme
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -229,9 +234,7 @@ def doc_redirect_include(file_path):
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = [
-    "_static"
-]
+# html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
@@ -240,7 +243,7 @@ def doc_redirect_include(file_path):
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-# html_last_updated_fmt = '%b %d, %Y'
+html_last_updated_fmt = "%Y-%m-%d"
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.

diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
@@ -42,6 +42,8 @@ Configuration of Request Options
 
 .. todo:: complete docs
 
+.. versionadded:: 1.8.0
+
 
 `request_options.yml.example`_
 

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -1,13 +1,2 @@
 .. include:: ../../README.rst
-
-.. toctree::
-   :maxdepth: 2
-
-   installation
-   configuration
-   running
-   tutorial
-   processes
-   package
-   changes
-   appendix
+.. include:: toc.rst
diff --git a/docs/source/package.rst b/docs/source/package.rst
@@ -1,31 +1,32 @@
-.. package
+.. package:
+.. application-package:
+.. include:: references.rst
 
 *************************
 Application Package
 *************************
 
-.. todo:: demo docs
-
-Based on |CWL|_ (CWL) |cwl-spec|_.
+The `Application Package` defines the internal script definition and configuration that will be executed by a process.
+This package is based on |CWL|_ (`CWL`) |cwl-spec|_. Using the extensive specification of `CWL` as backbone for
+internal execution of the process allows it to run multiple type of applications, whether they are referenced to by
+`docker image`, `bash script` or more.
 
+.. note::
+    The large community and use cases covered by `CWL` makes it extremely versatile. If you encounter any issue running
+    your `Application Package` in `Weaver` (such as file permissions for example), chances are that there exists a
+    workaround somewhere in the |cwl-spec|_. Most typical problems are usually handled by some flag or argument in the
+    `CWL` definition, so this reference should be explored first. Please also refer to `Common Problems and Solutions`_
+    section and existing `Weaver Issues`_. Ultimately if no solution can be found, open an new issue about your specific
+    problem.
 
-.. |CWL| replace:: Common Workflow Language
-.. _CWL: https://www.commonwl.org/
-.. |cwl-spec| replace:: specification
-.. _cwl-spec: https://www.commonwl.org/#Specification
-
+.. |pkg-req| replace:: ``GET /processes/{id}/package``
+.. _pkg-req: https://pavics-weaver.readthedocs.io/en/setup-docs/api.html#tag/Processes%2Fpaths%2F~1processes~1%7Bprocess_id%7D~1package%2Fget
 
 Typical CWL Package Definition
 ===========================================
 
 .. todo:: CommandLineTool
 
-Correspondance between CWL and WPS fields
-===========================================
-
-.. todo:: demo docs
-
-
 Correspondance between CWL and WPS fields
 ===========================================
 
@@ -52,3 +53,64 @@ Multiple Values
 -----------------------
 
 .. todo:: minOccurs/maxOccurs + array + WPS repeats IDs vs CWL as list
+
+Common Problems and Solutions
+===========================================
+
+This section present some commonly encountered use-cases and basic solutions.
+
+Permission error on input files
+----------------------------------------------
+
+Some processes expect their inputs to be writable (e.g.: ZIP files). When running an *Application Package* based on a
+`docker image`, `Weaver` mounts the input files as `volumes` in read-only mode for security reasons. This causes these
+processes to immediately fail as the running user cannot override nor write temporary files in the same directory
+(where the volume was mounted to), as it is marked with read permissions.
+
+To resolve this issue, the application developer should add the ``InitialWorkDirRequirement``
+(|cwl-wd-ref|_, |cwl-wd-ex|_) to his CWL package definition. This tells CWL to stage the files into the docker image
+into the running directory where the user will be allowed to generate outputs, and therefore, also allow edition of the
+inputs or generation of temporary files as when unpacking a compressed file.
+
+.. |cwl-wd-ref| replace:: reference
+.. _cwl-wd-ref: `cwl-workdir-req`_
+.. |cwl-wd-ex| replace:: example
+.. _cwl-wd-ex: `cwl-workdir-req`_
+
+As example, the CWL definition could be similar to the following:
+
+.. code-block:: json
+
+    {
+      "cwlVersion": "v1.0",
+      "class": "CommandLineTool",
+      "requirements": {
+        "DockerRequirement": {
+          "dockerPull": "<docker-url>"
+        },
+        "InitialWorkDirRequirement": {
+          "listing": [{
+             "entry": "$(inputs.input_file)",
+             "writable": true
+            }
+          ]
+        }
+      },
+      "arguments": ["$(runtime.outdir)"],
+      "inputs": {
+        "input_file": {
+        "type": "File"
+      }
+    }
+
+Note that ``$(inputs.input_file)`` is tells which input to resolve for staging in ``InitialWorkDirRequirement`` using
+the ``"writable": True`` parameter. This file will be mounted with write permissions into working runtime directory.
+
+
+Links
+-----------------------
+
+- |cwl-guide|_
+- |cwl-cmdtool|_
+- |cwl-workflow|_
+
diff --git a/docs/source/processes.rst b/docs/source/processes.rst
@@ -69,6 +69,7 @@ Managing processes included in Weaver ADES/EMS
 Following steps represent the typical steps applied to deploy a process, execute it and retrieve the results.
 
 .. _Deploy:
+
 Register a new process (Deploy)
 -----------------------------------------
 
@@ -105,6 +106,7 @@ interfaces, but it is recommended to use the WPS-REST one because of the added f
 
 .. _GetCapabilities:
 .. _DescribeProcess:
+
 Access registered process(es) (GetCapabilities, DescribeProcess)
 ------------------------------------------------------------------------
 
@@ -132,18 +134,21 @@ that define the process references and expected inputs/outputs.
 .. _`Process requests`: https://pavics-weaver.readthedocs.io/en/setup-docs/api.html#tag/Processes
 
 .. _Execute:
+
 Execution of a process (Execute, Job)
 ---------------------------------------------------------------------
 
 .. todo::
 
 .. _GetStatus:
+
 Monitoring of a process (GetStatus)
 ---------------------------------------------------------------------
 
 .. todo::
 
 .. _GetResult:
+
 Obtaining output results, logs or errors
 ---------------------------------------------------------------------
 

diff --git a/docs/source/references.rst b/docs/source/references.rst
@@ -0,0 +1,26 @@
+.. Listing of all useful references for the documentation
+.. Don't place any 'visible/rendered' documentation here (only links), or it will appear everywhere it is included
+
+.. |cwl| replace:: Common Workflow Language
+.. _cwl: `cwl-home`_
+.. |cwl-home| replace:: CWL Homepage
+.. _cwl-home: https://www.commonwl.org/
+.. |cwl-spec| replace:: CWL Specification
+.. _cwl-spec: https://www.commonwl.org/#Specification
+.. |cwl-guide| replace:: CWL User Guide
+.. _cwl-guide: http://www.commonwl.org/user_guide/
+.. |cwl-cmdtool| replace:: CWL CommandLineTool
+.. _cwl-cmdtool: https://www.commonwl.org/v1.1/CommandLineTool.html
+.. |cwl-workflow| replace:: CWL Workflow
+.. _cwl-workflow: https://www.commonwl.org/v1.1/Workflow.html
+.. _cwl-workdir-req: https://www.commonwl.org/v1.1/CommandLineTool.html#InitialWorkDirRequirement
+.. _cwl-workdir-ex: https://www.commonwl.org/user_guide/15-staging/
+.. _`Weaver Issues`: https://github.com/crim-ca/weaver/issues
+.. |oas| replace:: OpenAPI Specification
+.. _oas: https://pavics-weaver.readthedocs.io/en/latest/api.html
+.. |ogc| replace:: Open Geospatial Consortium (OGC)
+.. _ogc: https://www.ogc.org/
+.. |ogc-home| replace:: |ogc| Homepage
+.. _ogc-home: `ogc`_
+.. |ogc-proc-api| replace:: OGC API - Processes (WPS-REST bindings)
+.. _ogc-proc-api: https://github.com/opengeospatial/wps-rest-binding
diff --git a/docs/source/toc.rst b/docs/source/toc.rst
@@ -0,0 +1,18 @@
+-----------
+Contents
+-----------
+
+.. toctree::
+    :maxdepth: 2
+
+    Contents <index>
+    installation
+    configuration
+    running
+    tutorial
+    processes
+    package
+    OpenAPI Specification <api.rst>
+    Source Code Documentation <autoapi/index.rst>
+    changes
+    appendix
diff --git a/requirements-docs.txt b/requirements-docs.txt
@@ -2,6 +2,7 @@
 # we actually need to install all requirements during docs build because of OpenAPI generation
 # (see 'docs/source/conf.py')
 -r requirements.txt
+cloud_sptheme
 pycodestyle
 # required for pywps.ext_autodoc
 pywps
@@ -11,3 +12,4 @@ sphinx-paramlinks>=0.4.1
 # adds redoc OpenAPI directly served on readthedocs
 sphinxcontrib-redoc>=1.6.0
 sphinx_autodoc_typehints[type_comments]
+sphinx_rtd_theme
diff --git a/weaver/utils.py b/weaver/utils.py
@@ -571,7 +571,7 @@ def request_extra(method,                       # type: AnyStr
     Standard library :mod:`requests` with additional functional utilities.
 
     Retry operation
-    ---------------
+    ~~~~~~~~~~~~~~~~~~~~~~
 
     Implements request retry if the previous request failed, up to the specified number of retries.
     Using :paramref:`backoff` factor, you can control the interval between request attempts such as::
@@ -601,7 +601,7 @@ def request_extra(method,                       # type: AnyStr
         items as first request is done immediately, and following attempts are done with the appropriate delay.
 
     File Transport Scheme
-    ---------------------
+    ~~~~~~~~~~~~~~~~~~~~~~
 
     Any request with ``file://`` scheme or empty scheme (no scheme specified) will be automatically handled as potential
     local file path. The path should be absolute to ensure it to be correctly resolved.
@@ -613,7 +613,7 @@ def request_extra(method,                       # type: AnyStr
         - :class:`FileAdapter`
 
     SSL Verification
-    ----------------
+    ~~~~~~~~~~~~~~~~~~~~~~
 
     Allows SSL verify option to be enabled or disabled according to configuration settings or explicit parameters.
     Any variation of ``verify`` or ``ssl_verify`` keyword arguments are considered. If they all resolve to ``True``,