diff --git a/CHANGES.rst b/CHANGES.rst index 737616ca9..76a7cc497 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,7 @@ Changes: - Add some sections for documentation (`#61 `_). - Add support of documentation RST file redirection to generated HTML for reference resolution in both Github source and Readthedocs served pages. +- Improve documentation links, ReadTheDocs format and TOC references. Fixes: ------ diff --git a/README.rst b/README.rst index a15f2d1dc..9f0e013dc 100644 --- a/README.rst +++ b/README.rst @@ -96,7 +96,7 @@ a few. Because of this, not all features offered in `Weaver` are guaranteed to b behaving `ADES` and/or `EMS` instances. The reference specification is tracked to preserve the minimal conformance requirements and provide feedback to |ogc|_ in this effect. -Weaver can be launched either as an *EMS* or an *ADES* according to configuration values it is deployed with. +Weaver can be launched either as an `EMS` or an `ADES` according to configuration values it is deployed with. For more details, see `Configuration`_ and `Documentation`_ sections. .. |ogc| replace:: Open Geospatial Consortium (OGC) diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 000000000..079c0e729 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,6 @@ +.. This creates an auto-link to the OpenAPI specification as 'api.rst' matches generated 'api.html' +.. The title is the name employed in the TOC, but nothing is actually displayed as the page redirects to 'api.html' +.. _api: + +OpenAPI Specification +======================= diff --git a/docs/source/appendix.rst b/docs/source/appendix.rst index 265b3af79..6d336e94f 100644 --- a/docs/source/appendix.rst +++ b/docs/source/appendix.rst @@ -1,6 +1,29 @@ .. _appendix: +.. include:: references.rst + +************ +Glossary +************ + +.. glossary:: + :sorted: + + ADES: Application Deployment and Execution Service + `CWL`_: Common Workflow Language + EMS: Execution Management Service + `OGC`_: Open Geospatial Consortium + WPS: Web Processing Service ************ Useful Links ************ +- |cwl-home|_ +- |cwl-spec|_ +- |cwl-guide|_ +- |cwl-cmdtool|_ +- |cwl-workflow|_ +- |oas|_ +- |ogc|_ +- |ogc-proc-api|_ +- `Weaver Issues`_ diff --git a/docs/source/conf.py b/docs/source/conf.py index 2b1014cb0..5c53c586d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,16 +48,18 @@ # extensions coming with Sphinx (named "sphinx.ext.*") or your custom ones. sys.path.append(os.path.join(DOC_DIR_ROOT, "_extensions")) extensions = [ - "doc_redirect", - "sphinxcontrib.redoc", - "sphinx.ext.autodoc", - "sphinx.ext.todo", - "sphinx.ext.viewcode", - "sphinx.ext.intersphinx", - "autoapi.extension", - "sphinx_autodoc_typehints", - "pywps.ext_autodoc", - "sphinx_paramlinks", + "doc_redirect", # redirect literal RST references -> built HTML page + "sphinxcontrib.redoc", # generate live OpenAPI with this doc + "sphinx.ext.autodoc", # document code docstrings + "cloud_sptheme.ext.autodoc_sections", # allow sections in docstrings code + "sphinx.ext.githubpages", # for publishing the doc to github pages + "sphinx.ext.todo", # support directives + "sphinx.ext.viewcode", # add links to highlighted source code + "sphinx.ext.intersphinx", # add links to other projects documentation + "autoapi.extension", # generate source code documentation + "sphinx_autodoc_typehints", # support '# type: (...) -> ...' typing + "pywps.ext_autodoc", # extra autodoc for PyWPS processes + "sphinx_paramlinks", # allow ':paramref:`<>`' references in docstrings ] # note: see custom extension documentation @@ -144,9 +146,9 @@ def doc_redirect_include(file_path): # built documents. # # The short X.Y version. -version = "0.3" +version = __meta__.__version__ # The full version, including alpha/beta/rc tags. -release = "0.3" +release = __meta__.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -198,14 +200,17 @@ def doc_redirect_include(file_path): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = "alabaster" -html_theme = "nature" +# html_theme = "nature" +html_theme = "sphinx_rtd_theme" # otherwise, readthedocs.org uses their theme by default, so no need to specify it # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -# html_theme_options = {} +html_theme_options = { + "navigation_depth": 2, # TOC, RTD theme +} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] @@ -229,9 +234,7 @@ def doc_redirect_include(file_path): # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [ - "_static" -] +# html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -240,7 +243,7 @@ def doc_redirect_include(file_path): # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = "%Y-%m-%d" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index b21c56904..a50215fb9 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -42,6 +42,8 @@ Configuration of Request Options .. todo:: complete docs +.. versionadded:: 1.8.0 + `request_options.yml.example`_ diff --git a/docs/source/index.rst b/docs/source/index.rst index 52fd319bf..7830968d7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,13 +1,2 @@ .. include:: ../../README.rst - -.. toctree:: - :maxdepth: 2 - - installation - configuration - running - tutorial - processes - package - changes - appendix +.. include:: toc.rst diff --git a/docs/source/package.rst b/docs/source/package.rst index 73edc199f..80af42b92 100644 --- a/docs/source/package.rst +++ b/docs/source/package.rst @@ -1,31 +1,32 @@ -.. package +.. package: +.. application-package: +.. include:: references.rst ************************* Application Package ************************* -.. todo:: demo docs - -Based on |CWL|_ (CWL) |cwl-spec|_. +The `Application Package` defines the internal script definition and configuration that will be executed by a process. +This package is based on |CWL|_ (`CWL`) |cwl-spec|_. Using the extensive specification of `CWL` as backbone for +internal execution of the process allows it to run multiple type of applications, whether they are referenced to by +`docker image`, `bash script` or more. +.. note:: + The large community and use cases covered by `CWL` makes it extremely versatile. If you encounter any issue running + your `Application Package` in `Weaver` (such as file permissions for example), chances are that there exists a + workaround somewhere in the |cwl-spec|_. Most typical problems are usually handled by some flag or argument in the + `CWL` definition, so this reference should be explored first. Please also refer to `Common Problems and Solutions`_ + section and existing `Weaver Issues`_. Ultimately if no solution can be found, open an new issue about your specific + problem. -.. |CWL| replace:: Common Workflow Language -.. _CWL: https://www.commonwl.org/ -.. |cwl-spec| replace:: specification -.. _cwl-spec: https://www.commonwl.org/#Specification - +.. |pkg-req| replace:: ``GET /processes/{id}/package`` +.. _pkg-req: https://pavics-weaver.readthedocs.io/en/setup-docs/api.html#tag/Processes%2Fpaths%2F~1processes~1%7Bprocess_id%7D~1package%2Fget Typical CWL Package Definition =========================================== .. todo:: CommandLineTool -Correspondance between CWL and WPS fields -=========================================== - -.. todo:: demo docs - - Correspondance between CWL and WPS fields =========================================== @@ -52,3 +53,64 @@ Multiple Values ----------------------- .. todo:: minOccurs/maxOccurs + array + WPS repeats IDs vs CWL as list + +Common Problems and Solutions +=========================================== + +This section present some commonly encountered use-cases and basic solutions. + +Permission error on input files +---------------------------------------------- + +Some processes expect their inputs to be writable (e.g.: ZIP files). When running an *Application Package* based on a +`docker image`, `Weaver` mounts the input files as `volumes` in read-only mode for security reasons. This causes these +processes to immediately fail as the running user cannot override nor write temporary files in the same directory +(where the volume was mounted to), as it is marked with read permissions. + +To resolve this issue, the application developer should add the ``InitialWorkDirRequirement`` +(|cwl-wd-ref|_, |cwl-wd-ex|_) to his CWL package definition. This tells CWL to stage the files into the docker image +into the running directory where the user will be allowed to generate outputs, and therefore, also allow edition of the +inputs or generation of temporary files as when unpacking a compressed file. + +.. |cwl-wd-ref| replace:: reference +.. _cwl-wd-ref: `cwl-workdir-req`_ +.. |cwl-wd-ex| replace:: example +.. _cwl-wd-ex: `cwl-workdir-req`_ + +As example, the CWL definition could be similar to the following: + +.. code-block:: json + + { + "cwlVersion": "v1.0", + "class": "CommandLineTool", + "requirements": { + "DockerRequirement": { + "dockerPull": "" + }, + "InitialWorkDirRequirement": { + "listing": [{ + "entry": "$(inputs.input_file)", + "writable": true + } + ] + } + }, + "arguments": ["$(runtime.outdir)"], + "inputs": { + "input_file": { + "type": "File" + } + } + +Note that ``$(inputs.input_file)`` is tells which input to resolve for staging in ``InitialWorkDirRequirement`` using +the ``"writable": True`` parameter. This file will be mounted with write permissions into working runtime directory. + + +Links +----------------------- + +- |cwl-guide|_ +- |cwl-cmdtool|_ +- |cwl-workflow|_ + diff --git a/docs/source/processes.rst b/docs/source/processes.rst index 906354e12..260e5ebc1 100644 --- a/docs/source/processes.rst +++ b/docs/source/processes.rst @@ -69,6 +69,7 @@ Managing processes included in Weaver ADES/EMS Following steps represent the typical steps applied to deploy a process, execute it and retrieve the results. .. _Deploy: + Register a new process (Deploy) ----------------------------------------- @@ -105,6 +106,7 @@ interfaces, but it is recommended to use the WPS-REST one because of the added f .. _GetCapabilities: .. _DescribeProcess: + Access registered process(es) (GetCapabilities, DescribeProcess) ------------------------------------------------------------------------ @@ -132,18 +134,21 @@ that define the process references and expected inputs/outputs. .. _`Process requests`: https://pavics-weaver.readthedocs.io/en/setup-docs/api.html#tag/Processes .. _Execute: + Execution of a process (Execute, Job) --------------------------------------------------------------------- .. todo:: .. _GetStatus: + Monitoring of a process (GetStatus) --------------------------------------------------------------------- .. todo:: .. _GetResult: + Obtaining output results, logs or errors --------------------------------------------------------------------- diff --git a/docs/source/references.rst b/docs/source/references.rst new file mode 100644 index 000000000..c3efcba4e --- /dev/null +++ b/docs/source/references.rst @@ -0,0 +1,26 @@ +.. Listing of all useful references for the documentation +.. Don't place any 'visible/rendered' documentation here (only links), or it will appear everywhere it is included + +.. |cwl| replace:: Common Workflow Language +.. _cwl: `cwl-home`_ +.. |cwl-home| replace:: CWL Homepage +.. _cwl-home: https://www.commonwl.org/ +.. |cwl-spec| replace:: CWL Specification +.. _cwl-spec: https://www.commonwl.org/#Specification +.. |cwl-guide| replace:: CWL User Guide +.. _cwl-guide: http://www.commonwl.org/user_guide/ +.. |cwl-cmdtool| replace:: CWL CommandLineTool +.. _cwl-cmdtool: https://www.commonwl.org/v1.1/CommandLineTool.html +.. |cwl-workflow| replace:: CWL Workflow +.. _cwl-workflow: https://www.commonwl.org/v1.1/Workflow.html +.. _cwl-workdir-req: https://www.commonwl.org/v1.1/CommandLineTool.html#InitialWorkDirRequirement +.. _cwl-workdir-ex: https://www.commonwl.org/user_guide/15-staging/ +.. _`Weaver Issues`: https://github.com/crim-ca/weaver/issues +.. |oas| replace:: OpenAPI Specification +.. _oas: https://pavics-weaver.readthedocs.io/en/latest/api.html +.. |ogc| replace:: Open Geospatial Consortium (OGC) +.. _ogc: https://www.ogc.org/ +.. |ogc-home| replace:: |ogc| Homepage +.. _ogc-home: `ogc`_ +.. |ogc-proc-api| replace:: OGC API - Processes (WPS-REST bindings) +.. _ogc-proc-api: https://github.com/opengeospatial/wps-rest-binding diff --git a/docs/source/toc.rst b/docs/source/toc.rst new file mode 100644 index 000000000..f8d14b4d2 --- /dev/null +++ b/docs/source/toc.rst @@ -0,0 +1,18 @@ +----------- +Contents +----------- + +.. toctree:: + :maxdepth: 2 + + Contents + installation + configuration + running + tutorial + processes + package + OpenAPI Specification + Source Code Documentation + changes + appendix diff --git a/requirements-docs.txt b/requirements-docs.txt index 6a0f42000..93dccf8ce 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,6 +2,7 @@ # we actually need to install all requirements during docs build because of OpenAPI generation # (see 'docs/source/conf.py') -r requirements.txt +cloud_sptheme pycodestyle # required for pywps.ext_autodoc pywps @@ -11,3 +12,4 @@ sphinx-paramlinks>=0.4.1 # adds redoc OpenAPI directly served on readthedocs sphinxcontrib-redoc>=1.6.0 sphinx_autodoc_typehints[type_comments] +sphinx_rtd_theme diff --git a/weaver/utils.py b/weaver/utils.py index d61c1eb78..d8a7144a2 100644 --- a/weaver/utils.py +++ b/weaver/utils.py @@ -571,7 +571,7 @@ def request_extra(method, # type: AnyStr Standard library :mod:`requests` with additional functional utilities. Retry operation - --------------- + ~~~~~~~~~~~~~~~~~~~~~~ Implements request retry if the previous request failed, up to the specified number of retries. Using :paramref:`backoff` factor, you can control the interval between request attempts such as:: @@ -601,7 +601,7 @@ def request_extra(method, # type: AnyStr items as first request is done immediately, and following attempts are done with the appropriate delay. File Transport Scheme - --------------------- + ~~~~~~~~~~~~~~~~~~~~~~ Any request with ``file://`` scheme or empty scheme (no scheme specified) will be automatically handled as potential local file path. The path should be absolute to ensure it to be correctly resolved. @@ -613,7 +613,7 @@ def request_extra(method, # type: AnyStr - :class:`FileAdapter` SSL Verification - ---------------- + ~~~~~~~~~~~~~~~~~~~~~~ Allows SSL verify option to be enabled or disabled according to configuration settings or explicit parameters. Any variation of ``verify`` or ``ssl_verify`` keyword arguments are considered. If they all resolve to ``True``,