Merge pull request #34 from elixir-europe/dev

v0.5.0

README.md

@@ -6,7 +6,7 @@
 
 ## Description
 
-This microservice uses [Flask](http://flask.pocoo.org/) and [connexion](https://github.com/zalando/connexion) to render the [GA4GH WES OpenAPI specification](https://github.com/ga4gh/workflow-execution-service-schemas). It allows users to send their workflows for execution, list current and previous workflow runs, and get the status and/or detailed information on individual workflow runs. It interprets workflows and breaks them down to individual tasks, for each task emitting a request that is compatible with the [GA4GH Task Execution Service](https://github.com/ga4gh/task-execution-schemas) (TES) OpenAPI specification. Thus, for end-to-end execution of workflows, a local or remote instance of a TES service such as [TESK](https://github.com/EMBL-EBI-TSI/TESK) or [funnel](https://ohsu-comp-bio.github.io/funnel/) is required.
+This microservice uses [Flask](http://flask.pocoo.org/) and [Connexion](https://github.com/zalando/connexion) to render the [GA4GH WES OpenAPI specification](https://github.com/ga4gh/workflow-execution-service-schemas). It allows users to send their workflows for execution, list current and previous workflow runs, and get the status and/or detailed information on individual workflow runs. It interprets workflows and breaks them down to individual tasks, for each task emitting a request that is compatible with the [GA4GH Task Execution Service](https://github.com/ga4gh/task-execution-schemas) (TES) OpenAPI specification. Thus, for end-to-end execution of workflows, a local or remote instance of a TES service such as [TESK](https://github.com/EMBL-EBI-TSI/TESK) or [funnel](https://ohsu-comp-bio.github.io/funnel/) is required.
 
 The service will be backed by a [MongoDB](https://www.mongodb.com/) database and will use [ELIXIR AAI](https://www.elixir-europe.org/services/compute/aai) authentication. It will provide an abstract middleware layer providing support for various workflow languages. Once implemented, support for individual languages can be added in the form of pluggable modules. Initially, we are planning to take advantage of [CWL-TES](https://github.com/common-workflow-language/cwl-tes) to provide support for the [Common Workflow Language](https://github.com/common-workflow-language/common-workflow-language) (CWL).
 
@@ -85,13 +85,14 @@ Install service
 python setup.py develop
 ```
 
-Set config file environment variable and optionally edit config file
+Optional: set config file environment variable and edit config file
 
 ```bash
-export WES_CONFIG="$PWD/wes_elixir/config/config.yaml"
+export WES_CONFIG="$PWD/wes_elixir/config/app_config.yaml"
 ```
 
 Set your .netrc file under your $HOME directory accordingly. The .netrc file should look like the following:
+
 ```bash
 machine ftp-private.ebi.ac.uk
 login redacted_username

requirements.txt

@@ -1,61 +1,28 @@
 amqp==2.3.2
 astroid==2.0.4
-#attrs==18.2.0
-#avro-cwl==1.8.4
-#bagit==1.7.0
 billiard==3.5.0.4
-#CacheControl==0.11.7
 celery==4.2.1
-#certifi==2018.8.13
-#chardet==3.0.4
 click==6.7
 clickclick==1.2.2
-configloader==1.0.1
 connexion==1.5.2
-#cwl-tes==0.2.0
-#cwltool==1.0.20180711112827
-#decorator==4.3.0
 Flask==1.0.2
 Flask-Cors==3.0.6
 Flask-PyMongo==2.1.0
-#future==0.16.0
-#idna==2.7
 inflection==0.3.1
-#isodate==0.6.0
 isort==4.3.4
 itsdangerous==0.24
 Jinja2==2.10
 jsonschema==2.6.0
 kombu==4.2.1
 lazy-object-proxy==1.3.1
-#lockfile==0.12.2
-#lxml==4.2.4
 MarkupSafe==1.0
 mccabe==0.6.1
-#mistune==0.7.4
-#mypy-extensions==0.4.1
-#networkx==2.1
-#prov==1.5.1
-#psutil==5.4.7
-#py-tes==0.2.1
 pylint==2.1.1
 pymongo==3.7.1
-#pyparsing==2.2.0
-#python-dateutil==2.6.1
 pytz==2018.5
 PyYAML==3.13
-#rdflib==4.2.2
-#rdflib-jsonld==0.4.0
-#requests==2.19.1
-#ruamel.yaml==0.14.12
-#schema-salad==2.7.20180809223002
-#shellescape==3.4.1
-#six==1.11.0
-#subprocess32==3.5.2
 swagger-spec-validator==2.3.1
 typed-ast==1.1.0
-#typing==3.6.4
-#urllib3==1.23
 vine==1.1.4
 Werkzeug==0.14.1
 wrapt==1.10.11

wes_elixir/api/register_openapi.py

@@ -0,0 +1,44 @@
+import logging
+import os
+
+
+# Get logger instance
+logger = logging.getLogger(__name__)
+
+
+def register_openapi(
+    app=None,
+    specs=[]
+):
+
+    '''Register OpenAPI specs with Connexion app'''
+
+    # Iterate over list of APIspecs
+    for spec in specs:
+
+        # Extract path
+        path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), spec['path']))
+
+        # Generate API endpoints from OpenAPI specs
+        try:
+            app.add_api(
+                path, 
+                strict_validation=spec['strict_validation'],
+                validate_responses=spec['validate_responses'],
+                swagger_ui=spec['swagger_ui'],
+                swagger_json=spec['swagger_json'],
+            )
+
+            # Log info message
+            logger.info("API endpoints specified in '{path}' added.".format(path=path))
+
+        except (FileNotFoundError, PermissionError) as e:
+            logger.critical("API specification file not found or accessible at '{path}'. Execution aborted. Original error message: {type}: {msg}".format(
+                path=path,
+                type=type(e).__name__,
+                msg=e,
+            ))
+            raise SystemExit(1)
+
+    # Return Connexion app
+    return(app)

wes_elixir/app.py

@@ -1,21 +1,48 @@
-from flask_cors import CORS
-
-from wes_elixir.config.config import config_connexion_app
+from wes_elixir.api.register_openapi import register_openapi
+from wes_elixir.config.app_config import parse_app_config
+from wes_elixir.config.config_parser import get_conf, get_conf_type
+from wes_elixir.config.log_config import configure_logging
 from wes_elixir.database.register_mongodb import register_mongodb
 from wes_elixir.errors.errors import register_error_handlers
 from wes_elixir.factories.connexion_app import create_connexion_app
-from wes_elixir.monitoring.register_task_monitor import register_task_monitor
+from wes_elixir.tasks.register_celery import register_task_service
+from wes_elixir.security.cors import enable_cors
 
 
 def main():
-    connexion_app = create_connexion_app()
-    connexion_app = config_connexion_app(connexion_app)
+
+    # Configure logger
+    configure_logging(config_var='WES_CONFIG_LOG')
+
+    # Parse app configuration
+    config = parse_app_config(config_var='WES_CONFIG')
+
+    # Create Connexion app
+    connexion_app = create_connexion_app(config)
+
+    # Register MongoDB
     connexion_app = register_mongodb(connexion_app)
+
+    # Register error handlers
     connexion_app = register_error_handlers(connexion_app)
-    register_task_monitor(connexion_app)
-    CORS(connexion_app.app)
-    connexion_app.run(use_reloader=True)
+
+    # Create Celery app and register background task monitoring service
+    register_task_service(connexion_app)
+
+    # Register OpenAPI specs
+    connexion_app = register_openapi(
+        app=connexion_app,
+        specs=get_conf_type(config, 'api', 'specs', types=(list))
+    )
+
+    # Enable cross-origin resource sharing
+    enable_cors(connexion_app.app)
+
+    # Run app
+    connexion_app.run(
+        use_reloader=get_conf(config, 'server', 'use_reloader')
+    )
 
 
 if __name__ == '__main__':
-    main()
+    main()

wes_elixir/celery_worker.py

@@ -1,5 +1,10 @@
+from wes_elixir.config.app_config import parse_app_config
 from wes_elixir.factories.celery_app import create_celery_app
 from wes_elixir.factories.connexion_app import create_connexion_app
 
 
-celery = create_celery_app(create_connexion_app(add_api=False))
+# Parse app configuration
+config = parse_app_config(config_var='WES_CONFIG')
+
+# Create Celery app
+celery = create_celery_app(create_connexion_app(config))

wes_elixir/config/app_config.py

@@ -0,0 +1,39 @@
+import logging
+import os
+import sys
+
+from wes_elixir.config.config_parser import YAMLConfigParser
+
+
+# Get logger instance
+logger = logging.getLogger(__name__)
+
+
+def parse_app_config(
+    config_var=None,
+    default_path=os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'app_config.yaml'))
+):
+
+    '''Parse configuration file and add to Connexion app'''
+
+    # Create parser instance
+    config = YAMLConfigParser()
+
+    # Parse config
+    try:
+        path = config.update_from_file_or_env(config_var=config_var, config_path=default_path)
+
+    # Abort if no config file was found/accessible 
+    except (FileNotFoundError, PermissionError):
+        logger.critical("No config file found. A config file needs to be available at '{default_path}'. Alternatively, point the environment variable '{config_var}' to its location. Execution aborted.".format(
+                default_path=default_path,
+                config_var=config_var,
+        ))
+        raise SystemExit(1)
+
+    # Log info 
+    else:
+        logger.info("App config loaded from '{path}'.".format(path=path))
+
+    # Return config
+    return config

wes_elixir/config/config.yaml → wes_elixir/config/app_config.yaml

@@ -3,8 +3,9 @@ server:
     host: localhost
     port: 7777
     debug: True
-    env: development
+    environment: development
     testing: False
+    use_reloader: False
 
 # Database settings
 database:
@@ -23,14 +24,21 @@ storage:
 
 # Celery task queuer
 celery:
-    result_backend: rpc://
     broker_url: pyamqp://localhost:5672//
+    result_backend: rpc://
+    include:
+        - wes_elixir.ga4gh.wes.utils_bg_tasks
     monitor:
         timeout: 0.1
 
 # OpenAPI specs
-openapi:
-    wes_yaml_specs: "ga4gh/wes/ga4gh.wes.0_3_0.openapi.yaml"
+api:
+    specs: 
+        - path: "ga4gh.wes.0_3_0.openapi.yaml"
+          strict_validation: True
+          validate_responses: True
+          swagger_ui: True
+          swagger_json: True
 
 # WES service info settings
 service_info:
@@ -53,7 +61,7 @@ service_info:
           default_value: "5"
     tags:
         known_tes_endpoints: "https://tes-dev.tsi.ebi.ac.uk/"
-        wes_elixir_version: 0.4.0
+        wes_elixir_version: 0.5.0
 
 # Endpoint parameters
 api_endpoints: