Merge pull request #37 from elixir-europe/dev

v0.6.0

.gitignore

@@ -224,3 +224,4 @@ cwl-tes/
 .vscode/
 tests/tmp/*
 tests/output/*
+.netrc

Dockerfile

@@ -0,0 +1,52 @@
+##### BASE IMAGE #####
+FROM ubuntu:16.04
+
+##### METADATA #####
+LABEL base.image="ubuntu:16.04"
+LABEL version="1"
+LABEL software="WES-ELIXIR"
+LABEL software.version="1.0"
+LABEL software.description="Flask microservice implementing the Global Alliance for Genomics and Health (GA4GH) Workflow Execution Service (WES) API specification."
+LABEL software.website="https://github.com/elixir-europe/WES-ELIXIR"
+LABEL software.documentation="https://github.com/elixir-europe/WES-ELIXIR"
+LABEL software.license="https://github.com/elixir-europe/WES-ELIXIR/blob/master/LICENSE"
+LABEL software.tags="General"
+LABEL maintainer="[email protected]"
+LABEL maintainer.organisation="Biozentrum, University of Basel"
+LABEL maintainer.location="Klingelbergstrasse 50/70, CH-4056 Basel, Switzerland"
+LABEL maintainer.lab="Zavolan Lab"
+LABEL maintainer.license="https://spdx.org/licenses/Apache-2.0"
+
+## install dependencies
+RUN apt-get update \
+  && apt-get install -y build-essential checkinstall libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev zlib1g-dev openssl libffi-dev python3-dev python3-setuptools git wget curl
+
+## install python
+RUN wget https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz \
+  && tar xJf Python-3.6.0.tar.xz \
+  && cd Python-3.6.0 \
+  && ./configure \
+  && make altinstall \
+  && ln -s /Python-3.6.0/python /usr/local/bin \
+  && cd ../ \
+  && python -m pip install --upgrade pip setuptools wheel virtualenv
+
+COPY ./ $HOME/WES-ELIXIR/
+
+## install WES-ELIXIR and cwl-tes
+#RUN git clone -b dockerize_app https://github.com/elixir-europe/WES-ELIXIR.git \
+
+RUN cd WES-ELIXIR \
+  && git clone https://github.com/common-workflow-language/cwl-tes.git \
+  && cd cwl-tes \
+  && git checkout ftp \
+  && git checkout ab58d1822a027eff2a456db9d712f5295ac42eac \
+  && python setup.py install \
+  && cd .. \
+  && pip install -r requirements.txt \
+  && python setup.py develop \
+  && cd ../
+
+ENV WES_CONFIG="/WES-ELIXIR/wes_elixir/config/app_config.yaml"
+
+COPY .netrc /root

docker-compose.yaml

@@ -0,0 +1,59 @@
+version: '3.6'
+services:
+
+  # nginx:
+  #   image: nginx:latest
+  #   restart: always
+  #   ports:
+  #     - "443:443"
+  #   links:
+  #     - wes-elixir
+  #   volumes:
+  #     - /docker/nginx/etc/ssl/:/etc/ssl/
+  #     - /docker/nginx/etc/nginx/conf.d/:/etc/nginx/conf.d
+
+  wes-elixir-celery:
+    image: wes-elixir:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: always
+    links:
+      - rabbit1
+    command: bash -c "cd /WES-ELIXIR/wes_elixir; celery worker -A celery_worker -E --loglevel=info"
+    volumes:
+      - ../data/tests:/WES-ELIXIR/tests
+
+
+  wes-elixir:
+    image: wes-elixir:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: always
+    ports:
+      - "7777:7777"
+    links:
+      - mongo
+    command: bash -c "cd /WES-ELIXIR/; python wes_elixir/app.py"
+    volumes:
+      - ../data/tests:/WES-ELIXIR/tests
+
+  rabbit1:
+    image: "rabbitmq:3-management"
+    hostname: "rabbit1"
+    ports:
+      - "15672:15672"
+      - "5672:5672"
+    links:
+      - mongo
+
+  mongo:
+    image: mongo:3.2
+    restart: always
+    ports:
+      - "27017:27017"
+    expose:
+      - "27017"
+    volumes:
+      - ../data/db:/data/db

requirements.txt

@@ -1,28 +1,60 @@
 amqp==2.3.2
 astroid==2.0.4
+attrs==18.2.0
+avro-cwl==1.8.4
+bagit==1.7.0
 billiard==3.5.0.4
+CacheControl==0.11.7
 celery==4.2.1
+certifi==2018.8.24
+chardet==3.0.4
 click==6.7
 clickclick==1.2.2
 connexion==1.5.2
+decorator==4.3.0
 Flask==1.0.2
 Flask-Cors==3.0.6
 Flask-PyMongo==2.1.0
+future==0.16.0
+idna==2.7
 inflection==0.3.1
+isodate==0.6.0
 isort==4.3.4
 itsdangerous==0.24
 Jinja2==2.10
 jsonschema==2.6.0
 kombu==4.2.1
 lazy-object-proxy==1.3.1
+lockfile==0.12.2
+lxml==4.2.5
 MarkupSafe==1.0
 mccabe==0.6.1
+mistune==0.7.4
+mypy-extensions==0.4.1
+networkx==2.2
+prov==1.5.1
+psutil==5.4.7
+py-tes==0.2.1
 pylint==2.1.1
 pymongo==3.7.1
+pyparsing==2.2.1
+python-dateutil==2.6.1
 pytz==2018.5
 PyYAML==3.13
+rdflib==4.2.2
+rdflib-jsonld==0.4.0
+requests==2.19.1
+ruamel.yaml==0.15.66
+scandir==1.9.0
+schema-salad==2.7.20180905124720
+shellescape==3.4.1
+six==1.11.0
+subprocess32==3.5.2
 swagger-spec-validator==2.3.1
 typed-ast==1.1.0
+typing==3.6.6
+typing-extensions==3.6.5
+urllib3==1.23
 vine==1.1.4
 Werkzeug==0.14.1
 wrapt==1.10.11

wes_elixir/app.py

@@ -40,9 +40,10 @@ def main():
 
     # Run app
     connexion_app.run(
+        host='0.0.0.0',
         use_reloader=get_conf(config, 'server', 'use_reloader')
     )
 
 
 if __name__ == '__main__':
-    main()
+    main()

wes_elixir/celery_worker.py

@@ -7,4 +7,4 @@
 config = parse_app_config(config_var='WES_CONFIG')
 
 # Create Celery app
-celery = create_celery_app(create_connexion_app(config))
+celery = create_celery_app(create_connexion_app(config))

wes_elixir/config/app_config.yaml

@@ -1,15 +1,15 @@
 # General server/service settings
 server:
-    host: localhost
+    host: '0.0.0.0'
     port: 7777
     debug: True
     environment: development
     testing: False
-    use_reloader: False
+    use_reloader: True
 
 # Database settings
 database:
-    host: 'localhost'
+    host: 'mongo'
     port: 27017
     name: wes-elixir-db
     run_id:
@@ -24,7 +24,7 @@ storage:
 
 # Celery task queuer
 celery:
-    broker_url: pyamqp://localhost:5672//
+    broker_url: pyamqp://rabbit1:5672//
     result_backend: rpc://
     include:
         - wes_elixir.ga4gh.wes.utils_bg_tasks
@@ -33,7 +33,7 @@ celery:
 
 # OpenAPI specs
 api:
-    specs: 
+    specs:
         - path: "ga4gh.wes.0_3_0.openapi.yaml"
           strict_validation: True
           validate_responses: True
@@ -61,7 +61,7 @@ service_info:
           default_value: "5"
     tags:
         known_tes_endpoints: "https://tes-dev.tsi.ebi.ac.uk/"
-        wes_elixir_version: 0.5.0
+        wes_elixir_version: 0.6.0
 
 # Endpoint parameters
 api_endpoints:

wes_elixir/database/db_utils.py

@@ -2,25 +2,40 @@
 from pymongo.collection import ReturnDocument
 
 
-def find_one_by_id(collection, object_id):
+def find_one_by_id(
+    collection,
+    object_id
+):
     '''Returns single object by object id, stripped of object id, or None if object not found'''
     return collection.find_one({'_id': ObjectId(object_id)}, {'_id': False})
 
 
-def find_one_by_index(collection, index, value):
+def find_one_by_index(
+    collection,
+    index,
+    value
+):
     '''Returns single object by index field value, stripped of object id, or None if object not found'''
     return collection.find_one({index: value}, {'_id': False})
 
 
-def find_one_field_by_index(collection, index, value, select):
+def find_one_field_by_index(
+    collection,
+    index,
+    value,
+    select
+):
     '''Returns single field from single object by index field value, stripped of object id, or None if object not found'''
     result = collection.find_one({index: value}, {select: True, '_id': False})
     if result is not None and select in result:
         result = result[select]
     return result
 
 
-def find_fields(collection, projection):
+def find_fields(
+    collection,
+    projection
+):
     '''Returns selected fields from all objects, stripped of object id, or None if no objects found'''
     projection_dict = {key:True for key in projection}
     projection_dict['_id'] = False
@@ -43,10 +58,28 @@ def find_id_latest(collection):
         return None
 
 
-def update_run_state(collection, task_id, state="UNKNOWN"):
+def update_run_state(
+    collection,
+    task_id,
+    state="UNKNOWN"
+):
     '''Update state of workflow run'''
     return collection.find_one_and_update(
         {"task_id": task_id},
         {"$set": {"api.state": state}},
         return_document=ReturnDocument.AFTER
     )
+
+
+def upsert_fields_in_root_object(
+    collection,
+    task_id,
+    root,
+    **kwargs
+):
+    '''Insert (or update) fields in(to) the same root (object) field'''
+    return collection.find_one_and_update(
+        {"task_id": task_id},
+        {"$set": {".".join([root, key]):value for (key,value) in kwargs.items()}},
+        return_document=ReturnDocument.AFTER
+    )

wes_elixir/factories/celery_app.py

@@ -28,6 +28,11 @@ def create_celery_app(app):
         calling_module=':'.join([stack()[1].filename, stack()[1].function])
     ))
 
+    # Set Celery options
+    # TODO: Hotfix to get around message truncation problem when processing STDOUT/STDERR
+    # TODO: Solve this differently (via result backend) as this is not very robust & may slow down system
+    celery.Task.resultrepr_maxsize = 200000
+
     # Update Celery app configuration with Flask app configuration
     celery.conf.update(app.app.config)
     logger.info("Celery app configured.")
@@ -40,4 +45,4 @@ def __call__(self, *args, **kwargs):
     celery.Task = ContextTask
     logger.debug("App context added to celery.Task class.")
 
-    return celery
+    return celery

wes_elixir/ga4gh/wes/utils_bg_tasks.py

@@ -17,15 +17,16 @@ def add_command_to_task_queue(
     bg_proc = subprocess.run(
         command_list,
         cwd=tmp_dir,
-        # TODO: Capture stdout/stderr
-        # TODO: Ensure they're captured as or converted to string not byte
-        # TODO: Ensure they are wrapped properly for long strings ("""?""")
-        #stdout=subprocess.PIPE,
-        #stderr=subprocess.PIPE,
-        #universal_newlines=True
+        universal_newlines=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
         # TODO: Use this from Python 3.7 on instead of stdout & stderr
         #capture_output=True
     )
 
+    # Convert STDOUT and STDERR to lists of lines
+    bg_proc.stdout = bg_proc.stdout.splitlines()
+    bg_proc.stderr = bg_proc.stderr.splitlines()
+
     # Return returncode, stdout, stderr and command args as dictionary
     return vars(bg_proc)