Update by ialbert on Fri Nov 3 23:17:13 DST 2017 from beep

biostar/engine/management/commands/job.py

@@ -5,43 +5,35 @@
 from django.utils.text import force_text
 from django.conf import settings
 
-
-
 logger = logging.getLogger('engine')
 
 CURR_DIR = os.path.dirname(os.path.realpath(__file__))
 
 
-def summarize_parameters(data):
+def summarize(data):
     '''
-    Summarises job parameters.
+    Summarizes job parameters.
     '''
-    summary = dict()
-    parameters = []
 
-    for param, details in data.items():
-        try:
-            if 'display_type' in details.keys():
-                if 'path' in details.keys():
-                    summary[param] = data[param]['name']
+    # Keep only fields that can be displayed.
+    objs = [obj for obj in data.values() if 'display_type' in obj]
+
+    def get_value(obj):
+        # Extract the parameter value from the object.
+        value = obj.get('name', '?') if 'path' in obj else obj.get('value', '?')
+        return value
 
-                if 'value' in details.keys():
-                    summary[param] = data[param]['value']
-        except KeyError:
-            print("KeyError while parsing parameters for job summary")
+    pairs = [(obj.get('label', 'Label'), get_value(obj)) for obj in objs]
+    patts = [f'- {a} = `{b}`' for a, b in pairs]
 
-    # format summary as a string.
-    for key, value in summary.items():
-        outline = "=".join([key, str(value)])
-        parameters.append(outline)
+    summary = "\n".join(patts)
 
-    summary_string = "\n".join(parameters)
-    return summary_string
+    return summary
 
 
 def run(job, options={}):
-    ''''
-    Runs a json
+    '''
+    Runs a job
     '''
     # Options that cause early termination.
     show_json = options.get('show_json')
@@ -52,7 +44,6 @@ def run(job, options={}):
     use_json = options.get('use_json')
     verbosity = options.get('verbosity', 0)
 
-
     # Defined in case we bail on errors before setting it.
     script = command = proc = None
 
@@ -113,7 +104,6 @@ def extra_context(job):
             print(full_command)
             return
 
-
         template = Template(template)
         context = Context(json_data)
         script = template.render(context)
@@ -127,7 +117,7 @@ def extra_context(job):
             return
 
         # Logging should start after the early returns.
-        logger.info(f'job id={job.id} started.')
+        logger.info(f'job id={job.id}, name={job.name}')
 
         # Make the output directory
         logger.info(f'job id={job.id} work_dir: {work_dir}')
@@ -149,7 +139,7 @@ def extra_context(job):
         job.state = job.RUNNING
 
         # Summarize input parameters.
-        job.summary = summarize_parameters(json_data)
+        job.summary = f'{job.summary}\n\n{summarize(json_data)}'
 
         job.save()
 
@@ -195,8 +185,8 @@ def extra_context(job):
     # Use -v 2 to see the output of the command.
     if verbosity > 1:
         job = Job.objects.get(id=job.id)
-        print ("-" * 40)
-        print (job.stdout_log)
+        print("-" * 40)
+        print(job.stdout_log)
         print("-" * 40)
         print(job.stderr_log)
 
@@ -207,9 +197,9 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
 
         parser.add_argument('--next',
-                        action='store_true',
-                        default=False,
-                        help="Runs the oldest queued job")
+                            action='store_true',
+                            default=False,
+                            help="Runs the oldest queued job")
 
         parser.add_argument('--id',
                             type=int,
@@ -242,8 +232,6 @@ def add_arguments(self, parser):
                             action='store_true',
                             help="Show most recent 10 queued.")
 
-
-
     def handle(self, *args, **options):
 
         jobid = options['id']

biostar/engine/models.py

@@ -1,6 +1,7 @@
-import logging, hjson
-
+import hjson
+import logging
 import mistune
+
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
 from django.db import models
@@ -9,7 +10,8 @@
 from django.urls import reverse
 from django.utils import timezone
 
-from . import util, settings
+from biostar import settings
+from . import util
 from .const import *
 
 logger = logging.getLogger("engine")

biostar/engine/static/style.css

@@ -3,7 +3,6 @@ body {
     padding-top: 1rem;
 }
 
-
 .row:hover {
     text-decoration: underline;
 }
@@ -13,6 +12,14 @@ h1 {
     font-size: 1.4rem;
 }
 
+code {
+    color: #c7254e;
+    background: #f5f2f0;
+    margin: 2px;
+    padding: 2px;
+    border-radius: 3px;
+   line-height: 1.7;
+}
 
 .muted {
     color: #999999;

biostar/engine/templates/job_view.html

@@ -34,17 +34,17 @@
         </div>
 
         <div class="ui vertical segment">
-            <div class="ui center aligned header">Job Input Parameters</div>
-            <pre>{{ job.summary }}</pre>
+            <div class="ui center aligned header">Input Parameters</div>
+            {{ job.summary|markdown|safe }}
         </div>
 
         <div class="ui vertical segment">
-            <div class="ui center aligned header">Job Standard Out Log</div>
+            <div class="ui center aligned header">Standard Out Log</div>
             <pre>{{ job.stdout_log }}</pre>
         </div>
 
          <div class="ui vertical segment">
-            <div class="ui center aligned header">Job Standard Error Log</div>
+            <div class="ui center aligned header">Standard Error Log</div>
             <pre>{{ job.stderr_log }}</pre>
         </div>
 

biostar/engine/urls.py

@@ -1,38 +1,14 @@
-"""website URL Configuration
-
-The `urlpatterns` list routes URLs to views. For more information please see:
-    https://docs.djangoproject.com/en/1.11/topics/http/urls/
-Examples:
-Function views
-    1. Add an import:  from my_app import views
-    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
-Class-based views
-    1. Add an import:  from other_app.views import Home
-    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
-Including another URLconf
-    1. Import the include() function: from django.conf.urls import url, include
-    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
-"""
-from django.conf.urls import url, include
-from django.contrib import admin
-from django.conf.urls.static import static
-from django.conf import settings
-
-import biostar.accounts.urls as accounts_urls
+from django.conf.urls import url
 from . import views
 
 urlpatterns = [
     url(r'^$', views.index, name="index"),
+
     url(r'^info/$', views.info, name="info"),
 
     # Engine specific admin site.
     url(r'^site/admin/', views.site_admin, name='site_admin'),
 
-    # This is the django generated admin site.
-    url(r'^django/admin/', admin.site.urls, name='django_admin'),
-
-    url(r'^accounts/', include(accounts_urls)),
-
     url(r'^project/create/$', views.project_create, name='project_create'),
     url(r'^project/list/$', views.project_list, name='project_list'),
     url(r'^project/view/(?P<id>\d+)/$', views.project_view, name='project_view'),
@@ -58,6 +34,3 @@
 
 ]
 
-if settings.DEBUG:
-    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT, show_indexes=True)
-    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT, show_indexes=True)

biostar/engine/settings.py → biostar/settings.py

@@ -17,7 +17,8 @@
 SITE_ID = 1
 SITE_DOMAIN = "localhost"
 SITE_NAME = "Biostar Engine"
-SITE_HEADER = '<i class="barcode icon"></i> Metagenomics Barcode Data Repository'
+SITE_HEADER = '<i class="barcode icon"></i> Bioinformatics Recipes'
+
 
 def join(*args):
     return os.path.abspath(os.path.join(*args))
@@ -29,9 +30,6 @@ def join(*args):
 
 ADMIN_GROUP_NAME = "Admins"
 
-# The id of the demo project.
-DEMO_PROJECT_UID = "demo"
-
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(join(__file__))
 
@@ -75,7 +73,7 @@ def join(*args):
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
 
-ROOT_URLCONF = 'biostar.engine.urls'
+ROOT_URLCONF = 'biostar.urls'
 
 TEMPLATES = [
     {
@@ -96,17 +94,12 @@ def join(*args):
     },
 ]
 
-WSGI_APPLICATION = 'biostar.engine.wsgi.application'
-ACCOUNT_EMAIL_REQUIRED = True
-ACCOUNT_LOGIN_ON_EMAIL_CONFIRMATION = True
-
-#LOGIN_URL = 'accounts/login/'
-#LOGOUT_URL = 'accounts/logout/'
+WSGI_APPLICATION = 'biostar.wsgi.application'
 
 # Database
 # https://docs.djangoproject.com/en/1.11/ref/settings/#databases
 
-DATABASE_NAME = join(BASE_DIR, '..', '..', 'export', 'database', 'engine.db')
+DATABASE_NAME = join(BASE_DIR, '..', 'export', 'database', 'engine.db')
 
 DATABASES = {
     'default': {
@@ -143,16 +136,16 @@ def join(*args):
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/1.11/howto/static-files/
 
-MEDIA_ROOT = join(BASE_DIR, '..', '..', 'export', 'media')
+MEDIA_ROOT = join(BASE_DIR, '..', 'export', 'media')
 
 # The location of resusable data.
-LOCAL_ROOT = join(BASE_DIR, '..', '..', 'export', 'local')
+LOCAL_ROOT = join(BASE_DIR, '..', 'export', 'local')
 
 MEDIA_URL = '/media/'
 STATIC_URL = '/static/'
-STATIC_ROOT = join(BASE_DIR, '..', '..', 'export', 'static')
+STATIC_ROOT = join(BASE_DIR, '..', 'export', 'static')
 STATICFILES_DIRS = [
-    join(BASE_DIR, "static"),
+    join(BASE_DIR, "engine", "static"),
 ]
 
 STATICFILES_FINDERS = (

biostar/tools/fastqc/fastqc.hjson

@@ -2,6 +2,7 @@
 
 sequence : {
     path: export/local/input/test_reads.fastq.gz
+    label: Input
     data_type: FASTQ_TYPE
     display_type: DROPDOWN
   }

biostar/urls.py

@@ -0,0 +1,25 @@
+from django.conf.urls import url, include
+from django.contrib import admin
+from django.conf.urls.static import static
+from django.conf import settings
+
+import biostar.accounts.urls as accounts_urls
+import biostar.engine.urls as engine_urls
+
+urlpatterns = [
+
+    # The engine handler.
+    url(r'^', include(engine_urls)),
+
+    # The user account handler.
+    url(r'^accounts/', include(accounts_urls)),
+
+    # The django generated admin site.
+    url(r'^django/admin/', admin.site.urls, name='django_admin'),
+
+]
+
+if settings.DEBUG:
+    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT, show_indexes=True)
+    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT, show_indexes=True)
+

biostar/engine/wsgi.py → biostar/wsgi.py

@@ -11,6 +11,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "biostar.engine.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "biostar.settings")
 
 application = get_wsgi_application()

conf/devel/devel_settings.py

@@ -1,7 +1,12 @@
-from biostar.engine.settings import *
+from biostar.settings import *
 
-DEBUG=True
+DEBUG = True
 
 WSGI_APPLICATION = 'conf.devel.devel_wsgi.application'
 
-ALLOWED_HOSTS = [ 'localhost', 'www.lvh.me' ]
+ALLOWED_HOSTS = ['localhost', 'www.lvh.me']
+
+try:
+    from .devel_secrets import *
+except ImportError as exc:
+    print("No devel_secrets module could be imported")

conf/devel/devel_wsgi.py

@@ -4,7 +4,7 @@
 
 from django.core.wsgi import get_wsgi_application
 
-# Override the DJANGO SETTINGS MODULE
+# Override the DJANGO SETTINGS MODULE.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conf.devel.devel_settings")
 
 application = get_wsgi_application()

conf/main/main_settings.py

@@ -1,8 +1,18 @@
-from biostar.engine.settings import *
-from .main_secrets import *
+from biostar.settings import *
 
 DEBUG = False
 
+SITE_ID = 1
+SITE_DOMAIN = "www.metabarcode.com"
+SITE_NAME = "Metabarcode Site"
+
+ALLOWED_HOSTS = [SITE_DOMAIN]
+
 WSGI_APPLICATION = 'conf.main.main_wsgi.application'
 
-ALLOWED_HOSTS = ['metabarcode.com', 'www.metabarcode.com']
+SITE_HEADER = '<i class="barcode icon"></i> Metagenomics Barcode Data Repository'
+
+try:
+    from .main_secrets import *
+except ImportError as exc:
+    print("No main_secrets module could be imported")