diff --git a/paperboy.db b/paperboy.db index 8a2454d..26dc6f2 100644 Binary files a/paperboy.db and b/paperboy.db differ diff --git a/paperboy/assets/static/js/bundle.js b/paperboy/assets/static/js/bundle.js index 194e32f..f468312 100644 --- a/paperboy/assets/static/js/bundle.js +++ b/paperboy/assets/static/js/bundle.js @@ -40762,7 +40762,7 @@ exports.i(__webpack_require__(215), ""); exports.i(__webpack_require__(216), ""); // module -exports.push([module.i, "/*-----------------------------------------------------------------------------\n| Copyright (c) 2014-2017, PhosphorJS Contributors\n|\n| Distributed under the terms of the BSD 3-Clause License.\n|\n| The full license is in the file LICENSE, distributed with this software.\n|----------------------------------------------------------------------------*/\n\n\n/**********/\n/* common */\n/**********/\nbody {\n display: flex;\n flex-direction: column;\n position: absolute;\n font-family: 'Roboto';\n top: 0;\n left: 0;\n right: 0;\n bottom: 0;\n margin: 0;\n padding: 0;\n overflow: hidden;\n font-family: Arial;\n}\n\n::-webkit-scrollbar {\n width: 5px;\n background: transparent;\n}\n::-webkit-scrollbar-thumb {\n background: var(--highlight-blue);\n}\n\ntextarea.json {\n min-height: 500px;\n}\n\n/**********/\n/* /common */\n/**********/\n\nbody.dark {\n color: var(--dark-color);\n background-color: var(--dark-bg-color2);\n}\nbody.dark a {\n color: var(--dark-color);\n}\n\nbody.light {\n color: var(--light-color);\n background-color: var(--light-bg-color2);\n}\n\nbody.light a {\n color: var(--light-color);\n}\n\nbody div.footer {\n position: absolute;\n display: flex;\n flex-direction: row;\n align-items: center;\n justify-content: center;\n width: 100%;\n bottom: 10px;\n}\n\nbutton {\n width: 100px;\n}\n\nbutton:hover {\n color: white;\n background-color: var(--highlight-blue);\n}\n\n#menuBar {\n flex: 0 0 auto;\n height: 45px;\n margin-left: 85%;\n display:flex;\n justify-content: center;\n}\n\nbody.dark #menuBar {\n background-color: var(--dark-bg-color2);\n color: var(--dark-color);\n}\n\nbody.light #menuBar {\n background-color: var(--light-bg-color2);\n color: var(--light-color);\n}\n\n#main {\n flex: 1 1 auto;\n top: -45px;\n}\n\n\n#main > .p-TabBar > .p-TabBar-content {\n margin-left: 30%;\n margin-right: 30%;\n}\n\nbody.dark #main > .p-TabBar > .p-TabBar-content {\n border-bottom: 1px solid var(--dark-border);\n}\n\nbody.light #main > .p-TabBar > .p-TabBar-content {\n border-bottom: 1px solid var(--light-border);\n}\n\n.p-TabBar > .p-TabBar-content > .p-TabBar-tab {\n display: flex;\n align-items: center;\n text-align: center;\n}\n\nbody.dark #main > .p-TabBar > .p-TabBar-content > .p-mod-current,\nbody.light #main > .p-TabBar > .p-TabBar-content > .p-mod-current {\n border-bottom:3px solid var(--highlight-blue);\n}\n\n#palette {\n min-width: 300px;\n}\n\nbody.dark #palette {\n border-right: 1px solid var(--dark-border);\n}\n\nbody.light #palette {\n border-right: 1px solid var(--light-border);\n}\n\n\n#dock {\n padding: 4px;\n}\n\nbody.dark #dock {\n background-color: var(--dark-bg-color);\n}\n\nbody.light #dock {\n background-color: var(--light-bg-color);\n}\n\nbody.dark .p-SplitPanel {\n background-color: var(--dark-bg-color);\n}\n\n\nbody.light .p-SplitPanel {\n background-color: var(--light-bg-color);\n}\n\nbody.dark tr:hover > td {\n background-color: var(--dark-bg-color2)\n}\n\nbody.light tr:hover > td {\n background-color: var(--light-bg-color2)\n}\n\ndiv.details {\n overflow-y: scroll;\n}\n\ndiv.details > table {\n width: 500px;\n margin-left: auto;\n margin-right: auto;\n margin-top: 5px;\n}\n\n\n/****** Tables ******/\n.uploader, .scheduler, .configurator,\n.primary {\n display: flex;\n flex-direction: column;\n overflow-y: scroll;\n}\n\n.primary > table {\n height:95%;\n text-align: left;\n}\n\n.primary form > p {\n height:2.5%;\n margin: auto;\n margin-top: 0;\n margin-bottom: 0;\n}\n\nbody.dark .primary > p span.page:hover {\n color: white;\n}\n\nbody.light .primary > p span.page:hover {\n color: #999;\n}\n\n.primary > p span.page-active {\n color: var(--highlight-orange);\n}\n\n\n.primary > table > tr:first-child {\n height: 20px;\n max-height: 20px;\n}\n\n.primary > table > tr > th {\n padding: 5px;\n}\n\n.primary > p {\n margin-top:0px;\n margin-bottom:0px;\n}\n\nbody.dark .primary > table > tr > th {\n border-bottom: 1px solid var(--dark-border);\n border-right: 1px solid var(--dark-border);\n}\n\nbody.light .primary > table > tr > th {\n border-bottom: 1px solid var(--light-border);\n border-right: 1px solid var(--light-border);\n}\n\n.primary > table > tr > td {\n padding: 5px;\n}\n\nbody.dark .primary > table > tr > td {\n border-bottom: 1px solid var(--dark-border);\n border-right: 1px solid var(--dark-border);\n}\n\nbody.light .primary > table > tr > td {\n border-bottom: 1px solid var(--light-border);\n border-right: 1px solid var(--light-border);\n}\n\n/****** ******/\n\n\n/****** Forms ******/\n.p-DockPanel-widget > form {\n width: 500px;\n margin-left: auto;\n margin-right: auto;\n margin-top: 5px;\n}\n\n\nform > table {\n width:100%;\n}\n\nform > table > tr> td > * {\n min-height:20px;\n min-width: 250px !important;\n width: 250px;\n\n margin-bottom: 0px !important;\n border: none !important;\n}\n\nform > table > tr> td > textarea {\n min-width: 250px !important;\n min-height: 300px !important;\n width:100%;\n}\n\nform > table > tr> td > select {\n min-width: 256px !important;\n}\n\nform > table > tr> td > input[type=checkbox] {\n /*margin-left:10px;*/\n}\n\n\nform > table > tr> td > input[type=submit] {\n min-width: 200px !important;\n width: 200px;\n}\n\nform > table > tr> td > input[type=submit]:hover {\n color: white;\n background-color: var(--highlight-blue);\n}\n\n.notebooks .uploader form {\n min-height: 275px;\n}\n\n.jobs .scheduler form {\n min-height: 600px;\n}\n\n.reports .configurator form {\n min-height: 350px;\n}\n/****** ******/\n", ""]); +exports.push([module.i, "/*-----------------------------------------------------------------------------\n| Copyright (c) 2014-2017, PhosphorJS Contributors\n|\n| Distributed under the terms of the BSD 3-Clause License.\n|\n| The full license is in the file LICENSE, distributed with this software.\n|----------------------------------------------------------------------------*/\n\n\n/**********/\n/* common */\n/**********/\nbody {\n display: flex;\n flex-direction: column;\n position: absolute;\n font-family: 'Roboto';\n top: 0;\n left: 0;\n right: 0;\n bottom: 0;\n margin: 0;\n padding: 0;\n overflow: hidden;\n font-family: Arial;\n}\n\n::-webkit-scrollbar {\n width: 5px;\n background: transparent;\n}\n::-webkit-scrollbar-thumb {\n background: var(--highlight-blue);\n}\n\ntextarea.json {\n min-height: 500px;\n}\n\n/**********/\n/* /common */\n/**********/\n\nbody.dark {\n color: var(--dark-color);\n background-color: var(--dark-bg-color2);\n}\nbody.dark a {\n color: var(--dark-color);\n}\n\nbody.light {\n color: var(--light-color);\n background-color: var(--light-bg-color2);\n}\n\nbody.light a {\n color: var(--light-color);\n}\n\nbody div.footer {\n position: absolute;\n display: flex;\n flex-direction: row;\n align-items: center;\n justify-content: center;\n width: 100%;\n bottom: 10px;\n}\n\nbutton {\n width: 100px;\n}\n\nbutton:hover {\n color: white;\n background-color: var(--highlight-blue);\n}\n\n#menuBar {\n flex: 0 0 auto;\n height: 45px;\n margin-left: 85%;\n display:flex;\n justify-content: center;\n}\n\nbody.dark #menuBar {\n background-color: var(--dark-bg-color2);\n color: var(--dark-color);\n}\n\nbody.light #menuBar {\n background-color: var(--light-bg-color2);\n color: var(--light-color);\n}\n\n#main {\n flex: 1 1 auto;\n top: -45px;\n}\n\n\n#main > .p-TabBar > .p-TabBar-content {\n margin-left: 30%;\n margin-right: 30%;\n}\n\nbody.dark #main > .p-TabBar > .p-TabBar-content {\n border-bottom: 1px solid var(--dark-border);\n}\n\nbody.light #main > .p-TabBar > .p-TabBar-content {\n border-bottom: 1px solid var(--light-border);\n}\n\n.p-TabBar > .p-TabBar-content > .p-TabBar-tab {\n display: flex;\n align-items: center;\n text-align: center;\n}\n\nbody.dark #main > .p-TabBar > .p-TabBar-content > .p-mod-current,\nbody.light #main > .p-TabBar > .p-TabBar-content > .p-mod-current {\n border-bottom:3px solid var(--highlight-blue);\n}\n\n#palette {\n min-width: 300px;\n}\n\nbody.dark #palette {\n border-right: 1px solid var(--dark-border);\n}\n\nbody.light #palette {\n border-right: 1px solid var(--light-border);\n}\n\n\n#dock {\n padding: 4px;\n}\n\nbody.dark #dock {\n background-color: var(--dark-bg-color);\n}\n\nbody.light #dock {\n background-color: var(--light-bg-color);\n}\n\nbody.dark .p-SplitPanel {\n background-color: var(--dark-bg-color);\n}\n\n\nbody.light .p-SplitPanel {\n background-color: var(--light-bg-color);\n}\n\nbody.dark tr:hover > td {\n background-color: var(--dark-bg-color2)\n}\n\nbody.light tr:hover > td {\n background-color: var(--light-bg-color2)\n}\n\ndiv.details {\n overflow-y: scroll;\n}\n\ndiv.details > table {\n width: 500px;\n margin-left: auto;\n margin-right: auto;\n margin-top: 5px;\n}\n\n\n/****** Tables ******/\n.uploader, .scheduler, .configurator,\n.primary {\n display: flex;\n flex-direction: column;\n overflow-y: scroll;\n}\n\n.primary > table {\n height:95%;\n text-align: left;\n}\n\n.primary form > p {\n height:2.5%;\n margin: auto;\n margin-top: 0;\n margin-bottom: 0;\n}\n\nbody.dark .primary > p span.page:hover {\n color: white;\n}\n\nbody.light .primary > p span.page:hover {\n color: #999;\n}\n\n.primary > p span.page-active {\n color: var(--highlight-orange);\n}\n\n\n.primary > table > tr:first-child {\n height: 20px;\n max-height: 20px;\n}\n\n.primary > table > tr > th {\n padding: 5px;\n}\n\n.primary > p {\n margin-top:0px;\n margin-bottom:0px;\n}\n\nbody.dark .primary > table > tr > th {\n border-bottom: 1px solid var(--dark-border);\n border-right: 1px solid var(--dark-border);\n}\n\nbody.light .primary > table > tr > th {\n border-bottom: 1px solid var(--light-border);\n border-right: 1px solid var(--light-border);\n}\n\n.primary > table > tr > td {\n padding: 5px;\n}\n\nbody.dark .primary > table > tr > td {\n border-bottom: 1px solid var(--dark-border);\n border-right: 1px solid var(--dark-border);\n}\n\nbody.light .primary > table > tr > td {\n border-bottom: 1px solid var(--light-border);\n border-right: 1px solid var(--light-border);\n}\n\n/****** ******/\n\n\n/****** Forms ******/\n.p-DockPanel-widget > form {\n width: 40%;\n margin-left: auto;\n margin-right: auto;\n margin-top: 5px;\n}\n\n\nform > table {\n width:100%;\n height: 100%;\n}\n\nform > table > tr > td {\n min-height: 25px;\n}\n\nform > table > tr > td:nth-child(2) {\n background-color: transparent !important;\n}\n\nform > table > tr> td:first-child {\n width: 25%;\n text-align: center;\n}\n\nform > table > tr> td > * {\n min-height:35px;\n min-width: 250px !important;\n width: 250px;\n height: 35px;\n\n margin-bottom: 0px !important;\n border: none !important;\n}\n\nform > table > tr> td > textarea {\n min-width: 250px !important;\n min-height: 300px !important;\n width:100%;\n}\n\nform > table > tr> td > select {\n min-width: 256px !important;\n}\n\nform > table > tr> td > input[type=checkbox] {\n /*margin-left:10px;*/\n}\n\n\nform > table > tr> td > input[type=submit] {\n min-width: 200px !important;\n width: 200px;\n}\n\nform > table > tr> td > input[type=submit]:hover {\n color: white;\n background-color: var(--highlight-blue);\n}\n\n.notebooks .uploader form {\n height: 350px;\n}\n\n.notebooks-detail form {\n height: 600px;\n}\n\n.jobs .scheduler form {\n height: 850px;\n}\n\n.jobs-detail form {\n height: 450px;\n\n}\n\n.reports .configurator form {\n height: 700px;\n}\n\n.reports-detail form {\n height: 750px;\n}\n/****** ******/\n", ""]); // exports diff --git a/paperboy/config/base.py b/paperboy/config/base.py index 7e67aa1..22ddc77 100644 --- a/paperboy/config/base.py +++ b/paperboy/config/base.py @@ -1,7 +1,7 @@ from traitlets import HasTraits _INTERVAL_TYPES = ('minutely', '5 minutes', '10 minutes', '30 minutes', 'hourly', '2 hours', '3 hours', '6 hours', '12 hours', 'daily', 'weekly', 'monthly') -_REPORT_TYPES = ('run', 'publish') +_REPORT_TYPES = ('convert', 'publish') _OUTPUT_TYPES = ('notebook', 'pdf', 'html', 'email', 'script') _PRIVACY_LEVELS = ('public', 'private') _SERVICE_LEVELS = ('production', 'research', 'development', 'personal') diff --git a/paperboy/scheduler/airflow/airflow.py b/paperboy/scheduler/airflow/airflow.py index c327528..6f4f5f4 100644 --- a/paperboy/scheduler/airflow/airflow.py +++ b/paperboy/scheduler/airflow/airflow.py @@ -9,7 +9,7 @@ from sqlalchemy import create_engine from ..base import BaseScheduler, TIMING_MAP -with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'paperboy.airflow.py')), 'r') as fp: +with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'paperboy.convert.airflow.py')), 'r') as fp: TEMPLATE = fp.read() QUERY = ''' diff --git a/paperboy/scheduler/airflow/airflow_operators/__init__.py b/paperboy/scheduler/airflow/airflow_operators/__init__.py index 2abf77f..5dbbb96 100644 --- a/paperboy/scheduler/airflow/airflow_operators/__init__.py +++ b/paperboy/scheduler/airflow/airflow_operators/__init__.py @@ -1 +1,6 @@ -from .operators import JobOperator, JobCleanupOperator, ReportOperator, PapermillOperator, NBConvertOperator, ReportPostOperator +from .common import JobOperator, JobCleanupOperator, ReportOperator +from .dokku import DokkuOperator +from .nbconvert import NBConvertOperator +from .papermill import PapermillOperator +from .post import ReportPostOperator +from .voila import VoilaOperator diff --git a/paperboy/scheduler/airflow/airflow_operators/common.py b/paperboy/scheduler/airflow/airflow_operators/common.py new file mode 100644 index 0000000..db5c50d --- /dev/null +++ b/paperboy/scheduler/airflow/airflow_operators/common.py @@ -0,0 +1,53 @@ +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults +from paperboy.utils import name_to_class + + +class JobOperator(BaseOperator): + @apply_defaults + def __init__(self, job, *args, **kwargs): + super(JobOperator, self).__init__(*args, **kwargs) + self.job = job + + def execute(self, context): + self.log.critical('job') + + +class JobCleanupOperator(BaseOperator): + @apply_defaults + def __init__(self, job, *args, **kwargs): + super(JobCleanupOperator, self).__init__(*args, **kwargs) + self.job = job + + def execute(self, context): + self.log.critical('job-cleanup') + + +class ReportOperator(BaseOperator): + @apply_defaults + def __init__(self, report, *args, **kwargs): + super(ReportOperator, self).__init__(*args, **kwargs) + self.report = report + + def execute(self, context): + self.log.critical('report') + + +class ReportPostOperator(BaseOperator): + @apply_defaults + def __init__(self, report, config, *args, **kwargs): + super(ReportPostOperator, self).__init__(*args, **kwargs) + self.report = report + self.nbconvert_task_id = self.task_id.replace('ReportPost', 'ReportNBConvert') + + self.config = name_to_class(config.get('config')).from_json(config) + + def execute(self, context): + self.log.critical('report-post') + + task_instance = context['task_instance'] + output_nb = task_instance.xcom_pull(task_ids=self.nbconvert_task_id, key=self.task_id) + self.log.critical(output_nb) + + outputter = self.config.clazz(self.config) + outputter.write(self.report, output_nb, task_id=self.task_id) diff --git a/paperboy/scheduler/airflow/airflow_operators/dokku.py b/paperboy/scheduler/airflow/airflow_operators/dokku.py new file mode 100644 index 0000000..88c3314 --- /dev/null +++ b/paperboy/scheduler/airflow/airflow_operators/dokku.py @@ -0,0 +1,23 @@ +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults +from paperboy.utils import name_to_class + + +class DokkuOperator(BaseOperator): + @apply_defaults + def __init__(self, report, config, *args, **kwargs): + super(DokkuOperator, self).__init__(*args, **kwargs) + self.report = report + self.nbconvert_task_id = self.task_id.replace('ReportPost', 'ReportNBConvert') + + self.config = name_to_class(config.get('config')).from_json(config) + + def execute(self, context): + self.log.critical('report-post') + + task_instance = context['task_instance'] + output_nb = task_instance.xcom_pull(task_ids=self.nbconvert_task_id, key=self.task_id) + self.log.critical(output_nb) + + outputter = self.config.clazz(self.config) + outputter.write(self.report, output_nb, task_id=self.task_id) diff --git a/paperboy/scheduler/airflow/airflow_operators/nbconvert.py b/paperboy/scheduler/airflow/airflow_operators/nbconvert.py new file mode 100644 index 0000000..58f532e --- /dev/null +++ b/paperboy/scheduler/airflow/airflow_operators/nbconvert.py @@ -0,0 +1,33 @@ +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults + + +class NBConvertOperator(BaseOperator): + @apply_defaults + def __init__(self, report, *args, **kwargs): + super(NBConvertOperator, self).__init__(*args, **kwargs) + self.report = report + self.papermill_task_id = self.task_id.replace('ReportNBConvert', 'ReportPapermill') + self.report_post_task_iud = self.task_id.replace('ReportNBConvert', 'ReportPost') + + def execute(self, context): + self.log.critical('nbconvert') + + task_instance = context['task_instance'] + papermilled = task_instance.xcom_pull(task_ids=self.papermill_task_id, key=self.task_id) + + if self.report['meta']['output'] != 'notebook': + from paperboy.worker import run_nbconvert + + template = self.report['meta'].get('template', '') + + ret = run_nbconvert(self.report['meta']['notebook'], + papermilled, + self.report['meta']['output'], + template, + self.report['meta']['strip_code'], + ) + else: + ret = papermilled + + task_instance.xcom_push(key=self.report_post_task_iud, value=ret) diff --git a/paperboy/scheduler/airflow/airflow_operators/papermill.py b/paperboy/scheduler/airflow/airflow_operators/papermill.py new file mode 100644 index 0000000..54532eb --- /dev/null +++ b/paperboy/scheduler/airflow/airflow_operators/papermill.py @@ -0,0 +1,22 @@ +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults + + +class PapermillOperator(BaseOperator): + @apply_defaults + def __init__(self, report, *args, **kwargs): + super(PapermillOperator, self).__init__(*args, **kwargs) + self.report = report + self.nbconvert_task_id = self.task_id.replace('ReportPapermill', 'ReportNBConvert') + + def execute(self, context): + self.log.critical('papermill') + + from paperboy.worker import run_papermill + ret = run_papermill(self.report['meta']['notebook'], + self.report['meta']['notebook_text'], + self.report['meta']['parameters'], + self.report['meta']['strip_code']) + + task_instance = context['task_instance'] + task_instance.xcom_push(key=self.nbconvert_task_id, value=ret) diff --git a/paperboy/scheduler/airflow/airflow_operators/post.py b/paperboy/scheduler/airflow/airflow_operators/post.py new file mode 100644 index 0000000..3a8f19b --- /dev/null +++ b/paperboy/scheduler/airflow/airflow_operators/post.py @@ -0,0 +1,23 @@ +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults +from paperboy.utils import name_to_class + + +class ReportPostOperator(BaseOperator): + @apply_defaults + def __init__(self, report, config, *args, **kwargs): + super(ReportPostOperator, self).__init__(*args, **kwargs) + self.report = report + self.nbconvert_task_id = self.task_id.replace('ReportPost', 'ReportNBConvert') + + self.config = name_to_class(config.get('config')).from_json(config) + + def execute(self, context): + self.log.critical('report-post') + + task_instance = context['task_instance'] + output_nb = task_instance.xcom_pull(task_ids=self.nbconvert_task_id, key=self.task_id) + self.log.critical(output_nb) + + outputter = self.config.clazz(self.config) + outputter.write(self.report, output_nb, task_id=self.task_id) diff --git a/paperboy/scheduler/airflow/airflow_operators/voila.py b/paperboy/scheduler/airflow/airflow_operators/voila.py new file mode 100644 index 0000000..8f6b81c --- /dev/null +++ b/paperboy/scheduler/airflow/airflow_operators/voila.py @@ -0,0 +1,23 @@ +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults +from paperboy.utils import name_to_class + + +class VoilaOperator(BaseOperator): + @apply_defaults + def __init__(self, report, config, *args, **kwargs): + super(VoilaOperator, self).__init__(*args, **kwargs) + self.report = report + self.nbconvert_task_id = self.task_id.replace('ReportPost', 'ReportNBConvert') + + self.config = name_to_class(config.get('config')).from_json(config) + + def execute(self, context): + self.log.critical('report-post') + + task_instance = context['task_instance'] + output_nb = task_instance.xcom_pull(task_ids=self.nbconvert_task_id, key=self.task_id) + self.log.critical(output_nb) + + outputter = self.config.clazz(self.config) + outputter.write(self.report, output_nb, task_id=self.task_id) diff --git a/paperboy/scheduler/airflow/paperboy.airflow.py b/paperboy/scheduler/airflow/paperboy.airflow.py index 4c00eb3..a651d98 100644 --- a/paperboy/scheduler/airflow/paperboy.airflow.py +++ b/paperboy/scheduler/airflow/paperboy.airflow.py @@ -1,8 +1,18 @@ import json from base64 import b64decode + +# Base Job Operators from paperboy.scheduler.airflow_operators import JobOperator, JobCleanupOperator + +# Base Report Operators from paperboy.scheduler.airflow_operators import ReportOperator, ReportPostOperator + +# Convert Operators from paperboy.scheduler.airflow_operators import PapermillOperator, NBConvertOperator + +# Publish operators +from paperboy.scheduler.airflow_operators import VoilaOperator, DokkuOperator + from airflow import DAG from datetime import timedelta, datetime @@ -75,7 +85,6 @@ job_json = json.loads(b64decode({{job_json}})) reports_json = json.loads(b64decode({{report_json}})) - ################################### # Create dag from job and reports # ################################### @@ -94,31 +103,57 @@ # copy over notebook text (only store 1 copy in the job json) rep['meta']['notebook_text'] = job_json['meta']['notebook_text'] + type = rep['meta']['type'] + + # Job -> Report -> Papermill -> NBConvert -> ReportPost -\ + # \--> Report -> Papermill -> Voila -> Dokku -> ReportPost --\ + # \-> Report -> Papermill -> NBConvert -> ReportPost ----> Job Cleanup + # Report operator, performs the required # steps prior to running the report - r = ReportOperator(report=rep, task_id='Report-{}'.format(rep['id']), dag=dag) + r = ReportOperator(report=rep, + task_id='Report-{}'.format(rep['id']), + dag=dag) + job.set_downstream(r) # Papermill operator, performs the report creation # using papermill and the report's individual # parameters and configuration - pp = PapermillOperator(report=rep, task_id='ReportPapermill-{}'.format(rep['id']), dag=dag) - - # NBConvert operator, performs the NBConversion if - # required - nb = NBConvertOperator(report=rep, task_id='ReportNBConvert-{}'.format(rep['id']), dag=dag) + pp = PapermillOperator(report=rep, + task_id='ReportPapermill-{}'.format(rep['id']), + dag=dag) + r.set_downstream(pp) # The post-report operator, used for post-report - # tasks such as sending the report in an email, - # deploying the report to a webserver, etc + # tasks such as sending the report in an email rp = ReportPostOperator(report=rep, config=json.loads('{{output_config}}'), task_id='ReportPost-{}'.format(rep['id']), dag=dag) - - # Job -> Report -> ReportPost -\ - # \--> Report -> ReportPost --\ - # \-> Report -> ReportPost ----> Job Cleanup - job.set_downstream(r) - r.set_downstream(pp) - pp.set_downstream(nb) - nb.set_downstream(rp) rp.set_downstream(cleanup) + + if type == 'convert': + # NBConvert operator, performs the NBConversion if + # required + nb = NBConvertOperator(report=rep, + task_id='ReportNBConvert-{}'.format(rep['id']), + dag=dag) + pp.set_downstream(nb) + nb.set_downstream(rp) + + elif type == 'publish': + # Assemble a Voila Job from papermilled notebook + v = VoilaOperator(report=rep, + task_id='ReportVoila-{}'.format(rep['id']), + dag=dag) + pp.set_downstream(v) + + # Dokku deployment operator, performs the + # deploy to the dokku repo + d = DokkuOperator(report=rep, + task_id='ReportDokku-{}'.format(rep['id']), + dag=dag) + v.set_downstream(d) + d.set_downstream(rp) + + else: + raise NotImplementedError() diff --git a/ts/style/index.css b/ts/style/index.css index 28175bb..dc19ac8 100644 --- a/ts/style/index.css +++ b/ts/style/index.css @@ -265,7 +265,7 @@ body.light .primary > table > tr > td { /****** Forms ******/ .p-DockPanel-widget > form { - width: 500px; + width: 40%; margin-left: auto; margin-right: auto; margin-top: 5px; @@ -274,12 +274,27 @@ body.light .primary > table > tr > td { form > table { width:100%; + height: 100%; +} + +form > table > tr > td { + min-height: 25px; +} + +form > table > tr > td:nth-child(2) { + background-color: transparent !important; +} + +form > table > tr> td:first-child { + width: 25%; + text-align: center; } form > table > tr> td > * { - min-height:20px; + min-height:35px; min-width: 250px !important; width: 250px; + height: 35px; margin-bottom: 0px !important; border: none !important; @@ -311,14 +326,27 @@ form > table > tr> td > input[type=submit]:hover { } .notebooks .uploader form { - min-height: 275px; + height: 350px; +} + +.notebooks-detail form { + height: 600px; } .jobs .scheduler form { - min-height: 600px; + height: 850px; +} + +.jobs-detail form { + height: 450px; + } .reports .configurator form { - min-height: 350px; + height: 700px; +} + +.reports-detail form { + height: 750px; } /****** ******/