Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add tests for serializer using output_metadata field #134

Open
wants to merge 28 commits into
base: develop
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
d77be1d
add tests for base RoslinQcOperator class
stevekm Jan 29, 2020
563009c
add test to Makefile
stevekm Feb 4, 2020
ce2a2c5
Merge branch 'develop' into roslin-qc
stevekm Feb 4, 2020
da2449a
update fixture dump script for dumping Run's
stevekm Feb 5, 2020
bd4bf07
start Roslin QC operator and input parsing with tests and fixtures
stevekm Feb 5, 2020
a4816de
merge against develop
stevekm Feb 5, 2020
83bf428
add tests for serializer using output_metadata field
stevekm Feb 11, 2020
b1406d9
merge against develop
stevekm Feb 11, 2020
9e3419b
update demo run methods and input.json template
stevekm Feb 12, 2020
33a8c19
add fixture duplication script
stevekm Feb 15, 2020
6cd29d2
Merge branch 'duplication-script' into roslin-qc
stevekm Feb 15, 2020
d452748
add multiple run handling for roslin qc input, update fixtures and sc…
stevekm Feb 16, 2020
300bea2
add RoslinQCOperator to get by classname for Operator Factory
stevekm Feb 18, 2020
de58404
make Roslin operator data easier to read
stevekm Feb 19, 2020
e9670d1
update Makefile and operator tests
stevekm Feb 19, 2020
30284a2
add roslin qc pipeline fixture
stevekm Feb 19, 2020
84d182a
clean up Roslin operator
stevekm Feb 19, 2020
5eccbf1
add sorting to QC inputs for reproducibility
stevekm Feb 19, 2020
851ad77
update Roslin qc operator and tests
stevekm Feb 19, 2020
cdeb92d
disable bid to cwl conversion since its handled elsewhere
stevekm Feb 20, 2020
40869d4
remove CWL conversion
stevekm Feb 20, 2020
2787baa
refactor job data generation methods
stevekm Feb 24, 2020
0d2bc2b
update Roslin QC operator tests
stevekm Feb 24, 2020
e615bf4
update test for input generation
stevekm Feb 24, 2020
0a7c95a
merge against develop and fix conflicts
stevekm Feb 24, 2020
145fd99
merge against develop
stevekm Feb 25, 2020
39f13c8
merge against roslin-qc
stevekm Feb 25, 2020
c43cc7d
update serializer tests
stevekm Feb 25, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
merge against develop and fix conflicts
  • Loading branch information
stevekm committed Feb 24, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 0a7c95a60c34e32ce91296a95f7b607b6f46c4e7
7 changes: 3 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -318,11 +318,12 @@ export DJ_DEBUG_LOG:=$(LOG_DIR_ABS)/dj.debug.log
django-init:
python manage.py makemigrations # --merge
python manage.py migrate
$(MAKE) djano-load-fixtures
$(MAKE) django-load-fixtures
python manage.py createsuperuser

djano-load-fixtures:
django-load-fixtures:
python manage.py loaddata \
beagle_etl.operator.json \
file_system.filegroup.json \
file_system.filetype.json \
file_system.storage.json \
@@ -533,5 +534,3 @@ PORT=
port-check:
ss -lntup | grep ':$(PORT)'
endif


10 changes: 10 additions & 0 deletions beagle_etl/fixtures/beagle_etl.operator.json
Original file line number Diff line number Diff line change
@@ -18,5 +18,15 @@
"class_name": "AccessOperator",
"slug": "access"
}
},
{
"model": "beagle_etl.operator",
"pk": 3,
"fields": {
"active": true,
"recipes": "[]",
"class_name": "RoslinQcOperator",
"slug": "roslin-qc"
}
}
]
14 changes: 1 addition & 13 deletions runner/operator/operator_factory.py
Original file line number Diff line number Diff line change
@@ -18,16 +18,4 @@ def get_by_model(model, **kwargs):

return OperatorFactory.operators[model.class_name](model, **kwargs)

def factory(pipeline, request_id):
if pipeline in ('tempo',):
return TempoOperator(request_id)
elif pipeline in ('roslin',):
return RoslinOperator(request_id)
elif pipeline in ('access',):
return AccessOperator(request_id)
elif pipeline in ('roslin-qc',):
return RoslinQcOperator(request_id)
else:
raise Exception("Invalid pipeline")
factory = staticmethod(factory)
get_by_class_name = staticmethod(get_by_class_name)
get_by_model = staticmethod(get_by_model)
4 changes: 2 additions & 2 deletions runner/operator/roslin_qc_operator/roslin_qc_operator.py
Original file line number Diff line number Diff line change
@@ -12,15 +12,15 @@ class RoslinQcOperator(Operator):
Operator for Roslin QC pipeline
"""

def __init__(self, request_id = None, run_ids = None):
def __init__(self, model, request_id = None, run_ids = None):
self.run_ids = run_ids
self.runs = None
if self.run_ids != None:
self.runs = Run.objects.filter(id__in = self.run_ids)
# TODO: add support for things like FileMetadata's
# TODO: add support for multiple request ids

Operator.__init__(self, 'roslin-qc', request_id)
Operator.__init__(self, model, request_id = request_id, run_ids = run_ids)

def get_pipeline_id(self):
return "9b7f2ac8-03a5-4c44-ae87-1d9f6500d19a"
3 changes: 2 additions & 1 deletion runner/tests/operator/roslin_qc_operator/test_input.py
Original file line number Diff line number Diff line change
@@ -29,7 +29,8 @@ class TestInput(TestCase):
"file_system.filegroup.json",
"file_system.filetype.json",
"file_system.storage.json",
"runner.pipeline.json"
"runner.pipeline.json",
"beagle_etl.operator.json"
]

def test_true(self):
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@
from django.test import TestCase
from runner.operator.operator_factory import OperatorFactory
from runner.operator.roslin_qc_operator.roslin_qc_operator import RoslinQcOperator
from beagle_etl.models import Operator
from django.conf import settings
from django.core.management import call_command
from runner.models import Pipeline, Run
@@ -18,31 +19,19 @@ class TestRoslinQcOperator(TestCase):
"file_system.filegroup.json",
"file_system.filetype.json",
"file_system.storage.json",
"runner.pipeline.json"
"runner.pipeline.json",
"beagle_etl.operator.json"
]

def test_operator_factory_roslin_qc1(self):
def test_create_operator1(self):
"""
Test that a Roslin QC operator instance can be created with Operator.factory
Test that a Roslin QC operator instance can be created
"""
pipeline_type = "roslin-qc"
request_id = "foo"
operator = OperatorFactory.factory(pipeline_type, request_id)
roslin_qc_model = Operator.objects.get(slug="roslin-qc")
operator = RoslinQcOperator(roslin_qc_model, request_id = request_id)
self.assertTrue(isinstance(operator, RoslinQcOperator))
self.assertTrue( operator.pipeline_id == "9b7f2ac8-03a5-4c44-ae87-1d9f6500d19a")
self.assertTrue( operator.operator == "roslin-qc")
self.assertTrue( operator.request_id == "foo")
self.assertTrue( operator._jobs == [])
self.assertTrue( len(operator.files) == 0)

def test_operator_factory_get_by_class_name(self):
"""
Test that you can get the RoslinQC operator by Operator.get_by_class_name
"""
operator = OperatorFactory.get_by_class_name("RoslinQcOperator", "foo")
self.assertTrue(isinstance(operator, RoslinQcOperator))
self.assertTrue( operator.pipeline_id == "9b7f2ac8-03a5-4c44-ae87-1d9f6500d19a")
self.assertTrue( operator.operator == "roslin-qc")
self.assertTrue( operator.request_id == "foo")
self.assertTrue( operator._jobs == [])
self.assertTrue( len(operator.files) == 0)
@@ -60,7 +49,8 @@ def test_direct_operator_creation(self, submit_job):
call_command('loaddata', test_files_fixture, verbosity=0)

# create the operator instance
operator = RoslinQcOperator(request_id = None, run_ids = ['ca18b090-03ad-4bef-acd3-52600f8e62eb'])
roslin_qc_model = Operator.objects.get(slug="roslin-qc")
operator = RoslinQcOperator(roslin_qc_model, run_ids = ['ca18b090-03ad-4bef-acd3-52600f8e62eb'])

# check its attributes
self.assertEqual(operator.run_ids, ['ca18b090-03ad-4bef-acd3-52600f8e62eb'])
11 changes: 1 addition & 10 deletions runner/tests/operator/test_operator_factory.py
Original file line number Diff line number Diff line change
@@ -12,13 +12,4 @@ def test_operator_factory_invalid_pipeline1(self):
"""
pipeline_type = "foo"
request_id = "bar"
self.assertRaises(Exception, OperatorFactory.factory, pipeline_type, request_id)

def test_no_requestID(self):
"""
Test that an operator can be returned without a requestID
"""
pipeline_type = "roslin"
request_id = None
operator = OperatorFactory.factory(pipeline_type, request_id)
self.assertEqual(operator.request_id, request_id)
self.assertRaises(Exception, OperatorFactory.get_by_model, pipeline_type, request_id = request_id)
You are viewing a condensed version of this merge commit. You can view the full changes here.