Merge branch 'release/1.0.8'

BD2KGenomics · Aug 28, 2017 · 6f6eca4 · 6f6eca4
2 parents 9675909 + 3883bb9
commit 6f6eca4
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 23 deletions.
diff --git a/luigi_task_executor/CNV.py b/luigi_task_executor/CNV.py
@@ -36,7 +36,7 @@ class ConsonanceTask(luigi.Task):
 
     redwood_host = luigi.Parameter("storage.ucsc-cgl.org")
     redwood_token = luigi.Parameter("must_be_defined")
-    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.18")
+    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.19")
 
     workflow_version = luigi.Parameter(default="must be defined")
 
@@ -60,6 +60,7 @@ class ConsonanceTask(luigi.Task):
     cnv_reference_files_json = luigi.Parameter(default="must input reference file metadata")
 
     touch_file_path = luigi.Parameter(default='must input touch file path')
+    metadata_json_file_name = luigi.Parameter(default='must input metadata json file name')
 
     #Consonance will not be called in test mode
     test_mode = luigi.BoolParameter(default = False)
@@ -250,7 +251,7 @@ class CNVCoordinator(luigi.Task):
     redwood_token = luigi.Parameter("must_be_defined")
     redwood_host = luigi.Parameter(default='storage.ucsc-cgp.org')
     image_descriptor = luigi.Parameter("must be defined")
-    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.18")
+    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.19")
     tmp_dir = luigi.Parameter(default='/datastore')
     max_jobs = luigi.Parameter(default='-1')
     bundle_uuid_filename_to_file_uuid = {}
@@ -451,12 +452,12 @@ def requires(self):
                                 cnv_jobs['samples'][sample_name]["submitter_donor_primary_site"] = hit["_source"]["submitter_donor_primary_site"]
                             else:
                                 cnv_jobs['samples'][sample_name]["submitter_donor_primary_site"] = "not provided"
-                            #cnv_jobs['samples'][sample_name]["submitter_specimen_id"] = specimen["submitter_specimen_id"]
-                            #cnv_jobs['samples'][sample_name]["specimen_uuid"] = specimen["specimen_uuid"]
+                            cnv_jobs['samples'][sample_name]["submitter_specimen_id"] = specimen["submitter_specimen_id"]
+                            cnv_jobs['samples'][sample_name]["specimen_uuid"] = specimen["specimen_uuid"]
                             cnv_jobs['samples'][sample_name]["submitter_specimen_type"] = specimen["submitter_specimen_type"]
                             cnv_jobs['samples'][sample_name]["submitter_experimental_design"] = specimen["submitter_experimental_design"]
-                            #cnv_jobs['samples'][sample_name]["submitter_sample_id"] = sample["submitter_sample_id"]
-                            #cnv_jobs['samples'][sample_name]["sample_uuid"] = sample["sample_uuid"]
+                            cnv_jobs['samples'][sample_name]["submitter_sample_id"] = sample["submitter_sample_id"]
+                            cnv_jobs['samples'][sample_name]["sample_uuid"] = sample["sample_uuid"]
                             cnv_jobs['samples'][sample_name]["analysis_type"] = "cnv_variant_calling"
                             cnv_jobs['samples'][sample_name]["workflow_name"] = "https://dockstore.org/workflows/BD2KGenomics/dockstore_workflow_cnv"
                             cnv_jobs['samples'][sample_name]["workflow_version"] = self.workflow_version
@@ -525,6 +526,7 @@ def requires(self):
                     print("\nreference files meta data:")
                     print(cnv_reference_files_json)
 
+                    metadata_json_file_name = sample_name + "_" + specimen_type + '_meta_data.json'
 
                     listOfJobs.append(ConsonanceTask(redwood_host=self.redwood_host, \
                         vm_instance_type=self.vm_instance_type,
@@ -539,6 +541,7 @@ def requires(self):
                         #submitter_sample_id = cnv_jobs['samples'][sample_name]['submitter_sample_id'], \
                         cnv_job_json = cnv_job_json, \
                         cnv_reference_files_json = cnv_reference_files_json, \
+                        metadata_json_file_name = metadata_json_file_name, \
                         touch_file_path = full_touch_file_path, test_mode=self.test_mode))
 
 

diff --git a/luigi_task_executor/Dockerfile_decider b/luigi_task_executor/Dockerfile_decider
@@ -116,6 +116,14 @@ COPY RNA-Seq.py .
 RUN chown ubuntu:ubuntu RNA-Seq.py
 RUN chmod a+x RNA-Seq.py
 
+COPY CNV.py . 
+RUN chown ubuntu:ubuntu CNV.py
+RUN chmod a+x CNV.py
+
+COPY Protect.py . 
+RUN chown ubuntu:ubuntu Protect.py
+RUN chmod a+x Protect.py
+
 #allow ubuntu (and all other users) to execute sudo without a password
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 

diff --git a/luigi_task_executor/ProtectTask.py → luigi_task_executor/Protect.py b/luigi_task_executor/ProtectTask.py → luigi_task_executor/Protect.py
@@ -36,7 +36,7 @@ class ConsonanceTask(luigi.Task):
 
     redwood_host = luigi.Parameter("storage.ucsc-cgl.org")
     redwood_token = luigi.Parameter("must_be_defined")
-    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.18")
+    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.19")
 
     workflow_version = luigi.Parameter(default="must be defined")
 
@@ -48,6 +48,7 @@ class ConsonanceTask(luigi.Task):
     image_descriptor = luigi.Parameter("must be defined")
 
     vm_region = luigi.Parameter(default='us-west-2')
+    vm_instance_type = luigi.Parameter(default='c4.8xlarge')
 
     ''' 
     #so these aren't actually optional, but i can hardcode a path as default
@@ -81,6 +82,7 @@ class ConsonanceTask(luigi.Task):
     protect_reference_files_json = luigi.Parameter(default="must input reference file metadata")
 
     touch_file_path = luigi.Parameter(default='must input touch file path')
+    metadata_json_file_name = luigi.Parameter(default='must input metadata json file name')
 
     #Consonance will not be called in test mode
     test_mode = luigi.BoolParameter(default = False)
@@ -207,7 +209,7 @@ def run(self):
 
         dockstore_json["workflow_type"] = self.workflow_type
         dockstore_json["tmpdir"] = self.tmp_dir
-        dockstore_json["vm_instance_type"] = "c4.8xlarge"
+        dockstore_json["vm_instance_type"] = self.vm_instance_type
         dockstore_json["vm_region"] = self.vm_region
         dockstore_json["vm_location"] = "aws"
         dockstore_json["vm_instance_cores"] = 36
@@ -226,7 +228,7 @@ def run(self):
         # execute consonance run, parse the job UUID
 
         #cmd = ["consonance", "run", "--image-descriptor", self.image_descriptor, "--flavour", "c4.8xlarge", "--run-descriptor", self.save_dockstore_json_local().path]
-        cmd = ["consonance", "run",  "--tool-dockstore-id", self.dockstore_tool_running_dockstore_tool, "--flavour", "c4.8xlarge", "--run-descriptor", self.save_dockstore_json_local().path]
+        cmd = ["consonance", "run",  "--tool-dockstore-id", self.dockstore_tool_running_dockstore_tool, "--flavour", self.vm_instance_type, "--run-descriptor", self.save_dockstore_json_local().path]
         cmd_str = ' '.join(cmd)
         if self.test_mode == False:
             print("** SUBMITTING TO CONSONANCE **")
@@ -320,7 +322,7 @@ class ProtectCoordinator(luigi.Task):
     redwood_token = luigi.Parameter("must_be_defined")
     redwood_host = luigi.Parameter(default='storage.ucsc-cgp.org')
     image_descriptor = luigi.Parameter("must be defined")
-    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.18")
+    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.19")
     tmp_dir = luigi.Parameter(default='/datastore')
     max_jobs = luigi.Parameter(default='-1')
     bundle_uuid_filename_to_file_uuid = {}
@@ -330,6 +332,7 @@ class ProtectCoordinator(luigi.Task):
     touch_file_bucket = luigi.Parameter(default="must be input")
 
     vm_region = luigi.Parameter(default='us-west-2')
+    vm_instance_type = luigi.Parameter(default='c4.8xlarge')
 
     #Consonance will not be called in test mode
     test_mode = luigi.BoolParameter(default = False)
@@ -537,12 +540,12 @@ def requires(self):
                                 protect_jobs['samples'][sample_name]["submitter_donor_primary_site"] = hit["_source"]["submitter_donor_primary_site"]
                             else:
                                 protect_jobs['samples'][sample_name]["submitter_donor_primary_site"] = "not provided"
-                            #protect_jobs['samples'][sample_name]["submitter_specimen_id"] = specimen["submitter_specimen_id"]
-                            #protect_jobs['samples'][sample_name]["specimen_uuid"] = specimen["specimen_uuid"]
+                            protect_jobs['samples'][sample_name]["submitter_specimen_id"] = specimen["submitter_specimen_id"]
+                            protect_jobs['samples'][sample_name]["specimen_uuid"] = specimen["specimen_uuid"]
                             protect_jobs['samples'][sample_name]["submitter_specimen_type"] = specimen["submitter_specimen_type"]
                             protect_jobs['samples'][sample_name]["submitter_experimental_design"] = specimen["submitter_experimental_design"]
-                            #protect_jobs['samples'][sample_name]["submitter_sample_id"] = sample["submitter_sample_id"]
-                            #protect_jobs['samples'][sample_name]["sample_uuid"] = sample["sample_uuid"]
+                            protect_jobs['samples'][sample_name]["submitter_sample_id"] = sample["submitter_sample_id"]
+                            protect_jobs['samples'][sample_name]["sample_uuid"] = sample["sample_uuid"]
                             protect_jobs['samples'][sample_name]["analysis_type"] = "immuno_target_pipelines"
                             protect_jobs['samples'][sample_name]["workflow_name"] = "quay.io/ucsc_cgl/protect"
                             protect_jobs['samples'][sample_name]["workflow_version"] = self.workflow_version
@@ -561,9 +564,12 @@ def requires(self):
                 print("\nprotect reference files meta data:")
                 print(protect_reference_files_json)
 
+                metadata_json_file_name = sample_name + '_meta_data.json' 
+
 
                 listOfJobs.append(ConsonanceTask(redwood_host=self.redwood_host, \
                     vm_region = self.vm_region, \
+                    vm_instance_type=self.vm_instance_type,
                     redwood_token=self.redwood_token, \
                     image_descriptor=self.image_descriptor, \
                     dockstore_tool_running_dockstore_tool=self.dockstore_tool_running_dockstore_tool, \
@@ -573,6 +579,7 @@ def requires(self):
                     #submitter_sample_id = protect_jobs['samples'][sample_name]['submitter_sample_id'], \
                     protect_job_json = protect_job_json, \
                     protect_reference_files_json = protect_reference_files_json, \
+                    metadata_json_file_name = metadata_json_file_name, \
                     touch_file_path = touch_file_path, test_mode=self.test_mode))
 
 
@@ -598,7 +605,7 @@ def output(self):
         ts_str = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H:%M:%S')
         #return luigi.LocalTarget('%s/consonance-jobs/RNASeq_3_1_x_Coordinator/RNASeqTask-%s.txt' % (self.tmp_dir, ts_str))
         workflow_version_dir = self.workflow_version.replace('.', '_')
-        return S3Target('s3://'+self.touch_file_bucket+'/consonance-jobs/ProTECT_Coordinator/{}/ProtectTask-{}.txt'.format(workflow_version_dir, ts_str))
+        return S3Target('s3://'+self.touch_file_bucket+'/consonance-jobs/ProTECT_Coordinator/{}/Protect-{}.txt'.format(workflow_version_dir, ts_str))
 
     def fileToUUID(self, input, bundle_uuid):
         return self.bundle_uuid_filename_to_file_uuid[bundle_uuid+"_"+input]

diff --git a/luigi_task_executor/RNA-Seq.py b/luigi_task_executor/RNA-Seq.py
@@ -29,7 +29,7 @@
 class ConsonanceTask(luigi.Task):
     redwood_host = luigi.Parameter("storage.ucsc-cgp.org")
     redwood_token = luigi.Parameter("must_be_defined")
-    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.18")
+    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.19")
 
     workflow_version = luigi.Parameter(default="must be defined")
 
@@ -41,7 +41,7 @@ class ConsonanceTask(luigi.Task):
     image_descriptor = luigi.Parameter("must be defined")
 
     disable_cutadapt = luigi.Parameter(default="false")
-    save_bam = luigi.Parameter(default="false")
+    save_bam = luigi.Parameter(default="true")
     save_wiggle = luigi.Parameter(default="false")
     no_clean = luigi.Parameter(default="true")
     resume = luigi.Parameter(default="")
@@ -80,6 +80,8 @@ class ConsonanceTask(luigi.Task):
     submitter_sample_id = luigi.Parameter(default='must input submitter sample id')
     meta_data_json = luigi.Parameter(default="must input metadata")
     touch_file_path = luigi.Parameter(default='must input touch file path')
+    metadata_json_file_name = luigi.Parameter(default='must input metadata json file name')
+
 
     vm_instance_type = luigi.Parameter(default='c4.8xlarge')
     vm_region = luigi.Parameter(default='us-east-1')
@@ -450,7 +452,7 @@ class RNASeqCoordinator(luigi.Task):
     redwood_token = luigi.Parameter("must_be_defined")
     redwood_host = luigi.Parameter(default='storage.ucsc-cgl.org')
     image_descriptor = luigi.Parameter("must be defined")
-    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.18")
+    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.19")
     tmp_dir = luigi.Parameter(default='/datastore')
     max_jobs = luigi.Parameter(default='-1')
     bundle_uuid_filename_to_file_uuid = {}
@@ -638,6 +640,8 @@ def requires(self):
                             #touch_file_path = ''.join(touch_file_path.split())
 
                             submitter_sample_id = sample["submitter_sample_id"]
+                            metadata_json_file_name = submitter_sample_id + '_meta_data.json' 
+
 
                             #This metadata will be passed to the Consonance Task and some
                             #some of the meta data will be used in the Luigi status page for the job
@@ -770,6 +774,7 @@ def requires(self):
                                          paired_filenames=paired_files, paired_file_uuids = paired_file_uuids, paired_bundle_uuids = paired_bundle_uuids, \
                                          tar_filenames=tar_files, tar_file_uuids = tar_file_uuids, tar_bundle_uuids = tar_bundle_uuids, \
                                          tmp_dir=self.tmp_dir, submitter_sample_id = submitter_sample_id, meta_data_json = meta_data_json, \
+                                         metadata_json_file_name = metadata_json_file_name, \
                                          touch_file_path = touch_file_path, workflow_version = self.workflow_version, test_mode = self.test_mode))
 
         print("total of {} jobs; max jobs allowed is {}\n\n".format(str(len(listOfJobs)), self.max_jobs))

diff --git a/luigi_task_executor/RNA-Seq_3_0_2-3.py b/luigi_task_executor/RNA-Seq_3_0_2-3.py
@@ -29,7 +29,7 @@
 class ConsonanceTask(luigi.Task):
     redwood_host = luigi.Parameter("storage.ucsc-cgp.org")
     redwood_token = luigi.Parameter("must_be_defined")
-    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.18")
+    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.19")
 
     workflow_version = luigi.Parameter(default="must be defined")
 
@@ -80,6 +80,7 @@ class ConsonanceTask(luigi.Task):
     submitter_sample_id = luigi.Parameter(default='must input submitter sample id')
     meta_data_json = luigi.Parameter(default="must input metadata")
     touch_file_path = luigi.Parameter(default='must input touch file path')
+    metadata_json_file_name = luigi.Parameter(default='must input metadata json file name')
 
     vm_region = luigi.Parameter(default='us-east-1')
 
@@ -459,7 +460,7 @@ class RNASeqCoordinator(luigi.Task):
     redwood_token = luigi.Parameter("must_be_defined")
     redwood_host = luigi.Parameter(default='storage.ucsc-cgl.org')
     image_descriptor = luigi.Parameter("must be defined")
-    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.18")
+    dockstore_tool_running_dockstore_tool = luigi.Parameter(default="quay.io/ucsc_cgl/dockstore-tool-runner:1.0.19")
     tmp_dir = luigi.Parameter(default='/datastore')
     max_jobs = luigi.Parameter(default='-1')
     bundle_uuid_filename_to_file_uuid = {}
@@ -654,6 +655,7 @@ def requires(self):
                             #touch_file_path = ''.join(touch_file_path.split())
 
                             submitter_sample_id = sample["submitter_sample_id"]
+                            metadata_json_file_name = submitter_sample_id + '_meta_data.json' 
 
                             #This metadata will be passed to the Consonance Task and some
                             #some of the meta data will be used in the Luigi status page for the job
@@ -786,6 +788,7 @@ def requires(self):
                                          paired_filenames=paired_files, paired_file_uuids = paired_file_uuids, paired_bundle_uuids = paired_bundle_uuids, \
                                          tar_filenames=tar_files, tar_file_uuids = tar_file_uuids, tar_bundle_uuids = tar_bundle_uuids, \
                                          tmp_dir=self.tmp_dir, submitter_sample_id = submitter_sample_id, meta_data_json = meta_data_json, \
+                                         metadata_json_file_name = metadata_json_file_name, \
                                          touch_file_path = touch_file_path, workflow_version = self.workflow_version, test_mode = self.test_mode))
 
         print("total of {} jobs; max jobs allowed is {}\n\n".format(str(len(listOfJobs)), self.max_jobs))