From 701c5ea8d16d31bd05f8f47b8147b9bf4aabd71d Mon Sep 17 00:00:00 2001 From: luisas Date: Fri, 15 Mar 2024 19:28:59 +0100 Subject: [PATCH 01/12] test --- conf/modules.config | 5 ++- main.nf | 6 +++- modules/local/prepare_shiny.nf | 2 +- nextflow.config | 1 + nextflow_schema.json | 6 ++++ .../main.nf | 32 ++++++++++++++++++- workflows/multiplesequencealign.nf | 5 ++- 7 files changed, 52 insertions(+), 5 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index c46a26b3..1a530ab3 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -92,6 +92,7 @@ process { // ------------------------------------ withName: "FAMSA_GUIDETREE"{ + tag = { "${meta.id} args:${meta.args_tree}" } ext.prefix = { "${meta.id}_${meta.tree}-args-${meta.argstree_clean}" } ext.args = { "${meta.args_tree}" == "null" ? '' : "${meta.args_tree}" } publishDir = [ @@ -102,6 +103,7 @@ process { } withName: "CLUSTALO_GUIDETREE"{ + tag = { "${meta.id} args:${meta.args_tree}" } ext.prefix = { "${meta.id}_${meta.tree}-args-${meta.argstree_clean}" } ext.args = { "${meta.args_tree}" == "null" ? '' : "${meta.args_tree}" } publishDir = [ @@ -125,6 +127,7 @@ process { } withName: "MAFFT|CLUSTALO_ALIGN|TCOFFEE_ALIGN|MUSCLE5_SUPER5|TCOFFEE3D_ALIGN|FAMSA_ALIGN|LEARNMSA_ALIGN|REGRESSIVE"{ + tag = { "${meta.id} tree:${meta.tree} argstree:${args_tree} args:${meta.args_aligner}" } ext.prefix = { "${meta.id}_${meta.tree}-args-${meta.argstree_clean}_${meta.aligner}-args-${meta.args_aligner_clean}" } ext.args = { "${meta.args_aligner}" == "null" ? '' : "${meta.args_aligner}" } if ( params.skip_compress ) { @@ -231,7 +234,7 @@ process { // ------------------------------------ withName: 'PREPARE_SHINY' { publishDir = [ - path: { "${params.outdir}/shiny" }, + path: { "${params.shinydir}" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] diff --git a/main.nf b/main.nf index 791a99ff..b4689e1b 100644 --- a/main.nf +++ b/main.nf @@ -59,6 +59,7 @@ workflow NFCORE_MULTIPLESEQUENCEALIGN { emit: multiqc_report = MULTIPLESEQUENCEALIGN.out.multiqc + shiny_stats = MULTIPLESEQUENCEALIGN.out.shiny_stats @@ -95,6 +96,7 @@ workflow { PIPELINE_INITIALISATION.out.tools ) + // // SUBWORKFLOW: Run completion tasks // @@ -105,7 +107,9 @@ workflow { params.outdir, params.monochrome_logs, params.hook_url, - NFCORE_MULTIPLESEQUENCEALIGN.out.multiqc_report + NFCORE_MULTIPLESEQUENCEALIGN.out.multiqc_report, + params.shiny_dir, + params.outdir ) } diff --git a/modules/local/prepare_shiny.nf b/modules/local/prepare_shiny.nf index 3932d099..93224fea 100644 --- a/modules/local/prepare_shiny.nf +++ b/modules/local/prepare_shiny.nf @@ -11,7 +11,7 @@ process PREPARE_SHINY { path (app) output: - tuple val (meta), path("shiny_data.csv"), emit: data + path("shiny_data.csv"), emit: data path ("shiny_app.py"), emit: app path ("run.sh"), emit: run path "versions.yml", emit: versions diff --git a/nextflow.config b/nextflow.config index b6417c06..4b0177ff 100644 --- a/nextflow.config +++ b/nextflow.config @@ -39,6 +39,7 @@ params { skip_multiqc = false // Shiny options + shinydir = "${outdir}/shiny" shiny_app = "${projectDir}/bin/app.py" skip_shiny = false diff --git a/nextflow_schema.json b/nextflow_schema.json index b95512b3..8ed9e72a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -51,6 +51,12 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" + }, + "shinydir": { + "type": "string", + "format": "directory-path", + "description": "The output directory for the shiny app where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" } } }, diff --git a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf index 7a94db37..a6bbeb49 100644 --- a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf @@ -122,6 +122,8 @@ workflow PIPELINE_COMPLETION { monochrome_logs // boolean: Disable ANSI colour codes in log output hook_url // string: hook URL for notifications multiqc_report // string: Path to MultiQC report + shiny_dir // string: Path to shiny stats file + trace_infos // string: Path to trace file main: @@ -140,9 +142,35 @@ workflow PIPELINE_COMPLETION { if (hook_url) { imNotification(summary_params, hook_url) } + + print("Testing how to use the trace files in shiny") + + // move the trace file to the output directory + def trace_file = new File("${trace_infos}/pipeline_info/execution") + + if (trace_file.exists()) { + trace_infos = filterTraceForShiny(trace_file) + }else{ + print("No trace file found in the pipeline_info directory") + } } } + +def filterTraceForShiny(trace_file){ + def trace_lines = trace_file.readLines() + def shiny_trace_lines = [] + print("Filtering trace file for shiny") + for (line in trace_lines){ + if (line.contains("COMPLETED") && line.contains("MULTIPLESEQUENCEALIGN:ALIGN")){ + shiny_trace_lines.add(line) + } + } + return shiny_trace_lines +} + + + /* ======================================================================================== FUNCTIONS @@ -260,7 +288,7 @@ class Utils { public static cleanArgs(argString) { - def cleanArgs = argString.toString().trim().replace("-", "").replace(" ", " ").replace(" ", "_").replaceAll("==", "_").replaceAll("\\s+", "") + def cleanArgs = argString.toString().trim().replace(" ", " ").replace(" ", "_").replaceAll("==", "_").replaceAll("\\s+", "") // if clearnArgs is empty, return "" if (cleanArgs == null || cleanArgs == "") { @@ -304,4 +332,6 @@ class Utils { + + } diff --git a/workflows/multiplesequencealign.nf b/workflows/multiplesequencealign.nf index 7e225d96..64678edb 100644 --- a/workflows/multiplesequencealign.nf +++ b/workflows/multiplesequencealign.nf @@ -225,9 +225,11 @@ workflow MULTIPLESEQUENCEALIGN { // // MODULE: Shiny // + ch_shiny_stats = Channel.empty() if( !params.skip_shiny){ PREPARE_SHINY ( stats_and_evaluation_summary, file(params.shiny_app) ) ch_versions = ch_versions.mix(PREPARE_SHINY.out.versions) + ch_shiny_stats = PREPARE_SHINY.out.data.toList() } softwareVersionsToYAML(ch_versions) @@ -265,7 +267,8 @@ workflow MULTIPLESEQUENCEALIGN { emit: versions = ch_versions - multiqc = multiqc_out // channel: [ path(versions.yml) ] + multiqc = multiqc_out + shiny_stats = ch_shiny_stats // channel: [ path(versions.yml) ] } From d19ef4c21b2d881b4833290f2ce0a4c4c3dab7fc Mon Sep 17 00:00:00 2001 From: luisas Date: Mon, 18 Mar 2024 16:53:09 +0100 Subject: [PATCH 02/12] merging script --- bin/{ => shiny_app}/app.py | 0 bin/shiny_app/merge_score_and_trace.py | 103 ++++++++++++++++++ conf/test.config | 1 + main.nf | 2 +- nextflow.config | 14 ++- .../main.nf | 27 +++-- 6 files changed, 133 insertions(+), 14 deletions(-) rename bin/{ => shiny_app}/app.py (100%) create mode 100644 bin/shiny_app/merge_score_and_trace.py diff --git a/bin/app.py b/bin/shiny_app/app.py similarity index 100% rename from bin/app.py rename to bin/shiny_app/app.py diff --git a/bin/shiny_app/merge_score_and_trace.py b/bin/shiny_app/merge_score_and_trace.py new file mode 100644 index 00000000..7587fb98 --- /dev/null +++ b/bin/shiny_app/merge_score_and_trace.py @@ -0,0 +1,103 @@ +import pandas as pd + +data_file = 'shiny_data.csv' +trace_file = 'trace.txt' +out_file_name = 'shiny_data_with_trace.csv' + +# read in shiny_data.csv +data = pd.read_csv(data_file) +# read in trace +trace = pd.read_csv(trace_file, sep='\t') + + +def convert_time(time): + if time is not None: + if "ms" in time: + time = time.replace('ms', '') + time = float(time)/60000 + elif "s" in time: + time = time.replace('s', '') + time = float(time)/60 + elif "m" in time: + time = time.replace('m', '') + elif "h" in time: + time = time.replace('h', '') + time = float(time)*60 + return time + +def convert_memory(memory): + # from anything to GB + if memory is not None: + if "GB" in memory: + memory = memory.replace('GB', '') + elif "MB" in memory: + memory = memory.replace('MB', '') + memory = float(memory)/1000 + elif "KB" in memory: + memory = memory.replace('KB', '') + memory = float(memory)/1000000 + return memory + +def clean_trace(trace): + # Update trace file + def extract_element(row, nelement): + elements = row.split(':') + return elements[nelement] + + trace["tag"] = trace.name.str.split('(', expand = True)[1].str.split(')', expand = True)[0] + trace["id"] = trace.tag.str.split(expand = True)[0] + trace["args"] = trace.tag.str.split("args:", expand=True)[1] + trace["full_name"] = trace.name.str.split('(', expand = True)[0].str.strip() + trace["process"] = trace.full_name.apply(extract_element, nelement=-1) + trace["subworkflow"] = trace.full_name.apply(extract_element, nelement=-2) + trace.replace('null', pd.NA, inplace=True) + return trace + +def prep_tree_trace(trace): + trace_trees = trace[trace["subworkflow"] == "COMPUTE_TREES"] + # rename args to args_tree + trace_trees.rename(columns={"args": "args_tree"}, inplace=True) + # rename process to tree and remove _GUIDETREE + trace_trees["tree"] = trace_trees["process"].str.replace("_GUIDETREE", "") + # subselect only the columns we need + trace_trees = trace_trees[["id", "args_tree", "tree", "realtime", "rss", "cpus"]] + trace_trees.rename(columns={"realtime": "time_tree"}, inplace=True) + trace_trees.rename(columns={"rss": "memory_tree"}, inplace=True) + trace_trees.rename(columns={"cpus": "cpus_tree"}, inplace=True) + trace_trees.replace('null', pd.NA, inplace=True) + print(trace_trees) + # remove ms from time_tree and convert it to min + trace_trees["time_tree"] = trace_trees["time_tree"].apply(convert_time) + # convert memory to GB + trace_trees["memory_tree"] = trace_trees["memory_tree"].apply(convert_memory) + return trace_trees + +def prep_align_trace(trace): + trace_align = trace[trace["subworkflow"] == "ALIGN"] + # rename args to args_align + trace_align.rename(columns={"args": "args_aligner"}, inplace=True) + # rename process to align and remove _ALIGN + trace_align["aligner"] = trace_align["process"].str.replace("_ALIGN", "") + # subselect only the columns we need + trace_align = trace_align[["id", "args_aligner", "aligner", "realtime", "rss", "cpus"]] + trace_align.rename(columns={"realtime": "time_align"}, inplace=True) + trace_align.rename(columns={"rss": "memory_align"}, inplace=True) + trace_align.rename(columns={"cpus": "cpus_align"}, inplace=True) + trace_align.replace('null', pd.NA, inplace=True) + # remove ms from time_align and convert it to min + trace_align["time_align"] = trace_align["time_align"].apply(convert_time) + # convert memory to GB + trace_align["memory_align"] = trace_align["memory_align"].apply(convert_memory) + return trace_align + +clean_trace = clean_trace(trace) +trace_trees = prep_tree_trace(clean_trace) +trace_align = prep_align_trace(clean_trace) + +#merge data and trace_trees +data_tree = pd.merge(data, trace_trees, on=["id", "tree", "args_tree"], how="left") +data_tree_align = pd.merge(data_tree, trace_align, on=["id", "aligner", "args_aligner"], how="left") + + +# write to file +data_tree_align.to_csv(out_file_name, index=False) \ No newline at end of file diff --git a/conf/test.config b/conf/test.config index 1ff599b8..bd549c6a 100644 --- a/conf/test.config +++ b/conf/test.config @@ -26,5 +26,6 @@ params { // Output directory outdir = "./outdir/" + shinydir = "${params.outdir}/shiny_app" } diff --git a/main.nf b/main.nf index b4689e1b..0f91f0da 100644 --- a/main.nf +++ b/main.nf @@ -108,7 +108,7 @@ workflow { params.monochrome_logs, params.hook_url, NFCORE_MULTIPLESEQUENCEALIGN.out.multiqc_report, - params.shiny_dir, + params.shinydir, params.outdir ) } diff --git a/nextflow.config b/nextflow.config index 4b0177ff..e59fad8a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -38,11 +38,6 @@ params { multiqc_methods_description = null skip_multiqc = false - // Shiny options - shinydir = "${outdir}/shiny" - shiny_app = "${projectDir}/bin/app.py" - skip_shiny = false - // Boilerplate options outdir = null publish_dir_mode = 'copy' @@ -54,6 +49,11 @@ params { help = false version = false + // Shiny options + shinydir = "${params.outdir}/shiny_app" + shiny_app = "${projectDir}/bin/shiny_app" + skip_shiny = false + // Config options config_profile_name = null config_profile_description = null @@ -233,7 +233,9 @@ report { } trace { enabled = true - file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" + //file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" + file = "${params.outdir}/pipeline_info/trace.txt" + fields = 'task_id, hash,native_id,name,status,exit,realtime,%cpu,rss,peak_rss,vmem,peak_vmem,rchar,wchar,cpus' } dag { enabled = true diff --git a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf index a6bbeb49..c1befa82 100644 --- a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf @@ -122,7 +122,7 @@ workflow PIPELINE_COMPLETION { monochrome_logs // boolean: Disable ANSI colour codes in log output hook_url // string: hook URL for notifications multiqc_report // string: Path to MultiQC report - shiny_dir // string: Path to shiny stats file + shinydir // string: Path to shiny stats file trace_infos // string: Path to trace file main: @@ -143,13 +143,22 @@ workflow PIPELINE_COMPLETION { imNotification(summary_params, hook_url) } - print("Testing how to use the trace files in shiny") + print(shinydir) + print(trace_infos) - // move the trace file to the output directory - def trace_file = new File("${trace_infos}/pipeline_info/execution") + // TODO have an input parameter that specifies which tracefile to use + // or to use all and merge or to use the latest + + // Preprocess the trace file and move it to shiny directory + def trace_file = new File("${trace_infos}/pipeline_info/trace.txt") if (trace_file.exists()) { trace_infos = filterTraceForShiny(trace_file) + print(trace_infos) + // move trace_infos to shinydir + def shiny_trace_file = new File("${shinydir}/trace.txt") + shiny_trace_file.write(trace_infos.join("\n")) + }else{ print("No trace file found in the pipeline_info directory") } @@ -162,9 +171,13 @@ def filterTraceForShiny(trace_file){ def shiny_trace_lines = [] print("Filtering trace file for shiny") for (line in trace_lines){ - if (line.contains("COMPLETED") && line.contains("MULTIPLESEQUENCEALIGN:ALIGN")){ - shiny_trace_lines.add(line) - } + // if (line.contains("hash")){ + // shiny_trace_lines.add(line) + // } + // if (line.contains("COMPLETED") && line.contains("MULTIPLESEQUENCEALIGN:ALIGN")){ + // shiny_trace_lines.add(line) + // } + shiny_trace_lines.add(line) } return shiny_trace_lines } From 8808bf78e5bee3c594188212af3316806221022b Mon Sep 17 00:00:00 2001 From: luisas Date: Mon, 18 Mar 2024 17:46:13 +0100 Subject: [PATCH 03/12] Files moved correctly --- bin/shiny_app/{app.py => shiny_app.py} | 12 ++++++++++-- ...d_trace.py => shiny_app_merge_score_and_trace.py} | 0 modules/local/prepare_shiny.nf | 9 +++++---- nextflow_schema.json | 4 ++-- 4 files changed, 17 insertions(+), 8 deletions(-) rename bin/shiny_app/{app.py => shiny_app.py} (89%) rename bin/shiny_app/{merge_score_and_trace.py => shiny_app_merge_score_and_trace.py} (100%) diff --git a/bin/shiny_app/app.py b/bin/shiny_app/shiny_app.py similarity index 89% rename from bin/shiny_app/app.py rename to bin/shiny_app/shiny_app.py index f4a739cb..8d483d22 100644 --- a/bin/shiny_app/app.py +++ b/bin/shiny_app/shiny_app.py @@ -6,6 +6,7 @@ import pandas as pd import matplotlib.pyplot as plt import sys +import os # Style @@ -14,7 +15,10 @@ # Load file # ---------------------------------------------------------------------------- -summary_report = "./shiny_data.csv" +summary_report = "./shiny_data_with_trace.csv" + +if not os.path.exists(summary_report): + summary_report = "./shiny_data.csv" try: inputfile = pd.read_csv(summary_report) @@ -33,6 +37,10 @@ "tc": "total column score (TC)", "perc_sim": "sequences avg similarity", "seq_length_mean": "sequence length (mean)", + "time_tree": "tree time (min)", + "time_align": "alignment time (min)", + "memory_tree": "tree memory (GB)", + "memory_align": "alignment memory (GB)" } app_ui = ui.page_fluid( @@ -110,4 +118,4 @@ def scatter(): return ax -app = App(app_ui, server) +app = App(app_ui, server) \ No newline at end of file diff --git a/bin/shiny_app/merge_score_and_trace.py b/bin/shiny_app/shiny_app_merge_score_and_trace.py similarity index 100% rename from bin/shiny_app/merge_score_and_trace.py rename to bin/shiny_app/shiny_app_merge_score_and_trace.py diff --git a/modules/local/prepare_shiny.nf b/modules/local/prepare_shiny.nf index 93224fea..321486f5 100644 --- a/modules/local/prepare_shiny.nf +++ b/modules/local/prepare_shiny.nf @@ -11,8 +11,8 @@ process PREPARE_SHINY { path (app) output: - path("shiny_data.csv"), emit: data - path ("shiny_app.py"), emit: app + path ("shiny_data.csv"), emit: data + path ("shiny_app*"), emit: app path ("run.sh"), emit: run path "versions.yml", emit: versions @@ -23,8 +23,9 @@ process PREPARE_SHINY { def args = task.ext.args ?: '' prefix = task.ext.prefix ?: "${meta.id}" """ - mv $table shiny_data.csv - mv $app shiny_app.py + cp $table shiny_data.csv + cp $app/* . + rm $app echo "shiny run --reload shiny_app.py" > run.sh chmod +x run.sh diff --git a/nextflow_schema.json b/nextflow_schema.json index 8ed9e72a..08536bff 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -129,8 +129,8 @@ }, "shiny_app": { "type": "string", - "format": "file-path", - "fa_icon": "fas fa-fast-forward", + "format": "directory-path", + "fa_icon": "fas fa-folder-open", "description": "File containing the main shiny app." } } From b28d0251f84ba6ee9d49500b757777e991ac79b6 Mon Sep 17 00:00:00 2001 From: luisas Date: Tue, 19 Mar 2024 12:55:50 +0100 Subject: [PATCH 04/12] updte --- bin/shiny_app/shiny_app.py | 2 + conf/modules.config | 6 +- conf/test.config | 1 - main.nf | 5 +- nextflow.config | 7 +- nextflow_schema.json | 10 +- .../main.nf | 136 +++++++++++++----- 7 files changed, 114 insertions(+), 53 deletions(-) diff --git a/bin/shiny_app/shiny_app.py b/bin/shiny_app/shiny_app.py index 8d483d22..8b908cf1 100644 --- a/bin/shiny_app/shiny_app.py +++ b/bin/shiny_app/shiny_app.py @@ -19,6 +19,8 @@ if not os.path.exists(summary_report): summary_report = "./shiny_data.csv" + # run merge script here + try: inputfile = pd.read_csv(summary_report) diff --git a/conf/modules.config b/conf/modules.config index 1a530ab3..dc83242e 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -234,16 +234,12 @@ process { // ------------------------------------ withName: 'PREPARE_SHINY' { publishDir = [ - path: { "${params.shinydir}" }, + path: { "${params.outdir}/shiny_app" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - - - - withName: 'ZIP' { ext.prefix = { "${meta.id}_${meta.tree}-args-${meta.argstree_clean}_${meta.aligner}-args-${meta.args_aligner_clean}" } publishDir = [ diff --git a/conf/test.config b/conf/test.config index bd549c6a..1ff599b8 100644 --- a/conf/test.config +++ b/conf/test.config @@ -26,6 +26,5 @@ params { // Output directory outdir = "./outdir/" - shinydir = "${params.outdir}/shiny_app" } diff --git a/main.nf b/main.nf index 0f91f0da..dd4687cc 100644 --- a/main.nf +++ b/main.nf @@ -108,8 +108,9 @@ workflow { params.monochrome_logs, params.hook_url, NFCORE_MULTIPLESEQUENCEALIGN.out.multiqc_report, - params.shinydir, - params.outdir + "${params.outdir}/shiny_app", + "${params.outdir}/pipeline_info", + params.shiny_trace_mode ) } diff --git a/nextflow.config b/nextflow.config index e59fad8a..860d4265 100644 --- a/nextflow.config +++ b/nextflow.config @@ -50,9 +50,9 @@ params { version = false // Shiny options - shinydir = "${params.outdir}/shiny_app" shiny_app = "${projectDir}/bin/shiny_app" skip_shiny = false + shiny_trace_mode = "all" // all, latest // Config options config_profile_name = null @@ -233,9 +233,8 @@ report { } trace { enabled = true - //file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" - file = "${params.outdir}/pipeline_info/trace.txt" - fields = 'task_id, hash,native_id,name,status,exit,realtime,%cpu,rss,peak_rss,vmem,peak_vmem,rchar,wchar,cpus' + file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" + fields = 'task_id, hash,native_id,name,status,exit,realtime,%cpu,rss,peak_rss,vmem,peak_vmem,rchar,wchar,cpus,start' } dag { enabled = true diff --git a/nextflow_schema.json b/nextflow_schema.json index 08536bff..4af3696a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -51,12 +51,6 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" - }, - "shinydir": { - "type": "string", - "format": "directory-path", - "description": "The output directory for the shiny app where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", - "fa_icon": "fas fa-folder-open" } } }, @@ -132,6 +126,10 @@ "format": "directory-path", "fa_icon": "fas fa-folder-open", "description": "File containing the main shiny app." + }, + "shiny_trace_mode": { + "type": "string", + "description": "variable containing the shiny_trace mode to be used." } } }, diff --git a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf index c1befa82..6489a93e 100644 --- a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf @@ -115,15 +115,16 @@ workflow PIPELINE_INITIALISATION { workflow PIPELINE_COMPLETION { take: - email // string: email address - email_on_fail // string: email address sent on pipeline failure - plaintext_email // boolean: Send plain-text email instead of HTML - outdir // path: Path to output directory where results will be published - monochrome_logs // boolean: Disable ANSI colour codes in log output - hook_url // string: hook URL for notifications - multiqc_report // string: Path to MultiQC report - shinydir // string: Path to shiny stats file - trace_infos // string: Path to trace file + email // string: email address + email_on_fail // string: email address sent on pipeline failure + plaintext_email // boolean: Send plain-text email instead of HTML + outdir // path: Path to output directory where results will be published + monochrome_logs // boolean: Disable ANSI colour codes in log output + hook_url // string: hook URL for notifications + multiqc_report // string: Path to MultiQC report + shiny_dir_path // string: Path to shiny stats file + trace_dir_path // string: Path to trace file + shiny_trace_mode // string: Mode to use for shiny trace file (default: "latest", options: "latest", "all") main: @@ -143,45 +144,110 @@ workflow PIPELINE_COMPLETION { imNotification(summary_params, hook_url) } - print(shinydir) - print(trace_infos) - - // TODO have an input parameter that specifies which tracefile to use - // or to use all and merge or to use the latest - - // Preprocess the trace file and move it to shiny directory - def trace_file = new File("${trace_infos}/pipeline_info/trace.txt") - - if (trace_file.exists()) { - trace_infos = filterTraceForShiny(trace_file) - print(trace_infos) - // move trace_infos to shinydir - def shiny_trace_file = new File("${shinydir}/trace.txt") - shiny_trace_file.write(trace_infos.join("\n")) - - }else{ - print("No trace file found in the pipeline_info directory") + if (shiny_trace_mode) { + getTraceForShiny(trace_dir_path, shiny_dir_path, shiny_trace_mode) } + } } +def getHeader(trace_file){ + def trace_lines = trace_file.readLines() + def header = trace_lines[0] + return header +} + def filterTraceForShiny(trace_file){ def trace_lines = trace_file.readLines() def shiny_trace_lines = [] - print("Filtering trace file for shiny") for (line in trace_lines){ - // if (line.contains("hash")){ - // shiny_trace_lines.add(line) - // } - // if (line.contains("COMPLETED") && line.contains("MULTIPLESEQUENCEALIGN:ALIGN")){ - // shiny_trace_lines.add(line) - // } - shiny_trace_lines.add(line) + if (line.contains("COMPLETED") && line.contains("MULTIPLESEQUENCEALIGN:ALIGN")){ + shiny_trace_lines.add(line) + } } return shiny_trace_lines } +// if multiple lines have the same name column +// only the one with the latest timestamp will be kept +def takeLatestComplete(traceInfos) { + // Remove the first line + colnames = traceInfos.first().split('\t') + // strip all the white spaces + + // get index of name and submit + def name_index = colnames.indexOf("name") + def submit_index = colnames.indexOf("start") + traceInfos = traceInfos.drop(1) + + // Initialize a map to store entries by their names and latest submit timestamps + def latestEntries = [:] + + // Iterate over each line + traceInfos.each { line -> + // Split the line into values + def values = line.split('\t') + + // Extract necessary data from the line + def name = values[name_index] // Assuming 'name' is at index 3 + def submit = values[submit_index] // Assuming 'submit' is at index 6 + + // If the name is not present in the map or if the current submit timestamp is later than the stored one + if (!latestEntries.containsKey(name) || submit > latestEntries[name][6]) { + // Store the current entry + latestEntries[name] = values + } + } + + // Convert the map values back to a list + def filteredData = latestEntries.values() + + // Add the header line back to the filtered data + def result = [traceInfos.first()] + + // Append filtered entries to the result + result.addAll(filteredData) + + return result +} + +def getTraceForShiny(trace_dir_path, shiny_dir_path, shiny_trace_mode){ + // According to the mode selected, get either the latest trace file or all trace files + // If all trace files are selected, it is assumed that the trace files were generated with the "resume" mode + def trace_dir = new File("${trace_dir_path}") + def trace_files = [] + if (shiny_trace_mode == "all"){ + trace_files = trace_dir.listFiles().findAll { it.name.startsWith("execution_trace") } + } + else if(shiny_trace_mode == "latest"){ + trace_files = trace_dir.listFiles().findAll { it.name.startsWith("execution_trace") }.sort { -it.lastModified() }.take(1) + } + else{ + print("Invalid shiny trace mode. Please use either 'latest' or 'all'") + } + // Filter the trace files for shiny + // and move the trace file to the shiny directory + if (trace_files.size() > 0) { + def trace_infos = [] + def header_added = false + for (file in trace_files){ + if( !header_added ){ + trace_infos = trace_infos + getHeader(file) + header_added = true + } + trace_infos = trace_infos + filterTraceForShiny(file) + } + + print(takeLatestComplete(trace_infos)) + + def shiny_trace_file = new File("${shiny_dir_path}/trace.txt") + shiny_trace_file.write(trace_infos.join("\n")) + }else{ + print("No trace file found in the " + trace_dir_path + " directory.") + } +} + /* From 24ce5a199ca5d20cc2109d5122cc6d84228b2611 Mon Sep 17 00:00:00 2001 From: luisas Date: Tue, 19 Mar 2024 13:55:45 +0100 Subject: [PATCH 05/12] working time --- .../main.nf | 193 +++++++++--------- 1 file changed, 93 insertions(+), 100 deletions(-) diff --git a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf index 6489a93e..453bb35b 100644 --- a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf @@ -33,10 +33,10 @@ workflow PIPELINE_INITIALISATION { help // boolean: Display help text validate_params // boolean: Boolean whether to validate parameters against the schema at runtime monochrome_logs // boolean: Do not use coloured log outputs - nextflow_cli_args // array: List of positional nextflow CLI args + nextflow_cli_args // array: List of positional nextflow CLI args outdir // string: The output directory where the results will be saved input // string: Path to input samplesheet - tools // string: Path to input tools samplesheet + tools // string: Path to input tools samplesheet main: @@ -152,104 +152,6 @@ workflow PIPELINE_COMPLETION { } -def getHeader(trace_file){ - def trace_lines = trace_file.readLines() - def header = trace_lines[0] - return header -} - -def filterTraceForShiny(trace_file){ - def trace_lines = trace_file.readLines() - def shiny_trace_lines = [] - for (line in trace_lines){ - if (line.contains("COMPLETED") && line.contains("MULTIPLESEQUENCEALIGN:ALIGN")){ - shiny_trace_lines.add(line) - } - } - return shiny_trace_lines -} - -// if multiple lines have the same name column -// only the one with the latest timestamp will be kept -def takeLatestComplete(traceInfos) { - // Remove the first line - colnames = traceInfos.first().split('\t') - // strip all the white spaces - - // get index of name and submit - def name_index = colnames.indexOf("name") - def submit_index = colnames.indexOf("start") - traceInfos = traceInfos.drop(1) - - // Initialize a map to store entries by their names and latest submit timestamps - def latestEntries = [:] - - // Iterate over each line - traceInfos.each { line -> - // Split the line into values - def values = line.split('\t') - - // Extract necessary data from the line - def name = values[name_index] // Assuming 'name' is at index 3 - def submit = values[submit_index] // Assuming 'submit' is at index 6 - - // If the name is not present in the map or if the current submit timestamp is later than the stored one - if (!latestEntries.containsKey(name) || submit > latestEntries[name][6]) { - // Store the current entry - latestEntries[name] = values - } - } - - // Convert the map values back to a list - def filteredData = latestEntries.values() - - // Add the header line back to the filtered data - def result = [traceInfos.first()] - - // Append filtered entries to the result - result.addAll(filteredData) - - return result -} - -def getTraceForShiny(trace_dir_path, shiny_dir_path, shiny_trace_mode){ - // According to the mode selected, get either the latest trace file or all trace files - // If all trace files are selected, it is assumed that the trace files were generated with the "resume" mode - def trace_dir = new File("${trace_dir_path}") - def trace_files = [] - if (shiny_trace_mode == "all"){ - trace_files = trace_dir.listFiles().findAll { it.name.startsWith("execution_trace") } - } - else if(shiny_trace_mode == "latest"){ - trace_files = trace_dir.listFiles().findAll { it.name.startsWith("execution_trace") }.sort { -it.lastModified() }.take(1) - } - else{ - print("Invalid shiny trace mode. Please use either 'latest' or 'all'") - } - // Filter the trace files for shiny - // and move the trace file to the shiny directory - if (trace_files.size() > 0) { - def trace_infos = [] - def header_added = false - for (file in trace_files){ - if( !header_added ){ - trace_infos = trace_infos + getHeader(file) - header_added = true - } - trace_infos = trace_infos + filterTraceForShiny(file) - } - - print(takeLatestComplete(trace_infos)) - - def shiny_trace_file = new File("${shiny_dir_path}/trace.txt") - shiny_trace_file.write(trace_infos.join("\n")) - }else{ - print("No trace file found in the " + trace_dir_path + " directory.") - } -} - - - /* ======================================================================================== FUNCTIONS @@ -358,6 +260,97 @@ def methodsDescriptionText(mqc_methods_yaml) { return description_html.toString() } +def getHeader(trace_file){ + // Get the header of the trace file + def trace_lines = trace_file.readLines() + def header = trace_lines[0] + return header +} + +def filterTraceForShiny(trace_file){ + // Retain only the lines that contain "COMPLETED" and "MULTIPLESEQUENCEALIGN:ALIGN" + def trace_lines = trace_file.readLines() + def shiny_trace_lines = [] + for (line in trace_lines){ + if (line.contains("COMPLETED") && line.contains("MULTIPLESEQUENCEALIGN:ALIGN")){ + shiny_trace_lines.add(line) + } + } + return shiny_trace_lines +} + +// if multiple lines have the same name column +// only the one with the latest timestamp will be kept +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +def takeLatestComplete(traceInfos) { + + // colnames and the position of the columns name and start + colnames = traceInfos.first().split('\t').collect { it.trim() } + def name_index = colnames.indexOf("name") + def start_index = colnames.indexOf("start") + // remove the column name line + traceInfos = traceInfos.drop(1) + // Initialize a map to store entries by their names and latest submit timestamps + def latestEntries = [:] + def formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS") + + // Iterate over each line + traceInfos.each { line -> + def values = line.split('\t') + def name = values[name_index] + def submit = LocalDateTime.parse(values[start_index], formatter) + if (!latestEntries.containsKey(name) || submit.isBefore(latestEntries[name][start_index])) { + latestEntries[name] = values + } + } + def filteredData = latestEntries.values() + + def result = [traceInfos.first()] + result.addAll(filteredData) + + return result +} + +def getTraceForShiny(trace_dir_path, shiny_dir_path, shiny_trace_mode){ + // According to the mode selected, get either the latest trace file or all trace files + // If all trace files are selected, it is assumed that the trace files were generated with the "resume" mode + def trace_dir = new File("${trace_dir_path}") + def trace_files = [] + if (shiny_trace_mode == "all"){ + trace_files = trace_dir.listFiles().findAll { it.name.startsWith("execution_trace") } + } + else if(shiny_trace_mode == "latest"){ + trace_files = trace_dir.listFiles().findAll { it.name.startsWith("execution_trace") }.sort { -it.lastModified() }.take(1) + } + else{ + print("Invalid shiny trace mode. Please use either 'latest' or 'all'") + } + // Filter the trace files for shiny + // and move the trace file to the shiny directory + if (trace_files.size() > 0) { + def trace_infos = [] + def header_added = false + for (file in trace_files){ + if( !header_added ){ + trace_infos = trace_infos + getHeader(file) + header_added = true + } + trace_infos = trace_infos + filterTraceForShiny(file) + } + // if trace infos is empty then print a message + if(trace_infos.size() == 0){ + print("There is an issue with your trace file!") + } + print(takeLatestComplete(trace_infos)) + + def shiny_trace_file = new File("${shiny_dir_path}/trace.txt") + shiny_trace_file.write(trace_infos.join("\n")) + }else{ + print("No trace file found in the " + trace_dir_path + " directory.") + } +} import nextflow.Nextflow import groovy.text.SimpleTemplateEngine From 12894d1eb74597d2981b23ee03c10653b2c82868 Mon Sep 17 00:00:00 2001 From: luisas Date: Tue, 19 Mar 2024 13:58:00 +0100 Subject: [PATCH 06/12] fix --- nextflow_schema.json | 2 +- .../local/utils_nfcore_multiplesequencealign_pipeline/main.nf | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 4af3696a..d09ed489 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -124,7 +124,7 @@ "shiny_app": { "type": "string", "format": "directory-path", - "fa_icon": "fas fa-folder-open", + "fa_icon": "fas fa-folder-open", "description": "File containing the main shiny app." }, "shiny_trace_mode": { diff --git a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf index 453bb35b..a884a471 100644 --- a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf @@ -297,11 +297,12 @@ def takeLatestComplete(traceInfos) { def formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS") // Iterate over each line + // If the name is not in the map or the submit timestamp is after the latest one, update the map traceInfos.each { line -> def values = line.split('\t') def name = values[name_index] def submit = LocalDateTime.parse(values[start_index], formatter) - if (!latestEntries.containsKey(name) || submit.isBefore(latestEntries[name][start_index])) { + if (!latestEntries.containsKey(name) || submit.isAfter(latestEntries[name][start_index])) { latestEntries[name] = values } } From 85ccf519f16f9e5e90602ac83d9c75b7ade2dbfd Mon Sep 17 00:00:00 2001 From: luisas Date: Tue, 19 Mar 2024 17:12:50 +0100 Subject: [PATCH 07/12] Fix app --- bin/shiny_app/shiny_app.py | 11 ++++-- .../shiny_app_merge_score_and_trace.py | 34 ++++++++----------- .../main.nf | 7 ++-- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/bin/shiny_app/shiny_app.py b/bin/shiny_app/shiny_app.py index 8b908cf1..ae4344bc 100644 --- a/bin/shiny_app/shiny_app.py +++ b/bin/shiny_app/shiny_app.py @@ -7,7 +7,7 @@ import matplotlib.pyplot as plt import sys import os - +import shiny_app_merge_score_and_trace as ms # Style sns.set(context="talk", style="white", font_scale=0.8) @@ -16,12 +16,17 @@ # Load file # ---------------------------------------------------------------------------- summary_report = "./shiny_data_with_trace.csv" +trace = "./trace.txt" if not os.path.exists(summary_report): - summary_report = "./shiny_data.csv" + summary_report_no_trace = "./shiny_data.csv" # run merge script here - + if os.path.exists(trace): + ms.merge_data_and_trace(summary_report_no_trace, trace, summary_report) + else: + summary_report = summary_report_no_trace + try: inputfile = pd.read_csv(summary_report) except: diff --git a/bin/shiny_app/shiny_app_merge_score_and_trace.py b/bin/shiny_app/shiny_app_merge_score_and_trace.py index 7587fb98..c682e833 100644 --- a/bin/shiny_app/shiny_app_merge_score_and_trace.py +++ b/bin/shiny_app/shiny_app_merge_score_and_trace.py @@ -1,15 +1,5 @@ import pandas as pd -data_file = 'shiny_data.csv' -trace_file = 'trace.txt' -out_file_name = 'shiny_data_with_trace.csv' - -# read in shiny_data.csv -data = pd.read_csv(data_file) -# read in trace -trace = pd.read_csv(trace_file, sep='\t') - - def convert_time(time): if time is not None: if "ms" in time: @@ -38,7 +28,7 @@ def convert_memory(memory): memory = float(memory)/1000000 return memory -def clean_trace(trace): +def cleanTrace(trace): # Update trace file def extract_element(row, nelement): elements = row.split(':') @@ -90,14 +80,20 @@ def prep_align_trace(trace): trace_align["memory_align"] = trace_align["memory_align"].apply(convert_memory) return trace_align -clean_trace = clean_trace(trace) -trace_trees = prep_tree_trace(clean_trace) -trace_align = prep_align_trace(clean_trace) -#merge data and trace_trees -data_tree = pd.merge(data, trace_trees, on=["id", "tree", "args_tree"], how="left") -data_tree_align = pd.merge(data_tree, trace_align, on=["id", "aligner", "args_aligner"], how="left") +def merge_data_and_trace(data_file,trace_file,out_file_name): + # read in shiny_data.csv + data = pd.read_csv(data_file) + # read in trace + trace = pd.read_csv(trace_file, sep='\t') + clean_trace = cleanTrace(trace) + trace_trees = prep_tree_trace(clean_trace) + trace_align = prep_align_trace(clean_trace) + + #merge data and trace_trees + data_tree = pd.merge(data, trace_trees, on=["id", "tree", "args_tree"], how="left") + data_tree_align = pd.merge(data_tree, trace_align, on=["id", "aligner", "args_aligner"], how="left") + # write to file + data_tree_align.to_csv(out_file_name, index=False) -# write to file -data_tree_align.to_csv(out_file_name, index=False) \ No newline at end of file diff --git a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf index a884a471..0deb762f 100644 --- a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf @@ -306,9 +306,10 @@ def takeLatestComplete(traceInfos) { latestEntries[name] = values } } - def filteredData = latestEntries.values() + def filteredData = colnames.join('\t') + '\n' + filteredData = filteredData + latestEntries.values().collect { it.join('\t') }.join('\n') - def result = [traceInfos.first()] + def result = [] result.addAll(filteredData) return result @@ -344,7 +345,7 @@ def getTraceForShiny(trace_dir_path, shiny_dir_path, shiny_trace_mode){ if(trace_infos.size() == 0){ print("There is an issue with your trace file!") } - print(takeLatestComplete(trace_infos)) + trace_infos = takeLatestComplete(trace_infos) def shiny_trace_file = new File("${shiny_dir_path}/trace.txt") shiny_trace_file.write(trace_infos.join("\n")) From e7c0b58e98ee2896f1c5119fd3f7dd304e7f8219 Mon Sep 17 00:00:00 2001 From: luisas Date: Tue, 19 Mar 2024 17:26:42 +0100 Subject: [PATCH 08/12] Fix lint --- CHANGELOG.md | 1 + bin/shiny_app/shiny_app.py | 1 - bin/shiny_app/shiny_app_merge_score_and_trace.py | 2 +- main.nf | 3 --- .../utils_nfcore_multiplesequencealign_pipeline/main.nf | 8 +++----- workflows/multiplesequencealign.nf | 3 +-- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4486ae03..03cf51b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Initial release of nf-core/multiplesequencealign, created with the [nf-core](htt [#93](https://github.com/nf-core/multiplesequencealign/pull/93) - Add multiqc basic support. Add custom params validation. Add basic shiny app. [#100](https://github.com/nf-core/multiplesequencealign/pull/100) - Add support for optional stats and evals. Clean tests. [#111](https://github.com/nf-core/multiplesequencealign/pull/111) - Add Readme documentation. Add nf-test for the pipeline. +[#113](https://github.com/nf-core/multiplesequencealign/pull/113) - Add reading of trace files for shiny app. ### `Fixed` diff --git a/bin/shiny_app/shiny_app.py b/bin/shiny_app/shiny_app.py index ae4344bc..b87c67c5 100644 --- a/bin/shiny_app/shiny_app.py +++ b/bin/shiny_app/shiny_app.py @@ -26,7 +26,6 @@ else: summary_report = summary_report_no_trace - try: inputfile = pd.read_csv(summary_report) except: diff --git a/bin/shiny_app/shiny_app_merge_score_and_trace.py b/bin/shiny_app/shiny_app_merge_score_and_trace.py index c682e833..e3072991 100644 --- a/bin/shiny_app/shiny_app_merge_score_and_trace.py +++ b/bin/shiny_app/shiny_app_merge_score_and_trace.py @@ -41,7 +41,7 @@ def extract_element(row, nelement): trace["process"] = trace.full_name.apply(extract_element, nelement=-1) trace["subworkflow"] = trace.full_name.apply(extract_element, nelement=-2) trace.replace('null', pd.NA, inplace=True) - return trace + return trace def prep_tree_trace(trace): trace_trees = trace[trace["subworkflow"] == "COMPUTE_TREES"] diff --git a/main.nf b/main.nf index dd4687cc..5c8fad75 100644 --- a/main.nf +++ b/main.nf @@ -59,9 +59,6 @@ workflow NFCORE_MULTIPLESEQUENCEALIGN { emit: multiqc_report = MULTIPLESEQUENCEALIGN.out.multiqc - shiny_stats = MULTIPLESEQUENCEALIGN.out.shiny_stats - - } /* diff --git a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf index 0deb762f..94f475c6 100644 --- a/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_multiplesequencealign_pipeline/main.nf @@ -287,9 +287,9 @@ import java.time.format.DateTimeFormatter def takeLatestComplete(traceInfos) { // colnames and the position of the columns name and start - colnames = traceInfos.first().split('\t').collect { it.trim() } + colnames = traceInfos.first().split('\t').collect { it.trim() } def name_index = colnames.indexOf("name") - def start_index = colnames.indexOf("start") + def start_index = colnames.indexOf("start") // remove the column name line traceInfos = traceInfos.drop(1) // Initialize a map to store entries by their names and latest submit timestamps @@ -300,7 +300,7 @@ def takeLatestComplete(traceInfos) { // If the name is not in the map or the submit timestamp is after the latest one, update the map traceInfos.each { line -> def values = line.split('\t') - def name = values[name_index] + def name = values[name_index] def submit = LocalDateTime.parse(values[start_index], formatter) if (!latestEntries.containsKey(name) || submit.isAfter(latestEntries[name][start_index])) { latestEntries[name] = values @@ -308,10 +308,8 @@ def takeLatestComplete(traceInfos) { } def filteredData = colnames.join('\t') + '\n' filteredData = filteredData + latestEntries.values().collect { it.join('\t') }.join('\n') - def result = [] result.addAll(filteredData) - return result } diff --git a/workflows/multiplesequencealign.nf b/workflows/multiplesequencealign.nf index 64678edb..a83a5761 100644 --- a/workflows/multiplesequencealign.nf +++ b/workflows/multiplesequencealign.nf @@ -267,8 +267,7 @@ workflow MULTIPLESEQUENCEALIGN { emit: versions = ch_versions - multiqc = multiqc_out - shiny_stats = ch_shiny_stats // channel: [ path(versions.yml) ] + multiqc = multiqc_out } From 2939e616d1d785d0eb204d52535e1a11f863e501 Mon Sep 17 00:00:00 2001 From: luisas Date: Tue, 19 Mar 2024 17:29:10 +0100 Subject: [PATCH 09/12] Fix lint --- bin/shiny_app/shiny_app.py | 2 +- main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/shiny_app/shiny_app.py b/bin/shiny_app/shiny_app.py index b87c67c5..47f21ea5 100644 --- a/bin/shiny_app/shiny_app.py +++ b/bin/shiny_app/shiny_app.py @@ -124,4 +124,4 @@ def scatter(): return ax -app = App(app_ui, server) \ No newline at end of file +app = App(app_ui, server) diff --git a/main.nf b/main.nf index 5c8fad75..a67c57b4 100644 --- a/main.nf +++ b/main.nf @@ -104,7 +104,7 @@ workflow { params.outdir, params.monochrome_logs, params.hook_url, - NFCORE_MULTIPLESEQUENCEALIGN.out.multiqc_report, + NFCORE_MULTIPLESEQUENCEALIGN.out.multiqc_report, "${params.outdir}/shiny_app", "${params.outdir}/pipeline_info", params.shiny_trace_mode From 73ac167f120408ed26238b73c0f6ab5d5fbd4402 Mon Sep 17 00:00:00 2001 From: luisas Date: Wed, 20 Mar 2024 12:17:51 +0100 Subject: [PATCH 10/12] Update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03cf51b7..72394d91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,8 @@ Initial release of nf-core/multiplesequencealign, created with the [nf-core](htt [#77](https://github.com/nf-core/multiplesequencealign/issues/77) - Add module zip [#93](https://github.com/nf-core/multiplesequencealign/pull/93) - Add multiqc basic support. Add custom params validation. Add basic shiny app. [#100](https://github.com/nf-core/multiplesequencealign/pull/100) - Add support for optional stats and evals. Clean tests. -[#111](https://github.com/nf-core/multiplesequencealign/pull/111) - Add Readme documentation. Add nf-test for the pipeline. -[#113](https://github.com/nf-core/multiplesequencealign/pull/113) - Add reading of trace files for shiny app. +[#110](https://github.com/nf-core/multiplesequencealign/issues/110) - Add Readme documentation. Add nf-test for the pipeline. +[#76](https://github.com/nf-core/multiplesequencealign/issues/76) - Add reading of trace files for shiny app. ### `Fixed` From e7dfa7a4b098ba5c99289b913ea2bba5d5d4b471 Mon Sep 17 00:00:00 2001 From: Luisa Santus Date: Wed, 20 Mar 2024 13:22:24 +0100 Subject: [PATCH 11/12] Update main.nf Co-authored-by: Jose Espinosa-Carrasco --- main.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/main.nf b/main.nf index a67c57b4..63a099f4 100644 --- a/main.nf +++ b/main.nf @@ -93,7 +93,6 @@ workflow { PIPELINE_INITIALISATION.out.tools ) - // // SUBWORKFLOW: Run completion tasks // From 45c1cd569ecc6a218c1ce8e5ab101875a4316d12 Mon Sep 17 00:00:00 2001 From: Luisa Santus Date: Wed, 20 Mar 2024 13:22:34 +0100 Subject: [PATCH 12/12] Update workflows/multiplesequencealign.nf Co-authored-by: Jose Espinosa-Carrasco --- workflows/multiplesequencealign.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/multiplesequencealign.nf b/workflows/multiplesequencealign.nf index 2c3efad7..e7fb25f6 100644 --- a/workflows/multiplesequencealign.nf +++ b/workflows/multiplesequencealign.nf @@ -265,7 +265,7 @@ workflow MULTIPLESEQUENCEALIGN { } emit: - versions = ch_versions + versions = ch_versions // channel: [ path(versions.yml) ] multiqc = multiqc_out }