From 76634df517a7aaf0e27e7105afad64a77555e3ec Mon Sep 17 00:00:00 2001 From: Dawn Smith Date: Tue, 12 Nov 2024 15:15:00 -0500 Subject: [PATCH 1/6] [DOC] Fix capitalization on module name --- bin/dm_qc_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/dm_qc_report.py b/bin/dm_qc_report.py index 58dc0eed..4dfe408d 100755 --- a/bin/dm_qc_report.py +++ b/bin/dm_qc_report.py @@ -22,7 +22,7 @@ Requires: FSL/5.0.10 - MATLAB/R2014a - qa-dti phantom pipeline + matlab/R2014a - qa-dti phantom pipeline AFNI/2014.12.16 - abcd_fmri phantom pipeline """ From 5a7d378ebc83ab627f76e5677239bc38611983e4 Mon Sep 17 00:00:00 2001 From: Dawn Smith Date: Tue, 12 Nov 2024 15:25:45 -0500 Subject: [PATCH 2/6] [FIX] Use correct function name --- datman/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datman/metrics.py b/datman/metrics.py index 01bcb0a3..e31aef26 100644 --- a/datman/metrics.py +++ b/datman/metrics.py @@ -44,7 +44,7 @@ def write_manifest(self, overwrite=False): if os.path.exists(self.manifest_path): if not overwrite: return - orig = self.read_json() + orig = self.read_manifest() else: orig = None From dae19f53c403e05741bd02e5533298ae0ecbe19c Mon Sep 17 00:00:00 2001 From: Dawn Smith Date: Tue, 12 Nov 2024 16:55:49 -0500 Subject: [PATCH 3/6] [ENH] Add 'refresh' flag so header diffs can update when settings change --- bin/dm_qc_report.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/bin/dm_qc_report.py b/bin/dm_qc_report.py index 4dfe408d..1abdd42d 100755 --- a/bin/dm_qc_report.py +++ b/bin/dm_qc_report.py @@ -12,7 +12,11 @@ Datman name of session to process e.g. DTI_CMH_H001_01_01 Options: - --remake Delete and recreate all QC metrics. + --refresh Update dashboard metadata (e.g. header diffs, scan + lengths) and generate missing metrics, if any. Note + that existing QC metrics will not be modified. + --remake Delete and recreate all QC metrics. Also force update + of dashboard metadata (e.g. header diffs, scan lengths). --log-to-server If set, all log messages will also be sent to the configured logging server. This is useful when the script is run on the queue, since it swallows logging. @@ -47,15 +51,18 @@ logger = logging.getLogger(os.path.basename(__file__)) REMAKE = False +REFRESH = False def main(): global REMAKE + global REFRESH arguments = docopt(__doc__) study = arguments[""] session = arguments[""] REMAKE = arguments["--remake"] + REFRESH = arguments["--refresh"] use_server = arguments["--log-to-server"] verbose = arguments["--verbose"] debug = arguments["--debug"] @@ -141,7 +148,7 @@ def submit_subjects(config): subs = get_subids(config) for subject in subs: - if not (REMAKE or needs_qc(subject, config)): + if not (REMAKE or REFRESH or needs_qc(subject, config)): continue command = make_command(subject) @@ -333,7 +340,7 @@ def update_dashboard(nii_path, header_ignore=None, header_tolerance=None): """ db_record = datman.dashboard.get_scan(nii_path) - if REMAKE or db_record.is_outdated_header_diffs(): + if REMAKE or REFRESH or db_record.is_outdated_header_diffs(): try: db_record.update_header_diffs( standard=db_record.gold_standards[0], @@ -344,7 +351,7 @@ def update_dashboard(nii_path, header_ignore=None, header_tolerance=None): f"exception: {e}" ) - if REMAKE or not db_record.length: + if REMAKE or REFRESH or not db_record.length: add_scan_length(nii_path, db_record) From 59f882c6543b3fd53c5fa46ea022e186c1540d9e Mon Sep 17 00:00:00 2001 From: Dawn Smith Date: Tue, 12 Nov 2024 16:56:40 -0500 Subject: [PATCH 4/6] [FIX] Increase memory to stop job time outs for large scans --- bin/dm_qc_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/dm_qc_report.py b/bin/dm_qc_report.py index 1abdd42d..5a548b3a 100755 --- a/bin/dm_qc_report.py +++ b/bin/dm_qc_report.py @@ -157,7 +157,7 @@ def submit_subjects(config): logger.info(f"Submitting QC job for {subject}.") datman.utils.submit_job( command, job_name, "/tmp", system=config.system, - argslist="--mem=3G" + argslist="--mem=5G" ) From 3f6a409881cfabf7dc85798fc6451b138b4feb2e Mon Sep 17 00:00:00 2001 From: Dawn Smith Date: Tue, 12 Nov 2024 17:28:18 -0500 Subject: [PATCH 5/6] [FIX] Ensure outputs actually removed when --remake used --- bin/dm_qc_report.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/dm_qc_report.py b/bin/dm_qc_report.py index 5a548b3a..59cfd138 100755 --- a/bin/dm_qc_report.py +++ b/bin/dm_qc_report.py @@ -411,11 +411,12 @@ def add_scan_length(nii_path, scan): def remove_outputs(metric): - for item in metric.outputs: - try: - os.remove(item) - except FileNotFoundError: - pass + for command in metric.outputs: + for item in metric.outputs[command]: + try: + os.remove(item) + except FileNotFoundError: + pass if __name__ == "__main__": From bfadd08faef872e4f2a0a7ad39acf1cac97644ae Mon Sep 17 00:00:00 2001 From: Dawn Smith Date: Tue, 12 Nov 2024 18:06:12 -0500 Subject: [PATCH 6/6] [FIX] Ensure manifest files regenerate when --remake used too --- bin/dm_qc_report.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/dm_qc_report.py b/bin/dm_qc_report.py index 59cfd138..8571f012 100755 --- a/bin/dm_qc_report.py +++ b/bin/dm_qc_report.py @@ -411,6 +411,10 @@ def add_scan_length(nii_path, scan): def remove_outputs(metric): + try: + os.remove(metric.manifest_path) + except FileNotFoundError: + pass for command in metric.outputs: for item in metric.outputs[command]: try: