From 08a9cc29e3a8286834962c3ccd426ca7323495ac Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 19 Dec 2022 14:34:15 +0100 Subject: [PATCH] fix compute_checksums for literal files --- cwltool/command_line_tool.py | 2 +- cwltool/process.py | 15 ++++++++++----- tests/test_examples.py | 22 ++++++++++++++++++++++ tests/wf/extract_region_specs.cwl | 21 +++++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 tests/wf/extract_region_specs.cwl diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 0b0d7f3ec..5b2bea5b2 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -866,7 +866,7 @@ def calc_checksum(location: str) -> Optional[str]: and "checksum" in e and e["checksum"] != "sha1$hash" ): - return cast(Optional[str], e["checksum"]) + return cast(str, e["checksum"]) return None def remove_prefix(s: str, prefix: str) -> str: diff --git a/cwltool/process.py b/cwltool/process.py index c442bf249..4f046928b 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1342,10 +1342,15 @@ def compute_checksums(fs_access: StdFsAccess, fileobj: CWLObjectType) -> None: if "checksum" not in fileobj: checksum = hashlib.sha1() # nosec location = cast(str, fileobj["location"]) - with fs_access.open(location, "rb") as f: - contents = f.read(1024 * 1024) - while contents != b"": - checksum.update(contents) + if "contents" in fileobj: + contents = cast(str, fileobj["contents"]).encode("utf-8") + checksum.update(contents) + fileobj["size"] = len(contents) + else: + with fs_access.open(location, "rb") as f: contents = f.read(1024 * 1024) + while contents != b"": + checksum.update(contents) + contents = f.read(1024 * 1024) + fileobj["size"] = fs_access.size(location) fileobj["checksum"] = "sha1$%s" % checksum.hexdigest() - fileobj["size"] = fs_access.size(location) diff --git a/tests/test_examples.py b/tests/test_examples.py index 1e4e939c6..4d479e313 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1325,6 +1325,28 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None: assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists() +@pytest.mark.parametrize("factor", test_factors) +def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None: + """Confirm that running a CLT with a default literal file with caching succeeds.""" + test_file = "tests/wf/extract_region_specs.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out"), + "--cachedir", + cache_dir, + get_data(test_file), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + def test_write_summary(tmp_path: Path) -> None: """Test --write-summary.""" commands = [ diff --git a/tests/wf/extract_region_specs.cwl b/tests/wf/extract_region_specs.cwl new file mode 100644 index 000000000..279fa4400 --- /dev/null +++ b/tests/wf/extract_region_specs.cwl @@ -0,0 +1,21 @@ +{ +"cwlVersion": "v1.0", +"class": "CommandLineTool", +"inputs": [ + { + "type": "File", + "default": { + "class": "File", + "basename": "extract_regions.py", + "contents": "#!/usr/bin/env python3\n\nfrom __future__ import print_function, division\nimport sys\n\ninput_filename = sys.argv[1]\nif len(sys.argv) == 3:\n fuzz = int(sys.argv[2])\nelse:\n fuzz = 0\ninput_file = open(input_filename)\n\ncount = 0\nfor line in input_file:\n if not line.startswith(\">\"):\n continue\n count += 1\n contig_regions_file = open(\"contig_regions{}.txt\".format(count), \"w\")\n proteins_list_file = open(\"proteins{}.txt\".format(count), \"w\")\n fields = line.split(\"|\")\n protein_id = fields[0][1:]\n contig_id = fields[1]\n r_start = int(fields[6])\n if r_start > fuzz:\n r_start = r_start - fuzz\n r_end = int(fields[7]) + fuzz\n print(\"{}:{}-{}\".format(contig_id, r_start, r_end), file=contig_regions_file)\n print(protein_id, file=proteins_list_file)\n contig_regions_file.close()\n proteins_list_file.close()\n" + }, + "inputBinding": { + "position": 1 + }, + "id": "scripts" + } +], +"outputs": [ +], +"baseCommand": "cat" +}