From 35c357763634894ba76066e92f5a3e4fecb9170a Mon Sep 17 00:00:00 2001 From: Kevin Schneider Date: Tue, 12 Sep 2023 11:16:00 +0200 Subject: [PATCH 1/4] implement #188 - Add FileSystemTree.FilterFolders - FileSystemTree.FilterFiles now only filters files (returns empty folders instead of removing them) - fix some static member bindings - Add tests for FilterFolders - Tests now test both instance and static methods --- src/FileSystem/FileSystemTree.fs | 26 ++- tests/FileSystem/FileSystemTree.Tests.fs | 227 ++++++++++++++++------- 2 files changed, 182 insertions(+), 71 deletions(-) diff --git a/src/FileSystem/FileSystemTree.fs b/src/FileSystem/FileSystemTree.fs index cca25929..02027040 100644 --- a/src/FileSystem/FileSystemTree.fs +++ b/src/FileSystem/FileSystemTree.fs @@ -128,16 +128,28 @@ type FileSystemTree = | File n -> if predicate n then Some (File n) else None | Folder (n, children) -> - let filteredChildren = children |> Array.choose loop - if Array.isEmpty filteredChildren then - None - else - Folder (n, filteredChildren) - |> Some + Folder (n, children |> Array.choose loop) + |> Some + loop this static member filterFiles (predicate: string -> bool) = - fun (tree: FileSystemTree) -> tree.Filter predicate + fun (tree: FileSystemTree) -> tree.FilterFiles predicate + + member this.FilterFolders (predicate: string -> bool) = + let rec loop (parent: FileSystemTree) = + match parent with + | File n -> Some (File n) + | Folder (n, children) -> + if predicate n then + Folder (n, children |> Array.choose loop) + |> Some + else + None + loop this + + static member filterFolders (predicate: string -> bool) = + fun (tree: FileSystemTree) -> tree.FilterFolders predicate member this.Filter (predicate: string -> bool) = let rec loop (parent: FileSystemTree) = diff --git a/tests/FileSystem/FileSystemTree.Tests.fs b/tests/FileSystem/FileSystemTree.Tests.fs index 827acebc..ac55f734 100644 --- a/tests/FileSystem/FileSystemTree.Tests.fs +++ b/tests/FileSystem/FileSystemTree.Tests.fs @@ -33,57 +33,58 @@ let private newArcRelativePaths = [| @"studies\TestAssay1\resources\.gitkeep"|] -let newArcFST = Folder("root",[| - File "isa.investigation.xlsx"; - Folder(".arc", [|File ".gitkeep"|]); - Folder(".git",[| - File "config"; File "description"; File "HEAD"; - Folder("hooks",[| - File "applypatch-msg.sample"; File "commit-msg.sample"; - File "fsmonitor-watchman.sample"; File "post-update.sample"; - File "pre-applypatch.sample"; File "pre-commit.sample"; - File "pre-merge-commit.sample"; File "pre-push.sample"; - File "pre-rebase.sample"; File "pre-receive.sample"; - File "prepare-commit-msg.sample"; - File "push-to-checkout.sample"; File "update.sample" +let newArcFST = + Folder("root",[| + File "isa.investigation.xlsx"; + Folder(".arc", [|File ".gitkeep"|]); + Folder(".git",[| + File "config"; File "description"; File "HEAD"; + Folder("hooks",[| + File "applypatch-msg.sample"; File "commit-msg.sample"; + File "fsmonitor-watchman.sample"; File "post-update.sample"; + File "pre-applypatch.sample"; File "pre-commit.sample"; + File "pre-merge-commit.sample"; File "pre-push.sample"; + File "pre-rebase.sample"; File "pre-receive.sample"; + File "prepare-commit-msg.sample"; + File "push-to-checkout.sample"; File "update.sample" + |]); + Folder ("info", [|File "exclude"|]) |]); - Folder ("info", [|File "exclude"|]) - |]); - Folder("assays",[| - File ".gitkeep"; - Folder("est",[| - File "isa.assay.xlsx"; File "README.md"; - Folder ("dataset", [|File ".gitkeep"|]); - Folder ("protocols", [|File ".gitkeep"|]) - |]); - Folder - ("TestAssay1",[| - File "isa.assay.xlsx"; File "README.md"; - Folder ("dataset", [|File ".gitkeep"|]); - Folder ("protocols", [|File ".gitkeep"|]) - |]) - |]); - Folder("runs", [|File ".gitkeep"|]); - Folder("studies",[| - File ".gitkeep"; - Folder("est",[| - File "isa.study.xlsx"; File "README.md"; - Folder ("protocols", [|File ".gitkeep"|]); - Folder ("resources", [|File ".gitkeep"|]) + Folder("assays",[| + File ".gitkeep"; + Folder("est",[| + File "isa.assay.xlsx"; File "README.md"; + Folder ("dataset", [|File ".gitkeep"|]); + Folder ("protocols", [|File ".gitkeep"|]) + |]); + Folder + ("TestAssay1",[| + File "isa.assay.xlsx"; File "README.md"; + Folder ("dataset", [|File ".gitkeep"|]); + Folder ("protocols", [|File ".gitkeep"|]) + |]) |]); - Folder("MyStudy",[| - File "isa.study.xlsx"; File "README.md"; - Folder ("protocols", [|File ".gitkeep"|]); - Folder ("resources", [|File ".gitkeep"|]) + Folder("runs", [|File ".gitkeep"|]); + Folder("studies",[| + File ".gitkeep"; + Folder("est",[| + File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [|File ".gitkeep"|]); + Folder ("resources", [|File ".gitkeep"|]) + |]); + Folder("MyStudy",[| + File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [|File ".gitkeep"|]); + Folder ("resources", [|File ".gitkeep"|]) + |]); + Folder("TestAssay1",[| + File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [|File ".gitkeep"|]); + Folder ("resources", [|File ".gitkeep"|]) + |]) |]); - Folder("TestAssay1",[| - File "isa.study.xlsx"; File "README.md"; - Folder ("protocols", [|File ".gitkeep"|]); - Folder ("resources", [|File ".gitkeep"|]) - |]) - |]); - Folder ("workflows", [|File ".gitkeep"|]) -|]) + Folder ("workflows", [|File ".gitkeep"|]) + |]) let private tests_fromFilePaths = testList "fromFilePaths" [ @@ -118,27 +119,58 @@ let private tests_Filter_Files = testList "FilterFiles" [ test "new arc (2023-07-11), keep .xlsx files" { let expected = Some( - Folder("root",[| - File "isa.investigation.xlsx"; + Folder("root",[| + File "isa.investigation.xlsx"; + Folder(".arc", [||]); + Folder(".git",[| + Folder("hooks",[||]); + Folder ("info", [||]) + |]); Folder("assays",[| - Folder ("est", [|File "isa.assay.xlsx"|]); - Folder ("TestAssay1", [|File "isa.assay.xlsx"|]) + Folder("est",[| + File "isa.assay.xlsx" + Folder ("dataset", [||]); + Folder ("protocols", [||]) + |]); + Folder + ("TestAssay1",[| + File "isa.assay.xlsx" + Folder ("dataset", [||]); + Folder ("protocols", [||]) + |]) |]); + Folder("runs", [||]); Folder("studies",[| - Folder ("est", [|File "isa.study.xlsx"|]); - Folder ("MyStudy", [|File "isa.study.xlsx"|]); - Folder ("TestAssay1", [|File "isa.study.xlsx"|]) - |]) + Folder("est",[| + File "isa.study.xlsx"; + Folder ("protocols", [||]); + Folder ("resources", [||]) + |]); + Folder("MyStudy",[| + File "isa.study.xlsx"; + Folder ("protocols", [||]); + Folder ("resources", [||]) + |]); + Folder("TestAssay1",[| + File "isa.study.xlsx"; + Folder ("protocols", [||]); + Folder ("resources", [||]) + |]) + |]); + Folder ("workflows", [||]) |]) ) let filest = FileSystemTree.fromFilePaths newArcRelativePaths - let actual = filest.FilterFiles (fun n -> n.EndsWith ".xlsx") - Expect.equal actual expected "" + let actualInstanceMethod = filest.FilterFiles (fun n -> n.EndsWith ".xlsx") + let actualStaticMember = filest |> FileSystemTree.filterFiles (fun n -> n.EndsWith ".xlsx") + Expect.equal actualInstanceMethod expected "instance member result incorrect." + Expect.equal actualStaticMember expected "static member result incorrect." } - test "new arc (2023-07-11), filter startswith '.'" { + test "new arc (2023-07-11), filter not startswith '.'" { let expected = Some( Folder("root",[| File "isa.investigation.xlsx"; + Folder(".arc", [||]); Folder(".git",[| File "config"; File "description"; File "HEAD"; Folder("hooks",[| @@ -155,34 +187,98 @@ let private tests_Filter_Files = Folder("assays",[| Folder("est",[| File "isa.assay.xlsx"; File "README.md"; + Folder ("dataset", [||]); + Folder ("protocols", [||]) + |]); + Folder + ("TestAssay1",[| + File "isa.assay.xlsx"; File "README.md"; + Folder ("dataset", [||]); + Folder ("protocols", [||]) + |]) + |]); + Folder("runs", [||]); + Folder("studies",[| + Folder("est",[| + File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [||]); + Folder ("resources", [||]) + |]); + Folder("MyStudy",[| + File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [||]); + Folder ("resources", [||]) + |]); + Folder("TestAssay1",[| + File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [||]); + Folder ("resources", [||]) + |]) + |]); + Folder ("workflows", [||]) + |]) + ) + let filest = FileSystemTree.fromFilePaths newArcRelativePaths + let actualInstanceMethod = filest.FilterFiles (fun n -> not (n.StartsWith ".")) + let actualStaticMember = filest |> FileSystemTree.filterFiles (fun n -> not (n.StartsWith ".")) + Expect.equal actualInstanceMethod expected "instance member result incorrect." + Expect.equal actualStaticMember expected "static member result incorrect." + } + ] + +let private tests_Filter_Folders = + testList "FilterFolders" [ + test "new arc (2023-07-11), filter not startswith '.'" { + let expected = Some( + Folder("root",[| + File "isa.investigation.xlsx"; + Folder("assays",[| + File ".gitkeep"; + Folder("est",[| + File "isa.assay.xlsx"; File "README.md"; + Folder ("dataset", [|File ".gitkeep"|]); + Folder ("protocols", [|File ".gitkeep"|]) |]); Folder ("TestAssay1",[| File "isa.assay.xlsx"; File "README.md"; + Folder ("dataset", [|File ".gitkeep"|]); + Folder ("protocols", [|File ".gitkeep"|]) |]) |]); + Folder("runs", [|File ".gitkeep"|]); Folder("studies",[| + File ".gitkeep"; Folder("est",[| File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [|File ".gitkeep"|]); + Folder ("resources", [|File ".gitkeep"|]) |]); Folder("MyStudy",[| File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [|File ".gitkeep"|]); + Folder ("resources", [|File ".gitkeep"|]) |]); Folder("TestAssay1",[| File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [|File ".gitkeep"|]); + Folder ("resources", [|File ".gitkeep"|]) |]) |]); + Folder ("workflows", [|File ".gitkeep"|]) |]) ) let filest = FileSystemTree.fromFilePaths newArcRelativePaths - let actual = filest.FilterFiles (fun n -> not (n.StartsWith ".")) - Expect.equal actual expected "" + let actualInstanceMethod = filest.FilterFolders (fun n -> not (n.StartsWith ".")) + let actualStaticMember = filest |> FileSystemTree.filterFolders (fun n -> not (n.StartsWith ".")) + Expect.equal actualInstanceMethod expected "instance member result incorrect." + Expect.equal actualStaticMember expected "static member result incorrect." } ] let private tests_Filter = testList "Filter" [ - test "new arc (2023-07-11), filter startswith '.'" { + test "new arc (2023-07-11), filter not startswith '.'" { let expected = Some( Folder("root",[| File "isa.investigation.xlsx"; @@ -214,8 +310,10 @@ let private tests_Filter = |]) ) let filest = FileSystemTree.fromFilePaths newArcRelativePaths - let actual = filest.Filter (fun n -> not (n.StartsWith ".")) - Expect.equal actual expected "" + let actualInstanceMethod = filest.Filter (fun n -> not (n.StartsWith ".")) + let actualStaticMember = filest |> FileSystemTree.filter (fun n -> not (n.StartsWith ".")) + Expect.equal actualInstanceMethod expected "instance member result incorrect." + Expect.equal actualStaticMember expected "static member result incorrect." } ] @@ -393,6 +491,7 @@ let main = testList "FileSystemTree" [ tests_fromFilePaths tests_ToFilePaths tests_Filter_Files + tests_Filter_Folders tests_Filter tests_AddFile ] From 095114dab4404ee2e8d4f2dc81e8220f474a10f2 Mon Sep 17 00:00:00 2001 From: Kevin Schneider Date: Tue, 12 Sep 2023 14:50:53 +0200 Subject: [PATCH 2/4] implement #187 -Add `GetRegisteredPayload` and `GetAdditionalPayload` - Add tests --- src/ARCtrl/ARCtrl.fs | 97 +++++++++++++++++++++++ tests/ARCtrl/ARCtrl.Tests.fs | 149 +++++++++++++++++++++++++++++++++++ 2 files changed, 246 insertions(+) diff --git a/src/ARCtrl/ARCtrl.fs b/src/ARCtrl/ARCtrl.fs index 3b799d60..37dc2230 100644 --- a/src/ARCtrl/ARCtrl.fs +++ b/src/ARCtrl/ARCtrl.fs @@ -242,7 +242,104 @@ type ARC(?isa : ISA.ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSyste let fsCopy = _fs.Copy() new ARC(?isa = isaCopy, ?cwl = _cwl, fs = fsCopy) + /// + /// Returns the FileSystemTree of the ARC with only the registered files and folders included. + /// + /// Wether or not to ignore hidden files and folders starting with '.'. If true, no hidden files are included in the result. (default: true) + member this.GetRegisteredPayload(?IgnoreHidden:bool) = + + let isaCopy = _isa |> Option.map (fun i -> i.Copy()) // not sure if needed, but let's be safe + + let registeredStudies = + isaCopy + |> Option.map (fun isa -> isa.Studies.ToArray()) // to-do: isa.RegisteredStudies + |> Option.defaultValue [||] + + let registeredAssays = + registeredStudies + |> Array.map (fun s -> s.Assays.ToArray()) // to-do: s.RegisteredAssays + |> Array.concat + + let includeRootFiles : Set = + set [ + "isa.investigation.xlsx" + "README.md" + ] + + let includeStudyFiles = + registeredStudies + |> Array.map (fun s -> + let studyFoldername = $"studies/{s.Identifier}" + + set [ + yield $"{studyFoldername}/isa.study.xlsx" + yield $"{studyFoldername}/README.md" + + //just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now. + for (kv) in s.Tables[0].Values do + yield kv.Value.AsFreeText // from arc root + yield $"{studyFoldername}/resources/{kv.Value.AsFreeText}" // from study root > resources + yield $"{studyFoldername}/protocols/{kv.Value.AsFreeText}" // from study root > protocols + ] + ) + |> Set.unionMany + + let includeAssayFiles = + registeredAssays + |> Array.map (fun a -> + let assayFoldername = $"assays/{a.Identifier}" + + set [ + yield $"{assayFoldername}/isa.assay.xlsx" + yield $"{assayFoldername}/README.md" + + //just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now. + for (kv) in a.Tables[0].Values do + yield kv.Value.AsFreeText // from arc root + yield $"{assayFoldername}/dataset/{kv.Value.AsFreeText}" // from assay root > dataset + yield $"{assayFoldername}/protocols/{kv.Value.AsFreeText}" // from assay root > protocols + ] + ) + |> Set.unionMany + + let includeFiles = Set.unionMany [includeRootFiles; includeStudyFiles; includeAssayFiles] + + let ignoreHidden = defaultArg IgnoreHidden true + let fsCopy = _fs.Copy() // not sure if needed, but let's be safe + + fsCopy.Tree + |> FileSystemTree.toFilePaths() + |> Array.filter (fun p -> + p.StartsWith("workflows") + || p.StartsWith("runs") + || includeFiles.Contains(p) + ) + |> FileSystemTree.fromFilePaths + |> fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFiles (fun n -> not (n.StartsWith("."))) else Some tree + |> Option.bind (fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFolders (fun n -> not (n.StartsWith("."))) else Some tree) + |> Option.defaultValue (FileSystemTree.fromFilePaths [||]) + + /// + /// Returns the FileSystemTree of the ARC with only and folders included that are considered additional payload. + /// + /// Wether or not to ignore hidden files and folders starting with '.'. If true, no hidden files are included in the result. (default: true) + + member this.GetAdditionalPayload(?IgnoreHidden:bool) = + let ignoreHidden = defaultArg IgnoreHidden true + let registeredPayload = + this.GetRegisteredPayload() + |> FileSystemTree.toFilePaths() + |> set + + _fs.Copy().Tree + |> FileSystemTree.toFilePaths() + |> Array.filter (fun p -> not (registeredPayload.Contains(p))) + |> FileSystemTree.fromFilePaths + |> fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFiles (fun n -> not (n.StartsWith("."))) else Some tree + |> Option.bind (fun tree -> if ignoreHidden then tree |> FileSystemTree.filterFolders (fun n -> not (n.StartsWith("."))) else Some tree) + |> Option.defaultValue (FileSystemTree.fromFilePaths [||]) + //-Pseudo code-// //// Option 1 diff --git a/tests/ARCtrl/ARCtrl.Tests.fs b/tests/ARCtrl/ARCtrl.Tests.fs index a89e1377..12298f8f 100644 --- a/tests/ARCtrl/ARCtrl.Tests.fs +++ b/tests/ARCtrl/ARCtrl.Tests.fs @@ -249,9 +249,158 @@ let private test_updateFileSystem = testList "update_Filesystem" [ ) ] +open ARCtrl.FileSystem + +let private ``payload_file_filters`` = + + let orderFST (fs : FileSystemTree) = + fs + |> FileSystemTree.toFilePaths() + |> Array.sort + |> FileSystemTree.fromFilePaths + + testList "payload file filters" [ + let inv = ArcInvestigation("MyInvestigation", "BestTitle") + + let assay = ArcAssay("registered_assay") + let assayTable = assay.InitTable("MyAssayTable") + assayTable.AppendColumn(CompositeHeader.Input (IOType.RawDataFile), [|CompositeCell.createFreeText "registered_assay_input.txt"|]) + assayTable.AppendColumn(CompositeHeader.ProtocolREF, [|CompositeCell.createFreeText "assay_protocol.rtf"|]) + assayTable.AppendColumn(CompositeHeader.Output (IOType.DerivedDataFile), [|CompositeCell.createFreeText "registered_assay_output.txt"|]) + + let study = ArcStudy("registered_study") + let studyTable = study.InitTable("MyStudyTable") + studyTable.AppendColumn(CompositeHeader.Input (IOType.Sample), [|CompositeCell.createFreeText "some_study_input_material"|]) + studyTable.AppendColumn(CompositeHeader.FreeText "Some File", [|CompositeCell.createFreeText "xd/some_file_that_lies_in_slashxd.txt"|]) + studyTable.AppendColumn(CompositeHeader.ProtocolREF, [|CompositeCell.createFreeText "study_protocol.pdf"|]) + studyTable.AppendColumn(CompositeHeader.Output (IOType.RawDataFile), [|CompositeCell.createFreeText "registered_study_output.txt"|]) + study.AddAssay(assay) + + inv.AddStudy(study) + + let fs = + Folder("root",[| + File "isa.investigation.xlsx"; // this should be included + File "README.md"; // this should be included + Folder("xd", [|File "some_file_that_lies_in_slashxd.txt"|]); // this should be included + Folder(".arc", [|File ".gitkeep"|]); + Folder(".git",[| + File "config"; File "description"; File "HEAD"; + Folder("hooks",[| + File "applypatch-msg.sample"; File "commit-msg.sample"; + File "fsmonitor-watchman.sample"; File "post-update.sample"; + File "pre-applypatch.sample"; File "pre-commit.sample"; + File "pre-merge-commit.sample"; File "pre-push.sample"; + File "pre-rebase.sample"; File "pre-receive.sample"; + File "prepare-commit-msg.sample"; + File "push-to-checkout.sample"; File "update.sample" + |]); + Folder ("info", [|File "exclude"|]) + |]); + Folder("assays",[| + File ".gitkeep"; + Folder("registered_assay",[| + File "isa.assay.xlsx"; // this should be included + File "README.md"; // this should be included + Folder ("dataset", [| + File "registered_assay_input.txt" // this should be included + File "registered_assay_output.txt" // this should be included + File "unregistered_file.txt" + |]; ); + Folder ("protocols", [|File "assay_protocol.rtf"|]) // this should be included + |]); + Folder + ("unregistered_assay",[| + File "isa.assay.xlsx"; File "README.md"; + Folder ("dataset", [|File ".gitkeep"|]); + Folder ("protocols", [|File ".gitkeep"|]) + |]) + |]); + Folder("runs", [|File ".gitkeep"|]); // this folder should be included (empty) + Folder("studies",[| + File ".gitkeep"; + Folder("registered_study",[| + File "isa.study.xlsx"; // this should be included + File "README.md"; // this should be included + Folder ("protocols", [|File "study_protocol.pdf"|]); // this should be included + Folder ("resources", [|File "registered_study_output.txt"|]) // this should be included + |]); + Folder("unregistered_study",[| + File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [|File ".gitkeep"|]); + Folder ("resources", [|File ".gitkeep"|]) + |]); + |]); + Folder ("workflows", [|File ".gitkeep"|]) // this folder should be included (empty) + |]) + + let arc = ARC(isa = inv, fs = FileSystem.create(fs)) + + test "GetRegisteredPayload" { + let expected = + Folder("root",[| + File "isa.investigation.xlsx"; // this should be included + File "README.md"; // this should be included + Folder("xd", [|File "some_file_that_lies_in_slashxd.txt"|]); // this should be included + Folder("assays",[| + Folder("registered_assay",[| + File "isa.assay.xlsx"; // this should be included + File "README.md"; // this should be included + Folder ("dataset", [| + File "registered_assay_input.txt" // this should be included + File "registered_assay_output.txt" // this should be included + |]; ); + Folder ("protocols", [|File "assay_protocol.rtf"|]) // this should be included + |]); + |]); + Folder("runs", [||]); // this folder should be included (empty) + Folder("studies",[| + Folder("registered_study",[| + File "isa.study.xlsx"; // this should be included + File "README.md"; // this should be included + Folder ("protocols", [|File "study_protocol.pdf"|]); // this should be included + Folder ("resources", [|File "registered_study_output.txt"|]) // this should be included + |]); + |]); + Folder ("workflows", [||]) // this folder should be included (empty) + |]) + + let actual = arc.GetRegisteredPayload() + Expect.equal (orderFST actual) (orderFST expected) "incorrect payload." + } + test "GetAdditionalPayload" { + let expected = + Folder("root",[| + Folder("assays",[| + Folder("registered_assay",[| + Folder ("dataset", [| + File "unregistered_file.txt" + |]; ); + |]); + Folder + ("unregistered_assay",[| + File "isa.assay.xlsx"; File "README.md"; + Folder ("dataset", [||]); + Folder ("protocols", [||]) + |]) + |]); + Folder("studies",[| + Folder("unregistered_study",[| + File "isa.study.xlsx"; File "README.md"; + Folder ("protocols", [||]); + Folder ("resources", [||]) + |]); + |]); + |]) + let actual = arc.GetAdditionalPayload() + Expect.equal (orderFST actual) (orderFST expected) "incorrect payload." + } + ] + let main = testList "main" [ test_model test_updateFileSystem test_isaFromContracts test_writeContracts + payload_file_filters ] \ No newline at end of file From 907f3a2b6a07f3e22d7f851113034eec097511fe Mon Sep 17 00:00:00 2001 From: Kevin Schneider Date: Tue, 12 Sep 2023 20:30:28 +0200 Subject: [PATCH 3/4] fix GetRegisteredPayload cell conversion failure - iterate through all tables instead of the first only - use ToFreeTextCell(), which is a proper conversion instead of AsFreeTextCell which is a match that can fail --- src/ARCtrl/ARCtrl.fs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/ARCtrl/ARCtrl.fs b/src/ARCtrl/ARCtrl.fs index 37dc2230..10d1b6f3 100644 --- a/src/ARCtrl/ARCtrl.fs +++ b/src/ARCtrl/ARCtrl.fs @@ -276,10 +276,12 @@ type ARC(?isa : ISA.ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSyste yield $"{studyFoldername}/README.md" //just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now. - for (kv) in s.Tables[0].Values do - yield kv.Value.AsFreeText // from arc root - yield $"{studyFoldername}/resources/{kv.Value.AsFreeText}" // from study root > resources - yield $"{studyFoldername}/protocols/{kv.Value.AsFreeText}" // from study root > protocols + for table in s.Tables do + for kv in table.Values do + let textValue = kv.Value.ToFreeTextCell().AsFreeText + yield textValue // from arc root + yield $"{studyFoldername}/resources/{textValue}" // from study root > resources + yield $"{studyFoldername}/protocols/{textValue}" // from study root > protocols ] ) |> Set.unionMany @@ -294,10 +296,12 @@ type ARC(?isa : ISA.ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSyste yield $"{assayFoldername}/README.md" //just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now. - for (kv) in a.Tables[0].Values do - yield kv.Value.AsFreeText // from arc root - yield $"{assayFoldername}/dataset/{kv.Value.AsFreeText}" // from assay root > dataset - yield $"{assayFoldername}/protocols/{kv.Value.AsFreeText}" // from assay root > protocols + for table in a.Tables do + for kv in table.Values do + let textValue = kv.Value.ToFreeTextCell().AsFreeText + yield textValue // from arc root + yield $"{assayFoldername}/dataset/{textValue}" // from assay root > dataset + yield $"{assayFoldername}/protocols/{textValue}" // from assay root > protocols ] ) |> Set.unionMany From affab06b7748ab3634921dfbbf0753ebac955a4b Mon Sep 17 00:00:00 2001 From: Kevin Schneider Date: Wed, 13 Sep 2023 19:36:27 +0200 Subject: [PATCH 4/4] Use path literals for payload path filtering --- src/ARCtrl/ARCtrl.fs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/ARCtrl/ARCtrl.fs b/src/ARCtrl/ARCtrl.fs index 10d1b6f3..d790798a 100644 --- a/src/ARCtrl/ARCtrl.fs +++ b/src/ARCtrl/ARCtrl.fs @@ -262,26 +262,26 @@ type ARC(?isa : ISA.ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSyste let includeRootFiles : Set = set [ - "isa.investigation.xlsx" - "README.md" + Path.InvestigationFileName + Path.READMEFileName ] let includeStudyFiles = registeredStudies |> Array.map (fun s -> - let studyFoldername = $"studies/{s.Identifier}" + let studyFoldername = $"{Path.StudiesFolderName}/{s.Identifier}" set [ - yield $"{studyFoldername}/isa.study.xlsx" - yield $"{studyFoldername}/README.md" + yield $"{studyFoldername}/{Path.StudyFileName}" + yield $"{studyFoldername}/{Path.READMEFileName}" //just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now. for table in s.Tables do for kv in table.Values do let textValue = kv.Value.ToFreeTextCell().AsFreeText yield textValue // from arc root - yield $"{studyFoldername}/resources/{textValue}" // from study root > resources - yield $"{studyFoldername}/protocols/{textValue}" // from study root > protocols + yield $"{studyFoldername}/{Path.StudiesResourcesFolderName}/{textValue}" // from study root > resources + yield $"{studyFoldername}/{Path.StudiesProtocolsFolderName}/{textValue}" // from study root > protocols ] ) |> Set.unionMany @@ -289,19 +289,19 @@ type ARC(?isa : ISA.ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSyste let includeAssayFiles = registeredAssays |> Array.map (fun a -> - let assayFoldername = $"assays/{a.Identifier}" + let assayFoldername = $"{Path.AssaysFolderName}/{a.Identifier}" set [ - yield $"{assayFoldername}/isa.assay.xlsx" - yield $"{assayFoldername}/README.md" + yield $"{assayFoldername}/{Path.AssayFileName}" + yield $"{assayFoldername}/{Path.READMEFileName}" //just allow any constructed path from cell values. there may be occasions where this includes wrong files, but its good enough for now. for table in a.Tables do for kv in table.Values do let textValue = kv.Value.ToFreeTextCell().AsFreeText yield textValue // from arc root - yield $"{assayFoldername}/dataset/{textValue}" // from assay root > dataset - yield $"{assayFoldername}/protocols/{textValue}" // from assay root > protocols + yield $"{assayFoldername}/{Path.AssayDatasetFolderName}/{textValue}" // from assay root > dataset + yield $"{assayFoldername}/{Path.AssayProtocolsFolderName}/{textValue}" // from assay root > protocols ] ) |> Set.unionMany @@ -315,8 +315,8 @@ type ARC(?isa : ISA.ArcInvestigation, ?cwl : CWL.CWL, ?fs : FileSystem.FileSyste fsCopy.Tree |> FileSystemTree.toFilePaths() |> Array.filter (fun p -> - p.StartsWith("workflows") - || p.StartsWith("runs") + p.StartsWith(Path.WorkflowsFolderName) + || p.StartsWith(Path.RunsFolderName) || includeFiles.Contains(p) ) |> FileSystemTree.fromFilePaths