Merge pull request #181 from nfdi4plants/arcIONetMigration

Call migrated and improved functions into arcIO.Net
nfdi4plants · Apr 3, 2023 · 59f219b · 59f219b
2 parents 85a6c89 + 4a30252
commit 59f219b
Show file tree

Hide file tree

Showing 22 changed files with 302 additions and 242 deletions.
diff --git a/global.json b/global.json
@@ -1,6 +1,6 @@
 {
   "sdk": {
-    "version": "6.0.406",
+    "version": "6.0.100",
     "rollForward": "latestMinor"
   }
 }
diff --git a/src/ArcCommander/APIs/ArcAPI.fs b/src/ArcCommander/APIs/ArcAPI.fs
@@ -8,6 +8,22 @@ open ArcCommander.ArgumentProcessing
 
 open ISADotNet
 open ISADotNet.XLSX
+open arcIO.NET
+open arcIO.NET.Converter
+
+module API =
+
+    module Investigation = 
+
+        let getProcesses (investigation) =
+            investigation.Studies 
+            |> Option.defaultValue [] |> List.collect (fun s -> 
+                s.Assays
+                |> Option.defaultValue [] |> List.collect (fun a -> 
+                    a.ProcessSequence |> Option.defaultValue []
+                )
+            )
+
 
 /// ArcCommander API functions that get executed by top level subcommand verbs.
 module ArcAPI = 
@@ -98,52 +114,10 @@ module ArcAPI =
 
         log.Info("Start Arc Update")
 
-        let assayRootFolder = AssayConfiguration.getRootFolderPath arcConfiguration
-
-        let investigationFilePath = IsaModelConfiguration.getInvestigationFilePath arcConfiguration
-
-        let assayNames = 
-            DirectoryInfo(assayRootFolder).GetDirectories()
-            |> Array.map (fun d -> d.Name)
-
-        let investigation =
-            try Investigation.fromFile investigationFilePath 
-            with
-            | :? FileNotFoundException -> 
-                Investigation.empty
-            | err -> 
-                log.Fatal($"{err.ToString()}")
-                raise (Exception(""))
-
-        let rec updateInvestigationAssays (assayNames : string list) (investigation : Investigation) =
-            match assayNames with
-            | a :: t ->
-                let assayFilePath = IsaModelConfiguration.getAssayFilePath a arcConfiguration
-                let assayFileName = IsaModelConfiguration.getAssayFileName a arcConfiguration
-                let factors,protocols,persons,assay = AssayFile.Assay.fromFile assayFilePath
-                let studies = investigation.Studies
-
-                match studies with
-                | Some studies ->
-                    match studies |> Seq.tryFind (API.Study.getAssays >> Option.defaultValue [] >> API.Assay.existsByFileName assayFileName) with
-                    | Some study -> 
-                        study
-                        |> API.Study.mapAssays (API.Assay.updateByFileName API.Update.UpdateByExistingAppendLists assay)
-                        |> API.Study.mapFactors (List.append factors >> List.distinctBy (fun f -> f.Name))
-                        |> API.Study.mapProtocols (List.append protocols >> List.distinctBy (fun p -> p.Name))
-                        |> API.Study.mapContacts (List.append persons >> List.distinctBy (fun p -> p.FirstName,p.LastName))
-                        |> fun s -> API.Study.updateBy ((=) study) API.Update.UpdateAll s studies
-                    | None ->
-                        Study.fromParts (Study.StudyInfo.create a "" "" "" "" "" []) [] [] factors [assay] protocols persons
-                        |> API.Study.add studies
-                | None ->
-                    [Study.fromParts (Study.StudyInfo.create a "" "" "" "" "" []) [] [] factors [assay] protocols persons]
-                |> API.Investigation.setStudies investigation
-                |> updateInvestigationAssays t
-            | [] -> investigation
-
-        updateInvestigationAssays (assayNames |> List.ofArray) investigation
-        |> Investigation.toFile investigationFilePath
+        let arcDir = GeneralConfiguration.getWorkDirectory arcConfiguration
+
+        Investigation.fromArcFolder arcDir
+        |> Investigation.overWrite arcDir
 
     /// Export the complete ARC as a JSON object.
     let export (arcConfiguration : ArcConfiguration) (arcArgs : Map<string,Argument>) =
@@ -152,73 +126,82 @@ module ArcAPI =
 
         log.Info("Start Arc Export")
 
-        let investigationFilePath = IsaModelConfiguration.getInvestigationFilePath arcConfiguration
-
-        let assayNames = AssayConfiguration.getAssayNames arcConfiguration
-
-        let investigation =
-            try Investigation.fromFile investigationFilePath 
-            with
-            | :? FileNotFoundException -> 
-                Investigation.empty
-            | err -> 
-                log.Fatal($"{err.Message}")
-                raise (Exception(""))
-
-        let rec updateInvestigationAssays (assayNames : string list) (investigation : Investigation) =
-            match assayNames with
-            | a :: t ->
-                let assayFilePath = IsaModelConfiguration.getAssayFilePath a arcConfiguration
-                let assayFileName = (IsaModelConfiguration.getAssayFileName a arcConfiguration).Replace("\\","/")
-                let factors,protocols,persons,assay = AssayFile.Assay.fromFile assayFilePath
-                let studies = investigation.Studies
-
-                match studies with
-                | Some studies ->
-                    match studies |> Seq.tryFind (API.Study.getAssays >> Option.defaultValue [] >> API.Assay.existsByFileName assayFileName) with
-                    | Some study -> 
-                        study
-                        |> API.Study.mapAssays (API.Assay.updateByFileName API.Update.UpdateByExistingAppendLists assay)
-                        |> API.Study.mapFactors (List.append factors >> List.distinctBy (fun f -> f.Name))
-                        |> API.Study.mapProtocols (List.append protocols >> List.distinctBy (fun p -> p.Name))
-                        |> API.Study.mapContacts (List.append persons >> List.distinctBy (fun p -> p.FirstName,p.LastName))
-                        |> fun s -> API.Study.updateBy ((=) study) API.Update.UpdateAll s studies
-                    | None ->
-                        Study.fromParts (Study.StudyInfo.create a "" "" "" "" "" []) [] [] factors [assay] protocols persons
-                        |> API.Study.add studies
-                | None ->                   
-                    [Study.fromParts (Study.StudyInfo.create a "" "" "" "" "" []) [] [] factors [assay] protocols persons]
-                |> API.Investigation.setStudies investigation
-                |> updateInvestigationAssays t
-            | [] -> investigation
-
-        let output = updateInvestigationAssays (assayNames |> List.ofArray) investigation
+        let workDir = GeneralConfiguration.getWorkDirectory arcConfiguration
+
+        let investigation = arcIO.NET.Investigation.fromArcFolder workDir
 
         if containsFlag "ProcessSequence" arcArgs then
 
-            let output = 
-                output.Studies 
-                |> Option.defaultValue [] |> List.collect (fun s -> 
-                    s.Assays
-                    |> Option.defaultValue [] |> List.collect (fun a -> 
-                        a.ProcessSequence |> Option.defaultValue []
-                    )
-                )
+            let output = API.Investigation.getProcesses investigation
 
             match tryGetFieldValueByName "Output" arcArgs with
             | Some p -> ArgumentProcessing.serializeToFile p output
             | None -> ()
 
-            //System.Console.Write(ArgumentProcessing.serializeToString output)
             log.Debug(ArgumentProcessing.serializeToString output)
         else 
-               
+
             match tryGetFieldValueByName "Output" arcArgs with
-            | Some p -> ISADotNet.Json.Investigation.toFile p output
+            | Some p -> ISADotNet.Json.Investigation.toFile p investigation
             | None -> ()
 
-            //System.Console.Write(ISADotNet.Json.Investigation.toString output)
-            log.Debug(ISADotNet.Json.Investigation.toString output)
+            log.Debug(ISADotNet.Json.Investigation.toString investigation)
+
+    /// Convert the complete ARC to a target format.
+    let convert (arcConfiguration : ArcConfiguration) (arcArgs : Map<string,Argument>) =
+
+        let log = Logging.createLogger "ArcConvertLog"
+
+        log.Info("Start Arc Convert")
+
+        let workDir = GeneralConfiguration.getWorkDirectory arcConfiguration
+
+        let nameRoot = getFieldValueByName "Target" arcArgs
+        let converterName = $"arc-convert-{nameRoot}"
+        let repoOwner =  "nfdi4plants"
+        let repoName = "converters"
+
+        log.Info (converterName)
+
+        let assayIdentifier = tryGetFieldValueByName "AssayIdentifier" arcArgs
+        let studyIdentifier = tryGetFieldValueByName "StudyIdentifier" arcArgs
+
+        log.Info("Fetch converter dll")
+
+        let assembly = 
+            if nameRoot.Contains ".dll" then
+                System.Reflection.Assembly.LoadFile nameRoot
+            else
+                let dll = ArcConversion.getDll repoOwner repoName $"{converterName}.dll"
+                System.Reflection.Assembly.Load dll
+        let converter = 
+            ArcConversion.callMethodOfAssembly converterName "create" assembly :?> ARCconverter
+        log.Info("Load ARC")
+
+        let i,s,a = ArcConversion.getISA studyIdentifier assayIdentifier workDir
+
+        log.Info("Run conversion")
+
+        match converter with
+        | ARCtoCSV f -> 
+            ArcConversion.handleCSV i s a workDir nameRoot converter
+        | ARCtoTSV f -> 
+            ArcConversion.handleTSV i s a workDir nameRoot converter
+        | ARCtoXLSX f -> 
+            ArcConversion.handleXLSX i s a workDir nameRoot converter
+        | ARCtoJSON f -> 
+            ArcConversion.handleJSON i s a workDir nameRoot converter
+        | _ -> failwith "no other converter defined"
+        |> function 
+           | Ok messages ->
+                log.Info $"Successfully converted to {nameRoot}"
+                ArcConversion.writeMessages workDir messages
+           | Error messages ->
+                ArcConversion.writeMessages workDir messages
+                log.Error $"Arc could not be converted to {nameRoot}, as some required values could not be retreived"
+                if ArcConversion.promptYesNo "Do you want missing fields to be written back into ARC? (y/n)" then
+                    ArcConversion.handleTransformations workDir converterName messages
+
 
     /// Returns true if called anywhere in an ARC.
     let isArc (arcConfiguration : ArcConfiguration) (arcArgs : Map<string,Argument>) = raise (NotImplementedException())
diff --git a/src/ArcCommander/APIs/AssayAPI.fs b/src/ArcCommander/APIs/AssayAPI.fs
@@ -10,7 +10,8 @@ open ISADotNet
 open ISADotNet.XLSX
 open ISADotNet.XLSX.AssayFile
 
-open FSharpSpreadsheetML
+open FsSpreadsheet.ExcelIO
+open arcIO.NET
 
 /// ArcCommander Assay API functions that get executed by the assay focused subcommand verbs
 module AssayAPI =
@@ -205,7 +206,7 @@ module AssayAPI =
             assay1.Comments             = assay2.Comments
 
         // read assay metadata information from assay file
-        let _, _, _, oldAssayAssayFile = AssayFile.Assay.fromFile assayFilepath
+        let _, oldAssayAssayFile = AssayFile.Assay.fromFile assayFilepath
 
         let getNewAssay oldAssay =
             ArgumentProcessing.Prompt.createIsaItemQuery 
@@ -273,7 +274,7 @@ module AssayAPI =
 
         let assayFilePath = IsaModelConfiguration.getAssayFilePath assayIdentifier arcConfiguration
 
-        let _, _, _, assay = Assay.fromFile assayFilePath
+        let _, assay = Assay.fromFile assayFilePath
 
         let studyIdentifier = 
             match getFieldValueByName "StudyIdentifier" assayArgs with
@@ -309,6 +310,7 @@ module AssayAPI =
                 let info = Study.StudyInfo.create studyIdentifier "" "" "" "" (IsaModelConfiguration.getStudyFileName studyIdentifier arcConfiguration) []
                 Study.fromParts info [] [] [] [assay] [] []
                 |> API.Study.add studies
+                |> List.filter (fun s -> s <> Study.empty)
         | None ->
             log.Info($"Study with the identifier {studyIdentifier} does not exist yet, creating it now.")
             if StudyAPI.StudyFile.exists arcConfiguration studyIdentifier |> not then
@@ -657,7 +659,7 @@ module AssayAPI =
 
             if System.IO.File.Exists assayFilePath then
                 try
-                    let _, _, p, a = AssayFile.Assay.fromFile assayFilePath
+                    let p, a = AssayFile.Assay.fromFile assayFilePath
                     p, Some a
                 with
                 | err -> 
@@ -753,7 +755,7 @@ module AssayAPI =
 
                     if System.IO.File.Exists assayFilePath then
                         try
-                            let _,_,p,a = AssayFile.Assay.fromFile assayFilePath
+                            let p,a = AssayFile.Assay.fromFile assayFilePath
                             p, Some a
                         with
                         | err -> 

diff --git a/src/ArcCommander/APIs/ConfigurationAPI.fs b/src/ArcCommander/APIs/ConfigurationAPI.fs
@@ -2,6 +2,7 @@
 
 open ArcCommander
 open ArgumentProcessing
+open arcIO.NET
 
 /// ArcCommander Configuration API functions that get executed by the configuration focused subcommand verbs
 module ConfigurationAPI =

diff --git a/src/ArcCommander/APIs/ExternalExecutables.fs b/src/ArcCommander/APIs/ExternalExecutables.fs
@@ -6,6 +6,7 @@ open System
 open System.IO
 open System.Diagnostics
 open Argu
+open arcIO.NET
 
 /// Functions for trying to run external tools, given the command line arguments can not be parsed.
 module ExternalExecutables =
@@ -82,9 +83,9 @@ module ExternalExecutables =
                         let roev = reviseOutput ev.Data
                         if matchCmdErrMsg roev || matchBashErrMsg roev then 
                             log.Error("No executable, command or script file with given argument name known.") 
-                            handleExceptionMessage log e2
+                            Logging.handleExceptionMessage log e2
                             raise (Exception())
-                        else checkNonLog roev (sprintf "External Tool ERROR: %s" >> log.Error)
+                        else Logging.checkNonLog roev (sprintf "External Tool ERROR: %s" >> log.Error)
                 )
                 let sbOutput = Text.StringBuilder() // StringBuilder for TRACE output (verbosity 2)
                 sbOutput.Append("External tool: ") |> ignore
@@ -98,9 +99,9 @@ module ExternalExecutables =
                             sbOutput.Append(char charAsInt) |> ignore
                     p.WaitForExit()
                     log.Trace(sbOutput.ToString()) // it is fine that the logging occurs after the external tool has done its job
-                with e3 -> handleExceptionMessage log e3
+                with e3 -> Logging.handleExceptionMessage log e3
                 None
             // If neither parsing, nor external executable tool search led to success, just return the error message
             | None -> 
-                handleExceptionMessage log e2
+                Logging.handleExceptionMessage log e2
                 None
diff --git a/src/ArcCommander/APIs/GitAPI.fs b/src/ArcCommander/APIs/GitAPI.fs
@@ -4,7 +4,7 @@ open ArcCommander
 open ArgumentProcessing
 open Fake.IO
 open System.IO
-
+open arcIO.NET
 
 module GitAPI =
 
@@ -113,7 +113,7 @@ module GitAPI =
             allFilesPlusSizes 
             |> List.iter (fun (file,size) ->
 
-                    /// Track files larger than the git lfs threshold with git lfs. If no threshold is set, track no files with git lfs
+                    // Track files larger than the git lfs threshold with git lfs. If no threshold is set, track no files with git lfs
                     match gitLfsThreshold with
                     | Some thr when size > thr -> trackWithLFS file
                     | _ -> trackWithAdd file

diff --git a/src/ArcCommander/APIs/InvestigationAPI.fs b/src/ArcCommander/APIs/InvestigationAPI.fs
@@ -5,7 +5,7 @@ open ArcCommander.ArgumentProcessing
 
 open ISADotNet
 open ISADotNet.XLSX
-
+open arcIO.NET
 
 /// ArcCommander Investigation API functions that get executed by the investigation focused subcommand verbs.
 module InvestigationAPI =
@@ -66,7 +66,7 @@ module InvestigationAPI =
 
         let originalInvestigation = Investigation.fromFile investigationFilePath
 
-        API.Investigation.update updateOption originalInvestigation investigation
+        API.Investigation.updateBy updateOption originalInvestigation investigation
         |> Investigation.toFile investigationFilePath
 
     /// Opens the existing investigation info in the ARC with the text editor set in globalArgs.
@@ -87,7 +87,7 @@ module InvestigationAPI =
                 (Investigation.InvestigationInfo.fromRows 1 >> fun (_,_,_,item) -> Investigation.fromParts item [] [] [] [] []) 
                 investigation
 
-        API.Investigation.update API.Update.UpdateAllAppendLists investigation editedInvestigation
+        API.Investigation.updateBy API.Update.UpdateAllAppendLists investigation editedInvestigation
         |> Investigation.toFile investigationFilePath
 
     /// Deletes the existing investigation file in the ARC if the given identifier matches the identifier set in the investigation file.

diff --git a/src/ArcCommander/APIs/RemoteAccessAPI.fs b/src/ArcCommander/APIs/RemoteAccessAPI.fs
@@ -2,7 +2,7 @@
 
 open ArcCommander
 open ArcCommander.ArgumentProcessing
-
+open arcIO.NET
 
 
 module RemoteAccessAPI =