Merge branch 'develop' of https://github.com/HicServices/RDMP

HicServices · Oct 22, 2024 · f64d05b · f64d05b
2 parents 6fda4e7 + 637e866
commit f64d05b
Show file tree

Hide file tree

Showing 35 changed files with 823 additions and 671 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -71,7 +71,7 @@ jobs:
           mv `find coverage -type f` db-ui.lcov
           dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov
           mv `find coverage -type f` db-core.lcov
-      - uses: coverallsapp/[email protected].0
+      - uses: coverallsapp/[email protected].3
         with:
           github-token: ${{ secrets.github_token }}
           files: ./db-ui.lcov ./db-core.lcov
@@ -137,7 +137,7 @@ jobs:
           mv `find coverage -type f` fs-ui.lcov
           dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release  -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov
           mv `find coverage -type f` fs-core.lcov
-      - uses: coverallsapp/[email protected].0
+      - uses: coverallsapp/[email protected].3
         with:
           github-token: ${{ secrets.github_token }}
           files: ./fs-ui.lcov ./fs-core.lcov
@@ -176,7 +176,6 @@ jobs:
           cp -r Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/runtimes ./PublishWinForms
           cp -r Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/x64 ./PublishWinForms
           cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/D3DCompiler_47_cor3.dll ./PublishWinForms
-          cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/mongocrypt.dll ./PublishWinForms
           cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/PenImc_cor3.dll ./PublishWinForms
           cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/PresentationNative_cor3.dll ./PublishWinForms
           cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/vcruntime140_cor3.dll ./PublishWinForms

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -7,7 +7,7 @@ on:
 jobs:
   build:
     name: Build and push Docker image
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     permissions:
       contents: read
       packages: write

diff --git a/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs b/Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs
@@ -373,9 +373,24 @@ private T Activate<T, T2>(T2 databaseObject, Image<Rgba32> tabImage)
 
         var uiInstance = new T();
         var floatable = WindowFactory.Create(this, RefreshBus, uiInstance, tabImage, databaseObject);
+
+        int? insertIndex = null;
+        var panel = _mainDockPanel.Panes.Where(p => p.IsActiveDocumentPane).SingleOrDefault();
+        if (panel is not null)
+        {
+            var contents = panel.Contents;
+            insertIndex = contents.IndexOf(panel.ActiveContent)+1;
+        }
+
         floatable.Show(_mainDockPanel, DockState.Document);
+
         uiInstance.SetDatabaseObject(this, databaseObject);
 
+        if (insertIndex is not null)
+        {
+            _mainDockPanel.ActivePane.SetContentIndex(floatable, (int)insertIndex);
+        }
+
         SetTabText(floatable, uiInstance);
 
         return uiInstance;
@@ -436,32 +451,32 @@ public DockContent Activate(DeserializeInstruction instruction, IActivateItems a
             case IObjectCollectionControl uiCollection:
                 return Activate(uiCollection, instruction.ObjectCollection);
             case IRDMPSingleDatabaseObjectControl uiInstance:
-            {
-                var databaseObject = instruction.DatabaseObject;
-
-                //the database object is gone? deleted maybe
-                if (databaseObject == null)
-                    return null;
-
-                DockContent floatable = WindowFactory.Create(this, RefreshBus, uiInstance,
-                    CoreIconProvider.GetImage(databaseObject), databaseObject);
-
-                floatable.Show(_mainDockPanel, DockState.Document);
-                try
                 {
-                    uiInstance.SetDatabaseObject(this, (DatabaseEntity)databaseObject);
-                    SetTabText(floatable, uiInstance);
+                    var databaseObject = instruction.DatabaseObject;
+
+                    //the database object is gone? deleted maybe
+                    if (databaseObject == null)
+                        return null;
+
+                    DockContent floatable = WindowFactory.Create(this, RefreshBus, uiInstance,
+                        CoreIconProvider.GetImage(databaseObject), databaseObject);
+
+                    floatable.Show(_mainDockPanel, DockState.Document);
+                    try
+                    {
+                        uiInstance.SetDatabaseObject(this, (DatabaseEntity)databaseObject);
+                        SetTabText(floatable, uiInstance);
+                    }
+                    catch (Exception e)
+                    {
+                        floatable.Close();
+                        throw new Exception(
+                            $"SetDatabaseObject failed on Control of Type '{instruction.UIControlType.Name}', control closed, see inner Exception for details",
+                            e);
+                    }
+
+                    return floatable;
                 }
-                catch (Exception e)
-                {
-                    floatable.Close();
-                    throw new Exception(
-                        $"SetDatabaseObject failed on Control of Type '{instruction.UIControlType.Name}', control closed, see inner Exception for details",
-                        e);
-                }
-
-                return floatable;
-            }
             default:
                 return (DockContent)activator.ShowWindow(c, true);
         }
@@ -833,7 +848,7 @@ public void StartSession(string sessionName, IEnumerable<IMapsDirectlyToDatabase
     public override IPipelineRunner GetPipelineRunner(DialogArgs args, IPipelineUseCase useCase, IPipeline pipeline)
     {
 
-        if(useCase is not null && pipeline is not null)
+        if (useCase is not null && pipeline is not null)
         {
             return new PipelineRunner(useCase, pipeline);
         }
@@ -868,7 +883,7 @@ public override CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(Externa
             return _mainDockPanel.Invoke(() =>
                 GetCohortHoldoutLookupRequest(externalCohortTable, project, cic));
 
-        var ui = new Rdmp.UI.CohortUI.CreateHoldoutLookup.CreateHoldoutLookupUI(this, externalCohortTable,  cic);
+        var ui = new Rdmp.UI.CohortUI.CreateHoldoutLookup.CreateHoldoutLookupUI(this, externalCohortTable, cic);
 
         if (!string.IsNullOrWhiteSpace(cic.Description))
             ui.CohortDescription = $"{cic.Description} ({Environment.UserName} - {DateTime.Now})";
@@ -878,7 +893,7 @@ public override CohortHoldoutLookupRequest GetCohortHoldoutLookupRequest(Externa
     public override ICatalogue CreateAndConfigureCatalogue(ITableInfo tableInfo,
         ColumnInfo[] extractionIdentifierColumns, string initialDescription, IProject projectSpecific, string folder)
     {
-        if(extractionIdentifierColumns is not null)
+        if (extractionIdentifierColumns is not null)
         {
             return base.CreateAndConfigureCatalogue(tableInfo, extractionIdentifierColumns, initialDescription, projectSpecific, folder);
         }

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,14 +1,29 @@
+<<<<<<< HEAD
 
 
 
 
 
+||||||| 925afff47
+
+=======
+>>>>>>> 637e86636063993b0156997bee867d2a7f5ed9ba
 # Changelog
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+<<<<<<< HEAD
+||||||| 925afff47
+=======
+## [8.3.1] - 2024-10-22
+
+- Improve Performance of regenerating problems with child providers
+- Update UI Tab opening Logic
+- Add Filter to Left-Hand Tree View
+
+>>>>>>> 637e86636063993b0156997bee867d2a7f5ed9ba
 ## [8.3.0] - 2024-09-23
 
 - Add New Find & Replace, currently available via User Settings

diff --git a/Directory.Build.props b/Directory.Build.props
@@ -0,0 +1,7 @@
+<Project>
+  <PropertyGroup>
+    <LangVersion>11.0</LangVersion>
+    <Version>8.3.0</Version>
+		<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+  </PropertyGroup>
+</Project>
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -1,22 +1,22 @@
 <Project>
   <ItemGroup>
-    <PackageVersion Include="AWSSDK.S3" Version="3.7.403.1" />
-    <PackageVersion Include="AWSSDK.SecurityToken" Version="3.7.400.22" />
-    <PackageVersion Include="AWSSDK.SSO" Version="3.7.400.22" />
-    <PackageVersion Include="AWSSDK.SSOOIDC" Version="3.7.400.22" />
+    <PackageVersion Include="AWSSDK.S3" Version="3.7.405" />
+    <PackageVersion Include="AWSSDK.SecurityToken" Version="3.7.400.36" />
+    <PackageVersion Include="AWSSDK.SSO" Version="3.7.400.36" />
+    <PackageVersion Include="AWSSDK.SSOOIDC" Version="3.7.400.36" />
     <PackageVersion Include="CommandLineParser" Version="2.9.1" />
     <PackageVersion Include="Equ" Version="2.3.0" />
     <PackageVersion Include="ExcelNumberFormat" Version="1.1.0" />
     <PackageVersion Include="FluentFTP" Version="51.1.0" />
     <PackageVersion Include="HIC.SynthEHR" Version="2.0.1" />
-    <PackageVersion Include="HIC.FAnsiSql" Version="3.2.6" />
+    <PackageVersion Include="HIC.FAnsiSql" Version="3.2.7" />
     <PackageVersion Include="LibArchive.Net" Version="0.1.5" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
     <PackageVersion Include="Minio" Version="6.0.3" />
-    <PackageVersion Include="MongoDB.Driver" Version="2.29.0" />
+    <PackageVersion Include="MongoDB.Driver" Version="3.0.0" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="NPOI" Version="2.7.1" />
     <PackageVersion Include="NLog" Version="5.3.4" />
@@ -25,7 +25,7 @@
     <PackageVersion Include="Spectre.Console" Version="0.49.1" />
     <PackageVersion Include="SSH.NET" Version="2024.1.0" />
     <PackageVersion Include="Terminal.Gui" Version="1.17.1" />
-    <PackageVersion Include="YamlDotNet" Version="16.1.2" />
+    <PackageVersion Include="YamlDotNet" Version="16.1.3" />
     <PackageVersion Include="ConsoleControl" Version="1.3.0" />
     <PackageVersion Include="Autoupdater.NET.Official" Version="1.9.2" />
     <PackageVersion Include="DockPanelSuite.ThemeVS2015" Version="3.1.1" />
@@ -37,7 +37,7 @@
     <PackageVersion Include="System.Resources.Extensions" Version="8.0.0" />
     <PackageVersion Include="VPKSoft.ScintillaLexers.NET" Version="1.1.16" />
     <PackageVersion Include="HIC.System.Windows.Forms.DataVisualization" Version="1.0.1" />
-    <PackageVersion Include="NunitXml.TestLogger" Version="4.0.254" />
+    <PackageVersion Include="NunitXml.TestLogger" Version="4.1.0" />
     <PackageVersion Include="NUnit3TestAdapter" Version="4.6.0" />
     <PackageVersion Include="NUnit.Analyzers" Version="4.3.0" />
     <PackageVersion Include="NSubstitute" Version="5.1.0" />

diff --git a/Rdmp.Core/Logging/LogManager.cs b/Rdmp.Core/Logging/LogManager.cs
@@ -63,7 +63,8 @@ public string[] ListDataTasks(bool hideTests = false)
 
         using var con = Server.GetConnection();
         con.Open();
-        using var cmd = Server.GetCommand("SELECT * FROM DataLoadTask", con);
+        var sh = Server.GetQuerySyntaxHelper();
+        using var cmd = Server.GetCommand($"SELECT * FROM {sh.EnsureWrapped("DataLoadTask")}", con);
         using var r = cmd.ExecuteReader();
         while (r.Read())
             if (!hideTests || !(bool)r["isTest"])
@@ -113,7 +114,8 @@ public string[] ListDataSets()
         using var con = Server.GetConnection();
         con.Open();
 
-        using var cmd = Server.GetCommand("SELECT * FROM DataSet", con);
+        var sh = Server.GetQuerySyntaxHelper();
+        using var cmd = Server.GetCommand($"SELECT * FROM {sh.EnsureWrapped("DataSet")}", con);
         using var r = cmd.ExecuteReader();
         while (r.Read())
             tasks.Add(r["dataSetID"].ToString());
@@ -133,7 +135,8 @@ public IEnumerable<ArchivalDataLoadInfo> GetArchivalDataLoadInfos(string dataTas
         int? specificDataLoadRunIDOnly = null, int? topX = null)
     {
         var db = Server.GetCurrentDatabase();
-        var run = db.ExpectTable("DataLoadRun");
+        var sh = Server.GetQuerySyntaxHelper();
+        var run = db.ExpectTable(sh.EnsureWrapped("DataLoadRun"));
 
         using var con = Server.GetConnection();
         con.Open();
@@ -144,11 +147,11 @@ public IEnumerable<ArchivalDataLoadInfo> GetArchivalDataLoadInfos(string dataTas
         string where;
         if (specificDataLoadRunIDOnly != null)
         {
-            where = $"WHERE ID={specificDataLoadRunIDOnly.Value}";
+            where = $"WHERE {sh.EnsureWrapped("ID")}={specificDataLoadRunIDOnly.Value}";
         }
         else
         {
-            where = "WHERE dataLoadTaskID = @dataTaskId";
+            where = $"WHERE {sh.EnsureWrapped("dataLoadTaskID")} = @dataTaskId";
             var p = cmd.CreateParameter();
             p.ParameterName = "@dataTaskId";
             p.Value = dataTaskId;
@@ -158,13 +161,13 @@ public IEnumerable<ArchivalDataLoadInfo> GetArchivalDataLoadInfos(string dataTas
         TopXResponse top = null;
 
         if (topX.HasValue)
-            top = Server.GetQuerySyntaxHelper().HowDoWeAchieveTopX(topX.Value);
+            top = sh.HowDoWeAchieveTopX(topX.Value);
 
         var sb = new StringBuilder("SELECT ");
 
         if (top?.Location == QueryComponent.SELECT) sb.AppendLine(top.SQL);
 
-        sb.AppendLine($" * FROM {run.GetFullyQualifiedName()}  {where} ORDER BY ID desc");
+        sb.AppendLine($" * FROM {run.GetFullyQualifiedName()}  {where} ORDER BY {sh.EnsureWrapped("ID")} desc");
 
         if (top?.Location == QueryComponent.Postfix) sb.AppendLine(top.SQL);
 
@@ -204,7 +207,8 @@ public IEnumerable<ArchivalDataLoadInfo> GetArchivalDataLoadInfos(string dataTas
 
     private static int GetDataTaskId(string dataTask, DiscoveredServer server, DbConnection con)
     {
-        using var cmd = server.GetCommand("SELECT ID FROM DataLoadTask WHERE name = @name", con);
+        var sh = server.GetQuerySyntaxHelper();
+        using var cmd = server.GetCommand($"SELECT {sh.EnsureWrapped("ID")} FROM {sh.EnsureWrapped("DataLoadTask")} WHERE name = @name", con);
         var p = cmd.CreateParameter();
         p.ParameterName = "@name";
         p.Value = dataTask;
@@ -236,9 +240,10 @@ public IDataLoadInfo CreateDataLoadInfo(string dataLoadTaskName, string packageN
     public void CreateNewLoggingTask(int id, string dataSetID)
     {
         using var conn = Server.GetConnection();
+        var sh = Server.GetQuerySyntaxHelper();
         conn.Open();
         var sql =
-            $"INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES ({id}, @dataSetID, @dataSetID, @date, @username, 1, 0, @dataSetID)";
+            $"INSERT INTO {sh.EnsureWrapped("DataLoadTask")} ({sh.EnsureWrapped("ID")}, description, name, {sh.EnsureWrapped("createTime")}, {sh.EnsureWrapped("userAccount")}, {sh.EnsureWrapped("statusID")}, {sh.EnsureWrapped("isTest")}, {sh.EnsureWrapped("dataSetID")}) VALUES ({id}, @dataSetID, @dataSetID, @date, @username, 1, {sh.False}, @dataSetID)";
 
         using var cmd = Server.GetCommand(sql, conn);
         Server.AddParameterWithValueToCommand("@date", cmd, DateTime.Now);
@@ -251,9 +256,10 @@ public void CreateNewLoggingTask(int id, string dataSetID)
     private void CreateNewDataSet(string datasetName)
     {
         using var conn = Server.GetConnection();
+        var sh = Server.GetQuerySyntaxHelper();
         conn.Open();
         {
-            const string sql = "INSERT INTO DataSet (dataSetID,name) VALUES (@datasetName,@datasetName)";
+            var sql = $"INSERT INTO {sh.EnsureWrapped("DataSet")} ({sh.EnsureWrapped("dataSetID")},name) VALUES (@datasetName,@datasetName)";
 
             using var cmd = Server.GetCommand(sql, conn);
             Server.AddParameterWithValueToCommand("@datasetName", cmd, datasetName.Substring(Math.Max(0, datasetName.Length - 150)));
@@ -274,8 +280,9 @@ public void CreateNewLoggingTaskIfNotExists(string toCreate)
     private int GetMaxTaskID()
     {
         using var conn = Server.GetConnection();
+        var sh = Server.GetQuerySyntaxHelper();
         conn.Open();
-        const string sql = "SELECT MAX(ID) FROM DataLoadTask";
+        var sql = $"SELECT MAX({sh.EnsureWrapped("ID")}) FROM {sh.EnsureWrapped("DataLoadTask")}";
 
         using var cmd = Server.GetCommand(sql, conn);
         var result = cmd.ExecuteScalar();
@@ -285,9 +292,10 @@ private int GetMaxTaskID()
     public void ResolveFatalErrors(int[] ids, DataLoadInfo.FatalErrorStates newState, string newExplanation)
     {
         using var conn = Server.GetConnection();
+        var sh = Server.GetQuerySyntaxHelper();
         conn.Open();
         var sql =
-            $"UPDATE FatalError SET explanation =@explanation, statusID=@statusID where ID in ({string.Join(",", ids)})";
+            $"UPDATE {sh.EnsureWrapped("FatalError")} SET explanation =@explanation, {sh.EnsureWrapped("statusID")}=@statusID where {sh.EnsureWrapped("ID")} in ({string.Join(",", ids)})";
 
         using var cmd = Server.GetCommand(sql, conn);
         Server.AddParameterWithValueToCommand("@explanation", cmd, newExplanation);
@@ -298,4 +306,4 @@ public void ResolveFatalErrors(int[] ids, DataLoadInfo.FatalErrorStates newState
             throw new Exception(
                 $"Query {sql} resulted in {affectedRows}, we were expecting there to be {ids.Length} updates because that is how many FatalError IDs that were passed to this method");
     }
-}
+}
diff --git a/Rdmp.Core/Providers/CatalogueProblemProvider.cs b/Rdmp.Core/Providers/CatalogueProblemProvider.cs
@@ -6,6 +6,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
 using Rdmp.Core.CommandExecution.AtomicCommands;
@@ -43,18 +44,13 @@ public override void RefreshProblems(ICoreChildProvider childProvider)
         _childProvider = childProvider;
 
         //Take all the catalogue items which DON'T have an associated ColumnInfo (should hopefully be quite rare)
-        var orphans = _childProvider.AllCatalogueItems.Where(ci => ci.ColumnInfo_ID == null);
-
-        //now identify those which have an ExtractionInformation (that's a problem! they are extractable but orphaned)
-        _orphanCatalogueItems = new HashSet<int>(
-            orphans.Where(o => _childProvider.AllExtractionInformations.Any(ei => ei.CatalogueItem_ID == o.ID))
-
-                //store just the ID for performance
-                .Select(i => i.ID));
-
+        var catalogueIDs = _childProvider.AllCatalogueItems.Where(ci => ci.ColumnInfo_ID == null).Select(i => i.ID).ToList();
+        var extractionInfoIDs = _childProvider.AllExtractionInformations.Select(ei => ei.CatalogueItem_ID).ToList();
+        var orphans = catalogueIDs.Intersect(extractionInfoIDs);
+        _orphanCatalogueItems = orphans.ToHashSet<int>();
         _usedJoinables = new HashSet<int>(
-            childProvider.AllJoinableCohortAggregateConfigurationUse.Select(
-                ju => ju.JoinableCohortAggregateConfiguration_ID));
+       childProvider.AllJoinableCohortAggregateConfigurationUse.Select(
+           ju => ju.JoinableCohortAggregateConfiguration_ID));
 
         _joinsWithMismatchedCollations = childProvider.AllJoinInfos.Where(j =>
             !string.IsNullOrWhiteSpace(j.PrimaryKey.Collation) &&