Feature/rdmp 13 removable default logging server (#1589)

* make external serverremovable * no longer throwing out errors * add catalogue fix * added todo reminder * working startup option * remove fk drop * fix creation issue * add todo * working delete * invert checkbox ui * add throw for unexpected no logging server * rename variables to match descriptions * fix bad merge * fix autosave issue
HicServices · Aug 9, 2023 · 243b5d9 · 243b5d9
1 parent cc8e300
commit 243b5d9
Show file tree

Hide file tree

Showing 11 changed files with 87 additions and 40 deletions.
diff --git a/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs b/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs
@@ -28,8 +28,8 @@ public void Test_ExampleDatasetsCreation()
         Assert.AreEqual(0, CatalogueRepository.GetAllObjects<AggregateConfiguration>().Length);
 
         //create the pipelines
-        var pipes = new CataloguePipelinesAndReferencesCreation(RepositoryLocator, null, null);
-        pipes.CreatePipelines();
+        var pipes = new CataloguePipelinesAndReferencesCreation(RepositoryLocator,null,null);
+        pipes.CreatePipelines(new PlatformDatabaseCreationOptions {});
 
         //create all the stuff
         var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer);

diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelDatabaseTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelDatabaseTests.cs
@@ -28,7 +28,7 @@ public void TestLoadingFileWithTrailingDotsInHeader()
         // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline)
         var creator = new CataloguePipelinesAndReferencesCreation(
             RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString);
-        creator.CreatePipelines();
+        creator.CreatePipelines(new PlatformDatabaseCreationOptions {});
 
         // find the excel loading pipeline
         var pipe = CatalogueRepository.GetAllObjects<Pipeline>().OrderByDescending(p => p.ID)

diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs
@@ -49,30 +49,28 @@ private void DoStartup()
         var startup = new Startup.Startup(new EnvironmentInfo(), _repositoryLocator);
         startup.DoStartup(new IgnoreAllErrorsCheckNotifier());
     }
-
-    private void CreateServers()
+    private void CreateServers(PlatformDatabaseCreationOptions options)
     {
         var defaults = _repositoryLocator.CatalogueRepository;
+        if(options.CreateLoggingServer){
+            _edsLogging = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "Logging",new LoggingDatabasePatcher())
+                {
+                    Server = _logging?.DataSource ?? throw new InvalidOperationException("Null logging database provided"),
+                    Database = _logging.InitialCatalog
+                };
+
+            if(_logging.UserID != null)
+            {
+                _edsLogging.Username = _logging.UserID;
+                _edsLogging.Password = _logging.Password;
+            }
 
-        _edsLogging = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "Logging",
-            new LoggingDatabasePatcher())
-        {
-            Server = _logging.DataSource,
-            Database = _logging.InitialCatalog
-        };
-
-        if (_logging.UserID != null)
-        {
-            _edsLogging.Username = _logging.UserID;
-            _edsLogging.Password = _logging.Password;
+            _edsLogging.SaveToDatabase();
+            defaults.SetDefault(PermissableDefaults.LiveLoggingServer_ID, _edsLogging);
+            Console.WriteLine("Successfully configured default logging server");
         }
 
-        _edsLogging.SaveToDatabase();
-        defaults.SetDefault(PermissableDefaults.LiveLoggingServer_ID, _edsLogging);
-        Console.WriteLine("Successfully configured default logging server");
-
-        var edsDQE =
-            new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "DQE", new DataQualityEnginePatcher())
+        var edsDQE = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "DQE", new DataQualityEnginePatcher())
             {
                 Server = _dqe.DataSource,
                 Database = _dqe.InitialCatalog
@@ -94,7 +92,8 @@ private void CreateServers()
         };
 
         //We are expecting a single username/password for everything here, so just use the dqe one
-        if (_dqe.UserID != null)
+        bool hasLoggingDB = _logging != null && _logging.UserID != null;
+        if (_dqe.UserID != null && hasLoggingDB)
         {
             if (_logging.UserID != _dqe.UserID || _logging.Password != _dqe.Password)
                 throw new Exception(
@@ -109,7 +108,7 @@ private void CreateServers()
         Console.WriteLine("Successfully configured RAW server");
     }
 
-    public void CreatePipelines()
+    public void CreatePipelines(PlatformDatabaseCreationOptions options)
     {
         var bulkInsertCsvPipe =
             CreatePipeline("BULK INSERT: CSV Import File (manual column-type editing)",
@@ -124,11 +123,11 @@ public void CreatePipelines()
         SetComponentProperties(bulkInsertCsvPipewithAdjuster.Source, "Separator", ",");
         SetComponentProperties(bulkInsertCsvPipewithAdjuster.Source, "StronglyTypeInput", false);
 
-        SetComponentProperties(bulkInsertCsvPipe.Destination, "LoggingServer", _edsLogging);
-        SetComponentProperties(bulkInsertCsvPipewithAdjuster.Destination, "LoggingServer", _edsLogging);
-
-        var createCohortFromCSV = CreatePipeline("CREATE COHORT:From CSV File", typeof(DelimitedFlatFileDataFlowSource),
-            typeof(BasicCohortDestination));
+        if(options.CreateLoggingServer){
+            SetComponentProperties(bulkInsertCsvPipe.Destination, "LoggingServer", _edsLogging);
+            SetComponentProperties(bulkInsertCsvPipewithAdjuster.Destination, "LoggingServer", _edsLogging);
+        }
+        var createCohortFromCSV = CreatePipeline("CREATE COHORT:From CSV File",typeof (DelimitedFlatFileDataFlowSource), typeof (BasicCohortDestination));
         SetComponentProperties(createCohortFromCSV.Source, "Separator", ",");
 
         CreatePipeline("CREATE COHORT:By Executing Cohort Identification Configuration",
@@ -198,10 +197,10 @@ private Pipeline CreatePipeline(string nameOfPipe, Type sourceType, Type destina
         return pipe;
     }
 
-    public void Create()
+    public void Create(PlatformDatabaseCreationOptions options)
     {
         DoStartup();
-        CreateServers();
-        CreatePipelines();
+        CreateServers(options);
+        CreatePipelines(options);
     }
 }
diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs
@@ -38,16 +38,19 @@ public static void CreatePlatformDatabases(PlatformDatabaseCreationOptions optio
         Create(DefaultDataExportDatabaseName, new DataExportPatcher(), options);
 
         var dqe = Create(DefaultDQEDatabaseName, new DataQualityEnginePatcher(), options);
-        var logging = Create(DefaultLoggingDatabaseName, new LoggingDatabasePatcher(), options);
-
+
+        SqlConnectionStringBuilder logging = null;
+        if(options.CreateLoggingServer){
+            logging = Create(DefaultLoggingDatabaseName, new LoggingDatabasePatcher(), options);
+        }
         CatalogueRepository.SuppressHelpLoading = true;
 
         var repo = new PlatformDatabaseCreationRepositoryFinder(options);
 
         if (!options.SkipPipelines)
         {
             var creator = new CataloguePipelinesAndReferencesCreation(repo, logging, dqe);
-            creator.Create();
+            creator.Create(options);
         }
 
         if (options.ExampleDatasets || options.Nightmare)

diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs
@@ -43,6 +43,9 @@ public class PlatformDatabaseCreationOptions
             "Skips creating the default Pipelines and Managed Server References in the Catalogue database once created.")]
     public bool SkipPipelines { get; set; }
 
+    [Option('l', "Create Logging Server", Default = true, HelpText = "Create the default logging server in the Catalogue database once created. Is superseeded by 'Skip Pipelines'")]
+    public bool CreateLoggingServer { get; set; }
+
     [Option('e', "ExampleDatasets", Default = false,
         HelpText = "Create example datasets, projects, extraction configurations and cohort queries")]
     public bool ExampleDatasets { get; set; }
@@ -80,6 +83,7 @@ public class PlatformDatabaseCreationOptions
             "Optional connection string keywords to use e.g. \"Key1=Value1; Key2=Value2\".  When using this option you must manually specify IntegratedSecurity if required.")]
     public string OtherKeywords { get; set; }
 
+
     [Usage]
     public static IEnumerable<Example> Examples
     {

diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs
@@ -527,10 +527,17 @@ public LoadMetadata LoadMetadata
 
     /// <inheritdoc/>
     [NoMappingToDatabase]
-    public ExternalDatabaseServer LiveLoggingServer =>
-        LiveLoggingServer_ID == null
-            ? null
-            : Repository.GetObjectByID<ExternalDatabaseServer>((int)LiveLoggingServer_ID);
+     public ExternalDatabaseServer LiveLoggingServer {
+        get{
+            if(LiveLoggingServer_ID != null){  
+                ExternalDatabaseServer[] dbs = Repository.GetAllObjectsWhere<ExternalDatabaseServer>("id",(int)LiveLoggingServer_ID);
+                if(dbs.Length > 0){
+                    return dbs.First();
+                }                         
+            }
+            return null;
+        }
+     }
 
     /// <inheritdoc/>
     [NoMappingToDatabase]

diff --git a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs
@@ -275,6 +275,15 @@ public bool DiscoverExistence(DataAccessContext context, out string reason) =>
 
     public override void DeleteInDatabase()
     {
+
+         if (WasCreatedBy(new LoggingDatabasePatcher())){
+            //If you're trying to delete a logging server, remove all references to it first
+            Catalogue[] catalogues = Repository.GetAllObjectsWhere<Catalogue>("LiveLoggingServer_ID",base.ID);
+            foreach(Catalogue catalogue in catalogues){
+                catalogue.LiveLoggingServer_ID = null;
+                catalogue.SaveToDatabase();
+            }
+         }
         base.DeleteInDatabase();
 
         // normally in database schema deleting an ExternalDatabaseServer will cascade to clear defaults

diff --git a/Rdmp.Core/GlobalStrings.Designer.cs b/Rdmp.Core/GlobalStrings.Designer.cs
diff --git a/Rdmp.Core/GlobalStrings.resx b/Rdmp.Core/GlobalStrings.resx
@@ -174,4 +174,7 @@
   <data name="FromFile" xml:space="preserve">
     <value>Catalogue from File...</value>
   </data>
+   <data name="CreateALoggingServer" xml:space="preserve">
+    <value>Create a logging server</value>
+  </data>
 </root>
diff --git a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs
diff --git a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs
@@ -271,6 +271,7 @@ private void btnCreateSuite_Click(object sender, EventArgs e)
                 Username = tbUsername.Text,
                 Password = tbPassword.Text,
                 ExampleDatasets = cbCreateExampleDatasets.Checked,
+                CreateLoggingServer = cbCreateLoggingServer.Checked,
                 Seed = _seed,
                 NumberOfPeople = _peopleCount,
                 NumberOfRowsPerDataset = _rowCount,