add docs

HicServices · Jan 6, 2025 · 851416e · 851416e
1 parent 2d3a7fa
commit 851416e
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 - Build on and target .Net 9 rather than 8
+- Add DQE Updater Mutilator for Data Loads see [DQE Post Load runner](./Documentation/DataLoadEngine/DQEPostLoadRunner.md)
+
 ## [8.4.2] - 2024-12-18
 
 - Fix issue with MEF constructing Remote Table Attachers

diff --git a/Documentation/DataLoadEngine/DQEPostLoadRunner.md b/Documentation/DataLoadEngine/DQEPostLoadRunner.md
@@ -0,0 +1,12 @@
+# DQE Post Load Runner
+
+The DQE post-load runner can be used to automatically perform a DQE update once a data load completes.
+The runner attempts to reuse any existing DQE results that have been unaffected by the data load, however this process can still be slow if the catalogue data is large and/or complex.
+
+## Requirements
+The DQE post-load runner requires an existing DQE result to exist, otherwise it will fail.
+
+## Configuration
+The runner makes a number of queries to the database, the timeout for these commands is configurable via the timeout option.
+
+
diff --git a/Rdmp.Core/DataQualityEngine/Reports/CatalogueConstraintReport.cs b/Rdmp.Core/DataQualityEngine/Reports/CatalogueConstraintReport.cs
@@ -239,7 +239,7 @@ public void UpdateReport(ICatalogue c, int dataLoadID, int? commandTimeout, IDat
                 var pivotColumnInfo = _catalogue.CatalogueItems.Where(ci => ci.Name == _pivotCategory).FirstOrDefault();
                 if (pivotColumnInfo is null) throw new Exception("Can't find column infor for pivot category");
                 var tableInfo = pivotColumnInfo.ColumnInfo.TableInfo;
-                
+
                 var dataDiffFetcher = new DiffDatabaseDataFetcher(2147483647, tableInfo, (int)_dataLoadID, commandTimeout != null ? (int)commandTimeout : 30);
                 dataDiffFetcher.FetchData(new AcceptAllCheckNotifier());
                 //pivot categories that have been replaces 100%?
@@ -433,7 +433,7 @@ public void UpdateReport(ICatalogue c, int dataLoadID, int? commandTimeout, IDat
                         }
                     }
                 }
-                //what about the replacements?
+                //replacements
                 if (existingIncomingPivotCategories.Any())
                 {
                     var updatedRowsDataTable = new DataTable();