feat: Smarter CombineMCMCChains

CMakeLists.txt

@@ -3,7 +3,7 @@ set(CMAKE_VERBOSE_MAKEFILE ON)
 # CMake version check
 cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
-project(MaCh3 VERSION 1.3.0 LANGUAGES CXX)
+project(MaCh3 VERSION 1.3.1 LANGUAGES CXX)
 set(MaCh3_VERSION ${PROJECT_VERSION})
 
 #LP - This option name is confusing, but I wont change it now.

Diagnostics/CombineMaCh3Chains.cpp

@@ -15,25 +15,42 @@ std::string OutFileName = "";
 int targetCompression = 1;
 std::vector<std::string> inpFileList;
 bool forceOverwrite = false;
+bool forceMerge = false;
+
+/// @brief KS: This allow us to skip output name etc in config. We expect Output name will be different but this doesn't invalidate chain merging
+bool ShouldSkipLine(const std::string& line, const std::vector<std::string>& SkipVector) {
+  // Otherwise, check if the line contains any word from SkipVector
+  for (const auto& word : SkipVector) {
+    MACH3LOG_TRACE("{} : {}",line, word);
+    if (line.find(word) != std::string::npos) {
+      MACH3LOG_TRACE("Found matching word, therefore Skipping");
+      return true;
+    }
+  }
+  return false;
+}
 
-
-void CompareTwoConfigs(const std::string& File1, const std::string& File2) {
-  // Treat File1 and File2 as strings (not files) to read like streams
+bool CompareTwoConfigs(const std::string& File1, const std::string& File2, const std::vector<std::string>& SkipVector) {
   std::istringstream file1(File1);
   std::istringstream file2(File2);
 
   std::string line1, line2;
   int lineNumber = 1;
-  // Read and compare line by line
+  bool areEqual = true;
+
   while (std::getline(file1, line1) && std::getline(file2, line2)) {
+    if (ShouldSkipLine(line1, SkipVector) || ShouldSkipLine(line2, SkipVector)) {
+      ++lineNumber;
+      continue;
+    }
     if (line1 != line2) {
+      areEqual = false;
       MACH3LOG_WARN("Difference found on line {}:", lineNumber);
       MACH3LOG_WARN("Config1: {}", line1);
       MACH3LOG_WARN("Config2: {}", line2);
     }
     ++lineNumber;
   }
-
   // Check if one file has extra lines
   while (std::getline(file1, line1)) {
     MACH3LOG_WARN("Extra line in {} on line {}: {}", File1, lineNumber, line1);
@@ -43,43 +60,24 @@ void CompareTwoConfigs(const std::string& File1, const std::string& File2) {
     MACH3LOG_WARN("Extra line in {} on line {}: {}", File2, lineNumber, line2);
     ++lineNumber;
   }
+  return areEqual;
 }
 
 /// EM: Will compare the version header contained in the two provided files and shout if they don't match
-bool checkSoftwareVersions(TFile *file, TFile *prevFile, const std::string& ConfigName)
+bool checkSoftwareVersions(TFile *file, TFile *prevFile, const std::string& ConfigName, const std::vector<std::string>& SkipVector = {})
 {
   bool weirdFile = false;
 
   TMacro *versionHeader = file->Get<TMacro>(ConfigName.c_str());
   TMacro *prevVersionHeader = prevFile->Get<TMacro>(ConfigName.c_str());
 
   // EM: compare the digest of the version header file in this file, with the previous one
-  if((versionHeader == NULL) && (prevVersionHeader == NULL)){
-    MACH3LOG_WARN("files don't contain an embedded version header, indicating they were made before this feature was added");
-    MACH3LOG_WARN("  I can still combine them but I can't guarantee they were made with matching software versions");
-    MACH3LOG_WARN("  This is ok but you'll just have to be extra careful and make sure you check this yourself!");
-  }
-  else if((versionHeader != NULL) && (prevVersionHeader == NULL)){
-    MACH3LOG_ERROR("looks like file {} has a version header file but previous ones do not", file->GetName());
-    MACH3LOG_ERROR("This is odd and suggests this file was made with an MaCh3 version");
-    MACH3LOG_ERROR("from after this feature was added, whereas other files were made with an older MaCh3");
-    weirdFile = true;
-  }
-  else if((versionHeader == NULL) && (prevVersionHeader != NULL)){
-    MACH3LOG_ERROR("looks like file {} doesn't have a version header file but previous ones do", file->GetName());
-    MACH3LOG_ERROR("This is odd and suggests this file was made with an MaCh3 version");
-    MACH3LOG_ERROR("from before this feature was added, whereas other files were made with newer MaCh3");
+  if(!CompareTwoConfigs(TMacroToString(*versionHeader), TMacroToString(*prevVersionHeader), SkipVector)){
+    MACH3LOG_ERROR("Looks like the {} embedded config for file {} is different to the previous ones", ConfigName, file->GetName());
+    MACH3LOG_ERROR("This strongly suggests that this file was made with different software versions than the previous ones");
     weirdFile = true;
   }
-  else{
-    MACH3LOG_DEBUG("  Prev header digest: {} :: current: {}", (prevVersionHeader->Checksum())->AsString(), (versionHeader->Checksum())->AsString());
-    if(std::strcmp((versionHeader->Checksum())->AsString(), (prevVersionHeader->Checksum())->AsString()) != 0){
-      MACH3LOG_ERROR("Looks like the {} embedded config for file {} is different to the previous ones", ConfigName, file->GetName());
-      MACH3LOG_ERROR("This strongly suggests that this file was made with different software versions than the previous ones");
-      CompareTwoConfigs(TMacroToString(*versionHeader), TMacroToString(*prevVersionHeader));
-      weirdFile = true;
-    }
-  }
+
   return weirdFile;
 }
 
@@ -165,20 +163,16 @@ void CombineChain()
     MACH3LOG_DEBUG("############ File {} #############", fileId);
 
     bool weirdFile = false;
+    if(checkSoftwareVersions(file, prevFile, "MaCh3Engine/version_header")) weirdFile = true;
+    if(checkSoftwareVersions(file, prevFile, "MaCh3_Config", {"OutputFile:", "NSteps:"})) weirdFile = true;
 
-
-    if(checkSoftwareVersions(file, prevFile, "MaCh3Engine/version_header"))
-        weirdFile = true;
-
-    if(checkSoftwareVersions(file, prevFile, "MaCh3_Config"))
-      weirdFile = true;
-
-    if(weirdFile){
+    if(weirdFile && !forceMerge){
       MACH3LOG_ERROR("");
       MACH3LOG_ERROR("=====================================================================================");
       MACH3LOG_ERROR("This is not a great idea and could lead to weird outputs and cause some big headaches");
       MACH3LOG_ERROR("further down the road. But if you reeeeally wanna do it and you know what you're");
       MACH3LOG_ERROR("doing you can come here and remove the 'throw'");
+      MACH3LOG_ERROR("Or use -m option");
       MACH3LOG_ERROR("{}:{}", __FILE__, __LINE__ + 2);
       MACH3LOG_ERROR("=====================================================================================");
       throw MaCh3Exception(__FILE__ , __LINE__ );
@@ -203,8 +197,7 @@ void CombineChain()
   bool mergeSuccess = fileMerger->PartialMerge(TFileMerger::kRegular | TFileMerger::kAll | TFileMerger::kOnlyListed);
   if(mergeSuccess){
     MACH3LOG_INFO("Files merged successfully");
-  }
-  else{
+  } else{
     MACH3LOG_ERROR("Failed to merge files");
   }
   delete fileMerger;
@@ -232,6 +225,7 @@ void usage(){
   MACH3LOG_INFO("output file    : name of combined spline file. optional: if not specified, the app will just use the first input file as the output, the same as hadd'");
   MACH3LOG_INFO("-c             : target compression level for the combined file, default is 1, in line with hadd");
   MACH3LOG_INFO("-f             : force overwrite the output file if it exists already");
+  MACH3LOG_INFO("-m             : merge files in-spite of differences");
   MACH3LOG_INFO("-h             : print this message and exit");
 }
 
@@ -270,6 +264,10 @@ void ParseArg(int argc, char *argv[]){
           targetCompression = atoi(optarg);
           break;
         }
+        case 'm': {
+          forceMerge = true;
+          break;
+        }
         case 'h': {
           usage();
           exit(0);

Doc/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MaCh3"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.3.0
+PROJECT_NUMBER         = 1.3.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a