From 3e4ffd5a91a826162b9b439a3b064e63af2f50ea Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Tue, 17 Dec 2024 11:12:04 +0000 Subject: [PATCH] Now combine MCMC chain is smart enough to ignore stuff like different output name or step number --- CMakeLists.txt | 2 +- Diagnostics/CombineMaCh3Chains.cpp | 78 +++++++++++++++--------------- Doc/Doxyfile | 2 +- 3 files changed, 40 insertions(+), 42 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5da55516e..895349dc5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ set(CMAKE_VERBOSE_MAKEFILE ON) # CMake version check cmake_minimum_required(VERSION 3.14 FATAL_ERROR) -project(MaCh3 VERSION 1.3.0 LANGUAGES CXX) +project(MaCh3 VERSION 1.3.1 LANGUAGES CXX) set(MaCh3_VERSION ${PROJECT_VERSION}) #LP - This option name is confusing, but I wont change it now. diff --git a/Diagnostics/CombineMaCh3Chains.cpp b/Diagnostics/CombineMaCh3Chains.cpp index 8c246496e..628dbd6ed 100644 --- a/Diagnostics/CombineMaCh3Chains.cpp +++ b/Diagnostics/CombineMaCh3Chains.cpp @@ -15,25 +15,42 @@ std::string OutFileName = ""; int targetCompression = 1; std::vector inpFileList; bool forceOverwrite = false; +bool forceMerge = false; + +/// @brief KS: This allow us to skip output name etc in config. We expect Output name will be different but this doesn't invalidate chain merging +bool ShouldSkipLine(const std::string& line, const std::vector& SkipVector) { + // Otherwise, check if the line contains any word from SkipVector + for (const auto& word : SkipVector) { + MACH3LOG_TRACE("{} : {}",line, word); + if (line.find(word) != std::string::npos) { + MACH3LOG_TRACE("Found matching word, therefore Skipping"); + return true; + } + } + return false; +} - -void CompareTwoConfigs(const std::string& File1, const std::string& File2) { - // Treat File1 and File2 as strings (not files) to read like streams +bool CompareTwoConfigs(const std::string& File1, const std::string& File2, const std::vector& SkipVector) { std::istringstream file1(File1); std::istringstream file2(File2); std::string line1, line2; int lineNumber = 1; - // Read and compare line by line + bool areEqual = true; + while (std::getline(file1, line1) && std::getline(file2, line2)) { + if (ShouldSkipLine(line1, SkipVector) || ShouldSkipLine(line2, SkipVector)) { + ++lineNumber; + continue; + } if (line1 != line2) { + areEqual = false; MACH3LOG_WARN("Difference found on line {}:", lineNumber); MACH3LOG_WARN("Config1: {}", line1); MACH3LOG_WARN("Config2: {}", line2); } ++lineNumber; } - // Check if one file has extra lines while (std::getline(file1, line1)) { MACH3LOG_WARN("Extra line in {} on line {}: {}", File1, lineNumber, line1); @@ -43,10 +60,11 @@ void CompareTwoConfigs(const std::string& File1, const std::string& File2) { MACH3LOG_WARN("Extra line in {} on line {}: {}", File2, lineNumber, line2); ++lineNumber; } + return areEqual; } /// EM: Will compare the version header contained in the two provided files and shout if they don't match -bool checkSoftwareVersions(TFile *file, TFile *prevFile, const std::string& ConfigName) +bool checkSoftwareVersions(TFile *file, TFile *prevFile, const std::string& ConfigName, const std::vector& SkipVector = {}) { bool weirdFile = false; @@ -54,32 +72,12 @@ bool checkSoftwareVersions(TFile *file, TFile *prevFile, const std::string& Conf TMacro *prevVersionHeader = prevFile->Get(ConfigName.c_str()); // EM: compare the digest of the version header file in this file, with the previous one - if((versionHeader == NULL) && (prevVersionHeader == NULL)){ - MACH3LOG_WARN("files don't contain an embedded version header, indicating they were made before this feature was added"); - MACH3LOG_WARN(" I can still combine them but I can't guarantee they were made with matching software versions"); - MACH3LOG_WARN(" This is ok but you'll just have to be extra careful and make sure you check this yourself!"); - } - else if((versionHeader != NULL) && (prevVersionHeader == NULL)){ - MACH3LOG_ERROR("looks like file {} has a version header file but previous ones do not", file->GetName()); - MACH3LOG_ERROR("This is odd and suggests this file was made with an MaCh3 version"); - MACH3LOG_ERROR("from after this feature was added, whereas other files were made with an older MaCh3"); - weirdFile = true; - } - else if((versionHeader == NULL) && (prevVersionHeader != NULL)){ - MACH3LOG_ERROR("looks like file {} doesn't have a version header file but previous ones do", file->GetName()); - MACH3LOG_ERROR("This is odd and suggests this file was made with an MaCh3 version"); - MACH3LOG_ERROR("from before this feature was added, whereas other files were made with newer MaCh3"); + if(!CompareTwoConfigs(TMacroToString(*versionHeader), TMacroToString(*prevVersionHeader), SkipVector)){ + MACH3LOG_ERROR("Looks like the {} embedded config for file {} is different to the previous ones", ConfigName, file->GetName()); + MACH3LOG_ERROR("This strongly suggests that this file was made with different software versions than the previous ones"); weirdFile = true; } - else{ - MACH3LOG_DEBUG(" Prev header digest: {} :: current: {}", (prevVersionHeader->Checksum())->AsString(), (versionHeader->Checksum())->AsString()); - if(std::strcmp((versionHeader->Checksum())->AsString(), (prevVersionHeader->Checksum())->AsString()) != 0){ - MACH3LOG_ERROR("Looks like the {} embedded config for file {} is different to the previous ones", ConfigName, file->GetName()); - MACH3LOG_ERROR("This strongly suggests that this file was made with different software versions than the previous ones"); - CompareTwoConfigs(TMacroToString(*versionHeader), TMacroToString(*prevVersionHeader)); - weirdFile = true; - } - } + return weirdFile; } @@ -165,20 +163,16 @@ void CombineChain() MACH3LOG_DEBUG("############ File {} #############", fileId); bool weirdFile = false; + if(checkSoftwareVersions(file, prevFile, "MaCh3Engine/version_header")) weirdFile = true; + if(checkSoftwareVersions(file, prevFile, "MaCh3_Config", {"OutputFile:", "NSteps:"})) weirdFile = true; - - if(checkSoftwareVersions(file, prevFile, "MaCh3Engine/version_header")) - weirdFile = true; - - if(checkSoftwareVersions(file, prevFile, "MaCh3_Config")) - weirdFile = true; - - if(weirdFile){ + if(weirdFile && !forceMerge){ MACH3LOG_ERROR(""); MACH3LOG_ERROR("====================================================================================="); MACH3LOG_ERROR("This is not a great idea and could lead to weird outputs and cause some big headaches"); MACH3LOG_ERROR("further down the road. But if you reeeeally wanna do it and you know what you're"); MACH3LOG_ERROR("doing you can come here and remove the 'throw'"); + MACH3LOG_ERROR("Or use -m option"); MACH3LOG_ERROR("{}:{}", __FILE__, __LINE__ + 2); MACH3LOG_ERROR("====================================================================================="); throw MaCh3Exception(__FILE__ , __LINE__ ); @@ -203,8 +197,7 @@ void CombineChain() bool mergeSuccess = fileMerger->PartialMerge(TFileMerger::kRegular | TFileMerger::kAll | TFileMerger::kOnlyListed); if(mergeSuccess){ MACH3LOG_INFO("Files merged successfully"); - } - else{ + } else{ MACH3LOG_ERROR("Failed to merge files"); } delete fileMerger; @@ -232,6 +225,7 @@ void usage(){ MACH3LOG_INFO("output file : name of combined spline file. optional: if not specified, the app will just use the first input file as the output, the same as hadd'"); MACH3LOG_INFO("-c : target compression level for the combined file, default is 1, in line with hadd"); MACH3LOG_INFO("-f : force overwrite the output file if it exists already"); + MACH3LOG_INFO("-m : merge files in-spite of differences"); MACH3LOG_INFO("-h : print this message and exit"); } @@ -270,6 +264,10 @@ void ParseArg(int argc, char *argv[]){ targetCompression = atoi(optarg); break; } + case 'm': { + forceMerge = true; + break; + } case 'h': { usage(); exit(0); diff --git a/Doc/Doxyfile b/Doc/Doxyfile index 05c921405..5a7677e9a 100644 --- a/Doc/Doxyfile +++ b/Doc/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = "MaCh3" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.3.0 +PROJECT_NUMBER = 1.3.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a