Merge branch 'main' into dbarrow257/pr/chain_restart_fix

.github/workflows/README.md

@@ -0,0 +1,51 @@
+# Actions
+
+Summary of currently implemented bots
+
+## Benchmark
+* Performs several action like reweight and save output in nice chart
+
+## CDImage
+* Creates container after each commit to develop or after tag
+
+## CIBuild
+* Checks compilation on several resources
+
+## CIPythonValidations
+* Testin of pyMaCh3
+
+## CIValidations
+* Unit and integration test of MaCh3
+
+## CodeQL
+* Code scanning of MaCh3
+
+## Doxygen
+* Creates doxygen and spinhx documentation for MaCh3
+
+## Label
+* Label each PR based on modified files
+
+## Linter
+* Lint code base (currently C++ turned off)
+
+## MakeRelase
+* Make realse after tag is made
+
+## Meme
+* Nothing to add, cool meme for PR
+
+## Newsletter
+* Make weekly summary of
+
+## PRtittleChecker
+* Ensures each PR starts with suffix like tidy: for example
+
+## Stale
+* Tags stale PRs and issues
+
+## TagTutorial
+* Make tag of tutorial after MaCh3 is tagged to ensure both have corresponding tags
+
+## Telemetry
+* Add telemetry like RAM and CPU usage for each PR

CMakeLists.txt

@@ -3,7 +3,7 @@ set(CMAKE_VERBOSE_MAKEFILE ON)
 # CMake version check
 cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
-project(MaCh3 VERSION 1.3.4 LANGUAGES CXX)
+project(MaCh3 VERSION 1.4.0 LANGUAGES CXX)
 set(MaCh3_VERSION ${PROJECT_VERSION})
 
 #LP - This option name is confusing, but I wont change it now.

Doc/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MaCh3"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.3.4
+PROJECT_NUMBER         = 1.4.0
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

covariance/covarianceBase.cpp

@@ -1,4 +1,5 @@
 #include "covariance/covarianceBase.h"
+#include "regex"
 
 // ********************************************
 covarianceBase::covarianceBase(std::string name, std::string file, double threshold, int FirstPCA, int LastPCA) : inputFile(file), pca(false),
@@ -210,6 +211,7 @@ void covarianceBase::init(const std::vector<std::string>& YAMLFile) {
     _fGenerated[i] = Get<double>(param["Systematic"]["ParameterValues"]["Generated"], __FILE__ , __LINE__);
     _fIndivStepScale[i] = Get<double>(param["Systematic"]["StepScale"]["MCMC"], __FILE__ , __LINE__);
     _fError[i] = Get<double>(param["Systematic"]["Error"], __FILE__ , __LINE__);
+    _fDetID[i] = GetFromManager<std::vector<std::string>>(param["Systematic"]["DetID"], {}, __FILE__, __LINE__);
     if(_fError[i] <= 0) {
       MACH3LOG_ERROR("Error for param {}({}) is negative and eqaul to {}", _fFancyNames[i], i, _fError[i]);
       throw MaCh3Exception(__FILE__ , __LINE__ );
@@ -333,6 +335,7 @@ void covarianceBase::ReserveMemory(const int SizeVec) {
   _fUpBound = std::vector<double>(SizeVec);
   _fFlatPrior = std::vector<bool>(SizeVec);
   _fIndivStepScale = std::vector<double>(SizeVec);
+  _fDetID = std::vector<std::vector<std::string>>(_fNumPar);
 
   corr_throw = new double[SizeVec]();
   // set random parameter vector (for correlated steps)
@@ -1302,3 +1305,35 @@ void covarianceBase::SaveUpdatedMatrixConfig() {
   fout << copyNode;
   fout.close();
 }
+
+// ********************************************
+bool covarianceBase::AppliesToDetID(const int SystIndex, const std::string& DetID) const {
+// ********************************************
+  // Empty means apply to all
+  if (_fDetID[SystIndex].size() == 0) return true;
+
+  // Make a copy and to lower case to not be case sensitive
+  std::string DetIDCopy = DetID;
+  std::transform(DetIDCopy.begin(), DetIDCopy.end(), DetIDCopy.begin(), ::tolower);
+  bool Applies = false;
+
+  for (size_t i = 0; i < _fDetID[SystIndex].size(); i++) {
+    // Convert to low case to not be case sensitive
+    std::string pattern = _fDetID[SystIndex][i];
+    std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);
+
+    // Replace '*' in the pattern with '.*' for regex matching
+    std::string regexPattern = "^" + std::regex_replace(pattern, std::regex("\\*"), ".*") + "$";
+    try {
+      std::regex regex(regexPattern);
+      if (std::regex_match(DetIDCopy, regex)) {
+        Applies = true;
+        break;
+      }
+    } catch (const std::regex_error& e) {
+      // Handle regex error (for invalid patterns)
+      MACH3LOG_ERROR("Regex error: {}", e.what());
+    }
+  }
+  return Applies;
+}

covariance/covarianceBase.h

@@ -425,6 +425,11 @@ class covarianceBase {
   /// @cite haario2001adaptive
   void updateAdaptiveCovariance();
 
+  /// @brief Check if parameter is affecting given det ID
+  /// @param SystIndex number of parameter
+  /// @param DetID The Detector ID used to filter parameters.
+  bool AppliesToDetID(const int SystIndex, const std::string& DetID) const;
+
   /// The input root file we read in
   const std::string inputFile;
 
@@ -476,6 +481,8 @@ class covarianceBase {
   std::vector<double> _fIndivStepScale;
   /// Whether to apply flat prior or not
   std::vector<bool> _fFlatPrior;
+  /// Tells to which samples object param should be applied
+  std::vector<std::vector<std::string>> _fDetID;
 
   /// perform PCA or not
   bool pca;

covariance/covarianceOsc.cpp

@@ -81,11 +81,34 @@ void covarianceOsc::Print() {
   MACH3LOG_INFO("Number of pars: {}", _fNumPar);
   MACH3LOG_INFO("Current: {} parameters:", matrixName);
 
-  MACH3LOG_INFO("{:<5} | {:<25} | {:<10} | {:<15} | {:<15} | {:<10}",
-                "#", "Name", "Nom.", "IndivStepScale", "_fError", "FlatPrior");
+  MACH3LOG_INFO("=================================================================================================================================");
+  MACH3LOG_INFO("{:<5} {:2} {:<25} {:2} {:<10} {:2} {:<15} {:2} {:<15} {:2} {:<10} {:2} {:<10}",
+                "#", "|", "Name", "|", "Prior", "|", "IndivStepScale", "|", "Error", "|", "FlatPrior", "|", "DetID");
+  MACH3LOG_INFO("---------------------------------------------------------------------------------------------------------------------------------");
+  for (int i = 0; i < _fNumPar; i++) {
+    std::string detIdString = "";
+    for (const auto& detID : _fDetID[i]) {
+      if (!detIdString.empty()) {
+        detIdString += ", ";
+      }
+      detIdString += detID;
+    }
 
-  for(int i = 0; i < _fNumPar; i++) {
-    MACH3LOG_INFO("{:<5} | {:<25} | {:<10.4f} | {:<15.2f} | {:<15.4f} | {:<10}",
-                  i, _fNames[i].c_str(), _fPreFitValue[i], _fIndivStepScale[i], _fError[i], _fFlatPrior[i]);
+    MACH3LOG_INFO("{:<5} {:2} {:<25} {:2} {:<10.4f} {:2} {:<15.2f} {:2} {:<15.4f} {:2} {:<10} {:2} {:<10}",
+                  i, "|", _fNames[i].c_str(), "|", _fPreFitValue[i], "|", _fIndivStepScale[i], "|", _fError[i], "|", _fFlatPrior[i], "|", detIdString);
+  }
+  MACH3LOG_INFO("=================================================================================================================================");
+}
+
+// ********************************************
+// DB Grab the Normalisation parameters for the relevant DetID
+std::vector<const double*> covarianceOsc::GetOscParsFromDetID(const std::string& DetID) {
+// ********************************************
+  std::vector<const double*> returnVec;
+  for (int i = 0; i < _fNumPar; ++i) {
+    if (AppliesToDetID(i, DetID)) {
+      returnVec.push_back(retPointer(i));
+    }
   }
+  return returnVec;
 }

covariance/covarianceOsc.h

@@ -17,7 +17,8 @@ class covarianceOsc : public covarianceBase
   void proposeStep() override;
   /// @brief Sets whether to flip delta M23.
   void setFlipDeltaM23(bool flip){flipdelM = flip;}
-
+  /// @brief Get pointers to Osc params from detId
+  std::vector<const double*> GetOscParsFromDetID(const std::string& DetID);
   /// @brief KS: Print all useful information's after initialization
   void Print();
 

covariance/covarianceXsec.cpp

@@ -28,7 +28,6 @@ void covarianceXsec::InitXsecFromConfig() {
 // ********************************************
   _fSystToGlobalSystIndexMap.resize(SystType::kSystTypes);
 
-  _fDetID = std::vector<int>(_fNumPar);
   _fParamType = std::vector<SystType>(_fNumPar);
   _ParameterGroup = std::vector<std::string>(_fNumPar);
 
@@ -43,7 +42,6 @@ void covarianceXsec::InitXsecFromConfig() {
   //PreFitValues etc etc.
   for (auto const &param : _fYAMLDoc["Systematics"])
   {
-    _fDetID[i] = Get<int>(param["Systematic"]["DetID"], __FILE__ , __LINE__);
     _ParameterGroup[i] = Get<std::string>(param["Systematic"]["ParameterGroup"], __FILE__ , __LINE__);
 
     //Fill the map to get the correlations later as well
@@ -105,7 +103,7 @@ covarianceXsec::~covarianceXsec() {
 
 // ********************************************
 // DB Grab the Spline Names for the relevant DetID
-const std::vector<std::string> covarianceXsec::GetSplineParsNamesFromDetID(const int DetID) {
+const std::vector<std::string> covarianceXsec::GetSplineParsNamesFromDetID(const std::string& DetID) {
 // ********************************************
   std::vector<std::string> returnVec;
   for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kSpline]) {
@@ -120,7 +118,7 @@ const std::vector<std::string> covarianceXsec::GetSplineParsNamesFromDetID(const
 }
 
 // ********************************************
-const std::vector<SplineInterpolation> covarianceXsec::GetSplineInterpolationFromDetID(const int DetID) {
+const std::vector<SplineInterpolation> covarianceXsec::GetSplineInterpolationFromDetID(const std::string& DetID) {
 // ********************************************
   std::vector<SplineInterpolation> returnVec;
   for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kSpline]) {
@@ -136,15 +134,15 @@ const std::vector<SplineInterpolation> covarianceXsec::GetSplineInterpolationFro
 
 // ********************************************
 // DB Grab the Spline Modes for the relevant DetID
-const std::vector< std::vector<int> > covarianceXsec::GetSplineModeVecFromDetID(const int DetID) {
+const std::vector< std::vector<int> > covarianceXsec::GetSplineModeVecFromDetID(const std::string& DetID) {
 // ********************************************
   std::vector< std::vector<int> > returnVec;
   //Need a counter or something to correctly get the index in _fSplineModes since it's not of length nPars
   //Should probably just make a std::map<std::string, int> for param name to FD spline index
   for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kSpline]) {
     auto &SplineIndex = pair.first;
     auto &SystIndex = pair.second;
-    if ((GetParDetID(SystIndex) & DetID)) { //If parameter applies to required DetID
+    if (AppliesToDetID(SystIndex, DetID)) { //If parameter applies to required DetID
       returnVec.push_back(SplineParams.at(SplineIndex)._fSplineModes);
     }
   }
@@ -212,7 +210,7 @@ XsecNorms4 covarianceXsec::GetXsecNorm(const YAML::Node& param, const int Index)
 // ********************************************
 // Grab the global syst index for the relevant DetID
 // i.e. get a vector of size nSplines where each entry is filled with the global syst number
-const std::vector<int> covarianceXsec::GetGlobalSystIndexFromDetID(const int DetID, const SystType Type) {
+const std::vector<int> covarianceXsec::GetGlobalSystIndexFromDetID(const std::string& DetID, const SystType Type) {
 // ********************************************
   std::vector<int> returnVec;
   for (auto &pair : _fSystToGlobalSystIndexMap[Type]) {
@@ -227,7 +225,7 @@ const std::vector<int> covarianceXsec::GetGlobalSystIndexFromDetID(const int Det
 // ********************************************
 // Grab the global syst index for the relevant DetID
 // i.e. get a vector of size nSplines where each entry is filled with the global syst number
-const std::vector<int> covarianceXsec::GetSystIndexFromDetID(int DetID,  const SystType Type) {
+const std::vector<int> covarianceXsec::GetSystIndexFromDetID(const std::string& DetID,  const SystType Type) {
 // ********************************************
   std::vector<int> returnVec;
   for (auto &pair : _fSystToGlobalSystIndexMap[Type]) {
@@ -266,7 +264,7 @@ XsecSplines1 covarianceXsec::GetXsecSpline(const YAML::Node& param) {
 
 // ********************************************
 // DB Grab the Normalisation parameters for the relevant DetID
-const std::vector<XsecNorms4> covarianceXsec::GetNormParsFromDetID(const int DetID) {
+const std::vector<XsecNorms4> covarianceXsec::GetNormParsFromDetID(const std::string& DetID) {
 // ********************************************
   std::vector<XsecNorms4> returnVec;
   for (auto &pair : _fSystToGlobalSystIndexMap[SystType::kNorm]) {
@@ -281,7 +279,7 @@ const std::vector<XsecNorms4> covarianceXsec::GetNormParsFromDetID(const int Det
 
 // ********************************************
 // DB Grab the number of parameters for the relevant DetID
-int covarianceXsec::GetNumParamsFromDetID(const int DetID, const SystType Type) {
+int covarianceXsec::GetNumParamsFromDetID(const std::string& DetID, const SystType Type) {
 // ********************************************
   int returnVal = 0;
   IterateOverParams(DetID,
@@ -293,7 +291,7 @@ int covarianceXsec::GetNumParamsFromDetID(const int DetID, const SystType Type)
 
 // ********************************************
 // DB Grab the parameter names for the relevant DetID
-const std::vector<std::string> covarianceXsec::GetParsNamesFromDetID(const int DetID, const SystType Type) {
+const std::vector<std::string> covarianceXsec::GetParsNamesFromDetID(const std::string& DetID, const SystType Type) {
 // ********************************************
   std::vector<std::string> returnVec;
   IterateOverParams(DetID,
@@ -305,7 +303,7 @@ const std::vector<std::string> covarianceXsec::GetParsNamesFromDetID(const int D
 
 // ********************************************
 // DB DB Grab the parameter indices for the relevant DetID
-const std::vector<int> covarianceXsec::GetParsIndexFromDetID(const int DetID, const SystType Type) {
+const std::vector<int> covarianceXsec::GetParsIndexFromDetID(const std::string& DetID, const SystType Type) {
 // ********************************************
   std::vector<int> returnVec;
   IterateOverParams(DetID,
@@ -317,7 +315,7 @@ const std::vector<int> covarianceXsec::GetParsIndexFromDetID(const int DetID, co
 
 // ********************************************
 template <typename FilterFunc, typename ActionFunc>
-void covarianceXsec::IterateOverParams(const int DetID, FilterFunc filter, ActionFunc action) {
+void covarianceXsec::IterateOverParams(const std::string& DetID, FilterFunc filter, ActionFunc action) {
 // ********************************************
   for (int i = 0; i < _fNumPar; ++i) {
     if ((AppliesToDetID(i, DetID)) && filter(i)) { // Common filter logic
@@ -382,11 +380,18 @@ void covarianceXsec::Print() {
 void covarianceXsec::PrintGlobablInfo() {
 // ********************************************
   MACH3LOG_INFO("============================================================================================================================================================");
-  MACH3LOG_INFO("{:<5} {:2} {:<40} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<5} {:2} {:<10}", "#", "|", "Name", "|", "Gen.", "|", "Prior", "|", "Error", "|", "Lower", "|", "Upper", "|", "StepScale", "|", "DetID", "|", "Type");
+  MACH3LOG_INFO("{:<5} {:2} {:<40} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10}", "#", "|", "Name", "|", "Gen.", "|", "Prior", "|", "Error", "|", "Lower", "|", "Upper", "|", "StepScale", "|", "DetID", "|", "Type");
   MACH3LOG_INFO("------------------------------------------------------------------------------------------------------------------------------------------------------------");
   for (int i = 0; i < GetNumParams(); i++) {
     std::string ErrString = fmt::format("{:.2f}", _fError[i]);
-    MACH3LOG_INFO("{:<5} {:2} {:<40} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<5} {:2} {:<10}", i, "|", GetParFancyName(i), "|", _fGenerated[i], "|", _fPreFitValue[i], "|", "+/- " + ErrString, "|", _fLowBound[i], "|", _fUpBound[i], "|", _fIndivStepScale[i], "|", _fDetID[i], "|", SystType_ToString(_fParamType[i]));
+    std::string detIdString = "";
+    for (const auto& detID : _fDetID[i]) {
+      if (!detIdString.empty()) {
+        detIdString += ", ";
+      }
+      detIdString += detID;
+    }
+    MACH3LOG_INFO("{:<5} {:2} {:<40} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10} {:2} {:<10}", i, "|", GetParFancyName(i), "|", _fGenerated[i], "|", _fPreFitValue[i], "|", "+/- " + ErrString, "|", _fLowBound[i], "|", _fUpBound[i], "|", _fIndivStepScale[i], "|", detIdString, "|", SystType_ToString(_fParamType[i]));
   }
   MACH3LOG_INFO("============================================================================================================================================================");
 }
@@ -589,9 +594,7 @@ void covarianceXsec::DumpMatrixToFile(const std::string& Name) {
 
   TVectorD* xsec_param_knot_weight_lb = new TVectorD(_fNumPar);
   TVectorD* xsec_param_knot_weight_ub = new TVectorD(_fNumPar);
-
   TVectorD* xsec_error = new TVectorD(_fNumPar);
-  TVectorD* xsec_param_id = new TVectorD(_fNumPar);
 
   for(int i = 0; i < _fNumPar; ++i)
   {
@@ -609,7 +612,6 @@ void covarianceXsec::DumpMatrixToFile(const std::string& Name) {
     (*xsec_flat_prior)[i] = _fFlatPrior[i];
     (*xsec_stepscale)[i] = _fIndivStepScale[i];
     (*xsec_error)[i] = _fError[i];
-    (*xsec_param_id)[i] = _fDetID[i];
 
     (*xsec_param_lb)[i] = _fLowBound[i];
     (*xsec_param_ub)[i] = _fUpBound[i];
@@ -656,9 +658,6 @@ void covarianceXsec::DumpMatrixToFile(const std::string& Name) {
   delete xsec_param_knot_weight_lb;
   xsec_param_knot_weight_ub->Write("xsec_param_knot_weight_ub");
   delete xsec_param_knot_weight_ub;
-
-  xsec_param_id->Write("xsec_param_id");
-  delete xsec_param_id;
   xsec_error->Write("xsec_error");
   delete xsec_error;