From b4bdd79c9520724a663440c1bf6ab90bb120e401 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Thu, 30 Jun 2022 18:12:48 -0600 Subject: [PATCH 01/53] added multiweight functions to RegionSelection and Cutflows --- .../SampleAnalyzer/Process/Counter/Counter.h | 27 +++++++++++++++++++ .../Process/Counter/CounterManager.h | 6 +++++ .../Process/RegionSelection/RegionSelection.h | 23 ++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/tools/SampleAnalyzer/Process/Counter/Counter.h b/tools/SampleAnalyzer/Process/Counter/Counter.h index dde27d9f..47cfbf27 100644 --- a/tools/SampleAnalyzer/Process/Counter/Counter.h +++ b/tools/SampleAnalyzer/Process/Counter/Counter.h @@ -34,6 +34,12 @@ #include #include +struct multiWeightEntry { + std::pair nentries_; + std::pair sumweight_; + std::pair sumweight2_; +}; + namespace MA5 { @@ -64,6 +70,8 @@ class Counter /// first = positive weight ; second = negative weight std::pair sumweight2_; + std::map multiweight_; + // ------------------------------------------------------------- // method members @@ -108,6 +116,25 @@ class Counter } } + void Increment(const std::map &multiweights){ + for(const auto &weight : multiweights){ + if(multiweight_.find(weight.first) == multiweight_.end()){ + multiweight_[weight.first] = new multiWeightEntry; + } + if(weight.second > 0){ + multiweight_[weight.first]->nentries_.first++; + multiweight_[weight.first]->sumweight_.first+=weight.second; + multiweight_[weight.first]->sumweight2_.first+=weight.second*weight.second; + } + else if (weight.second < 0){ + multiweight_[weight.first]->nentries_.second++; + multiweight_[weight.first]->sumweight_.second+=weight.second; + multiweight_[weight.first]->sumweight2_.second+=weight.second*weight.second; + } + } + } + + }; } diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.h b/tools/SampleAnalyzer/Process/Counter/CounterManager.h index 654abed9..45e65c92 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.h +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.h @@ -30,6 +30,7 @@ #include #include #include +#include // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/Counter.h" @@ -108,6 +109,11 @@ class CounterManager void Finalize() { Reset(); } + void IncrementNInitial(const std::map &multiweight){ + initial_.Increment(multiweight); + } + + }; } diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h index 2f491436..805d299f 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h @@ -29,6 +29,7 @@ // STL headers #include #include +#include // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/CounterManager.h" @@ -48,6 +49,8 @@ class RegionSelection MAbool surviving_; MAuint32 NumberOfCutsAppliedSoFar_; MAfloat64 weight_; + std::map multiWeight_; + CounterManager cutflow_; @@ -117,6 +120,26 @@ class RegionSelection weight_=weight; } + //Multiweight Integration implementation below + void SetWeight(const std::map &multiweight) { + multiWeight_ = multiweight; + } + + std::map GetWeights() {return multiWeight_;} + + void IncrementCutFlow(const std::map &multiweight){ + cutflow_[NumberOfCutsAppliedSoFar_].Increment(multiweight); + NumberOfCutsAppliedSoFar_++; + } + + void InitializeForNewEvent(const std::map &multiweight){ + SetSurvivingTest(true); + SetNumberOfCutsAppliedSoFar(0); + cutflow_.IncrementNInitial(multiweight); + multiWeight_=multiweight; + } + + }; } From d62b55e22e98551902b56ae11f214cdadd4822b8 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Thu, 7 Jul 2022 17:40:32 -0600 Subject: [PATCH 02/53] added initialize for multiweight to regionselectionmanager --- .../Process/RegionSelection/RegionSelectionManager.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h index 2e250199..a7d89279 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h @@ -30,6 +30,7 @@ #include #include #include +#include // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/MultiRegionCounterManager.h" @@ -145,6 +146,16 @@ class RegionSelectionManager plotmanager_.GetHistos()[i]->SetFreshEvent(true); } + /// Initialize for multiweight + void InitializeForNewEvent(const std::map &weights){ + NumberOfSurvivingRegions_ = regions_.size(); + for (MAuint32 i=0; iInitializeForNewEvent(weights); + } + + } + /// This method associates all regions with a cut void AddCut(const std::string&name) { From 20a2e6c3c9e5426765c9c749e1e6113136bfffce Mon Sep 17 00:00:00 2001 From: kfan326 Date: Sat, 23 Jul 2022 10:41:51 -0600 Subject: [PATCH 03/53] fixed regionSelectionManager destructor destructor now deletes allocated region selection objects when going out of scope. --- .../Process/RegionSelection/RegionSelectionManager.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h index 2e250199..6be43755 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h @@ -73,7 +73,12 @@ class RegionSelectionManager RegionSelectionManager() {}; /// Destructor - ~RegionSelectionManager() { }; + // + ~RegionSelectionManager() { + for(auto ®ion_pointer : regions_){ + delete region_pointer; + } + }; /// Reset void Reset() From 5b68fbe738856c0d0836cc6b614893b1bd144e21 Mon Sep 17 00:00:00 2001 From: kfan326 Date: Sun, 24 Jul 2022 09:45:40 -0600 Subject: [PATCH 04/53] Update tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h Co-authored-by: Jack Y. Araz --- .../Process/RegionSelection/RegionSelectionManager.h | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h index 6be43755..049f2e19 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h @@ -73,7 +73,6 @@ class RegionSelectionManager RegionSelectionManager() {}; /// Destructor - // ~RegionSelectionManager() { for(auto ®ion_pointer : regions_){ delete region_pointer; From fecbffcf5d88fbddeb4f971e249394b61987ea3e Mon Sep 17 00:00:00 2001 From: kfan326 Date: Sun, 24 Jul 2022 10:16:46 -0600 Subject: [PATCH 05/53] Update changelog-dev.md --- doc/releases/changelog-dev.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md index a60fbbba..029827b5 100644 --- a/doc/releases/changelog-dev.md +++ b/doc/releases/changelog-dev.md @@ -4,8 +4,15 @@ ## Improvements +RegionSelectionManager Object should not longer leak memory + ## Bug fixes +Fixed destructor in RegionSelectionManager so that RegionSelection objects allocated inside the region_ vector are properly destructed upon existing scope/destruction of RegionSelectionManager. + + ## Contributors -This release contains contributions from (in alphabetical order): \ No newline at end of file +This release contains contributions from (in alphabetical order): + +Jack Araz, Kyle Fan, Benjamin Fuks From 8451685eaa9dea1e9d3c529009fc4024c154b82d Mon Sep 17 00:00:00 2001 From: jackaraz Date: Mon, 25 Jul 2022 09:32:12 +0100 Subject: [PATCH 06/53] update changelog-dev.md --- doc/releases/changelog-dev.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md index 029827b5..4e9eb059 100644 --- a/doc/releases/changelog-dev.md +++ b/doc/releases/changelog-dev.md @@ -4,15 +4,16 @@ ## Improvements -RegionSelectionManager Object should not longer leak memory - ## Bug fixes -Fixed destructor in RegionSelectionManager so that RegionSelection objects allocated inside the region_ vector are properly destructed upon existing scope/destruction of RegionSelectionManager. + * Fixed destructor in `RegionSelectionManager` so that `RegionSelection` + objects allocated inside the `region_vector` are properly destructed upon + existing `scope/destruction` of `RegionSelectionManager`. + ([#113](https://github.com/MadAnalysis/madanalysis5/pull/113)) ## Contributors This release contains contributions from (in alphabetical order): -Jack Araz, Kyle Fan, Benjamin Fuks +[Kyle Fan](https://github.com/kfan326) From beb3b7aee59dd8c01ba0becb668fb0fc303c6806 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Sun, 31 Jul 2022 14:42:35 -0600 Subject: [PATCH 07/53] commented out debug for cutflow --- .../SampleAnalyzer/Process/Counter/Counter.h | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tools/SampleAnalyzer/Process/Counter/Counter.h b/tools/SampleAnalyzer/Process/Counter/Counter.h index 47cfbf27..8cfdcb84 100644 --- a/tools/SampleAnalyzer/Process/Counter/Counter.h +++ b/tools/SampleAnalyzer/Process/Counter/Counter.h @@ -33,6 +33,7 @@ #include #include #include +#include struct multiWeightEntry { std::pair nentries_; @@ -114,10 +115,30 @@ class Counter sumweight_.second+=weight; sumweight2_.second+=weight*weight; } + + } + + void Debug(int debugCount, int weightprocessed){ + for(auto &p : multiweight_){ + std::ofstream output; + output.open("/Users/kfan/desktop/output.txt", std::fstream::app); + output << "ID : " << p.first << " increment multiweight call number : " << debugCount << " weight Processed :"<< weightprocessed; + output << "\n"; + output << "pos entries : " << p.second->nentries_.first << " -- " << nentries_.first << " neg entries : " << p.second->nentries_.second << " -- " << nentries_.second << "\n"; + output << "pos sum : " << p.second->sumweight_.first << " -- " << sumweight_.first << " neg sum : " << p.second->sumweight_.second << " -- " << sumweight_.second << "\n"; + output << "pos squared : " <sumweight2_.first << " -- " << sumweight2_.first << " neg squared : " << p.second->sumweight2_.second << " -- " << sumweight2_.second << "\n"; + output << "----------------------------------------------------------"; + output << "\n"; + output.close(); + + } + } + void Increment(const std::map &multiweights){ - for(const auto &weight : multiweights){ + // static int incrementDebugCount = 0; + for(const auto &weight : multiweights){ if(multiweight_.find(weight.first) == multiweight_.end()){ multiweight_[weight.first] = new multiWeightEntry; } @@ -132,6 +153,12 @@ class Counter multiweight_[weight.first]->sumweight2_.second+=weight.second*weight.second; } } + + //Debug testing +// Debug(incrementDebugCount, multiweights.begin()->second); +// incrementDebugCount++; + + } From 855248088e7c3eb5a8cf72db864d3e462eae88a5 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Mon, 1 Aug 2022 22:25:37 -0600 Subject: [PATCH 08/53] integrated SQLite3 output format for cutflows --- madanalysis/build/makefile_writer.py | 3 +- .../Process/Core/SampleAnalyzer.cpp | 67 ++++++++++++++++++- .../SampleAnalyzer/Process/Counter/Counter.h | 54 +++++++++------ .../Process/Counter/CounterManager.cpp | 64 ++++++++++++++++-- .../Process/Counter/CounterManager.h | 14 +++- .../Process/RegionSelection/RegionSelection.h | 24 +++++-- .../RegionSelectionManager.cpp | 7 +- .../RegionSelection/RegionSelectionManager.h | 40 +++++++++++ 8 files changed, 234 insertions(+), 39 deletions(-) diff --git a/madanalysis/build/makefile_writer.py b/madanalysis/build/makefile_writer.py index f3ff61f4..8a075b54 100644 --- a/madanalysis/build/makefile_writer.py +++ b/madanalysis/build/makefile_writer.py @@ -347,7 +347,8 @@ def Makefile( # - general libs=[] - file.write('LIBFLAGS = \n') + # added SQL + file.write('LIBFLAGS = -l sqlite3\n') # - commons if options.has_commons: diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index 19d05390..fccef27e 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -38,6 +38,7 @@ #include "SampleAnalyzer/Commons/Base/Configuration.h" #include "SampleAnalyzer/Commons/Service/ExceptionService.h" #include "SampleAnalyzer/Commons/Service/Physics.h" +#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" using namespace MA5; @@ -814,7 +815,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, RegionSelection *myRS = myanalysis->Manager()->Regions()[j]; std::string safname = myanalysis->Output() + "/Cutflows/" + CleanName(myRS->GetName()) + ".saf"; - out.Initialize(&cfg_, safname.c_str()); + out.Initialize(&cfg_, safname.c_str()); out.WriteHeader(); myRS->WriteCutflow(out); out.WriteFoot(); @@ -822,6 +823,68 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, } } + + //save multi-weight cutflows to SQL - Kyle Fan + for(int i = 0; i < analyzers_.size(); ++i){ + std::string path = analyzers_[i]->Output() + "/Cutflows/cutflows.db"; + DatabaseManager dbManager(path); + dbManager.createTables(); + dbManager.addCut("initial", "event"); + bool addInitial = true; + + AnalyzerBase* myanalysis = analyzers_[i]; + //insert region,cut pair to cutflow table and region,cut,weight_id (weight data) to weights table + for(int j = 0; j < myanalysis->Manager()->Regions().size(); ++j){ + RegionSelection *myRS = myanalysis->Manager()->Regions()[j]; + std::string region_name = myRS->GetName(); + + //add initial events to db + if(addInitial){ + Counter initial = myRS->GetCutflow().GetInitial(); + for(const auto &p : initial.multiweight_){ + int id, pos_entries, neg_entries; + double pos_sum, neg_sum, pos_2sum, neg_2sum; + id = p.first; + pos_entries = p.second->nentries_.first; + neg_entries = p.second->nentries_.second; + pos_sum = p.second->sumweight_.first; + neg_sum = p.second->sumweight_.second; + pos_2sum = p.second->sumweight2_.first; + neg_2sum = p.second->sumweight2_.second; + dbManager.addWeight("initial", "event", id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); + } + addInitial = false; + } + + for(const auto &cut : myRS->GetCutflow().GetCounters()){ + std::string cut_name = cut.name_; + dbManager.addCut(region_name, cut_name); + for(const auto &p : cut.multiweight_){ + int id, pos_entries, neg_entries; + double pos_sum, neg_sum, pos_2sum, neg_2sum; + id = p.first; + pos_entries = p.second->nentries_.first; + neg_entries = p.second->nentries_.second; + pos_sum = p.second->sumweight_.first; + neg_sum = p.second->sumweight_.second; + pos_2sum = p.second->sumweight2_.first; + neg_2sum = p.second->sumweight2_.second; + dbManager.addWeight(region_name, cut.name_, id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); + } + + + } + } + + dbManager.closeDB(); + } + + //end of multi-weight cutflow code + + + + + // The user-defined stuff for(MAuint32 i=0; iFinalize(summary,mySamples); @@ -1316,4 +1379,4 @@ void SampleAnalyzer::AddDefaultInvisible() PHYSICS->mcConfig().AddInvisibleId(16); PHYSICS->mcConfig().AddInvisibleId(1000022); PHYSICS->mcConfig().AddInvisibleId(1000039); -} \ No newline at end of file +} diff --git a/tools/SampleAnalyzer/Process/Counter/Counter.h b/tools/SampleAnalyzer/Process/Counter/Counter.h index 8cfdcb84..fcd43202 100644 --- a/tools/SampleAnalyzer/Process/Counter/Counter.h +++ b/tools/SampleAnalyzer/Process/Counter/Counter.h @@ -56,6 +56,8 @@ class Counter // ------------------------------------------------------------- public : + + /// name of the analysis std::string name_; @@ -90,7 +92,7 @@ class Counter /// Destructor ~Counter() - { } + { } /// Reset void Reset() @@ -98,24 +100,10 @@ class Counter nentries_ = std::make_pair(0,0); sumweight_ = std::make_pair(0.,0.); sumweight2_ = std::make_pair(0.,0.); - } - - /// Increment the counter - void Increment(const MAfloat32& weight=1.) - { - if (weight>0) - { - nentries_.first++; - sumweight_.first+=weight; - sumweight2_.first+=weight*weight; - } - else if (weight<0) - { - nentries_.second++; - sumweight_.second+=weight; - sumweight2_.second+=weight*weight; - } - + for(auto &p : multiweight_){ + delete p.second; + } + multiweight_.clear(); } void Debug(int debugCount, int weightprocessed){ @@ -155,13 +143,35 @@ class Counter } //Debug testing -// Debug(incrementDebugCount, multiweights.begin()->second); -// incrementDebugCount++; - + // Debug(incrementDebugCount, multiweights.begin()->second); + // incrementDebugCount++; } + /// Increment the counter + void Increment(const MAfloat32& weight) + { + if (weight>0) + { + nentries_.first++; + sumweight_.first+=weight; + sumweight2_.first+=weight*weight; + } + else if (weight<0) + { + nentries_.second++; + sumweight_.second+=weight; + sumweight2_.second+=weight*weight; + } + + } + + + + + + }; } diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp b/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp index a780da8c..0a0e49a9 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp @@ -24,6 +24,8 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/CounterManager.h" +#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + using namespace MA5; @@ -77,13 +79,15 @@ void CounterManager::Write_RootFormat(TFile* output) const /// Write the counters in a TEXT file void CounterManager::Write_TextFormat(SAFWriter& output) const { + + // header *output.GetStream() << "" << std::endl; // name *output.GetStream() << "\"Initial number of events\" #" << std::endl; - // nentries + // nentries output.GetStream()->width(15); *output.GetStream() << std::left << std::scientific << initial_.nentries_.first; *output.GetStream() << " "; @@ -111,12 +115,10 @@ void CounterManager::Write_TextFormat(SAFWriter& output) const *output.GetStream() << "" << std::endl; *output.GetStream() << std::endl; - - // Loop over the counters for (MAuint32 i=0;i" << std::endl; // name @@ -154,4 +156,58 @@ void CounterManager::Write_TextFormat(SAFWriter& output) const *output.GetStream() << "" << std::endl; *output.GetStream() << std::endl; } + + + + /* + //initialize database file and add cuts/weights to DB + DatabaseManager dbManager(folder_path + "Cutflow.db"); + dbManager.createTables(); + + //push initial to database + + dbManager.addCut("initial", "events"); + for(const auto &p : initial_.multiweight_){ + + int id, pos_entries, neg_entries; + double pos_sum, neg_sum, pos_2sum, neg_2sum; + id = p.first; + pos_entries = p.second->nentries_.first; + neg_entries = p.second->nentries_.second; + pos_sum = p.second->sumweight_.first; + neg_sum = p.second->sumweight_.second; + pos_2sum = p.second->sumweight2_.first; + neg_2sum = p.second->sumweight2_.second; + + dbManager.addWeight("initial", "events", id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); + } + + //push cutflow weights to database + for(const auto &cut : counters_){ + +// std::cout << region_name << " " << cut.name_ << " " << counters_.size() << std::endl; + dbManager.addCut(region_name, cut.name_); + + for(const auto &p : cut.multiweight_){ + int id, pos_entries, neg_entries; + double pos_sum, neg_sum, pos_2sum, neg_2sum; + id = p.first; + pos_entries = p.second->nentries_.first; + neg_entries = p.second->nentries_.second; + pos_sum = p.second->sumweight_.first; + neg_sum = p.second->sumweight_.second; + pos_2sum = p.second->sumweight2_.first; + neg_2sum = p.second->sumweight2_.second; +// std::cout << "id : " << id << " pos_entry " << pos_entries << " neg_entry " << neg_entries << " pos_sum " < #include #include -#include + // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/Counter.h" @@ -90,8 +90,10 @@ class CounterManager { return counters_[index];} /// Incrementing the initial number of events - void IncrementNInitial(MAfloat32 weight=1.0) - { initial_.Increment(weight); } + void IncrementNInitial(MAfloat32 weight) + { + initial_.Increment(weight); + } /// Incrementing the initial number of events Counter& GetInitial() @@ -109,9 +111,15 @@ class CounterManager void Finalize() { Reset(); } + void IncrementNInitial(const std::map &multiweight){ initial_.Increment(multiweight); } + + + std::vector GetCounters(){ + return counters_; + } }; diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h index 805d299f..f95991e0 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h @@ -48,6 +48,7 @@ class RegionSelection std::string name_; MAbool surviving_; MAuint32 NumberOfCutsAppliedSoFar_; + MAuint32 MultiNumberOfCutsAppliedSoFar_; MAfloat64 weight_; std::map multiWeight_; @@ -67,10 +68,13 @@ class RegionSelection /// Destructor ~RegionSelection() { }; + /// Get methods std::string GetName() { return name_; } + CounterManager GetCutflow(){ return cutflow_; } + MAbool IsSurviving() { return surviving_; } @@ -82,7 +86,7 @@ class RegionSelection /// Printing the cutflow void WriteCutflow(SAFWriter& output) - { cutflow_.Write_TextFormat(output); } + { cutflow_.Write_TextFormat(output);} /// Set methods void SetName(std::string name) @@ -100,11 +104,16 @@ class RegionSelection void SetNumberOfCutsAppliedSoFar(MAuint32 NumberOfCutsAppliedSoFar) { NumberOfCutsAppliedSoFar_ = NumberOfCutsAppliedSoFar; } + void SetMultiNumberOfCutsAppliedSoFar(MAuint32 MultiNumberOfCutsAppliedSoFar){ + MultiNumberOfCutsAppliedSoFar_ = MultiNumberOfCutsAppliedSoFar; + } + // Increment CutFlow (when this region passes a cut) void IncrementCutFlow(MAfloat64 weight) { cutflow_[NumberOfCutsAppliedSoFar_].Increment(weight); NumberOfCutsAppliedSoFar_++; + } // Add a cut to the CutFlow @@ -127,18 +136,21 @@ class RegionSelection std::map GetWeights() {return multiWeight_;} + void IncrementCutFlow(const std::map &multiweight){ - cutflow_[NumberOfCutsAppliedSoFar_].Increment(multiweight); - NumberOfCutsAppliedSoFar_++; + cutflow_[MultiNumberOfCutsAppliedSoFar_].Increment(multiweight); + MultiNumberOfCutsAppliedSoFar_++; } - + + + void InitializeForNewEvent(const std::map &multiweight){ SetSurvivingTest(true); - SetNumberOfCutsAppliedSoFar(0); + SetMultiNumberOfCutsAppliedSoFar(0); cutflow_.IncrementNInitial(multiweight); multiWeight_=multiweight; } - + }; diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp index b666a4af..aabe4f63 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp @@ -24,6 +24,7 @@ // STL headers #include +#include // SampleAnalyzer headers #include "SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h" @@ -69,7 +70,11 @@ MAbool RegionSelectionManager::ApplyCut(MAbool condition, std::string const &cut if(!ThisRegion->IsSurviving() ) { continue; } /// Check the current cut: - if(condition) { ThisRegion->IncrementCutFlow(ThisRegion->GetWeight()); } + if(condition) { + ThisRegion->IncrementCutFlow(ThisRegion->GetWeight()); + //multiweight + ThisRegion->IncrementCutFlow(ThisRegion->GetWeights()); + } else { ThisRegion->SetSurvivingTest(false); diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h index a7d89279..9f7419d9 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h @@ -31,6 +31,9 @@ #include #include #include +#include + + // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/MultiRegionCounterManager.h" @@ -40,6 +43,7 @@ #include "SampleAnalyzer/Process/Writer/SAFWriter.h" #include "SampleAnalyzer/Commons/Service/ExceptionService.h" +using namespace std; namespace MA5 { @@ -66,6 +70,11 @@ class RegionSelectionManager /// Weight associated with the processed event MAfloat64 weight_; + + //multiweight + std::map multiweight_; + + // ------------------------------------------------------------- // method members // ------------------------------------------------------------- @@ -102,6 +111,10 @@ class RegionSelectionManager MAfloat64 GetCurrentEventWeight() { return weight_; } + std::map GetCurrentEventMultiWeight(){ + return multiweight_; + } + /// Set method void SetCurrentEventWeight(MAfloat64 weight) { @@ -121,6 +134,24 @@ class RegionSelectionManager } } + //set methods for multiweight + void SetCurrentEventWeight(const std::map &multiweight){ + multiweight_ = multiweight; + for(auto ®ion : regions_){ + region->SetWeight(multiweight); + } + } + + void SetRegionWeight(std::string name, std::map &multiweight){ + for(auto ®ion : regions_){ + if(region->GetName() == name) { + region->SetWeight(multiweight); + break; + } + } + } + + /// Adding a RegionSelection to the manager void AddRegionSelection(const std::string& name) { @@ -142,11 +173,13 @@ class RegionSelectionManager NumberOfSurvivingRegions_ = regions_.size(); for (MAuint32 i=0; iInitializeForNewEvent(EventWeight); + for (MAuint32 i=0; i < plotmanager_.GetNplots(); i++) plotmanager_.GetHistos()[i]->SetFreshEvent(true); } /// Initialize for multiweight + void InitializeForNewEvent(const std::map &weights){ NumberOfSurvivingRegions_ = regions_.size(); for (MAuint32 i=0; i Date: Sat, 6 Aug 2022 00:08:01 -0600 Subject: [PATCH 09/53] added database manager header file --- .../Process/Writer/DatabaseManager.h | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 tools/SampleAnalyzer/Process/Writer/DatabaseManager.h diff --git a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h new file mode 100644 index 00000000..8da1ecca --- /dev/null +++ b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h @@ -0,0 +1,151 @@ +#pragma once + + +#include +#include +#include + +using namespace std; + +class DatabaseManager { + + private: + // Pointer to SQLite connection + sqlite3 *db; + + // Save any error messages + char *zErrMsg; + + // Save the result of opening the file + int rc; + + // Saved SQL + string sql; + + // Create a callback function + static int callback(void *NotUsed, int argc, char **argv, char **azColName) { + + // int argc: holds the number of results + // (array) azColName: holds each column returned + // (array) argv: holds each value + + for(int i = 0; i < argc; i++) { + + // Show column name, value, and newline + cout << azColName[i] << ": " << argv[i] << endl; + + } + + // Insert a newline + cout << endl; + + // Return successful + return 0; + } + + bool checkDBErrors(string msg) { + + if( rc ){ + // Show an error message + cout << "DB Error: " << sqlite3_errmsg(db) << " " << msg << endl; + return false; + + } + else { + cout << " success @ " << msg << endl; + return true; + } + + } + + + public: + + DatabaseManager(string path) { + // Save the result of opening the file + rc = sqlite3_open(path.c_str(), &db); + // enable foreign key constraint + sqlite3_exec(db, "PRAGMA foreign_keys = ON;", 0 ,0 ,0); + bool open_success = checkDBErrors("opening DB"); + if(!open_success) { + sqlite3_close(db); + cout << "open DB failed!" << endl; + } + } + + void createTables() { + + // Save SQL to create a table + sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \ + "region_name TEXT NOT NULL,"\ + "cut_name TEXT NOT NULL," \ + "primary key (region_name, cut_name));"; + + // Run the SQL + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating cutflow table"); + + sql = "CREATE TABLE IF NOT EXISTS Weights(" \ + "r_name TEXT NOT NULL," \ + "c_name TEXT NOT NULL," \ + "id INTEGER NOT NULL," \ + "pos_entries INTEGER NOT NULL," \ + "neg_entries INTEGER NOT NULL," \ + "pos_sum DOUBLE NOT NULL," \ + "neg_sum DOUBLE NOT NULL," \ + "pos_squared_sum DOUBLE NOT NULL,"\ + "neg_squared_sum DOUBLE NOT NULL,"\ + "primary key (r_name, c_name, id)" \ + "foreign key (r_name, c_name) references Cutflow(region_name, cut_name) ON DELETE CASCADE);"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating weights table"); + } + + //add cut to databse with primary keys region name and cut name + void addCut(string r_name, string c_name) { + + + sql = "INSERT INTO Cutflow VALUES ('" + r_name + "'" + "," + "'" + c_name + "')"; + //cout << sql << endl; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + checkDBErrors("inserting cutflow: " + r_name + " " + c_name); + + } + + //add weight to database with primary keys region name, cut name, and weight id + void addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { + sql = "INSERT INTO Weights VALUES ('" + r_name + "'" + ",'" + c_name + "','" + to_string(id) + "','" + to_string(pos) + "','" + to_string(neg) + "','" + to_string(pos_sum) + "','" + to_string(neg_sum) \ + + "','" + to_string(pos_2sum) + "','" + to_string(neg_2sum) + "')"; + //cout << sql << endl; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id)); + + } + + void closeDB() { + + // Close the SQL connection + sqlite3_close(db); + + } + + + +}; + +/* +int main() { + DatabaseManager dbManager("test.db"); + dbManager.createTables(); + dbManager.addCut("region1", "cut1"); + dbManager.addWeight("region1", "cut1", 1, 2, 2, 2, -2, 4, 4); + dbManager.addWeight("region1", "cut1", 10, 2, 3, 4, 5, 6, 7); + dbManager.closeDB(); + +} + +*/ + + + From 01cb86af9757f0d85a8d5dac51638cdb9f5907f5 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Sun, 7 Aug 2022 22:36:49 -0600 Subject: [PATCH 10/53] delegated WriteSQL to CounterManager from SampleAnalyzer::Finalize --- .../Process/Core/SampleAnalyzer.cpp | 46 +--------- .../Process/Core/xdr_istream.cpp | 4 +- .../SampleAnalyzer/Process/Counter/Counter.h | 16 +++- .../Process/Counter/CounterManager.cpp | 84 ++++++++----------- .../Process/Counter/CounterManager.h | 6 +- .../Process/RegionSelection/RegionSelection.h | 6 +- 6 files changed, 62 insertions(+), 100 deletions(-) diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index fccef27e..27a58c6c 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -828,63 +828,21 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, for(int i = 0; i < analyzers_.size(); ++i){ std::string path = analyzers_[i]->Output() + "/Cutflows/cutflows.db"; DatabaseManager dbManager(path); - dbManager.createTables(); - dbManager.addCut("initial", "event"); + dbManager.createTables(); bool addInitial = true; AnalyzerBase* myanalysis = analyzers_[i]; //insert region,cut pair to cutflow table and region,cut,weight_id (weight data) to weights table for(int j = 0; j < myanalysis->Manager()->Regions().size(); ++j){ RegionSelection *myRS = myanalysis->Manager()->Regions()[j]; - std::string region_name = myRS->GetName(); - - //add initial events to db - if(addInitial){ - Counter initial = myRS->GetCutflow().GetInitial(); - for(const auto &p : initial.multiweight_){ - int id, pos_entries, neg_entries; - double pos_sum, neg_sum, pos_2sum, neg_2sum; - id = p.first; - pos_entries = p.second->nentries_.first; - neg_entries = p.second->nentries_.second; - pos_sum = p.second->sumweight_.first; - neg_sum = p.second->sumweight_.second; - pos_2sum = p.second->sumweight2_.first; - neg_2sum = p.second->sumweight2_.second; - dbManager.addWeight("initial", "event", id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); - } - addInitial = false; - } - - for(const auto &cut : myRS->GetCutflow().GetCounters()){ - std::string cut_name = cut.name_; - dbManager.addCut(region_name, cut_name); - for(const auto &p : cut.multiweight_){ - int id, pos_entries, neg_entries; - double pos_sum, neg_sum, pos_2sum, neg_2sum; - id = p.first; - pos_entries = p.second->nentries_.first; - neg_entries = p.second->nentries_.second; - pos_sum = p.second->sumweight_.first; - neg_sum = p.second->sumweight_.second; - pos_2sum = p.second->sumweight2_.first; - neg_2sum = p.second->sumweight2_.second; - dbManager.addWeight(region_name, cut.name_, id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); - } - - - } + myRS->WriteSQL(dbManager, addInitial); } - dbManager.closeDB(); } //end of multi-weight cutflow code - - - // The user-defined stuff for(MAuint32 i=0; iFinalize(summary,mySamples); diff --git a/tools/SampleAnalyzer/Process/Core/xdr_istream.cpp b/tools/SampleAnalyzer/Process/Core/xdr_istream.cpp index cd3bc81c..3b65f1a6 100644 --- a/tools/SampleAnalyzer/Process/Core/xdr_istream.cpp +++ b/tools/SampleAnalyzer/Process/Core/xdr_istream.cpp @@ -50,8 +50,8 @@ xdr_istream& xdr_istream::operator>>(std::string &s) MAchar* dummy = new MAchar[pad]; sb_->sgetn(dummy,pad); - delete line; - delete dummy; + delete [] line; + delete [] dummy; return *this; } diff --git a/tools/SampleAnalyzer/Process/Counter/Counter.h b/tools/SampleAnalyzer/Process/Counter/Counter.h index fcd43202..d028c78f 100644 --- a/tools/SampleAnalyzer/Process/Counter/Counter.h +++ b/tools/SampleAnalyzer/Process/Counter/Counter.h @@ -39,6 +39,14 @@ struct multiWeightEntry { std::pair nentries_; std::pair sumweight_; std::pair sumweight2_; + multiWeightEntry(){ + nentries_.first = 0; + nentries_.second = 0; + sumweight_.first = 0.; + sumweight_.second = 0.; + sumweight2_.first = 0.; + sumweight2_.second = 0.; + } }; @@ -82,7 +90,7 @@ class Counter public : /// Constructor without argument - Counter(const std::string& name = "unkwown") + Counter(const std::string& name = "unknown") { name_ = name; nentries_ = std::make_pair(0,0); @@ -92,7 +100,9 @@ class Counter /// Destructor ~Counter() - { } + { + + } /// Reset void Reset() @@ -128,7 +138,7 @@ class Counter // static int incrementDebugCount = 0; for(const auto &weight : multiweights){ if(multiweight_.find(weight.first) == multiweight_.end()){ - multiweight_[weight.first] = new multiWeightEntry; + multiweight_[weight.first] = new multiWeightEntry(); } if(weight.second > 0){ multiweight_[weight.first]->nentries_.first++; diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp b/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp index 0a0e49a9..4149ee68 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp @@ -24,7 +24,7 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/CounterManager.h" -#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + @@ -158,56 +158,46 @@ void CounterManager::Write_TextFormat(SAFWriter& output) const } - - /* - //initialize database file and add cuts/weights to DB - DatabaseManager dbManager(folder_path + "Cutflow.db"); - dbManager.createTables(); - - //push initial to database - - dbManager.addCut("initial", "events"); - for(const auto &p : initial_.multiweight_){ - - int id, pos_entries, neg_entries; - double pos_sum, neg_sum, pos_2sum, neg_2sum; - id = p.first; - pos_entries = p.second->nentries_.first; - neg_entries = p.second->nentries_.second; - pos_sum = p.second->sumweight_.first; - neg_sum = p.second->sumweight_.second; - pos_2sum = p.second->sumweight2_.first; - neg_2sum = p.second->sumweight2_.second; - - dbManager.addWeight("initial", "events", id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); - } +} - //push cutflow weights to database - for(const auto &cut : counters_){ - -// std::cout << region_name << " " << cut.name_ << " " << counters_.size() << std::endl; - dbManager.addCut(region_name, cut.name_); - - for(const auto &p : cut.multiweight_){ - int id, pos_entries, neg_entries; - double pos_sum, neg_sum, pos_2sum, neg_2sum; - id = p.first; - pos_entries = p.second->nentries_.first; - neg_entries = p.second->nentries_.second; - pos_sum = p.second->sumweight_.first; - neg_sum = p.second->sumweight_.second; - pos_2sum = p.second->sumweight2_.first; - neg_2sum = p.second->sumweight2_.second; -// std::cout << "id : " << id << " pos_entry " << pos_entries << " neg_entry " << neg_entries << " pos_sum " <nentries_.first; + neg_entries = p.second->nentries_.second; + pos_sum = p.second->sumweight_.first; + neg_sum = p.second->sumweight_.second; + pos_2sum = p.second->sumweight2_.first; + neg_2sum = p.second->sumweight2_.second; + db.addWeight("initial", "event", id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); + } + AddInitial = false; + } + for(const auto &cut : counters_){ + std::string cut_name = cut.name_; + db.addCut(region_name, cut_name); + for(const auto &p : cut.multiweight_){ + int id, pos_entries, neg_entries; + double pos_sum, neg_sum, pos_2sum, neg_2sum; + id = p.first; + pos_entries = p.second->nentries_.first; + neg_entries = p.second->nentries_.second; + pos_sum = p.second->sumweight_.first; + neg_sum = p.second->sumweight_.second; + pos_2sum = p.second->sumweight2_.first; + neg_2sum = p.second->sumweight2_.second; + db.addWeight(region_name, cut_name, id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); + } - dbManager.addWeight(region_name, cut.name_, id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); - } - dbManager.closeDB(); + } + - } - */ - } + diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.h b/tools/SampleAnalyzer/Process/Counter/CounterManager.h index 8b80c71a..02b45565 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.h +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.h @@ -35,6 +35,7 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/Counter.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" +#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" namespace MA5 @@ -104,6 +105,8 @@ class CounterManager /// Write the counters in a Text file void Write_TextFormat(SAFWriter& output) const; + void WriteSQL(DatabaseManager &db, bool &AddInitial, std::string region_name); + /// Write the counters in a ROOT file // void Write_RootFormat(TFile* output) const; @@ -117,9 +120,6 @@ class CounterManager } - std::vector GetCounters(){ - return counters_; - } }; diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h index f95991e0..2ae83948 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h @@ -34,6 +34,7 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/CounterManager.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" +#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" namespace MA5 @@ -73,7 +74,6 @@ class RegionSelection std::string GetName() { return name_; } - CounterManager GetCutflow(){ return cutflow_; } MAbool IsSurviving() { return surviving_; } @@ -88,6 +88,10 @@ class RegionSelection void WriteCutflow(SAFWriter& output) { cutflow_.Write_TextFormat(output);} + void WriteSQL(DatabaseManager &db, bool &AddInitial){ + cutflow_.WriteSQL(db, AddInitial, name_); + } + /// Set methods void SetName(std::string name) { name_ = name; } From fad5125796fde05c83342abedcd4f26035ba52fb Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Mon, 22 Aug 2022 23:58:56 -0600 Subject: [PATCH 11/53] added operators to weight collections --- .../Commons/DataFormat/WeightCollection.h | 48 +++++++++++++++++++ .../SampleAnalyzer/Process/Counter/Counter.h | 4 +- .../Process/Counter/CounterManager.cpp | 11 +++-- .../Process/Counter/CounterManager.h | 2 + .../Process/RegionSelection/RegionSelection.h | 1 + .../RegionSelection/RegionSelectionManager.h | 7 +-- 6 files changed, 66 insertions(+), 7 deletions(-) diff --git a/tools/SampleAnalyzer/Commons/DataFormat/WeightCollection.h b/tools/SampleAnalyzer/Commons/DataFormat/WeightCollection.h index 55a51c63..8b3cc431 100644 --- a/tools/SampleAnalyzer/Commons/DataFormat/WeightCollection.h +++ b/tools/SampleAnalyzer/Commons/DataFormat/WeightCollection.h @@ -66,6 +66,13 @@ class WeightCollection ~WeightCollection() { } + //copy constructor + WeightCollection(const WeightCollection &rhs){ + for(const auto &id_weights :rhs.weights_){ + weights_[id_weights.first]=id_weights.second; + } + } + /// Clear all the content void Reset() { weights_.clear(); } @@ -162,6 +169,47 @@ class WeightCollection } } + //multiply operator + WeightCollection& operator*=(const MAfloat64 multiple){ + for(auto &id_value : weights_){ + id_value.second *= multiple; + } + return *this; + } + + //add operator + WeightCollection& operator+=(const MAfloat64 input){ + for(auto &id_value : weights_){ + id_value.second += input; + } + return *this; + } + + //subtract operator + WeightCollection& operator-=(const MAfloat64 input){ + for(auto &id_value : weights_){ + id_value.second -= input; + } + return *this; + } + + //divide operator + WeightCollection& operator/=(const MAfloat64 input){ + for(auto &id_value : weights_){ + id_value.second /= input; + } + return *this; + } + + + //assignment operator + WeightCollection& operator=(const MAfloat64 input){ + for(auto &id_value : weights_){ + id_value.second = input; + } + return *this; + } + }; diff --git a/tools/SampleAnalyzer/Process/Counter/Counter.h b/tools/SampleAnalyzer/Process/Counter/Counter.h index d028c78f..7838885e 100644 --- a/tools/SampleAnalyzer/Process/Counter/Counter.h +++ b/tools/SampleAnalyzer/Process/Counter/Counter.h @@ -101,7 +101,9 @@ class Counter /// Destructor ~Counter() { - + for(auto &p : multiweight_){ + delete p.second; + } } /// Reset diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp b/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp index 4149ee68..a295cbab 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp @@ -160,8 +160,13 @@ void CounterManager::Write_TextFormat(SAFWriter& output) const } + +//pass in database handler object, flag to check if initial events has been added, no need to repeatedly insert +//initial since each region has an identical copy, pass in region name void CounterManager::WriteSQL(DatabaseManager &db, bool &AddInitial, std::string region_name){ if(AddInitial){ + //if add initial is true, insert the initial event cut, and for each weight id, insert the values into + //the database, set flag to false afterwards db.addCut("initial","event"); for(const auto &p : initial_.multiweight_){ int id, pos_entries, neg_entries; @@ -178,6 +183,8 @@ void CounterManager::WriteSQL(DatabaseManager &db, bool &AddInitial, std::string AddInitial = false; } + //for each cut in the region, add region and cut names to cutflow table + //for each weight id in each cut, insert the values into the database for(const auto &cut : counters_){ std::string cut_name = cut.name_; db.addCut(region_name, cut_name); @@ -192,12 +199,10 @@ void CounterManager::WriteSQL(DatabaseManager &db, bool &AddInitial, std::string pos_2sum = p.second->sumweight2_.first; neg_2sum = p.second->sumweight2_.second; db.addWeight(region_name, cut_name, id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); - } - - } + } } diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.h b/tools/SampleAnalyzer/Process/Counter/CounterManager.h index 02b45565..6e2acacd 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.h +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.h @@ -105,6 +105,8 @@ class CounterManager /// Write the counters in a Text file void Write_TextFormat(SAFWriter& output) const; + + //Write to SQL database void WriteSQL(DatabaseManager &db, bool &AddInitial, std::string region_name); /// Write the counters in a ROOT file diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h index 2ae83948..ce96ccdc 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h @@ -88,6 +88,7 @@ class RegionSelection void WriteCutflow(SAFWriter& output) { cutflow_.Write_TextFormat(output);} + //write to SQL database void WriteSQL(DatabaseManager &db, bool &AddInitial){ cutflow_.WriteSQL(db, AddInitial, name_); } diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h index 5b2c47cf..d248daf2 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h @@ -42,6 +42,7 @@ #include "SampleAnalyzer/Commons/Service/LogService.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" #include "SampleAnalyzer/Commons/Service/ExceptionService.h" +#include "SampleAnalyzer/Commons/DataFormat/WeightCollection.h" using namespace std; @@ -139,10 +140,10 @@ class RegionSelectionManager } //set methods for multiweight - void SetCurrentEventWeight(const std::map &multiweight){ - multiweight_ = multiweight; + void SetCurrentEventWeight(const WeightCollection &multiweight){ + multiweight_ = multiweight.GetWeights(); for(auto ®ion : regions_){ - region->SetWeight(multiweight); + region->SetWeight(multiweight_); } } From f39ceb6762d2b5bc6149a207a44ea2b1407b8ae5 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Wed, 24 Aug 2022 02:31:20 -0600 Subject: [PATCH 12/53] added histogramming to multiweight integration with SQLite3 output --- .../Process/Core/SampleAnalyzer.cpp | 11 ++- tools/SampleAnalyzer/Process/Plot/Histo.cpp | 34 ++++++++ tools/SampleAnalyzer/Process/Plot/Histo.h | 74 ++++++++++++++++ tools/SampleAnalyzer/Process/Plot/PlotBase.h | 70 ++++++++++++++- .../SampleAnalyzer/Process/Plot/PlotManager.h | 9 +- .../RegionSelectionManager.cpp | 8 +- .../RegionSelection/RegionSelectionManager.h | 5 +- .../Process/Writer/DatabaseManager.h | 87 ++++++++++++++++++- 8 files changed, 289 insertions(+), 9 deletions(-) diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index 27a58c6c..221f1848 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -805,6 +805,15 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, out.Finalize(); } + // Create histo SQlite file + for(MAuint32 i = 0; i < analyzers_.size(); ++i){ + std::string path = analyzers_[i]->Output() + "/Histograms/histo.db"; + DatabaseManager dbManager(path); + dbManager.createHistoTables(); + analyzers_[i]->Manager()->GetPlotManager()->WriteSQL(dbManager); + dbManager.closeDB(); + } + // Saving the cut flows for(MAuint32 i=0; i& mySamples, for(int i = 0; i < analyzers_.size(); ++i){ std::string path = analyzers_[i]->Output() + "/Cutflows/cutflows.db"; DatabaseManager dbManager(path); - dbManager.createTables(); + dbManager.createCutflowTables(); bool addInitial = true; AnalyzerBase* myanalysis = analyzers_[i]; diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.cpp b/tools/SampleAnalyzer/Process/Plot/Histo.cpp index c02d778e..abe6b1f5 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.cpp +++ b/tools/SampleAnalyzer/Process/Plot/Histo.cpp @@ -42,6 +42,40 @@ void Histo::Write_TextFormat(std::ostream* output) *output << std::endl; } +void Histo::WriteSQL(DatabaseManager &db){ + std::string regionNames = ""; + for(MAuint32 i = 0; i < regions_.size(); ++i){ + if(i != 0) {regionNames += " ";} + regionNames += regions_[i]->GetName(); + } + db.addHisto(name_, nbins_, xmin_, xmax_, regionNames); + db.addStatistic(name_, + nevents_.first, + nevents_.second, + nevents_w_.first, + nevents_w_.second, + nentries_.first, + nentries_.second, + sum_w_.first, + sum_w_.second, + sum_ww_.first, + sum_ww_.second, + sum_xw_.first, + sum_xw_.second, + sum_xxw_.first, + sum_xxw_.second); + std::cout << MultiweightHistoData.size() << std::endl; + for(const auto &weight_id : MultiweightHistoData){ + db.addData(name_, weight_id.first, "underflow", weight_id.second->underflow_.first, weight_id.second->underflow_.second); + db.addData(name_, weight_id.first, "overflow", weight_id.second->overflow_.first, weight_id.second->overflow_.second); + for(int i = 0; i < nbins_; ++i){ + db.addData(name_, weight_id.first, "bin " + to_string(i+1), weight_id.second->histo_[i].first, weight_id.second->histo_[i].second); + } + } + + +} + /// Write the plot in a Text file void Histo::Write_TextFormatBody(std::ostream* output) diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.h b/tools/SampleAnalyzer/Process/Plot/Histo.h index 6945d25b..3fec2fff 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.h +++ b/tools/SampleAnalyzer/Process/Plot/Histo.h @@ -35,6 +35,26 @@ #include "SampleAnalyzer/Process/Plot/PlotBase.h" #include "SampleAnalyzer/Process/RegionSelection/RegionSelection.h" #include "SampleAnalyzer/Commons/Service/ExceptionService.h" +#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + + +struct MultiWeightHisto { + std::vector< std::pair > histo_; + std::pair underflow_; + std::pair overflow_; + + /// Sum of event-weights over entries + std::pair sum_w_; + + /// Sum of squared weights + std::pair sum_ww_; + + /// Sum of value * weight + std::pair sum_xw_; + + /// Sum of value * value * weight + std::pair sum_xxw_; +}; namespace MA5 @@ -74,6 +94,8 @@ class Histo : public PlotBase /// RegionSelections attached to the histo std::vector regions_; + std::map MultiweightHistoData; + // ------------------------------------------------------------- // method members // ------------------------------------------------------------- @@ -204,8 +226,60 @@ class Histo : public PlotBase } } + void Fill(MAfloat64 value, std::map &weights){ + // Safety : nan or isinf + try + { + if (std::isnan(value)) throw EXCEPTION_WARNING("Skipping a NaN (Not a Number) value in an histogram.","",0); + if (std::isinf(value)) throw EXCEPTION_WARNING("Skipping a Infinity value in an histogram.","",0); + } + catch (const std::exception& e) + { + MANAGE_EXCEPTION(e); + } + + for(const auto &id_weight : weights){ + if(MultiweightHistoData.find(id_weight.first) == MultiweightHistoData.end()){ + MultiweightHistoData[id_weight.first] = new MultiWeightHisto(); + MultiweightHistoData[id_weight.first]->histo_.resize(nbins_); + } + //Positive weight + if(id_weight.second >= 0){ + multiweight_event_info[id_weight.first]->nentries_.first++; + MultiweightHistoData[id_weight.first]->sum_w_.first +=id_weight.second; + MultiweightHistoData[id_weight.first]->sum_ww_.first +=id_weight.second * id_weight.second; + MultiweightHistoData[id_weight.first]->sum_xw_.first +=value * id_weight.second; + MultiweightHistoData[id_weight.first]->sum_xxw_.first +=value*value*id_weight.second; + if (value < xmin_) MultiweightHistoData[id_weight.first]->underflow_.first+=id_weight.second; + else if (value >= xmax_) MultiweightHistoData[id_weight.first]->overflow_.first+=id_weight.second; + else + { + MultiweightHistoData[id_weight.first]->histo_[std::floor((value-xmin_)/step_)].first+=id_weight.second; + } + } else { + int absWeight = std::abs(id_weight.second); + multiweight_event_info[id_weight.first]->nentries_.second++; + MultiweightHistoData[id_weight.first]->sum_w_.second += absWeight; + MultiweightHistoData[id_weight.first]->sum_ww_.second += absWeight * absWeight; + MultiweightHistoData[id_weight.first]->sum_xw_.second +=value * absWeight; + MultiweightHistoData[id_weight.first]->sum_xxw_.second +=value*value*absWeight; + if (value < xmin_) MultiweightHistoData[id_weight.first]->underflow_.second+=absWeight; + else if (value >= xmax_) MultiweightHistoData[id_weight.first]->overflow_.second+=absWeight; + else + { + MultiweightHistoData[id_weight.first]->histo_[std::floor((value-xmin_)/step_)].second+=absWeight; + } + + } + } + + + + } + /// Write the plot in a ROOT file virtual void Write_TextFormat(std::ostream* output); + virtual void WriteSQL(DatabaseManager &db); /// Write the plot in a ROOT file // virtual void Write_RootFormat(std::pair& histos); diff --git a/tools/SampleAnalyzer/Process/Plot/PlotBase.h b/tools/SampleAnalyzer/Process/Plot/PlotBase.h index cd883247..22ad692c 100644 --- a/tools/SampleAnalyzer/Process/Plot/PlotBase.h +++ b/tools/SampleAnalyzer/Process/Plot/PlotBase.h @@ -34,6 +34,22 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Commons/Service/LogService.h" +#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + + +struct MultiweightEvents { + /// Number of events + std::pair nevents_; + + /// Number of entries + std::pair nentries_; + + /// Sum of event-weight over events + std::pair nevents_w_; + + MAbool fresh_event_; + +}; namespace MA5 @@ -62,6 +78,9 @@ class PlotBase /// Flag telling whether a given histo has already been modified for an event MAbool fresh_event_; + //map containing event meta data for each weight id + std::map multiweight_event_info; + // ------------------------------------------------------------- // method members @@ -76,6 +95,7 @@ class PlotBase nentries_ = std::make_pair(0,0); nevents_w_ = std::make_pair(0,0); fresh_event_ = true; + } /// Constructor with argument @@ -86,21 +106,42 @@ class PlotBase nevents_w_ = std::make_pair(0,0); nentries_ = std::make_pair(0,0); fresh_event_ = true; + } /// Destructor virtual ~PlotBase() - { } + { + for(auto &p : multiweight_event_info){ + delete p.second; + } + } /// Accesor for fresh_event MAbool FreshEvent() { return fresh_event_;} + /// Accessor for multiweight fresh_event + MAbool MultiweightFreshEvent(int index) { + return multiweight_event_info[index]->fresh_event_; + } + /// Modifier for fresh_event - void SetFreshEvent(MAbool tag) { fresh_event_ = tag;} + void SetFreshEvent(MAbool tag) { + fresh_event_ = tag; + } + + /// Modifier for Multiweight fresh_event + void SetMultiweightFreshEvent(MAbool tag) { + for(auto &weight_id : multiweight_event_info){ + weight_id.second->fresh_event_ = tag; + } + } /// Write the plot in a ROOT file virtual void Write_TextFormat(std::ostream* output) = 0; + virtual void WriteSQL(DatabaseManager &db) {}; + /// Increment number of events void IncrementNEvents(MAfloat64 weight=1.0) { @@ -118,10 +159,35 @@ class PlotBase SetFreshEvent(false); } + void IncrementNEvents(std::map &weights){ + for(const auto &id_weight : weights){ + if(multiweight_event_info.find(id_weight.first) == multiweight_event_info.end()){ + multiweight_event_info[id_weight.first] = new MultiweightEvents(); + multiweight_event_info[id_weight.first]->fresh_event_ = true; + } + + if(MultiweightFreshEvent(id_weight.first)){ + if(id_weight.second >= 0){ + multiweight_event_info[id_weight.first]->nevents_.first++; + multiweight_event_info[id_weight.first]->nevents_w_.first+=id_weight.second; + } else { + multiweight_event_info[id_weight.first]->nevents_.second++; + multiweight_event_info[id_weight.first]->nevents_w_.second+=std::abs(id_weight.second); + } + + } + multiweight_event_info[id_weight.first]->fresh_event_ = false; + } + } + /// Return Number of events const std::pair& GetNEvents() { return nevents_; } + const std::pair& GetNEvents(int index){ + return multiweight_event_info[index]->nevents_; + } + // Return the name std::string GetName() { return name_; } diff --git a/tools/SampleAnalyzer/Process/Plot/PlotManager.h b/tools/SampleAnalyzer/Process/Plot/PlotManager.h index 737c7030..94307884 100644 --- a/tools/SampleAnalyzer/Process/Plot/PlotManager.h +++ b/tools/SampleAnalyzer/Process/Plot/PlotManager.h @@ -38,7 +38,7 @@ #include "SampleAnalyzer/Process/Plot/HistoFrequency.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" #include "SampleAnalyzer/Process/RegionSelection/RegionSelection.h" - +#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" namespace MA5 { @@ -139,6 +139,13 @@ class PlotManager /// Write the counters in a Text file void Write_TextFormat(SAFWriter& output); + void WriteSQL(DatabaseManager &db){ + for(auto &pl : plots_){ + pl->WriteSQL(db); + } + } + + /// Finalizing void Finalize() { Reset(); } diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp index aabe4f63..0ca6e193 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp @@ -142,8 +142,14 @@ void RegionSelectionManager::FillHisto(std::string const&histname, MAfloat64 val } catch (const std::exception& e) { MANAGE_EXCEPTION(e); } // Filling the histo - if (myhisto->FreshEvent()) myhisto->IncrementNEvents(weight_); + if (myhisto->FreshEvent()) { + myhisto->IncrementNEvents(weight_); + } + + //std::cout << "multiweight size in regionManager is : " << multiweight_.size() << std::endl; myhisto->Fill(val,weight_); + myhisto->IncrementNEvents(multiweight_); + myhisto->Fill(val, multiweight_); } break; } diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h index d248daf2..176d08e1 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h @@ -186,12 +186,15 @@ class RegionSelectionManager /// Initialize for multiweight void InitializeForNewEvent(const std::map &weights){ + multiweight_ = weights; NumberOfSurvivingRegions_ = regions_.size(); for (MAuint32 i=0; iInitializeForNewEvent(weights); } - + for(MAuint32 i = 0; i < plotmanager_.GetNplots(); ++i){ + plotmanager_.GetHistos()[i]->SetMultiweightFreshEvent(true); + } } diff --git a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h index 8da1ecca..311b3862 100644 --- a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h +++ b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h @@ -73,7 +73,7 @@ class DatabaseManager { } } - void createTables() { + void createCutflowTables() { // Save SQL to create a table sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \ @@ -101,6 +101,87 @@ class DatabaseManager { rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); checkDBErrors("creating weights table"); } + + void createHistoTables(){ + sql = "CREATE TABLE IF NOT EXISTS HistoDescription("\ + "name TEXT NOT NULL,"\ + "num_of_bins INTEGER NOT NULL,"\ + "xmin DOUBLE NOT NULL,"\ + "xmas DOUBLE NOT NULL,"\ + "regions TEXT NOT NULL,"\ + "primary key(name) );"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating HistoDescription table"); + + sql = "CREATE TABLE IF NOT EXISTS Statistics("\ + "name TEXT NOT NULL,"\ + "pos_num_events INTEGER NOT NULL,"\ + "neg_num_events INTEGER NOT NULL,"\ + "pos_sum_event_weights_over_events DOUBLE NOT NULL,"\ + "neg_sum_event_weights_over_events DOUBLE NOT NULL,"\ + "pos_entries INTEGER NOT NULL,"\ + "neg_entries INTEGER NOT NULL,"\ + "pos_sum_event_weights_over_entries DOUBLE NOT NULL,"\ + "neg_sum_event_weights_over_entries DOUBLE NOT NULL,"\ + "pos_sum_squared_weights DOUBLE NOT NULL,"\ + "neg_sum_squared_weights DOUBLE NOT NULL,"\ + "pos_value_times_weight DOUBLE NOT NULL,"\ + "neg_value_times_weight DOUBLE NOT NULL,"\ + "pos_value_squared_times_weight DOUBLE NOT NULL,"\ + "neg_value_squared_times_weight DOUBLE NOT NULL,"\ + "primary key(name) );"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Statistics table"); + + sql = "CREATE TABLE IF NOT EXISTS Data("\ + "name TEXT NOT NULL,"\ + "id INTERGER NOT NULL,"\ + "bin TEXT NOT NULL,"\ + "positive DOUBLE NOT NULL,"\ + "negative DOUBLE NOT NULL,"\ + "primary key (name, id, bin) );"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Data table"); + + } + + void addHisto(string name, int bins, double xmin, double xmax, string regions){ + sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + checkDBErrors("inserting into Histo: " + name); + } + + void addStatistic(string name, + int pos_events, + int neg_events, + double pos_sum_events, + double neg_sum_events, + int pos_entries, + int neg_entries, + double pos_sum_entries, + double neg_sum_entries, + double pos_sum_squared, + double neg_sum_squared, + double pos_val_weight, + double neg_val_weight, + double pos_val2_weight, + double neg_val2_weight){ + + sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + checkDBErrors("inserting into Statistics: " + name); + + } + + void addData(string name, int id, string bin, double positive, double negative){ + sql = "INSERT INTO Data VALUES ('" + name + "'" + "," + "'" + to_string(id) + "'" + "," + "'" + bin + "'" + "," + "'" + to_string(positive) + "'" + "," + "'" + to_string(negative) + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + checkDBErrors("inserting into Data: " + name + " " + to_string(id)); + + } //add cut to databse with primary keys region name and cut name void addCut(string r_name, string c_name) { @@ -109,7 +190,7 @@ class DatabaseManager { sql = "INSERT INTO Cutflow VALUES ('" + r_name + "'" + "," + "'" + c_name + "')"; //cout << sql << endl; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - checkDBErrors("inserting cutflow: " + r_name + " " + c_name); + //checkDBErrors("inserting cutflow: " + r_name + " " + c_name); } @@ -119,7 +200,7 @@ class DatabaseManager { + "','" + to_string(pos_2sum) + "','" + to_string(neg_2sum) + "')"; //cout << sql << endl; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id)); + //checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id)); } From 48eac29081cef2b4933b11d54ed76ae1db5b1458 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Wed, 24 Aug 2022 10:50:03 -0600 Subject: [PATCH 13/53] fixed Histogramming Statistics table/Histo WriteSQL to write unique statistics for each weight id --- tools/SampleAnalyzer/Process/Plot/Histo.cpp | 41 +++++++++++-------- tools/SampleAnalyzer/Process/Plot/Histo.h | 6 ++- .../RegionSelectionManager.cpp | 3 +- .../Process/Writer/DatabaseManager.h | 8 ++-- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.cpp b/tools/SampleAnalyzer/Process/Plot/Histo.cpp index abe6b1f5..e4a3ab34 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.cpp +++ b/tools/SampleAnalyzer/Process/Plot/Histo.cpp @@ -42,35 +42,42 @@ void Histo::Write_TextFormat(std::ostream* output) *output << std::endl; } +//db class passed in from SampleAnalyzer execute function, adds histo, statistics and data for each weight id to +//SQlite3 output file void Histo::WriteSQL(DatabaseManager &db){ + + //create string of regions associated with the Histo std::string regionNames = ""; for(MAuint32 i = 0; i < regions_.size(); ++i){ if(i != 0) {regionNames += " ";} regionNames += regions_[i]->GetName(); } + //add general histo info to the Histo Description table db.addHisto(name_, nbins_, xmin_, xmax_, regionNames); - db.addStatistic(name_, - nevents_.first, - nevents_.second, - nevents_w_.first, - nevents_w_.second, - nentries_.first, - nentries_.second, - sum_w_.first, - sum_w_.second, - sum_ww_.first, - sum_ww_.second, - sum_xw_.first, - sum_xw_.second, - sum_xxw_.first, - sum_xxw_.second); - std::cout << MultiweightHistoData.size() << std::endl; + //for each histo, weight id pair: add statistics info to Statistics table for(const auto &weight_id : MultiweightHistoData){ + db.addStatistic(name_, + weight_id.first, + multiweight_event_info[weight_id.first]->nevents_.first, + multiweight_event_info[weight_id.first]->nevents_.second, + multiweight_event_info[weight_id.first]->nevents_w_.first, + multiweight_event_info[weight_id.first]->nevents_w_.second, + multiweight_event_info[weight_id.first]->nentries_.first, + multiweight_event_info[weight_id.first]->nentries_.second, + weight_id.second->sum_w_.first, + weight_id.second->sum_w_.second, + weight_id.second->sum_ww_.first, + weight_id.second->sum_ww_.second, + weight_id.second->sum_xw_.first, + weight_id.second->sum_xw_.second, + weight_id.second->sum_xxw_.first, + weight_id.second->sum_xxw_.second); + //for each weight histo,weight id pair: add bucket data to Data table db.addData(name_, weight_id.first, "underflow", weight_id.second->underflow_.first, weight_id.second->underflow_.second); - db.addData(name_, weight_id.first, "overflow", weight_id.second->overflow_.first, weight_id.second->overflow_.second); for(int i = 0; i < nbins_; ++i){ db.addData(name_, weight_id.first, "bin " + to_string(i+1), weight_id.second->histo_[i].first, weight_id.second->histo_[i].second); } + db.addData(name_, weight_id.first, "overflow", weight_id.second->overflow_.first, weight_id.second->overflow_.second); } diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.h b/tools/SampleAnalyzer/Process/Plot/Histo.h index 3fec2fff..075e0269 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.h +++ b/tools/SampleAnalyzer/Process/Plot/Histo.h @@ -158,7 +158,11 @@ class Histo : public PlotBase /// Destructor virtual ~Histo() - { } + { + for(auto &p : MultiweightHistoData){ + delete p.second; + } + } /// Setting the linked regions void SetSelectionRegions(std::vector myregions) diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp index 0ca6e193..c138dd4b 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp @@ -145,8 +145,7 @@ void RegionSelectionManager::FillHisto(std::string const&histname, MAfloat64 val if (myhisto->FreshEvent()) { myhisto->IncrementNEvents(weight_); } - - //std::cout << "multiweight size in regionManager is : " << multiweight_.size() << std::endl; + myhisto->Fill(val,weight_); myhisto->IncrementNEvents(multiweight_); myhisto->Fill(val, multiweight_); diff --git a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h index 311b3862..01ce8305 100644 --- a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h +++ b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h @@ -115,6 +115,7 @@ class DatabaseManager { sql = "CREATE TABLE IF NOT EXISTS Statistics("\ "name TEXT NOT NULL,"\ + "id TEXT NOT NULL,"\ "pos_num_events INTEGER NOT NULL,"\ "neg_num_events INTEGER NOT NULL,"\ "pos_sum_event_weights_over_events DOUBLE NOT NULL,"\ @@ -129,7 +130,7 @@ class DatabaseManager { "neg_value_times_weight DOUBLE NOT NULL,"\ "pos_value_squared_times_weight DOUBLE NOT NULL,"\ "neg_value_squared_times_weight DOUBLE NOT NULL,"\ - "primary key(name) );"; + "primary key(name, id) );"; rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); checkDBErrors("creating Statistics table"); @@ -153,7 +154,8 @@ class DatabaseManager { checkDBErrors("inserting into Histo: " + name); } - void addStatistic(string name, + void addStatistic(string name, + int id, int pos_events, int neg_events, double pos_sum_events, @@ -169,7 +171,7 @@ class DatabaseManager { double pos_val2_weight, double neg_val2_weight){ - sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; + sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(id) + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); checkDBErrors("inserting into Statistics: " + name); From b65998610ce2327790e30050b98fc5baa24d8098 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Wed, 31 Aug 2022 19:57:06 -0600 Subject: [PATCH 14/53] fixed bug with missing 0 entries --- requirements.txt | 1 + .../Process/Analyzer/MergingPlots.cpp | 2 +- .../Process/Core/SampleAnalyzer.cpp | 2 +- .../SampleAnalyzer/Process/Counter/Counter.h | 75 +++++++++++-------- .../Process/Counter/CounterManager.cpp | 27 +++---- .../Process/Counter/CounterManager.h | 10 ++- .../Process/RegionSelection/RegionSelection.h | 26 ++----- .../RegionSelectionManager.cpp | 5 +- .../RegionSelection/RegionSelectionManager.h | 23 ++++-- 9 files changed, 95 insertions(+), 76 deletions(-) diff --git a/requirements.txt b/requirements.txt index 352d7e20..f7820720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ scipy>=1.7.1 numpy>=1.19.5 pyhf==0.6.3 lxml>=4.6.2 + diff --git a/tools/SampleAnalyzer/Process/Analyzer/MergingPlots.cpp b/tools/SampleAnalyzer/Process/Analyzer/MergingPlots.cpp index af133836..e4bf6f13 100644 --- a/tools/SampleAnalyzer/Process/Analyzer/MergingPlots.cpp +++ b/tools/SampleAnalyzer/Process/Analyzer/MergingPlots.cpp @@ -109,7 +109,7 @@ MAbool MergingPlots::Execute(SampleFormat& mySample, const EventFormat& myEvent) WARNING << "Found one event with a zero weight. Skipping..." << endmsg; return false; } - Manager()->InitializeForNewEvent(myEventWeight); + Manager()->InitializeForNewEvent(myEventWeight, myEvent.mc()->multiweights().GetWeights()); // Getting number of extra jets in the event MAuint32 njets = 0; diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index 221f1848..3096efab 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -835,7 +835,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, //save multi-weight cutflows to SQL - Kyle Fan for(int i = 0; i < analyzers_.size(); ++i){ - std::string path = analyzers_[i]->Output() + "/Cutflows/cutflows.db"; + std::string path = analyzers_[i]->Output() + "/Cutflows/Analysis-" + to_string(i) + "-cutflows.db"; DatabaseManager dbManager(path); dbManager.createCutflowTables(); bool addInitial = true; diff --git a/tools/SampleAnalyzer/Process/Counter/Counter.h b/tools/SampleAnalyzer/Process/Counter/Counter.h index 7838885e..a3b213ff 100644 --- a/tools/SampleAnalyzer/Process/Counter/Counter.h +++ b/tools/SampleAnalyzer/Process/Counter/Counter.h @@ -36,17 +36,26 @@ #include struct multiWeightEntry { - std::pair nentries_; - std::pair sumweight_; - std::pair sumweight2_; - multiWeightEntry(){ - nentries_.first = 0; - nentries_.second = 0; - sumweight_.first = 0.; - sumweight_.second = 0.; - sumweight2_.first = 0.; - sumweight2_.second = 0.; - } + std::pair nentries_; + std::pair sumweight_; + std::pair sumweight2_; + multiWeightEntry(double input = 0){ + nentries_.first = input; + nentries_.second = input; + sumweight_.first = input; + sumweight_.second = input; + sumweight2_.first = input; + sumweight2_.second = input; + } + + multiWeightEntry(const multiWeightEntry &rhs){ + nentries_.first = rhs.nentries_.first; + nentries_.second = rhs.nentries_.second; + sumweight_.first = rhs.sumweight_.first; + sumweight_.second = rhs.sumweight_.second; + sumweight2_.first = rhs.sumweight2_.first; + sumweight2_.second = rhs.sumweight2_.second; + } }; @@ -81,7 +90,7 @@ class Counter /// first = positive weight ; second = negative weight std::pair sumweight2_; - std::map multiweight_; + std::map multiweight_; // ------------------------------------------------------------- @@ -100,11 +109,7 @@ class Counter /// Destructor ~Counter() - { - for(auto &p : multiweight_){ - delete p.second; - } - } + { } /// Reset void Reset() @@ -112,12 +117,10 @@ class Counter nentries_ = std::make_pair(0,0); sumweight_ = std::make_pair(0.,0.); sumweight2_ = std::make_pair(0.,0.); - for(auto &p : multiweight_){ - delete p.second; - } multiweight_.clear(); } + /* void Debug(int debugCount, int weightprocessed){ for(auto &p : multiweight_){ std::ofstream output; @@ -134,32 +137,34 @@ class Counter } } + */ + void Increment(const std::map &multiweights){ // static int incrementDebugCount = 0; - for(const auto &weight : multiweights){ - if(multiweight_.find(weight.first) == multiweight_.end()){ - multiweight_[weight.first] = new multiWeightEntry(); - } + + for(const auto &weight : multiweights){ + if(weight.second > 0){ - multiweight_[weight.first]->nentries_.first++; - multiweight_[weight.first]->sumweight_.first+=weight.second; - multiweight_[weight.first]->sumweight2_.first+=weight.second*weight.second; + multiweight_[weight.first].nentries_.first++; + multiweight_[weight.first].sumweight_.first+=weight.second; + multiweight_[weight.first].sumweight2_.first+=weight.second*weight.second; } else if (weight.second < 0){ - multiweight_[weight.first]->nentries_.second++; - multiweight_[weight.first]->sumweight_.second+=weight.second; - multiweight_[weight.first]->sumweight2_.second+=weight.second*weight.second; + multiweight_[weight.first].nentries_.second++; + multiweight_[weight.first].sumweight_.second+=weight.second; + multiweight_[weight.first].sumweight2_.second+=weight.second*weight.second; } } + //Debug testing // Debug(incrementDebugCount, multiweights.begin()->second); // incrementDebugCount++; } - + /// Increment the counter void Increment(const MAfloat32& weight) @@ -179,6 +184,14 @@ class Counter } + void Init(const std::map &weights){ + for(auto w : weights){ + if(multiweight_.find(w.first) == multiweight_.end()) {multiweight_[w.first] = multiWeightEntry(0) ;} + } + } + + + diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp b/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp index a295cbab..84100561 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.cpp @@ -172,12 +172,12 @@ void CounterManager::WriteSQL(DatabaseManager &db, bool &AddInitial, std::string int id, pos_entries, neg_entries; double pos_sum, neg_sum, pos_2sum, neg_2sum; id = p.first; - pos_entries = p.second->nentries_.first; - neg_entries = p.second->nentries_.second; - pos_sum = p.second->sumweight_.first; - neg_sum = p.second->sumweight_.second; - pos_2sum = p.second->sumweight2_.first; - neg_2sum = p.second->sumweight2_.second; + pos_entries = p.second.nentries_.first; + neg_entries = p.second.nentries_.second; + pos_sum = p.second.sumweight_.first; + neg_sum = p.second.sumweight_.second; + pos_2sum = p.second.sumweight2_.first; + neg_2sum = p.second.sumweight2_.second; db.addWeight("initial", "event", id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); } AddInitial = false; @@ -187,17 +187,18 @@ void CounterManager::WriteSQL(DatabaseManager &db, bool &AddInitial, std::string //for each weight id in each cut, insert the values into the database for(const auto &cut : counters_){ std::string cut_name = cut.name_; - db.addCut(region_name, cut_name); + db.addCut(region_name, cut_name); + // std::cout << "number of multiweight entries in cut : " << region_name << " " << cut_name << " is : " << cut.multiweight_.size() << std::endl; for(const auto &p : cut.multiweight_){ int id, pos_entries, neg_entries; double pos_sum, neg_sum, pos_2sum, neg_2sum; id = p.first; - pos_entries = p.second->nentries_.first; - neg_entries = p.second->nentries_.second; - pos_sum = p.second->sumweight_.first; - neg_sum = p.second->sumweight_.second; - pos_2sum = p.second->sumweight2_.first; - neg_2sum = p.second->sumweight2_.second; + pos_entries = p.second.nentries_.first; + neg_entries = p.second.nentries_.second; + pos_sum = p.second.sumweight_.first; + neg_sum = p.second.sumweight_.second; + pos_2sum = p.second.sumweight2_.first; + neg_2sum = p.second.sumweight2_.second; db.addWeight(region_name, cut_name, id, pos_entries, neg_entries, pos_sum, neg_sum, pos_2sum, neg_2sum); } diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.h b/tools/SampleAnalyzer/Process/Counter/CounterManager.h index 6e2acacd..6a23179a 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.h +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.h @@ -91,9 +91,13 @@ class CounterManager { return counters_[index];} /// Incrementing the initial number of events - void IncrementNInitial(MAfloat32 weight) + void IncrementNInitial(MAfloat32 weight, const std::map &multiweight) { - initial_.Increment(weight); + initial_.Increment(weight); + initial_.Increment(multiweight); + for(auto &cut : counters_){ + cut.Init(multiweight); + } } /// Incrementing the initial number of events @@ -117,9 +121,11 @@ class CounterManager { Reset(); } + /* void IncrementNInitial(const std::map &multiweight){ initial_.Increment(multiweight); } + */ diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h index ce96ccdc..11f8d3c1 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h @@ -49,7 +49,6 @@ class RegionSelection std::string name_; MAbool surviving_; MAuint32 NumberOfCutsAppliedSoFar_; - MAuint32 MultiNumberOfCutsAppliedSoFar_; MAfloat64 weight_; std::map multiWeight_; @@ -109,14 +108,12 @@ class RegionSelection void SetNumberOfCutsAppliedSoFar(MAuint32 NumberOfCutsAppliedSoFar) { NumberOfCutsAppliedSoFar_ = NumberOfCutsAppliedSoFar; } - void SetMultiNumberOfCutsAppliedSoFar(MAuint32 MultiNumberOfCutsAppliedSoFar){ - MultiNumberOfCutsAppliedSoFar_ = MultiNumberOfCutsAppliedSoFar; - } // Increment CutFlow (when this region passes a cut) - void IncrementCutFlow(MAfloat64 weight) + void IncrementCutFlow(MAfloat64 weight, const std::map &multiweight) { cutflow_[NumberOfCutsAppliedSoFar_].Increment(weight); + cutflow_[NumberOfCutsAppliedSoFar_].Increment(multiweight); NumberOfCutsAppliedSoFar_++; } @@ -126,11 +123,12 @@ class RegionSelection { cutflow_.InitCut(CutName); } /// Getting ready for a new event - void InitializeForNewEvent(const MAfloat64 &weight) + void InitializeForNewEvent(const MAfloat64 &weight,const std::map &multiweight) { SetSurvivingTest(true); SetNumberOfCutsAppliedSoFar(0); - cutflow_.IncrementNInitial(weight); + cutflow_.IncrementNInitial(weight, multiweight); + multiWeight_=multiweight; weight_=weight; } @@ -142,20 +140,6 @@ class RegionSelection std::map GetWeights() {return multiWeight_;} - void IncrementCutFlow(const std::map &multiweight){ - cutflow_[MultiNumberOfCutsAppliedSoFar_].Increment(multiweight); - MultiNumberOfCutsAppliedSoFar_++; - } - - - - void InitializeForNewEvent(const std::map &multiweight){ - SetSurvivingTest(true); - SetMultiNumberOfCutsAppliedSoFar(0); - cutflow_.IncrementNInitial(multiweight); - multiWeight_=multiweight; - } - }; diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp index c138dd4b..c38a8ddd 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp @@ -64,6 +64,7 @@ MAbool RegionSelectionManager::ApplyCut(MAbool condition, std::string const &cut std::vector RegionsForThisCut = mycut->Regions(); for (MAuint32 i=0; iIncrementCutFlow(ThisRegion->GetWeight()); + ThisRegion->IncrementCutFlow(ThisRegion->GetWeight(), ThisRegion->GetWeights()); //multiweight - ThisRegion->IncrementCutFlow(ThisRegion->GetWeights()); + //ThisRegion->IncrementCutFlow(ThisRegion->GetWeights()); } else { diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h index 176d08e1..236e0077 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h @@ -120,6 +120,7 @@ class RegionSelectionManager return multiweight_; } + /// Set method void SetCurrentEventWeight(MAfloat64 weight) { @@ -139,6 +140,7 @@ class RegionSelectionManager } } + //set methods for multiweight void SetCurrentEventWeight(const WeightCollection &multiweight){ multiweight_ = multiweight.GetWeights(); @@ -156,6 +158,7 @@ class RegionSelectionManager } } + /// Adding a RegionSelection to the manager void AddRegionSelection(const std::string& name) @@ -172,22 +175,31 @@ class RegionSelectionManager } /// Getting ready for a new event - void InitializeForNewEvent(MAfloat64 EventWeight) + void InitializeForNewEvent(MAfloat64 EventWeight, const std::map &weights) { weight_ = EventWeight; + multiweight_ = weights; NumberOfSurvivingRegions_ = regions_.size(); for (MAuint32 i=0; iInitializeForNewEvent(EventWeight); - + { + regions_[i]->InitializeForNewEvent(EventWeight, weights); + + + } for (MAuint32 i=0; i < plotmanager_.GetNplots(); i++) - plotmanager_.GetHistos()[i]->SetFreshEvent(true); + { + plotmanager_.GetHistos()[i]->SetFreshEvent(true); + plotmanager_.GetHistos()[i]->SetMultiweightFreshEvent(true); + } } /// Initialize for multiweight + + /* void InitializeForNewEvent(const std::map &weights){ multiweight_ = weights; - NumberOfSurvivingRegions_ = regions_.size(); + //NumberOfSurvivingRegions_ = regions_.size(); for (MAuint32 i=0; iInitializeForNewEvent(weights); @@ -196,6 +208,7 @@ class RegionSelectionManager plotmanager_.GetHistos()[i]->SetMultiweightFreshEvent(true); } } + */ /// This method associates all regions with a cut From 5e3a36c0c80ca819b090ec8f89a4bf8935ee02de Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Wed, 31 Aug 2022 21:42:45 -0600 Subject: [PATCH 15/53] fixed missing histogram data when entries are 0 --- tools/SampleAnalyzer/Process/Plot/Histo.cpp | 34 ++++----- tools/SampleAnalyzer/Process/Plot/Histo.h | 73 ++++++++++++++----- tools/SampleAnalyzer/Process/Plot/PlotBase.h | 53 +++++++++++--- .../RegionSelectionManager.cpp | 2 +- .../RegionSelection/RegionSelectionManager.h | 1 + 5 files changed, 113 insertions(+), 50 deletions(-) diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.cpp b/tools/SampleAnalyzer/Process/Plot/Histo.cpp index e4a3ab34..5af388f2 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.cpp +++ b/tools/SampleAnalyzer/Process/Plot/Histo.cpp @@ -58,26 +58,26 @@ void Histo::WriteSQL(DatabaseManager &db){ for(const auto &weight_id : MultiweightHistoData){ db.addStatistic(name_, weight_id.first, - multiweight_event_info[weight_id.first]->nevents_.first, - multiweight_event_info[weight_id.first]->nevents_.second, - multiweight_event_info[weight_id.first]->nevents_w_.first, - multiweight_event_info[weight_id.first]->nevents_w_.second, - multiweight_event_info[weight_id.first]->nentries_.first, - multiweight_event_info[weight_id.first]->nentries_.second, - weight_id.second->sum_w_.first, - weight_id.second->sum_w_.second, - weight_id.second->sum_ww_.first, - weight_id.second->sum_ww_.second, - weight_id.second->sum_xw_.first, - weight_id.second->sum_xw_.second, - weight_id.second->sum_xxw_.first, - weight_id.second->sum_xxw_.second); + multiweight_event_info[weight_id.first].nevents_.first, + multiweight_event_info[weight_id.first].nevents_.second, + multiweight_event_info[weight_id.first].nevents_w_.first, + multiweight_event_info[weight_id.first].nevents_w_.second, + multiweight_event_info[weight_id.first].nentries_.first, + multiweight_event_info[weight_id.first].nentries_.second, + weight_id.second.sum_w_.first, + weight_id.second.sum_w_.second, + weight_id.second.sum_ww_.first, + weight_id.second.sum_ww_.second, + weight_id.second.sum_xw_.first, + weight_id.second.sum_xw_.second, + weight_id.second.sum_xxw_.first, + weight_id.second.sum_xxw_.second); //for each weight histo,weight id pair: add bucket data to Data table - db.addData(name_, weight_id.first, "underflow", weight_id.second->underflow_.first, weight_id.second->underflow_.second); + db.addData(name_, weight_id.first, "underflow", weight_id.second.underflow_.first, weight_id.second.underflow_.second); for(int i = 0; i < nbins_; ++i){ - db.addData(name_, weight_id.first, "bin " + to_string(i+1), weight_id.second->histo_[i].first, weight_id.second->histo_[i].second); + db.addData(name_, weight_id.first, "bin " + to_string(i+1), weight_id.second.histo_[i].first, weight_id.second.histo_[i].second); } - db.addData(name_, weight_id.first, "overflow", weight_id.second->overflow_.first, weight_id.second->overflow_.second); + db.addData(name_, weight_id.first, "overflow", weight_id.second.overflow_.first, weight_id.second.overflow_.second); } diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.h b/tools/SampleAnalyzer/Process/Plot/Histo.h index 075e0269..1e385f88 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.h +++ b/tools/SampleAnalyzer/Process/Plot/Histo.h @@ -54,6 +54,26 @@ struct MultiWeightHisto { /// Sum of value * value * weight std::pair sum_xxw_; + + MultiWeightHisto() {} + + MultiWeightHisto(const MultiWeightHisto &rhs){ + histo_ = rhs.histo_; + underflow_.first = rhs.underflow_.first; + underflow_.second = rhs.underflow_.second; + overflow_.first = rhs.overflow_.first; + overflow_.second = rhs.overflow_.second; + sum_w_.first = rhs.sum_w_.first; + sum_w_.second = rhs.sum_w_.second; + sum_ww_.first = rhs.sum_ww_.first; + sum_ww_.second = rhs.sum_ww_.second; + sum_xw_.first = rhs.sum_xw_.first; + sum_xw_.second = rhs.sum_xw_.second; + sum_xxw_.first = rhs.sum_xxw_.first; + sum_xxw_.second = rhs.sum_xxw_.second; + } + + }; @@ -94,7 +114,7 @@ class Histo : public PlotBase /// RegionSelections attached to the histo std::vector regions_; - std::map MultiweightHistoData; + std::map MultiweightHistoData; // ------------------------------------------------------------- // method members @@ -158,12 +178,22 @@ class Histo : public PlotBase /// Destructor virtual ~Histo() - { - for(auto &p : MultiweightHistoData){ - delete p.second; + { } + + virtual void InitMultiweights(const std::map &weights){ + for(const auto &id_weight : weights){ + if(multiweight_event_info.find(id_weight.first) == multiweight_event_info.end()){ + multiweight_event_info[id_weight.first] = MultiweightEvents(); + } + if(MultiweightHistoData.find(id_weight.first) == MultiweightHistoData.end()){ + MultiweightHistoData[id_weight.first] = MultiWeightHisto(); + MultiweightHistoData[id_weight.first].histo_.resize(nbins_); + } + } } + /// Setting the linked regions void SetSelectionRegions(std::vector myregions) { regions_.insert(regions_.end(), myregions.begin(), myregions.end()); } @@ -243,35 +273,38 @@ class Histo : public PlotBase } for(const auto &id_weight : weights){ + + /* if(MultiweightHistoData.find(id_weight.first) == MultiweightHistoData.end()){ MultiweightHistoData[id_weight.first] = new MultiWeightHisto(); MultiweightHistoData[id_weight.first]->histo_.resize(nbins_); } + */ //Positive weight if(id_weight.second >= 0){ - multiweight_event_info[id_weight.first]->nentries_.first++; - MultiweightHistoData[id_weight.first]->sum_w_.first +=id_weight.second; - MultiweightHistoData[id_weight.first]->sum_ww_.first +=id_weight.second * id_weight.second; - MultiweightHistoData[id_weight.first]->sum_xw_.first +=value * id_weight.second; - MultiweightHistoData[id_weight.first]->sum_xxw_.first +=value*value*id_weight.second; - if (value < xmin_) MultiweightHistoData[id_weight.first]->underflow_.first+=id_weight.second; - else if (value >= xmax_) MultiweightHistoData[id_weight.first]->overflow_.first+=id_weight.second; + multiweight_event_info[id_weight.first].nentries_.first++; + MultiweightHistoData[id_weight.first].sum_w_.first +=id_weight.second; + MultiweightHistoData[id_weight.first].sum_ww_.first +=id_weight.second * id_weight.second; + MultiweightHistoData[id_weight.first].sum_xw_.first +=value * id_weight.second; + MultiweightHistoData[id_weight.first].sum_xxw_.first +=value*value*id_weight.second; + if (value < xmin_) MultiweightHistoData[id_weight.first].underflow_.first+=id_weight.second; + else if (value >= xmax_) MultiweightHistoData[id_weight.first].overflow_.first+=id_weight.second; else { - MultiweightHistoData[id_weight.first]->histo_[std::floor((value-xmin_)/step_)].first+=id_weight.second; + MultiweightHistoData[id_weight.first].histo_[std::floor((value-xmin_)/step_)].first+=id_weight.second; } } else { int absWeight = std::abs(id_weight.second); - multiweight_event_info[id_weight.first]->nentries_.second++; - MultiweightHistoData[id_weight.first]->sum_w_.second += absWeight; - MultiweightHistoData[id_weight.first]->sum_ww_.second += absWeight * absWeight; - MultiweightHistoData[id_weight.first]->sum_xw_.second +=value * absWeight; - MultiweightHistoData[id_weight.first]->sum_xxw_.second +=value*value*absWeight; - if (value < xmin_) MultiweightHistoData[id_weight.first]->underflow_.second+=absWeight; - else if (value >= xmax_) MultiweightHistoData[id_weight.first]->overflow_.second+=absWeight; + multiweight_event_info[id_weight.first].nentries_.second++; + MultiweightHistoData[id_weight.first].sum_w_.second += absWeight; + MultiweightHistoData[id_weight.first].sum_ww_.second += absWeight * absWeight; + MultiweightHistoData[id_weight.first].sum_xw_.second +=value * absWeight; + MultiweightHistoData[id_weight.first].sum_xxw_.second +=value*value*absWeight; + if (value < xmin_) MultiweightHistoData[id_weight.first].underflow_.second+=absWeight; + else if (value >= xmax_) MultiweightHistoData[id_weight.first].overflow_.second+=absWeight; else { - MultiweightHistoData[id_weight.first]->histo_[std::floor((value-xmin_)/step_)].second+=absWeight; + MultiweightHistoData[id_weight.first].histo_[std::floor((value-xmin_)/step_)].second+=absWeight; } } diff --git a/tools/SampleAnalyzer/Process/Plot/PlotBase.h b/tools/SampleAnalyzer/Process/Plot/PlotBase.h index 22ad692c..b7eaf1ec 100644 --- a/tools/SampleAnalyzer/Process/Plot/PlotBase.h +++ b/tools/SampleAnalyzer/Process/Plot/PlotBase.h @@ -49,6 +49,27 @@ struct MultiweightEvents { MAbool fresh_event_; + MultiweightEvents() { + fresh_event_ = true; + nevents_.first = 0; + nevents_.second = 0; + nentries_.first = 0; + nentries_.second = 0; + nevents_w_.first = 0.; + nevents_w_.second = 0.; + + } + + MultiweightEvents(const MultiweightEvents &rhs){ + nevents_.first = rhs.nevents_.first; + nevents_.second = rhs.nevents_.second; + nentries_.first = rhs.nentries_.first; + nentries_.second = rhs.nentries_.second; + nevents_w_.first = rhs.nevents_w_.first; + nevents_w_.second = rhs.nevents_w_.second; + fresh_event_ = rhs.fresh_event_; + } + }; @@ -79,7 +100,7 @@ class PlotBase MAbool fresh_event_; //map containing event meta data for each weight id - std::map multiweight_event_info; + std::map multiweight_event_info; // ------------------------------------------------------------- @@ -112,9 +133,6 @@ class PlotBase /// Destructor virtual ~PlotBase() { - for(auto &p : multiweight_event_info){ - delete p.second; - } } /// Accesor for fresh_event @@ -122,7 +140,7 @@ class PlotBase /// Accessor for multiweight fresh_event MAbool MultiweightFreshEvent(int index) { - return multiweight_event_info[index]->fresh_event_; + return multiweight_event_info[index].fresh_event_; } /// Modifier for fresh_event @@ -133,7 +151,16 @@ class PlotBase /// Modifier for Multiweight fresh_event void SetMultiweightFreshEvent(MAbool tag) { for(auto &weight_id : multiweight_event_info){ - weight_id.second->fresh_event_ = tag; + weight_id.second.fresh_event_ = tag; + } + } + + virtual void InitMultiweights(const std::map &weights){ + for(const auto &id_weight : weights){ + if(multiweight_event_info.find(id_weight.first) == multiweight_event_info.end()){ + multiweight_event_info[id_weight.first] = MultiweightEvents(); + } + } } @@ -161,22 +188,24 @@ class PlotBase void IncrementNEvents(std::map &weights){ for(const auto &id_weight : weights){ + /* if(multiweight_event_info.find(id_weight.first) == multiweight_event_info.end()){ multiweight_event_info[id_weight.first] = new MultiweightEvents(); multiweight_event_info[id_weight.first]->fresh_event_ = true; } + */ if(MultiweightFreshEvent(id_weight.first)){ if(id_weight.second >= 0){ - multiweight_event_info[id_weight.first]->nevents_.first++; - multiweight_event_info[id_weight.first]->nevents_w_.first+=id_weight.second; + multiweight_event_info[id_weight.first].nevents_.first++; + multiweight_event_info[id_weight.first].nevents_w_.first+=id_weight.second; } else { - multiweight_event_info[id_weight.first]->nevents_.second++; - multiweight_event_info[id_weight.first]->nevents_w_.second+=std::abs(id_weight.second); + multiweight_event_info[id_weight.first].nevents_.second++; + multiweight_event_info[id_weight.first].nevents_w_.second+=std::abs(id_weight.second); } } - multiweight_event_info[id_weight.first]->fresh_event_ = false; + multiweight_event_info[id_weight.first].fresh_event_ = false; } } @@ -185,7 +214,7 @@ class PlotBase { return nevents_; } const std::pair& GetNEvents(int index){ - return multiweight_event_info[index]->nevents_; + return multiweight_event_info[index].nevents_; } // Return the name diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp index c38a8ddd..77ac7d89 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp @@ -149,7 +149,7 @@ void RegionSelectionManager::FillHisto(std::string const&histname, MAfloat64 val myhisto->Fill(val,weight_); myhisto->IncrementNEvents(multiweight_); - myhisto->Fill(val, multiweight_); + myhisto->Fill(val,multiweight_); } break; } diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h index 236e0077..4cc74c98 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.h @@ -190,6 +190,7 @@ class RegionSelectionManager { plotmanager_.GetHistos()[i]->SetFreshEvent(true); plotmanager_.GetHistos()[i]->SetMultiweightFreshEvent(true); + plotmanager_.GetHistos()[i]->InitMultiweights(weights); } } From 922ece85508aede8d947a630a14b2b8dc89b9ccc Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Thu, 1 Sep 2022 17:31:12 -0600 Subject: [PATCH 16/53] removed databse entry insertion debug messages --- tools/SampleAnalyzer/Process/Plot/PlotBase.h | 1 + tools/SampleAnalyzer/Process/Writer/DatabaseManager.h | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/SampleAnalyzer/Process/Plot/PlotBase.h b/tools/SampleAnalyzer/Process/Plot/PlotBase.h index b7eaf1ec..6d99c5ef 100644 --- a/tools/SampleAnalyzer/Process/Plot/PlotBase.h +++ b/tools/SampleAnalyzer/Process/Plot/PlotBase.h @@ -155,6 +155,7 @@ class PlotBase } } + //initialize histograms multiweights (without initialization, 0 entries will not show up in histogram) virtual void InitMultiweights(const std::map &weights){ for(const auto &id_weight : weights){ if(multiweight_event_info.find(id_weight.first) == multiweight_event_info.end()){ diff --git a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h index 01ce8305..4727522a 100644 --- a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h +++ b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h @@ -151,7 +151,7 @@ class DatabaseManager { void addHisto(string name, int bins, double xmin, double xmax, string regions){ sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')"; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - checkDBErrors("inserting into Histo: " + name); + //checkDBErrors("inserting into Histo: " + name); } void addStatistic(string name, @@ -174,14 +174,14 @@ class DatabaseManager { sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(id) + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - checkDBErrors("inserting into Statistics: " + name); + //checkDBErrors("inserting into Statistics: " + name); } void addData(string name, int id, string bin, double positive, double negative){ sql = "INSERT INTO Data VALUES ('" + name + "'" + "," + "'" + to_string(id) + "'" + "," + "'" + bin + "'" + "," + "'" + to_string(positive) + "'" + "," + "'" + to_string(negative) + "')"; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - checkDBErrors("inserting into Data: " + name + " " + to_string(id)); + //checkDBErrors("inserting into Data: " + name + " " + to_string(id)); } From 1e39fd2c52a400880e3f808bf13af8c62bebb92b Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Fri, 2 Sep 2022 00:13:58 -0600 Subject: [PATCH 17/53] added weight names to DB --- .../Commons/DataFormat/MCSampleFormat.h | 17 ++++++++++++++++- .../Process/Core/SampleAnalyzer.cpp | 2 +- .../Process/Reader/HEPMCReader.cpp | 6 ++++-- .../SampleAnalyzer/Process/Reader/HEPMCReader.h | 2 +- .../Process/Writer/DatabaseManager.h | 14 ++++++++++++++ 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h b/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h index 9a09bd05..dba54e50 100644 --- a/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h +++ b/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h @@ -36,7 +36,8 @@ #include "SampleAnalyzer/Commons/DataFormat/GeneratorInfo.h" #include "SampleAnalyzer/Commons/DataFormat/MCProcessFormat.h" #include "SampleAnalyzer/Commons/DataFormat/WeightDefinition.h" -#include "SampleAnalyzer/Commons/Service/LogService.h" +#include "SampleAnalyzer/Commons/Service/LogService.h" +#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" namespace MA5 @@ -80,6 +81,8 @@ class MCSampleFormat // ----------------------- multiweights ------------------------ WeightDefinition weight_definition_; + std::map weight_names; + // ----------------------- file info --------------------------- MAfloat64 xsection_; @@ -125,6 +128,18 @@ class MCSampleFormat sumweight_negative_ = 0.; } + //set weight names + void SetName(int id, std::string name) { + weight_names[id] = name; + } + + void WriteWeightNames(DatabaseManager &db){ + db.createWeightNamesTable(); + for(auto &p : weight_names){ + db.addWeightDefinition(p.first, p.second); + } + } + /// Accessoir to the generator type const MA5GEN::GeneratorType* GeneratorType() const { return sample_generator_; } diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index 3096efab..ef104eb0 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -814,7 +814,6 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, dbManager.closeDB(); } - // Saving the cut flows for(MAuint32 i=0; i& mySamples, bool addInitial = true; AnalyzerBase* myanalysis = analyzers_[i]; + mySamples[i].mc()->WriteWeightNames(dbManager); //insert region,cut pair to cutflow table and region,cut,weight_id (weight data) to weights table for(int j = 0; j < myanalysis->Manager()->Regions().size(); ++j){ RegionSelection *myRS = myanalysis->Manager()->Regions()[j]; diff --git a/tools/SampleAnalyzer/Process/Reader/HEPMCReader.cpp b/tools/SampleAnalyzer/Process/Reader/HEPMCReader.cpp index 9ef07ab5..6e854d77 100644 --- a/tools/SampleAnalyzer/Process/Reader/HEPMCReader.cpp +++ b/tools/SampleAnalyzer/Process/Reader/HEPMCReader.cpp @@ -228,7 +228,7 @@ MAbool HEPMCReader::FinalizeEvent(SampleFormat& mySample, EventFormat& myEvent) //------------------------------------------------------------------------------ // FillWeightNames //------------------------------------------------------------------------------ -MAbool HEPMCReader::FillWeightNames(const std::string& line) +MAbool HEPMCReader::FillWeightNames(const std::string& line, SampleFormat& mySample) { // Splitting line in words std::stringstream str; @@ -254,6 +254,8 @@ MAbool HEPMCReader::FillWeightNames(const std::string& line) if (tmp[0]=='"' && tmp[tmp.size()-1]=='"') tmp=tmp.substr(1,tmp.size()-2); weight_names[i]=tmp; + mySample.mc()->SetName(i+1, tmp); + } // Ok @@ -298,7 +300,7 @@ MAbool HEPMCReader::FillEvent(const std::string& line, if(firstWord=="E") FillEventInformations(line, myEvent); // Weight names - else if (firstWord=="N") FillWeightNames(line); + else if (firstWord=="N") FillWeightNames(line, mySample); // Event units else if (firstWord=="U") FillUnits(line); diff --git a/tools/SampleAnalyzer/Process/Reader/HEPMCReader.h b/tools/SampleAnalyzer/Process/Reader/HEPMCReader.h index fc2b42e3..82c65403 100644 --- a/tools/SampleAnalyzer/Process/Reader/HEPMCReader.h +++ b/tools/SampleAnalyzer/Process/Reader/HEPMCReader.h @@ -111,7 +111,7 @@ class HEPMCReader : public ReaderTextBase void FillEventPDFInfo(const std::string& line, SampleFormat& mySample, EventFormat& myEvent); void FillEventParticleLine(const std::string& line, EventFormat& myEvent); void FillEventVertexLine(const std::string& line, EventFormat& myEvent); - MAbool FillWeightNames(const std::string& line); + MAbool FillWeightNames(const std::string& line, SampleFormat& mySample); MAbool FillHeavyIons(const std::string& line); }; diff --git a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h index 4727522a..fbc78816 100644 --- a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h +++ b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h @@ -148,6 +148,20 @@ class DatabaseManager { } + void createWeightNamesTable(){ + sql = "CREATE TABLE IF NOT EXISTS WeightDefinition("\ + "id INTERGER NOT NULL,"\ + "definition TEXT NOT NULL,"\ + "primary key (id) );"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Weight Names table"); + } + + void addWeightDefinition(int id, string def){ + sql = "INSERT INTO WeightDefinition VALUES ('" + to_string(id) + "','" + def + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + } + void addHisto(string name, int bins, double xmin, double xmax, string regions){ sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')"; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); From 8a3b4d7d5251c3de3610dcee75c41722f67dbbb2 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Fri, 2 Sep 2022 09:39:10 -0600 Subject: [PATCH 18/53] get weight names from first sample only, weight names should be identical for all samples within one analysis --- tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index ef104eb0..e726f708 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -840,7 +840,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, bool addInitial = true; AnalyzerBase* myanalysis = analyzers_[i]; - mySamples[i].mc()->WriteWeightNames(dbManager); + mySamples[0].mc()->WriteWeightNames(dbManager); //insert region,cut pair to cutflow table and region,cut,weight_id (weight data) to weights table for(int j = 0; j < myanalysis->Manager()->Regions().size(); ++j){ RegionSelection *myRS = myanalysis->Manager()->Regions()[j]; From 2882fc5fcda5336025019e5546909e102e700096 Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Fri, 2 Sep 2022 10:01:23 -0600 Subject: [PATCH 19/53] fixed typo in database HistoDescription table xmax --- tools/SampleAnalyzer/Process/Writer/DatabaseManager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h index fbc78816..1e4d62e2 100644 --- a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h +++ b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h @@ -107,7 +107,7 @@ class DatabaseManager { "name TEXT NOT NULL,"\ "num_of_bins INTEGER NOT NULL,"\ "xmin DOUBLE NOT NULL,"\ - "xmas DOUBLE NOT NULL,"\ + "xmax DOUBLE NOT NULL,"\ "regions TEXT NOT NULL,"\ "primary key(name) );"; rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); From 9b879e0f8c600b2c95d726a122a925528d0fdaaf Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Thu, 8 Sep 2022 01:51:33 -0600 Subject: [PATCH 20/53] changed cutflow db output file name --- tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index e726f708..dc5abbce 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -834,7 +834,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, //save multi-weight cutflows to SQL - Kyle Fan for(int i = 0; i < analyzers_.size(); ++i){ - std::string path = analyzers_[i]->Output() + "/Cutflows/Analysis-" + to_string(i) + "-cutflows.db"; + std::string path = analyzers_[i]->Output() + "/Cutflows/cutflows.db"; DatabaseManager dbManager(path); dbManager.createCutflowTables(); bool addInitial = true; From c37c293bebb8f536cf774c5555afc9f6aeef329d Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Sun, 11 Sep 2022 23:16:09 -0600 Subject: [PATCH 21/53] added detect script for sqlite3 --- madanalysis/system/architecture_info.py | 1 + madanalysis/system/detect_manager.py | 3 + madanalysis/system/detect_sqlite.py | 95 +++++++++++++++++++++++++ madanalysis/system/session_info.py | 1 + madanalysis/system/user_info.py | 2 + 5 files changed, 102 insertions(+) create mode 100644 madanalysis/system/detect_sqlite.py diff --git a/madanalysis/system/architecture_info.py b/madanalysis/system/architecture_info.py index 53b363b8..ea19ce1b 100644 --- a/madanalysis/system/architecture_info.py +++ b/madanalysis/system/architecture_info.py @@ -46,6 +46,7 @@ def __init__(self): self.has_zlib = False self.has_delphes = False self.has_delphesMA5tune = False + # Library to put before all the others? self.root_priority = False diff --git a/madanalysis/system/detect_manager.py b/madanalysis/system/detect_manager.py index 318399bf..aadcf1e4 100644 --- a/madanalysis/system/detect_manager.py +++ b/madanalysis/system/detect_manager.py @@ -126,6 +126,9 @@ def Execute(self, rawpackage): elif package=='latex': from madanalysis.system.detect_latex import DetectLatex checker=DetectLatex(self.archi_info, self.user_info, self.session_info, self.debug) + elif package=='sqlite': + from madanalysis.system.detect_sqlite import DetectSQLite + checker=DetectSQLite(self.archi_info, self.user_info, self.session_info, self.debug) else: self.logger.error('the package "'+rawpackage+'" is unknown') return False diff --git a/madanalysis/system/detect_sqlite.py b/madanalysis/system/detect_sqlite.py new file mode 100644 index 00000000..61489ce5 --- /dev/null +++ b/madanalysis/system/detect_sqlite.py @@ -0,0 +1,95 @@ +################################################################################ +# +# Copyright (C) 2012-2022 Jack Araz, Eric Conte & Benjamin Fuks +# The MadAnalysis development team, email: +# +# This file is part of MadAnalysis 5. +# Official website: +# +# MadAnalysis 5 is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# MadAnalysis 5 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with MadAnalysis 5. If not, see +# +################################################################################ + + +from __future__ import absolute_import +import logging +import glob +import os +import sys +import re +import platform +from shell_command import ShellCommand +from madanalysis.enumeration.detect_status_type import DetectStatusType + + +class DetectSQLite: + + def __init__(self, archi_info, user_info, session_info, debug): + # mandatory options + self.archi_info = archi_info + self.user_info = user_info + self.session_info = session_info + self.debug = debug + self.name = 'sqlite3' + self.mandatory = False + self.log = [] + self.logger = logging.getLogger('MA5') + + # adding what you want here + + def PrintDisableMessage(self): + self.logger.warning("sqlite3 not detected, multiweight output format will not be supported!") + + + def IsItVetoed(self): + if self.user_info.sqlite_veto: + self.logger.debug("user setting: veto on SQLite3") + return True + else: + self.logger.debug("no user veto") + return False + + + def AutoDetection(self): + # Which + result = ShellCommand.Which('sqlite3',all=False,mute=True) + if len(result)==0: + return DetectStatusType.UNFOUND,'' + if self.debug: + self.logger.debug(" which: " + str(result[0])) + + # Ok + return DetectStatusType.FOUND,'' + + + def ExtractInfo(self): + # Which all + if self.debug: + result = ShellCommand.Which('sqlite3',all=True,mute=True) + if len(result)==0: + return False + self.logger.debug(" which-all: ") + for file in result: + self.logger.debug(" - "+str(file)) + + # Ok + return True + + + def SaveInfo(self): + self.session_info.has_sqlite3=True + return True + + + diff --git a/madanalysis/system/session_info.py b/madanalysis/system/session_info.py index 5016b3be..15b8f6d9 100644 --- a/madanalysis/system/session_info.py +++ b/madanalysis/system/session_info.py @@ -44,6 +44,7 @@ def __init__(self): self.has_pad = False self.has_padsfs = False self.has_padma5 = False + self.has_sqlite3 = False self.gcc_header_search_path = [] self.gcc_library_search_path = [] self.padma5_build_path = "" diff --git a/madanalysis/system/user_info.py b/madanalysis/system/user_info.py index b13da86d..5d985040 100644 --- a/madanalysis/system/user_info.py +++ b/madanalysis/system/user_info.py @@ -88,6 +88,8 @@ def __init__(self): # dvipdf self.dvipdf_veto = None + self.sqlite_veto = None + # logger self.logger = logging.getLogger('MA5') From d48cae3fc3a97103dec35f9e8ffb04bbadbf25da Mon Sep 17 00:00:00 2001 From: Kyle Fan Date: Sun, 11 Sep 2022 23:57:47 -0600 Subject: [PATCH 22/53] add checkup.py modification --- madanalysis/system/checkup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/madanalysis/system/checkup.py b/madanalysis/system/checkup.py index 4629f6f7..1c788c18 100644 --- a/madanalysis/system/checkup.py +++ b/madanalysis/system/checkup.py @@ -328,6 +328,8 @@ def CheckOptionalProcessingPackages(self): return False if not self.checker.Execute('root'): return False + if not self.checker.Execute('sqlite'): + return False self.archi_info.has_delphes = checker2.checkDelphes() self.archi_info.has_delphesMA5tune = checker2.checkDelphesMA5tune() From 4b7b8a7e3776fe5f008b6440ed58973ef74d9b47 Mon Sep 17 00:00:00 2001 From: Kyle Date: Sun, 18 Sep 2022 15:21:37 -0600 Subject: [PATCH 23/53] update detect sqlite --- madanalysis/system/detect_sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/madanalysis/system/detect_sqlite.py b/madanalysis/system/detect_sqlite.py index 61489ce5..920dbd09 100644 --- a/madanalysis/system/detect_sqlite.py +++ b/madanalysis/system/detect_sqlite.py @@ -41,7 +41,7 @@ def __init__(self, archi_info, user_info, session_info, debug): self.user_info = user_info self.session_info = session_info self.debug = debug - self.name = 'sqlite3' + self.name = 'SQLite3' self.mandatory = False self.log = [] self.logger = logging.getLogger('MA5') From 698ea225230192609132a16926832fad7dd00ca9 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 22 Sep 2022 00:45:19 -0600 Subject: [PATCH 24/53] added multiweight to execute function writer --- madanalysis/job/job_execute.py | 2 +- madanalysis/system/architecture_info.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/madanalysis/job/job_execute.py b/madanalysis/job/job_execute.py index 6380d211..9137612b 100644 --- a/madanalysis/job/job_execute.py +++ b/madanalysis/job/job_execute.py @@ -43,7 +43,7 @@ def WriteExecute(file,main,part_list): file.write(' if (weighted_events_ && event.mc()!=0) ' +\ '__event_weight__ = event.mc()->weight();\n\n') file.write(' if (sample.mc()!=0) sample.mc()->addWeightedEvents(__event_weight__);\n') - file.write(' Manager()->InitializeForNewEvent(__event_weight__);\n') + file.write(' Manager()->InitializeForNewEvent(__event_weight__, event.mc()->multiweights().GetWeights());\n') file.write('\n') # Reseting instance name diff --git a/madanalysis/system/architecture_info.py b/madanalysis/system/architecture_info.py index ea19ce1b..982d026c 100644 --- a/madanalysis/system/architecture_info.py +++ b/madanalysis/system/architecture_info.py @@ -46,6 +46,7 @@ def __init__(self): self.has_zlib = False self.has_delphes = False self.has_delphesMA5tune = False + # Library to put before all the others? From dc1d3bd1bab5fa122f74a8f611fc96d8cdc49513 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 22 Sep 2022 21:17:46 -0600 Subject: [PATCH 25/53] edited makefile writers for sqlite3 --- madanalysis/IOinterface/job_writer.py | 1 + madanalysis/IOinterface/library_writer.py | 9 +++++++++ madanalysis/build/makefile_writer.py | 23 +++++++++++++++++++++-- madanalysis/core/main.py | 3 ++- madanalysis/system/architecture_info.py | 1 + madanalysis/system/detect_sqlite.py | 1 + 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/madanalysis/IOinterface/job_writer.py b/madanalysis/IOinterface/job_writer.py index 07941a7d..cb72197c 100644 --- a/madanalysis/IOinterface/job_writer.py +++ b/madanalysis/IOinterface/job_writer.py @@ -750,6 +750,7 @@ def WriteMakefiles(self, option="", **kwargs): options.has_root_inc = self.main.archi_info.has_root options.has_root_lib = self.main.archi_info.has_root + options.has_sqlite = self.main.archi_info.has_sqlite3 #options.has_userpackage = True toRemove=['Log/compilation.log','Log/linking.log','Log/cleanup.log','Log/mrproper.log'] diff --git a/madanalysis/IOinterface/library_writer.py b/madanalysis/IOinterface/library_writer.py index 39812f29..491c06f0 100644 --- a/madanalysis/IOinterface/library_writer.py +++ b/madanalysis/IOinterface/library_writer.py @@ -131,6 +131,8 @@ def WriteMakefileForInterfaces(self,package): filename = self.path+"/SampleAnalyzer/Test/Makefile_delphesMA5tune" elif package=='test_root': filename = self.path+"/SampleAnalyzer/Test/Makefile_root" + elif package=='test_sqlite': + filename = self.path+"/SampleAnalyzer/Test/Makefile_sqlite" # Header title='' @@ -158,6 +160,8 @@ def WriteMakefileForInterfaces(self,package): title='*delphesMA5tune-interface* test' elif package=='test_root': title='*root-interface* test' + elif package=='test+sqlite': + title='*sqlite* test' else: title='interface to '+package @@ -342,6 +346,8 @@ def WriteMakefileForInterfaces(self,package): # options.has_delphesMA5tune_tag = self.main.archi_info.has_delphesMA5tune # options.has_zlib_tag = self.main.archi_info.has_zlib toRemove.extend(['compilation_process.log','linking_process.log','cleanup_process.log','mrproper_process.log','../Bin/TestSampleAnalyzer.log']) + elif package=='sqlite': + options.has_sqlite = self.main.archi_info.has_sqlite3 # file pattern if package in ['commons','process','configuration']: @@ -374,6 +380,9 @@ def WriteMakefileForInterfaces(self,package): elif package=='test_root': cppfiles = ['Root/*.cpp'] hfiles = ['Root/*.h'] + elif package=='test_sqlite': + cppfiles = ['SQLite/*.cpp'] + hfiles = ['SQLite/*.h'] else: cppfiles = [package+'/*.cpp'] hfiles = [package+'/*.h'] diff --git a/madanalysis/build/makefile_writer.py b/madanalysis/build/makefile_writer.py index 8a075b54..926bb281 100644 --- a/madanalysis/build/makefile_writer.py +++ b/madanalysis/build/makefile_writer.py @@ -41,6 +41,7 @@ def __init__(self): self.has_fastjet = False self.has_delphes = False self.has_delphesMA5tune = False + self.has_sqlite3 = False @staticmethod @@ -98,7 +99,10 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options): file.write('\tcd Test && $(MAKE) -f Makefile_delphesMA5tune\n') if options.has_process: file.write('\tcd Process && $(MAKE) -f Makefile\n') - file.write('\tcd Test && $(MAKE) -f Makefile_process\n') + file.write('\tcd Test && $(MAKE) -f Makefile_process\n') + if options.has_sqlite3: + file.write('\tcd Interfaces && $(MAKE) -f Makefile_sqlite\n') + file.write('\tcd Test && $(MAKE) -f Makefile_sqlite\n') file.write('\n') # Clean @@ -125,6 +129,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options): if options.has_process: file.write('\tcd Process && $(MAKE) -f Makefile clean\n') file.write('\tcd Test && $(MAKE) -f Makefile_process clean\n') + if options.has_sqlite3: + file.write('\tcd Interfaces && $(MAKE) -f Makefile_sqlite clean\n') + file.write('\tcd Test && $(MAKE) -f Makefile_sqlite clean\n') file.write('\n') # Mrproper @@ -152,6 +159,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options): if options.has_process: file.write('\tcd Process && $(MAKE) -f Makefile mrproper\n') file.write('\tcd Test && $(MAKE) -f Makefile_process mrproper\n') + if options.has_sqlite3: + file.write('\tcd Interfaces && $(MAKE) -f Makefile_sqlite mrproper\n') + file.write('\tcd Test && $(MAKE) -f Makefile_sqlite mrproper\n') file.write('\n') # Closing the file @@ -194,6 +204,7 @@ def __init__(self): self.has_root_tag = False self.has_root_lib = False self.has_root_ma5lib = False + self.has_sqlite = False @@ -321,7 +332,8 @@ def Makefile( for header in archi_info.delphesMA5tune_inc_paths: cxxflags.extend(['-I'+header]) file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n') - + + # - tags cxxflags=[] if options.has_root_tag: @@ -347,6 +359,7 @@ def Makefile( # - general libs=[] + # added SQL file.write('LIBFLAGS = -l sqlite3\n') @@ -430,6 +443,10 @@ def Makefile( if options.has_heptoptagger: file.write('LIBFLAGS += -lHEPTopTagger_for_ma5\n') + # SQLite3 + #if options.has_sqlite: + # file.write('LIBFLAGS += -l sqlite3\n') + # - Commons if options.has_commons: libs=[] @@ -465,6 +482,8 @@ def Makefile( libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libsubstructure_for_ma5.so') if options.has_heptoptagger: libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libHEPTopTagger_for_ma5.so') + if options.has_sqlite: + libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libsqlite_for_ma5.so') if len(libs)!=0: file.write('# Requirements to check before building\n') for ind in range(0,len(libs)): diff --git a/madanalysis/core/main.py b/madanalysis/core/main.py index 570b539a..90c6cadb 100644 --- a/madanalysis/core/main.py +++ b/madanalysis/core/main.py @@ -659,7 +659,8 @@ def BuildLibrary(self,forced=False): options.has_zlib = self.archi_info.has_zlib options.has_fastjet = self.archi_info.has_fastjet options.has_delphes = self.archi_info.has_delphes - options.has_delphesMA5tune = self.archi_info.has_delphesMA5tune + options.has_delphesMA5tune = self.archi_info.has_delphesMA5tune + options.has_sqlite3 = self.archi_info.has_sqlite3 #MakefileWriter.UserfriendlyMakefileForSampleAnalyzer(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Makefile',options) # Writing the setup diff --git a/madanalysis/system/architecture_info.py b/madanalysis/system/architecture_info.py index 982d026c..a0fcd393 100644 --- a/madanalysis/system/architecture_info.py +++ b/madanalysis/system/architecture_info.py @@ -46,6 +46,7 @@ def __init__(self): self.has_zlib = False self.has_delphes = False self.has_delphesMA5tune = False + self.has_sqlite3 = False diff --git a/madanalysis/system/detect_sqlite.py b/madanalysis/system/detect_sqlite.py index 920dbd09..a8431026 100644 --- a/madanalysis/system/detect_sqlite.py +++ b/madanalysis/system/detect_sqlite.py @@ -89,6 +89,7 @@ def ExtractInfo(self): def SaveInfo(self): self.session_info.has_sqlite3=True + self.archi_info.has_sqlite3 = True return True From 4e5abe030a10b4fb038eba145cc526944f4d9d56 Mon Sep 17 00:00:00 2001 From: Kyle Date: Wed, 28 Sep 2022 02:40:17 -0600 Subject: [PATCH 26/53] interface currently links global version of SQLite3 if detected, Multiweight SQLite3 output only compiled if defined --- madanalysis/IOinterface/library_writer.py | 8 +- madanalysis/build/makefile_writer.py | 17 +- madanalysis/core/library_builder.py | 4 + madanalysis/core/main.py | 11 + madanalysis/system/architecture_info.py | 1 + madanalysis/system/detect_sqlite.py | 1 + .../Commons/DataFormat/MCSampleFormat.h | 8 +- .../Process/Core/SampleAnalyzer.cpp | 27 +- .../Process/Counter/CounterManager.h | 8 +- tools/SampleAnalyzer/Process/Plot/Histo.cpp | 2 +- tools/SampleAnalyzer/Process/Plot/Histo.h | 7 +- tools/SampleAnalyzer/Process/Plot/PlotBase.h | 6 +- .../SampleAnalyzer/Process/Plot/PlotManager.h | 7 +- .../Process/RegionSelection/RegionSelection.h | 7 +- .../Process/Writer/DatabaseManager.h | 248 ------------------ 15 files changed, 89 insertions(+), 273 deletions(-) delete mode 100644 tools/SampleAnalyzer/Process/Writer/DatabaseManager.h diff --git a/madanalysis/IOinterface/library_writer.py b/madanalysis/IOinterface/library_writer.py index 491c06f0..ce661263 100644 --- a/madanalysis/IOinterface/library_writer.py +++ b/madanalysis/IOinterface/library_writer.py @@ -160,7 +160,7 @@ def WriteMakefileForInterfaces(self,package): title='*delphesMA5tune-interface* test' elif package=='test_root': title='*root-interface* test' - elif package=='test+sqlite': + elif package=='test_sqlite': title='*sqlite* test' else: title='interface to '+package @@ -243,10 +243,13 @@ def WriteMakefileForInterfaces(self,package): options.ma5_fastjet_mode = self.main.archi_info.has_fastjet options.has_fastjet_inc = self.main.archi_info.has_fastjet options.has_fastjet_lib = self.main.archi_info.has_fastjet + #options.has_sqlite_lib = self.main.archi_info.has_sqlite3 + options.has_sqlite_tag = self.main.archi_info.has_sqlite3 # options.has_fastjet_ma5lib = self.main.archi_info.has_fastjet toRemove.extend(['compilation.log','linking.log','cleanup.log','mrproper.log']) elif package=='test_commons': options.has_commons = True + options.has_sqlite_tag = self.main.archi_info.has_sqlite3 toRemove.extend(['compilation_commons.log','linking_commons.log','cleanup_commons.log','mrproper_commons.log','../Bin/TestCommons.log']) elif package=='zlib': options.has_commons = True @@ -256,6 +259,7 @@ def WriteMakefileForInterfaces(self,package): elif package=='test_zlib': options.has_commons = True options.has_zlib_ma5lib = True + options.has_sqlite_tag = self.main.archi_info.has_sqlite3 # options.has_zlib_lib = True toRemove.extend(['compilation_zlib.log','linking_zlib.log','cleanup_zlib.log','mrproper_zlib.log','../Bin/TestZlib.log']) elif package=='delphes': @@ -328,6 +332,8 @@ def WriteMakefileForInterfaces(self,package): options.has_fastjet_lib = self.main.archi_info.has_fastjet options.ma5_fastjet_mode = self.main.archi_info.has_fastjet options.has_substructure = self.main.archi_info.has_fjcontrib and self.main.archi_info.has_fastjet + options.has_sqlite_tag = self.main.archi_info.has_sqlite3 + options.has_sqlite_lib = self.main.archi_info.has_sqlite3 toRemove.extend(['compilation.log','linking.log','cleanup.log','mrproper.log']) elif package=='test_process': diff --git a/madanalysis/build/makefile_writer.py b/madanalysis/build/makefile_writer.py index 926bb281..24a83751 100644 --- a/madanalysis/build/makefile_writer.py +++ b/madanalysis/build/makefile_writer.py @@ -205,6 +205,8 @@ def __init__(self): self.has_root_lib = False self.has_root_ma5lib = False self.has_sqlite = False + self.has_sqlite_tag = False + self.has_sqlite_lib = False @@ -332,6 +334,7 @@ def Makefile( for header in archi_info.delphesMA5tune_inc_paths: cxxflags.extend(['-I'+header]) file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n') + # - tags @@ -350,6 +353,8 @@ def Makefile( cxxflags.extend(['-DDELPHES_USE']) if options.has_delphesMA5tune_tag: cxxflags.extend(['-DDELPHESMA5TUNE_USE']) + if options.has_sqlite_tag: + cxxflags.extend(['-DSQLITE3_USE']) if len(cxxflags)!=0: file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n') file.write('\n') @@ -361,7 +366,7 @@ def Makefile( libs=[] # added SQL - file.write('LIBFLAGS = -l sqlite3\n') + #file.write('LIBFLAGS = -l sqlite3\n') # - commons if options.has_commons: @@ -444,8 +449,12 @@ def Makefile( file.write('LIBFLAGS += -lHEPTopTagger_for_ma5\n') # SQLite3 - #if options.has_sqlite: - # file.write('LIBFLAGS += -l sqlite3\n') + if options.has_sqlite: + file.write('LIBFLAGS += -l sqlite3\n') + + if options.has_sqlite_lib: + file.write('LIBFLAGS += -l sqlite3\n') + # - Commons if options.has_commons: @@ -482,7 +491,7 @@ def Makefile( libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libsubstructure_for_ma5.so') if options.has_heptoptagger: libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libHEPTopTagger_for_ma5.so') - if options.has_sqlite: + if options.has_sqlite_lib: libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libsqlite_for_ma5.so') if len(libs)!=0: file.write('# Requirements to check before building\n') diff --git a/madanalysis/core/library_builder.py b/madanalysis/core/library_builder.py index cdb2c685..d6f476f0 100644 --- a/madanalysis/core/library_builder.py +++ b/madanalysis/core/library_builder.py @@ -80,6 +80,10 @@ def checkMA5(self): libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libdelphes_for_ma5.so') if self.archi_info.has_delphesMA5tune: libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libdelphesMA5tune_for_ma5.so') + if self.archi_info.has_sqlite3: + libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libsqlite_for_ma5.so') + + for library in libraries: if not os.path.isfile(library): self.logger.debug('\t-> library '+ library + " not found.") diff --git a/madanalysis/core/main.py b/madanalysis/core/main.py index 90c6cadb..3b8c224f 100644 --- a/madanalysis/core/main.py +++ b/madanalysis/core/main.py @@ -626,6 +626,16 @@ def BuildLibrary(self,forced=False): self.archi_info.ma5dir + '/tools/SampleAnalyzer/Bin/TestRoot', self.archi_info.ma5dir + '/tools/SampleAnalyzer/Test/', True]) + # SQLite3 + if self.archi_info.has_sqlite3: + libraries.append(['SQLite3', 'interface to SQLite3', 'sqlite', + self.archi_info.ma5dir + '/tools/SampleAnalyzer/Lib/libsqlite_for_ma5.so', + self.archi_info.ma5dir + '/tools/SampleAnalyzer/Interfaces', False]) + #libraries.append(['test_sqlite', 'interface to SQLite3', 'test_sqlite', + # self.archi_info.ma5dir + '/tools/SampleAnalyzer/Bin/TestSQLite', + # self.archi_info.ma5dir + '/tools/SampleAnalyzer/Test/', True]) + + # Process libraries.append(['process', 'SampleAnalyzer core', 'process', self.archi_info.ma5dir + '/tools/SampleAnalyzer/Lib/libprocess_for_ma5.so', @@ -634,6 +644,7 @@ def BuildLibrary(self,forced=False): self.archi_info.ma5dir + '/tools/SampleAnalyzer/Bin/TestSampleAnalyzer', self.archi_info.ma5dir + '/tools/SampleAnalyzer/Test/', True]) + # Writing the Makefiles self.logger.info("") diff --git a/madanalysis/system/architecture_info.py b/madanalysis/system/architecture_info.py index a0fcd393..9faf243f 100644 --- a/madanalysis/system/architecture_info.py +++ b/madanalysis/system/architecture_info.py @@ -101,6 +101,7 @@ def __init__(self): self.delphesMA5tune_lib="" self.fastjet_bin_path="" self.fastjet_lib_paths=[] + # C++ compiler versions self.cpp11 = False diff --git a/madanalysis/system/detect_sqlite.py b/madanalysis/system/detect_sqlite.py index a8431026..5436db4c 100644 --- a/madanalysis/system/detect_sqlite.py +++ b/madanalysis/system/detect_sqlite.py @@ -47,6 +47,7 @@ def __init__(self, archi_info, user_info, session_info, debug): self.logger = logging.getLogger('MA5') # adding what you want here + self.headernames=['DatabaseManager.h'] def PrintDisableMessage(self): self.logger.warning("sqlite3 not detected, multiweight output format will not be supported!") diff --git a/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h b/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h index dba54e50..b1ccb3ce 100644 --- a/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h +++ b/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h @@ -37,7 +37,10 @@ #include "SampleAnalyzer/Commons/DataFormat/MCProcessFormat.h" #include "SampleAnalyzer/Commons/DataFormat/WeightDefinition.h" #include "SampleAnalyzer/Commons/Service/LogService.h" -#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + +#ifdef SQLITE3_USE + #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" +#endif namespace MA5 @@ -133,13 +136,14 @@ class MCSampleFormat weight_names[id] = name; } +#ifdef SQLITE3_USE void WriteWeightNames(DatabaseManager &db){ db.createWeightNamesTable(); for(auto &p : weight_names){ db.addWeightDefinition(p.first, p.second); } } - +#endif /// Accessoir to the generator type const MA5GEN::GeneratorType* GeneratorType() const { return sample_generator_; } diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index dc5abbce..cafe90f6 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -38,7 +38,10 @@ #include "SampleAnalyzer/Commons/Base/Configuration.h" #include "SampleAnalyzer/Commons/Service/ExceptionService.h" #include "SampleAnalyzer/Commons/Service/Physics.h" -#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + +#ifdef SQLITE3_USE + #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" +#endif using namespace MA5; @@ -805,15 +808,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, out.Finalize(); } - // Create histo SQlite file - for(MAuint32 i = 0; i < analyzers_.size(); ++i){ - std::string path = analyzers_[i]->Output() + "/Histograms/histo.db"; - DatabaseManager dbManager(path); - dbManager.createHistoTables(); - analyzers_[i]->Manager()->GetPlotManager()->WriteSQL(dbManager); - dbManager.closeDB(); - } - + // Saving the cut flows for(MAuint32 i=0; i& mySamples, } } +#ifdef SQLITE3_USE + + // Create histo SQlite file + for(MAuint32 i = 0; i < analyzers_.size(); ++i){ + std::string path = analyzers_[i]->Output() + "/Histograms/histo.db"; + DatabaseManager dbManager(path); + dbManager.createHistoTables(); + analyzers_[i]->Manager()->GetPlotManager()->WriteSQL(dbManager); + dbManager.closeDB(); + } //save multi-weight cutflows to SQL - Kyle Fan for(int i = 0; i < analyzers_.size(); ++i){ @@ -848,7 +853,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, } dbManager.closeDB(); } - +#endif //end of multi-weight cutflow code diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.h b/tools/SampleAnalyzer/Process/Counter/CounterManager.h index 6a23179a..65dc068e 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.h +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.h @@ -35,7 +35,10 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/Counter.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" -#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + +#ifdef SQLITE3_USE + #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" +#endif namespace MA5 @@ -109,9 +112,10 @@ class CounterManager /// Write the counters in a Text file void Write_TextFormat(SAFWriter& output) const; - +#ifdef SQLITE3_USE //Write to SQL database void WriteSQL(DatabaseManager &db, bool &AddInitial, std::string region_name); +#endif /// Write the counters in a ROOT file // void Write_RootFormat(TFile* output) const; diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.cpp b/tools/SampleAnalyzer/Process/Plot/Histo.cpp index 5af388f2..9cb51674 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.cpp +++ b/tools/SampleAnalyzer/Process/Plot/Histo.cpp @@ -75,7 +75,7 @@ void Histo::WriteSQL(DatabaseManager &db){ //for each weight histo,weight id pair: add bucket data to Data table db.addData(name_, weight_id.first, "underflow", weight_id.second.underflow_.first, weight_id.second.underflow_.second); for(int i = 0; i < nbins_; ++i){ - db.addData(name_, weight_id.first, "bin " + to_string(i+1), weight_id.second.histo_[i].first, weight_id.second.histo_[i].second); + db.addData(name_, weight_id.first, to_string(i+1), weight_id.second.histo_[i].first, weight_id.second.histo_[i].second); } db.addData(name_, weight_id.first, "overflow", weight_id.second.overflow_.first, weight_id.second.overflow_.second); } diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.h b/tools/SampleAnalyzer/Process/Plot/Histo.h index 1e385f88..dd61dec7 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.h +++ b/tools/SampleAnalyzer/Process/Plot/Histo.h @@ -35,7 +35,9 @@ #include "SampleAnalyzer/Process/Plot/PlotBase.h" #include "SampleAnalyzer/Process/RegionSelection/RegionSelection.h" #include "SampleAnalyzer/Commons/Service/ExceptionService.h" -#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" +#ifdef SQLITE3_USE + #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" +#endif struct MultiWeightHisto { @@ -316,7 +318,10 @@ class Histo : public PlotBase /// Write the plot in a ROOT file virtual void Write_TextFormat(std::ostream* output); + +#ifdef SQLITE3_USE virtual void WriteSQL(DatabaseManager &db); +#endif /// Write the plot in a ROOT file // virtual void Write_RootFormat(std::pair& histos); diff --git a/tools/SampleAnalyzer/Process/Plot/PlotBase.h b/tools/SampleAnalyzer/Process/Plot/PlotBase.h index 6d99c5ef..2bfd39bd 100644 --- a/tools/SampleAnalyzer/Process/Plot/PlotBase.h +++ b/tools/SampleAnalyzer/Process/Plot/PlotBase.h @@ -34,7 +34,9 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Commons/Service/LogService.h" -#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" +#ifdef SQLITE3_USE + #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" +#endif struct MultiweightEvents { @@ -168,7 +170,9 @@ class PlotBase /// Write the plot in a ROOT file virtual void Write_TextFormat(std::ostream* output) = 0; +#ifdef SQLITE3_USE virtual void WriteSQL(DatabaseManager &db) {}; +#endif /// Increment number of events void IncrementNEvents(MAfloat64 weight=1.0) diff --git a/tools/SampleAnalyzer/Process/Plot/PlotManager.h b/tools/SampleAnalyzer/Process/Plot/PlotManager.h index 94307884..a71a19cf 100644 --- a/tools/SampleAnalyzer/Process/Plot/PlotManager.h +++ b/tools/SampleAnalyzer/Process/Plot/PlotManager.h @@ -38,7 +38,10 @@ #include "SampleAnalyzer/Process/Plot/HistoFrequency.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" #include "SampleAnalyzer/Process/RegionSelection/RegionSelection.h" -#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + +#ifdef SQLITE3_USE + #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" +#endif namespace MA5 { @@ -139,11 +142,13 @@ class PlotManager /// Write the counters in a Text file void Write_TextFormat(SAFWriter& output); +#ifdef SQLITE3_USE void WriteSQL(DatabaseManager &db){ for(auto &pl : plots_){ pl->WriteSQL(db); } } +#endif /// Finalizing diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h index 11f8d3c1..d0c07185 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h @@ -34,7 +34,10 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Process/Counter/CounterManager.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" -#include "SampleAnalyzer/Process/Writer/DatabaseManager.h" + +#ifdef SQLITE3_USE + #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" +#endif namespace MA5 @@ -87,10 +90,12 @@ class RegionSelection void WriteCutflow(SAFWriter& output) { cutflow_.Write_TextFormat(output);} +#ifdef SQLITE3_USE //write to SQL database void WriteSQL(DatabaseManager &db, bool &AddInitial){ cutflow_.WriteSQL(db, AddInitial, name_); } +#endif /// Set methods void SetName(std::string name) diff --git a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h b/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h deleted file mode 100644 index 1e4d62e2..00000000 --- a/tools/SampleAnalyzer/Process/Writer/DatabaseManager.h +++ /dev/null @@ -1,248 +0,0 @@ -#pragma once - - -#include -#include -#include - -using namespace std; - -class DatabaseManager { - - private: - // Pointer to SQLite connection - sqlite3 *db; - - // Save any error messages - char *zErrMsg; - - // Save the result of opening the file - int rc; - - // Saved SQL - string sql; - - // Create a callback function - static int callback(void *NotUsed, int argc, char **argv, char **azColName) { - - // int argc: holds the number of results - // (array) azColName: holds each column returned - // (array) argv: holds each value - - for(int i = 0; i < argc; i++) { - - // Show column name, value, and newline - cout << azColName[i] << ": " << argv[i] << endl; - - } - - // Insert a newline - cout << endl; - - // Return successful - return 0; - } - - bool checkDBErrors(string msg) { - - if( rc ){ - // Show an error message - cout << "DB Error: " << sqlite3_errmsg(db) << " " << msg << endl; - return false; - - } - else { - cout << " success @ " << msg << endl; - return true; - } - - } - - - public: - - DatabaseManager(string path) { - // Save the result of opening the file - rc = sqlite3_open(path.c_str(), &db); - // enable foreign key constraint - sqlite3_exec(db, "PRAGMA foreign_keys = ON;", 0 ,0 ,0); - bool open_success = checkDBErrors("opening DB"); - if(!open_success) { - sqlite3_close(db); - cout << "open DB failed!" << endl; - } - } - - void createCutflowTables() { - - // Save SQL to create a table - sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \ - "region_name TEXT NOT NULL,"\ - "cut_name TEXT NOT NULL," \ - "primary key (region_name, cut_name));"; - - // Run the SQL - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating cutflow table"); - - sql = "CREATE TABLE IF NOT EXISTS Weights(" \ - "r_name TEXT NOT NULL," \ - "c_name TEXT NOT NULL," \ - "id INTEGER NOT NULL," \ - "pos_entries INTEGER NOT NULL," \ - "neg_entries INTEGER NOT NULL," \ - "pos_sum DOUBLE NOT NULL," \ - "neg_sum DOUBLE NOT NULL," \ - "pos_squared_sum DOUBLE NOT NULL,"\ - "neg_squared_sum DOUBLE NOT NULL,"\ - "primary key (r_name, c_name, id)" \ - "foreign key (r_name, c_name) references Cutflow(region_name, cut_name) ON DELETE CASCADE);"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating weights table"); - } - - void createHistoTables(){ - sql = "CREATE TABLE IF NOT EXISTS HistoDescription("\ - "name TEXT NOT NULL,"\ - "num_of_bins INTEGER NOT NULL,"\ - "xmin DOUBLE NOT NULL,"\ - "xmax DOUBLE NOT NULL,"\ - "regions TEXT NOT NULL,"\ - "primary key(name) );"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating HistoDescription table"); - - sql = "CREATE TABLE IF NOT EXISTS Statistics("\ - "name TEXT NOT NULL,"\ - "id TEXT NOT NULL,"\ - "pos_num_events INTEGER NOT NULL,"\ - "neg_num_events INTEGER NOT NULL,"\ - "pos_sum_event_weights_over_events DOUBLE NOT NULL,"\ - "neg_sum_event_weights_over_events DOUBLE NOT NULL,"\ - "pos_entries INTEGER NOT NULL,"\ - "neg_entries INTEGER NOT NULL,"\ - "pos_sum_event_weights_over_entries DOUBLE NOT NULL,"\ - "neg_sum_event_weights_over_entries DOUBLE NOT NULL,"\ - "pos_sum_squared_weights DOUBLE NOT NULL,"\ - "neg_sum_squared_weights DOUBLE NOT NULL,"\ - "pos_value_times_weight DOUBLE NOT NULL,"\ - "neg_value_times_weight DOUBLE NOT NULL,"\ - "pos_value_squared_times_weight DOUBLE NOT NULL,"\ - "neg_value_squared_times_weight DOUBLE NOT NULL,"\ - "primary key(name, id) );"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating Statistics table"); - - sql = "CREATE TABLE IF NOT EXISTS Data("\ - "name TEXT NOT NULL,"\ - "id INTERGER NOT NULL,"\ - "bin TEXT NOT NULL,"\ - "positive DOUBLE NOT NULL,"\ - "negative DOUBLE NOT NULL,"\ - "primary key (name, id, bin) );"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating Data table"); - - } - - void createWeightNamesTable(){ - sql = "CREATE TABLE IF NOT EXISTS WeightDefinition("\ - "id INTERGER NOT NULL,"\ - "definition TEXT NOT NULL,"\ - "primary key (id) );"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating Weight Names table"); - } - - void addWeightDefinition(int id, string def){ - sql = "INSERT INTO WeightDefinition VALUES ('" + to_string(id) + "','" + def + "')"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - } - - void addHisto(string name, int bins, double xmin, double xmax, string regions){ - sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting into Histo: " + name); - } - - void addStatistic(string name, - int id, - int pos_events, - int neg_events, - double pos_sum_events, - double neg_sum_events, - int pos_entries, - int neg_entries, - double pos_sum_entries, - double neg_sum_entries, - double pos_sum_squared, - double neg_sum_squared, - double pos_val_weight, - double neg_val_weight, - double pos_val2_weight, - double neg_val2_weight){ - - sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(id) + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting into Statistics: " + name); - - } - - void addData(string name, int id, string bin, double positive, double negative){ - sql = "INSERT INTO Data VALUES ('" + name + "'" + "," + "'" + to_string(id) + "'" + "," + "'" + bin + "'" + "," + "'" + to_string(positive) + "'" + "," + "'" + to_string(negative) + "')"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting into Data: " + name + " " + to_string(id)); - - } - - //add cut to databse with primary keys region name and cut name - void addCut(string r_name, string c_name) { - - - sql = "INSERT INTO Cutflow VALUES ('" + r_name + "'" + "," + "'" + c_name + "')"; - //cout << sql << endl; - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting cutflow: " + r_name + " " + c_name); - - } - - //add weight to database with primary keys region name, cut name, and weight id - void addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { - sql = "INSERT INTO Weights VALUES ('" + r_name + "'" + ",'" + c_name + "','" + to_string(id) + "','" + to_string(pos) + "','" + to_string(neg) + "','" + to_string(pos_sum) + "','" + to_string(neg_sum) \ - + "','" + to_string(pos_2sum) + "','" + to_string(neg_2sum) + "')"; - //cout << sql << endl; - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id)); - - } - - void closeDB() { - - // Close the SQL connection - sqlite3_close(db); - - } - - - -}; - -/* -int main() { - DatabaseManager dbManager("test.db"); - dbManager.createTables(); - dbManager.addCut("region1", "cut1"); - dbManager.addWeight("region1", "cut1", 1, 2, 2, 2, -2, 4, 4); - dbManager.addWeight("region1", "cut1", 10, 2, 3, 4, 5, 6, 7); - dbManager.closeDB(); - -} - -*/ - - - From 3777adb350af68b5aa88eab28fa5011c6616936b Mon Sep 17 00:00:00 2001 From: Kyle Date: Wed, 28 Sep 2022 02:45:36 -0600 Subject: [PATCH 27/53] readded databasemanager to interfaces --- .../Interfaces/SQLite3/.DS_Store | Bin 0 -> 6148 bytes .../Interfaces/SQLite3/DatabaseManager.h | 248 ++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 tools/SampleAnalyzer/Interfaces/SQLite3/.DS_Store create mode 100644 tools/SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h diff --git a/tools/SampleAnalyzer/Interfaces/SQLite3/.DS_Store b/tools/SampleAnalyzer/Interfaces/SQLite3/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..855d85c3ba838e5a31e4cb09c618262448dbee27 GIT binary patch literal 6148 zcmeHKJx>Bb5S>v95*zF+_ZOJ(4>sf|D6ISgdP=Y$k|^}sQShVn&4&hZF`=>`?PDLgnGKLw;-rNxO+-!iWj3^j}GkSTJPg%+X_&iwct#ruW!j9|T zv0ve=54o4O#d`D7uQxri?WZ1K?JI#bK?SG)6`%rCfC`+efc0Krd=pGh1*iZO_*KBZ z4+UmilUuNVI^cW+0FMxMVC}O6xL5#OlUpz%fJP}eN;O9ejdH|G=GEjD9Oa^ZGtSAI zH76AH+Yv9GE?R@>sQ?u?Rp2_d6YKwT*dO!%X%Tl+fC^lc0y^p*yBS_7d+Xrktk)LU s6YK}b^>jFI#b9s6*jOvRJIX7z#`Buog1sDZF9-9FfayX|f&Wn88_KyK$N&HU literal 0 HcmV?d00001 diff --git a/tools/SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h b/tools/SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h new file mode 100644 index 00000000..1e4d62e2 --- /dev/null +++ b/tools/SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h @@ -0,0 +1,248 @@ +#pragma once + + +#include +#include +#include + +using namespace std; + +class DatabaseManager { + + private: + // Pointer to SQLite connection + sqlite3 *db; + + // Save any error messages + char *zErrMsg; + + // Save the result of opening the file + int rc; + + // Saved SQL + string sql; + + // Create a callback function + static int callback(void *NotUsed, int argc, char **argv, char **azColName) { + + // int argc: holds the number of results + // (array) azColName: holds each column returned + // (array) argv: holds each value + + for(int i = 0; i < argc; i++) { + + // Show column name, value, and newline + cout << azColName[i] << ": " << argv[i] << endl; + + } + + // Insert a newline + cout << endl; + + // Return successful + return 0; + } + + bool checkDBErrors(string msg) { + + if( rc ){ + // Show an error message + cout << "DB Error: " << sqlite3_errmsg(db) << " " << msg << endl; + return false; + + } + else { + cout << " success @ " << msg << endl; + return true; + } + + } + + + public: + + DatabaseManager(string path) { + // Save the result of opening the file + rc = sqlite3_open(path.c_str(), &db); + // enable foreign key constraint + sqlite3_exec(db, "PRAGMA foreign_keys = ON;", 0 ,0 ,0); + bool open_success = checkDBErrors("opening DB"); + if(!open_success) { + sqlite3_close(db); + cout << "open DB failed!" << endl; + } + } + + void createCutflowTables() { + + // Save SQL to create a table + sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \ + "region_name TEXT NOT NULL,"\ + "cut_name TEXT NOT NULL," \ + "primary key (region_name, cut_name));"; + + // Run the SQL + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating cutflow table"); + + sql = "CREATE TABLE IF NOT EXISTS Weights(" \ + "r_name TEXT NOT NULL," \ + "c_name TEXT NOT NULL," \ + "id INTEGER NOT NULL," \ + "pos_entries INTEGER NOT NULL," \ + "neg_entries INTEGER NOT NULL," \ + "pos_sum DOUBLE NOT NULL," \ + "neg_sum DOUBLE NOT NULL," \ + "pos_squared_sum DOUBLE NOT NULL,"\ + "neg_squared_sum DOUBLE NOT NULL,"\ + "primary key (r_name, c_name, id)" \ + "foreign key (r_name, c_name) references Cutflow(region_name, cut_name) ON DELETE CASCADE);"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating weights table"); + } + + void createHistoTables(){ + sql = "CREATE TABLE IF NOT EXISTS HistoDescription("\ + "name TEXT NOT NULL,"\ + "num_of_bins INTEGER NOT NULL,"\ + "xmin DOUBLE NOT NULL,"\ + "xmax DOUBLE NOT NULL,"\ + "regions TEXT NOT NULL,"\ + "primary key(name) );"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating HistoDescription table"); + + sql = "CREATE TABLE IF NOT EXISTS Statistics("\ + "name TEXT NOT NULL,"\ + "id TEXT NOT NULL,"\ + "pos_num_events INTEGER NOT NULL,"\ + "neg_num_events INTEGER NOT NULL,"\ + "pos_sum_event_weights_over_events DOUBLE NOT NULL,"\ + "neg_sum_event_weights_over_events DOUBLE NOT NULL,"\ + "pos_entries INTEGER NOT NULL,"\ + "neg_entries INTEGER NOT NULL,"\ + "pos_sum_event_weights_over_entries DOUBLE NOT NULL,"\ + "neg_sum_event_weights_over_entries DOUBLE NOT NULL,"\ + "pos_sum_squared_weights DOUBLE NOT NULL,"\ + "neg_sum_squared_weights DOUBLE NOT NULL,"\ + "pos_value_times_weight DOUBLE NOT NULL,"\ + "neg_value_times_weight DOUBLE NOT NULL,"\ + "pos_value_squared_times_weight DOUBLE NOT NULL,"\ + "neg_value_squared_times_weight DOUBLE NOT NULL,"\ + "primary key(name, id) );"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Statistics table"); + + sql = "CREATE TABLE IF NOT EXISTS Data("\ + "name TEXT NOT NULL,"\ + "id INTERGER NOT NULL,"\ + "bin TEXT NOT NULL,"\ + "positive DOUBLE NOT NULL,"\ + "negative DOUBLE NOT NULL,"\ + "primary key (name, id, bin) );"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Data table"); + + } + + void createWeightNamesTable(){ + sql = "CREATE TABLE IF NOT EXISTS WeightDefinition("\ + "id INTERGER NOT NULL,"\ + "definition TEXT NOT NULL,"\ + "primary key (id) );"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Weight Names table"); + } + + void addWeightDefinition(int id, string def){ + sql = "INSERT INTO WeightDefinition VALUES ('" + to_string(id) + "','" + def + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + } + + void addHisto(string name, int bins, double xmin, double xmax, string regions){ + sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting into Histo: " + name); + } + + void addStatistic(string name, + int id, + int pos_events, + int neg_events, + double pos_sum_events, + double neg_sum_events, + int pos_entries, + int neg_entries, + double pos_sum_entries, + double neg_sum_entries, + double pos_sum_squared, + double neg_sum_squared, + double pos_val_weight, + double neg_val_weight, + double pos_val2_weight, + double neg_val2_weight){ + + sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(id) + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting into Statistics: " + name); + + } + + void addData(string name, int id, string bin, double positive, double negative){ + sql = "INSERT INTO Data VALUES ('" + name + "'" + "," + "'" + to_string(id) + "'" + "," + "'" + bin + "'" + "," + "'" + to_string(positive) + "'" + "," + "'" + to_string(negative) + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting into Data: " + name + " " + to_string(id)); + + } + + //add cut to databse with primary keys region name and cut name + void addCut(string r_name, string c_name) { + + + sql = "INSERT INTO Cutflow VALUES ('" + r_name + "'" + "," + "'" + c_name + "')"; + //cout << sql << endl; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting cutflow: " + r_name + " " + c_name); + + } + + //add weight to database with primary keys region name, cut name, and weight id + void addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { + sql = "INSERT INTO Weights VALUES ('" + r_name + "'" + ",'" + c_name + "','" + to_string(id) + "','" + to_string(pos) + "','" + to_string(neg) + "','" + to_string(pos_sum) + "','" + to_string(neg_sum) \ + + "','" + to_string(pos_2sum) + "','" + to_string(neg_2sum) + "')"; + //cout << sql << endl; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id)); + + } + + void closeDB() { + + // Close the SQL connection + sqlite3_close(db); + + } + + + +}; + +/* +int main() { + DatabaseManager dbManager("test.db"); + dbManager.createTables(); + dbManager.addCut("region1", "cut1"); + dbManager.addWeight("region1", "cut1", 1, 2, 2, 2, -2, 4, 4); + dbManager.addWeight("region1", "cut1", 10, 2, 3, 4, 5, 6, 7); + dbManager.closeDB(); + +} + +*/ + + + From b393229ed89ca84ffc41c5d74ac63e8b698431a9 Mon Sep 17 00:00:00 2001 From: Kyle Date: Wed, 28 Sep 2022 02:51:02 -0600 Subject: [PATCH 28/53] removed .DS file --- .../SampleAnalyzer/Interfaces/SQLite3/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tools/SampleAnalyzer/Interfaces/SQLite3/.DS_Store diff --git a/tools/SampleAnalyzer/Interfaces/SQLite3/.DS_Store b/tools/SampleAnalyzer/Interfaces/SQLite3/.DS_Store deleted file mode 100644 index 855d85c3ba838e5a31e4cb09c618262448dbee27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKJx>Bb5S>v95*zF+_ZOJ(4>sf|D6ISgdP=Y$k|^}sQShVn&4&hZF`=>`?PDLgnGKLw;-rNxO+-!iWj3^j}GkSTJPg%+X_&iwct#ruW!j9|T zv0ve=54o4O#d`D7uQxri?WZ1K?JI#bK?SG)6`%rCfC`+efc0Krd=pGh1*iZO_*KBZ z4+UmilUuNVI^cW+0FMxMVC}O6xL5#OlUpz%fJP}eN;O9ejdH|G=GEjD9Oa^ZGtSAI zH76AH+Yv9GE?R@>sQ?u?Rp2_d6YKwT*dO!%X%Tl+fC^lc0y^p*yBS_7d+Xrktk)LU s6YK}b^>jFI#b9s6*jOvRJIX7z#`Buog1sDZF9-9FfayX|f&Wn88_KyK$N&HU From c505e01cf80a9db67730cc5206421f83c2a7b741 Mon Sep 17 00:00:00 2001 From: kfan326 Date: Wed, 19 Oct 2022 21:58:42 -0600 Subject: [PATCH 29/53] Update madanalysis/system/architecture_info.py Co-authored-by: Jack Y. Araz --- madanalysis/system/architecture_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/madanalysis/system/architecture_info.py b/madanalysis/system/architecture_info.py index 9faf243f..640a82e3 100644 --- a/madanalysis/system/architecture_info.py +++ b/madanalysis/system/architecture_info.py @@ -46,9 +46,10 @@ def __init__(self): self.has_zlib = False self.has_delphes = False self.has_delphesMA5tune = False + + # Flag for SQlite3 self.has_sqlite3 = False - # Library to put before all the others? self.root_priority = False From 9c22841df3123fdb382db90dfd85b7a3c5985428 Mon Sep 17 00:00:00 2001 From: kfan326 Date: Wed, 19 Oct 2022 21:59:15 -0600 Subject: [PATCH 30/53] Update madanalysis/IOinterface/library_writer.py Co-authored-by: Jack Y. Araz --- madanalysis/IOinterface/library_writer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/madanalysis/IOinterface/library_writer.py b/madanalysis/IOinterface/library_writer.py index ce661263..87ea8e6d 100644 --- a/madanalysis/IOinterface/library_writer.py +++ b/madanalysis/IOinterface/library_writer.py @@ -160,8 +160,6 @@ def WriteMakefileForInterfaces(self,package): title='*delphesMA5tune-interface* test' elif package=='test_root': title='*root-interface* test' - elif package=='test_sqlite': - title='*sqlite* test' else: title='interface to '+package From 430bf6311223101c46349db12d4b3fedc8f211f3 Mon Sep 17 00:00:00 2001 From: kfan326 Date: Wed, 19 Oct 2022 21:59:56 -0600 Subject: [PATCH 31/53] Update madanalysis/core/main.py Co-authored-by: Jack Y. Araz --- madanalysis/core/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/madanalysis/core/main.py b/madanalysis/core/main.py index 3b8c224f..85ec8876 100644 --- a/madanalysis/core/main.py +++ b/madanalysis/core/main.py @@ -631,9 +631,6 @@ def BuildLibrary(self,forced=False): libraries.append(['SQLite3', 'interface to SQLite3', 'sqlite', self.archi_info.ma5dir + '/tools/SampleAnalyzer/Lib/libsqlite_for_ma5.so', self.archi_info.ma5dir + '/tools/SampleAnalyzer/Interfaces', False]) - #libraries.append(['test_sqlite', 'interface to SQLite3', 'test_sqlite', - # self.archi_info.ma5dir + '/tools/SampleAnalyzer/Bin/TestSQLite', - # self.archi_info.ma5dir + '/tools/SampleAnalyzer/Test/', True]) # Process From 3d94e2daf2a53c062f76425a82f0d04b8f978cf6 Mon Sep 17 00:00:00 2001 From: kfan326 Date: Wed, 19 Oct 2022 22:00:10 -0600 Subject: [PATCH 32/53] Update madanalysis/system/detect_sqlite.py Co-authored-by: Jack Y. Araz --- madanalysis/system/detect_sqlite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/madanalysis/system/detect_sqlite.py b/madanalysis/system/detect_sqlite.py index 5436db4c..d74e85a6 100644 --- a/madanalysis/system/detect_sqlite.py +++ b/madanalysis/system/detect_sqlite.py @@ -89,7 +89,6 @@ def ExtractInfo(self): def SaveInfo(self): - self.session_info.has_sqlite3=True self.archi_info.has_sqlite3 = True return True From af5fb18af3621c846d7be9d239680e1c9840bdf1 Mon Sep 17 00:00:00 2001 From: kfan326 Date: Wed, 19 Oct 2022 22:00:27 -0600 Subject: [PATCH 33/53] Update madanalysis/system/session_info.py Co-authored-by: Jack Y. Araz --- madanalysis/system/session_info.py | 1 - 1 file changed, 1 deletion(-) diff --git a/madanalysis/system/session_info.py b/madanalysis/system/session_info.py index 15b8f6d9..5016b3be 100644 --- a/madanalysis/system/session_info.py +++ b/madanalysis/system/session_info.py @@ -44,7 +44,6 @@ def __init__(self): self.has_pad = False self.has_padsfs = False self.has_padma5 = False - self.has_sqlite3 = False self.gcc_header_search_path = [] self.gcc_library_search_path = [] self.padma5_build_path = "" From 274e0d94328cdac880c0ee4fd4ae43b6017ef850 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 20 Oct 2022 21:46:54 -0600 Subject: [PATCH 34/53] made base class for SQL --- .../Commons/Base/DatabaseManager.h | 84 ++++++++++++++++++ tools/SampleAnalyzer/Commons/Base/SQLbase.h | 48 ++++++++++ .../Commons/DataFormat/MCSampleFormat.h | 8 +- .../Interfaces/sqlite/.DS_Store | Bin 0 -> 6148 bytes .../DatabaseManager.h => sqlite/SQLiteDB.h} | 30 ++----- .../Process/Counter/CounterManager.h | 7 +- tools/SampleAnalyzer/Process/Plot/Histo.h | 7 +- tools/SampleAnalyzer/Process/Plot/PlotBase.h | 7 +- .../SampleAnalyzer/Process/Plot/PlotManager.h | 7 +- .../Process/RegionSelection/RegionSelection.h | 7 +- 10 files changed, 152 insertions(+), 53 deletions(-) create mode 100644 tools/SampleAnalyzer/Commons/Base/DatabaseManager.h create mode 100644 tools/SampleAnalyzer/Commons/Base/SQLbase.h create mode 100644 tools/SampleAnalyzer/Interfaces/sqlite/.DS_Store rename tools/SampleAnalyzer/Interfaces/{SQLite3/DatabaseManager.h => sqlite/SQLiteDB.h} (95%) diff --git a/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h b/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h new file mode 100644 index 00000000..7d02a5c1 --- /dev/null +++ b/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h @@ -0,0 +1,84 @@ +#pragma once + + +#include +#include "SampleAnalyzer/Commons/Base/SQLbase.h" + +#ifdef SQLITE3_USE + #include "SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h" +#endif + +class DatabaseManager { + private: + SQLiteBase *manager; + + public: + DatabaseManager(std::string path) { + #ifdef SQLITE3_USE + manager = new SQLiteDB(path); + #else + manager = new SQLiteBase(path); + #endif + } + + ~DatabaseManager() { + delete manager; + } + + void createCutflowTables() { + manager->createCutflowTables(); + } + + void createHistoTables() { + manager->createHistoTables(); + } + + void createWeightNamesTable() { + manager->createWeightNamesTable(); + } + + void addWeightDefinition(int id, std::string def) { + manager->addWeightDefinition(id, def); + } + + void addHisto(std::string name, int bins, double xmin, double xmax, std::string regions) { + manager->addHisto(name, bins, xmin, xmax, regions); + } + + void addStatistic(std::string name, + int id, + int pos_events, + int neg_events, + double pos_sum_events, + double neg_sum_events, + int pos_entries, + int neg_entries, + double pos_sum_entries, + double neg_sum_entries, + double pos_sum_squared, + double neg_sum_squared, + double pos_val_weight, + double neg_val_weight, + double pos_val2_weight, + double neg_val2_weight) { + manager->addStatistic(name,id,pos_events, neg_events, pos_sum_events, neg_sum_events, pos_entries, neg_entries, pos_sum_entries, neg_sum_entries, pos_sum_squared, neg_sum_squared, pos_val_weight, neg_val_weight, pos_val2_weight, neg_val2_weight); + } + + void addData(std::string name, int id, std::string bin, double positive, double negative) { + manager->addData(name, id, bin, positive, negative); + } + + void addCut(std::string r_name, std::string c_name) { + manager->addCut(r_name, c_name); + } + + void addWeight(std::string r_name, std::string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { + manager->addWeight(r_name,c_name, id, pos, neg, pos_sum, neg_sum, pos_2sum, neg_2sum); + } + + void closeDB() { + manager->closeDB(); + } + + +}; diff --git a/tools/SampleAnalyzer/Commons/Base/SQLbase.h b/tools/SampleAnalyzer/Commons/Base/SQLbase.h new file mode 100644 index 00000000..13674313 --- /dev/null +++ b/tools/SampleAnalyzer/Commons/Base/SQLbase.h @@ -0,0 +1,48 @@ +#pragma once + +#include + + +class SQLiteBase { + public : + SQLiteBase(std::string path){} + + virtual ~SQLiteBase() {} + + virtual void createCutflowTables() {} + + virtual void createHistoTables() {} + + virtual void createWeightNamesTable() {} + + virtual void addWeightDefinition(int id, std::string def) {} + + virtual void addHisto(std::string name, int bins, double xmin, double xmax, std::string regions) {} + + virtual void addStatistic(std::string name, + int id, + int pos_events, + int neg_events, + double pos_sum_events, + double neg_sum_events, + int pos_entries, + int neg_entries, + double pos_sum_entries, + double neg_sum_entries, + double pos_sum_squared, + double neg_sum_squared, + double pos_val_weight, + double neg_val_weight, + double pos_val2_weight, + double neg_val2_weight) {} + + virtual void addData(std::string name, int id, std::string bin, double positive, double negative) {} + + virtual void addCut(std::string r_name, std::string c_name) {} + + virtual void addWeight(std::string r_name, std::string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) {} + + virtual void closeDB() {} + + +}; diff --git a/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h b/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h index b1ccb3ce..67b17881 100644 --- a/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h +++ b/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h @@ -38,9 +38,7 @@ #include "SampleAnalyzer/Commons/DataFormat/WeightDefinition.h" #include "SampleAnalyzer/Commons/Service/LogService.h" -#ifdef SQLITE3_USE - #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" -#endif +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" namespace MA5 @@ -136,14 +134,14 @@ class MCSampleFormat weight_names[id] = name; } -#ifdef SQLITE3_USE + void WriteWeightNames(DatabaseManager &db){ db.createWeightNamesTable(); for(auto &p : weight_names){ db.addWeightDefinition(p.first, p.second); } } -#endif + /// Accessoir to the generator type const MA5GEN::GeneratorType* GeneratorType() const { return sample_generator_; } diff --git a/tools/SampleAnalyzer/Interfaces/sqlite/.DS_Store b/tools/SampleAnalyzer/Interfaces/sqlite/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 #include #include -using namespace std; +#include "SampleAnalyzer/Commons/Base/SQLbase.h" -class DatabaseManager { +using namespace std; +class SQLiteDB : public SQLiteBase { private: // Pointer to SQLite connection sqlite3 *db; @@ -61,7 +61,7 @@ class DatabaseManager { public: - DatabaseManager(string path) { + SQLiteDB(string path) : SQLiteBase(path){ // Save the result of opening the file rc = sqlite3_open(path.c_str(), &db); // enable foreign key constraint @@ -220,29 +220,15 @@ class DatabaseManager { } - void closeDB() { + void closeDB() + + { // Close the SQL connection sqlite3_close(db); } - + }; - -/* -int main() { - DatabaseManager dbManager("test.db"); - dbManager.createTables(); - dbManager.addCut("region1", "cut1"); - dbManager.addWeight("region1", "cut1", 1, 2, 2, 2, -2, 4, 4); - dbManager.addWeight("region1", "cut1", 10, 2, 3, 4, 5, 6, 7); - dbManager.closeDB(); - -} - -*/ - - - diff --git a/tools/SampleAnalyzer/Process/Counter/CounterManager.h b/tools/SampleAnalyzer/Process/Counter/CounterManager.h index 65dc068e..e281b151 100644 --- a/tools/SampleAnalyzer/Process/Counter/CounterManager.h +++ b/tools/SampleAnalyzer/Process/Counter/CounterManager.h @@ -36,9 +36,8 @@ #include "SampleAnalyzer/Process/Counter/Counter.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" -#ifdef SQLITE3_USE - #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" -#endif +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" + namespace MA5 @@ -112,10 +111,8 @@ class CounterManager /// Write the counters in a Text file void Write_TextFormat(SAFWriter& output) const; -#ifdef SQLITE3_USE //Write to SQL database void WriteSQL(DatabaseManager &db, bool &AddInitial, std::string region_name); -#endif /// Write the counters in a ROOT file // void Write_RootFormat(TFile* output) const; diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.h b/tools/SampleAnalyzer/Process/Plot/Histo.h index dd61dec7..b65ce583 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.h +++ b/tools/SampleAnalyzer/Process/Plot/Histo.h @@ -35,9 +35,8 @@ #include "SampleAnalyzer/Process/Plot/PlotBase.h" #include "SampleAnalyzer/Process/RegionSelection/RegionSelection.h" #include "SampleAnalyzer/Commons/Service/ExceptionService.h" -#ifdef SQLITE3_USE - #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" -#endif + +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" struct MultiWeightHisto { @@ -319,9 +318,7 @@ class Histo : public PlotBase /// Write the plot in a ROOT file virtual void Write_TextFormat(std::ostream* output); -#ifdef SQLITE3_USE virtual void WriteSQL(DatabaseManager &db); -#endif /// Write the plot in a ROOT file // virtual void Write_RootFormat(std::pair& histos); diff --git a/tools/SampleAnalyzer/Process/Plot/PlotBase.h b/tools/SampleAnalyzer/Process/Plot/PlotBase.h index 2bfd39bd..89dfa795 100644 --- a/tools/SampleAnalyzer/Process/Plot/PlotBase.h +++ b/tools/SampleAnalyzer/Process/Plot/PlotBase.h @@ -34,9 +34,8 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Commons/Service/LogService.h" -#ifdef SQLITE3_USE - #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" -#endif + +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" struct MultiweightEvents { @@ -170,9 +169,7 @@ class PlotBase /// Write the plot in a ROOT file virtual void Write_TextFormat(std::ostream* output) = 0; -#ifdef SQLITE3_USE virtual void WriteSQL(DatabaseManager &db) {}; -#endif /// Increment number of events void IncrementNEvents(MAfloat64 weight=1.0) diff --git a/tools/SampleAnalyzer/Process/Plot/PlotManager.h b/tools/SampleAnalyzer/Process/Plot/PlotManager.h index a71a19cf..76aa8442 100644 --- a/tools/SampleAnalyzer/Process/Plot/PlotManager.h +++ b/tools/SampleAnalyzer/Process/Plot/PlotManager.h @@ -39,9 +39,8 @@ #include "SampleAnalyzer/Process/Writer/SAFWriter.h" #include "SampleAnalyzer/Process/RegionSelection/RegionSelection.h" -#ifdef SQLITE3_USE - #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" -#endif +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" + namespace MA5 { @@ -142,13 +141,11 @@ class PlotManager /// Write the counters in a Text file void Write_TextFormat(SAFWriter& output); -#ifdef SQLITE3_USE void WriteSQL(DatabaseManager &db){ for(auto &pl : plots_){ pl->WriteSQL(db); } } -#endif /// Finalizing diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h index d0c07185..9841fe79 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelection.h @@ -35,10 +35,7 @@ #include "SampleAnalyzer/Process/Counter/CounterManager.h" #include "SampleAnalyzer/Process/Writer/SAFWriter.h" -#ifdef SQLITE3_USE - #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" -#endif - +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" namespace MA5 { @@ -90,12 +87,10 @@ class RegionSelection void WriteCutflow(SAFWriter& output) { cutflow_.Write_TextFormat(output);} -#ifdef SQLITE3_USE //write to SQL database void WriteSQL(DatabaseManager &db, bool &AddInitial){ cutflow_.WriteSQL(db, AddInitial, name_); } -#endif /// Set methods void SetName(std::string name) From 0ce8e891daed11261b338dd78e3a4fd2d537ea67 Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 21 Oct 2022 00:56:34 -0600 Subject: [PATCH 35/53] fixed interface for SQLite --- madanalysis/IOinterface/library_writer.py | 9 +- madanalysis/build/makefile_writer.py | 3 +- .../Interfaces/sqlite/SQLiteDB.cpp | 212 ++++++++++++++++++ .../Interfaces/sqlite/SQLiteDB.h | 200 ++--------------- .../Process/Core/SampleAnalyzer.cpp | 9 +- 5 files changed, 238 insertions(+), 195 deletions(-) create mode 100644 tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.cpp diff --git a/madanalysis/IOinterface/library_writer.py b/madanalysis/IOinterface/library_writer.py index 87ea8e6d..bdc74271 100644 --- a/madanalysis/IOinterface/library_writer.py +++ b/madanalysis/IOinterface/library_writer.py @@ -131,9 +131,7 @@ def WriteMakefileForInterfaces(self,package): filename = self.path+"/SampleAnalyzer/Test/Makefile_delphesMA5tune" elif package=='test_root': filename = self.path+"/SampleAnalyzer/Test/Makefile_root" - elif package=='test_sqlite': - filename = self.path+"/SampleAnalyzer/Test/Makefile_sqlite" - + # Header title='' if package=='commons': @@ -383,10 +381,7 @@ def WriteMakefileForInterfaces(self,package): hfiles = ['DelphesMA5tune/*.h'] elif package=='test_root': cppfiles = ['Root/*.cpp'] - hfiles = ['Root/*.h'] - elif package=='test_sqlite': - cppfiles = ['SQLite/*.cpp'] - hfiles = ['SQLite/*.h'] + hfiles = ['Root/*.h'] else: cppfiles = [package+'/*.cpp'] hfiles = [package+'/*.h'] diff --git a/madanalysis/build/makefile_writer.py b/madanalysis/build/makefile_writer.py index 24a83751..2fdaa10b 100644 --- a/madanalysis/build/makefile_writer.py +++ b/madanalysis/build/makefile_writer.py @@ -453,7 +453,8 @@ def Makefile( file.write('LIBFLAGS += -l sqlite3\n') if options.has_sqlite_lib: - file.write('LIBFLAGS += -l sqlite3\n') + ##file.write('LIBFLAGS += -l sqlite3\n') + file.write('LIBFLAGS += -l sqlite_for_ma5\n') # - Commons diff --git a/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.cpp b/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.cpp new file mode 100644 index 00000000..6345908f --- /dev/null +++ b/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.cpp @@ -0,0 +1,212 @@ +#include +#include +#include + +#include "SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h" + + +using namespace std; + +int SQLiteDB::callback(void *NotUsed, int argc, char **argv, char **azColName) { + + // int argc: holds the number of results + // (array) azColName: holds each column returned + // (array) argv: holds each value + + for(int i = 0; i < argc; i++) { + + // Show column name, value, and newline + cout << azColName[i] << ": " << argv[i] << endl; + + } + + // Insert a newline + cout << endl; + + // Return successful + return 0; +} + +bool SQLiteDB::checkDBErrors(string msg){ + if( rc ){ + // Show an error message + cout << "DB Error: " << sqlite3_errmsg(db) << " " << msg << endl; + return false; + } + else { + cout << " success @ " << msg << endl; + return true; + } + +} + +SQLiteDB::SQLiteDB(string path) : SQLiteBase(path){ + // Save the result of opening the file + rc = sqlite3_open(path.c_str(), &db); + // enable foreign key constraint + sqlite3_exec(db, "PRAGMA foreign_keys = ON;", 0 ,0 ,0); + bool open_success = checkDBErrors("opening DB"); + if(!open_success) { + sqlite3_close(db); + cout << "open DB failed!" << endl; + } +} + +void SQLiteDB::createCutflowTables() { + + // Save SQL to create a table + sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \ + "region_name TEXT NOT NULL,"\ + "cut_name TEXT NOT NULL," \ + "primary key (region_name, cut_name));"; + + // Run the SQL + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating cutflow table"); + + sql = "CREATE TABLE IF NOT EXISTS Weights(" \ + "r_name TEXT NOT NULL," \ + "c_name TEXT NOT NULL," \ + "id INTEGER NOT NULL," \ + "pos_entries INTEGER NOT NULL," \ + "neg_entries INTEGER NOT NULL," \ + "pos_sum DOUBLE NOT NULL," \ + "neg_sum DOUBLE NOT NULL," \ + "pos_squared_sum DOUBLE NOT NULL,"\ + "neg_squared_sum DOUBLE NOT NULL,"\ + "primary key (r_name, c_name, id)" \ + "foreign key (r_name, c_name) references Cutflow(region_name, cut_name) ON DELETE CASCADE);"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating weights table"); +} + +void SQLiteDB::createHistoTables(){ + sql = "CREATE TABLE IF NOT EXISTS HistoDescription("\ + "name TEXT NOT NULL,"\ + "num_of_bins INTEGER NOT NULL,"\ + "xmin DOUBLE NOT NULL,"\ + "xmax DOUBLE NOT NULL,"\ + "regions TEXT NOT NULL,"\ + "primary key(name) );"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating HistoDescription table"); + + sql = "CREATE TABLE IF NOT EXISTS Statistics("\ + "name TEXT NOT NULL,"\ + "id TEXT NOT NULL,"\ + "pos_num_events INTEGER NOT NULL,"\ + "neg_num_events INTEGER NOT NULL,"\ + "pos_sum_event_weights_over_events DOUBLE NOT NULL,"\ + "neg_sum_event_weights_over_events DOUBLE NOT NULL,"\ + "pos_entries INTEGER NOT NULL,"\ + "neg_entries INTEGER NOT NULL,"\ + "pos_sum_event_weights_over_entries DOUBLE NOT NULL,"\ + "neg_sum_event_weights_over_entries DOUBLE NOT NULL,"\ + "pos_sum_squared_weights DOUBLE NOT NULL,"\ + "neg_sum_squared_weights DOUBLE NOT NULL,"\ + "pos_value_times_weight DOUBLE NOT NULL,"\ + "neg_value_times_weight DOUBLE NOT NULL,"\ + "pos_value_squared_times_weight DOUBLE NOT NULL,"\ + "neg_value_squared_times_weight DOUBLE NOT NULL,"\ + "primary key(name, id) );"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Statistics table"); + + sql = "CREATE TABLE IF NOT EXISTS Data("\ + "name TEXT NOT NULL,"\ + "id INTERGER NOT NULL,"\ + "bin TEXT NOT NULL,"\ + "positive DOUBLE NOT NULL,"\ + "negative DOUBLE NOT NULL,"\ + "primary key (name, id, bin) );"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Data table"); + + } + + void SQLiteDB::createWeightNamesTable(){ + sql = "CREATE TABLE IF NOT EXISTS WeightDefinition("\ + "id INTERGER NOT NULL,"\ + "definition TEXT NOT NULL,"\ + "primary key (id) );"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating Weight Names table"); + } + + void SQLiteDB::addWeightDefinition(int id, string def){ + sql = "INSERT INTO WeightDefinition VALUES ('" + to_string(id) + "','" + def + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + } + + void SQLiteDB::addHisto(string name, int bins, double xmin, double xmax, string regions){ + sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting into Histo: " + name); + } + + void SQLiteDB::addStatistic(string name, + int id, + int pos_events, + int neg_events, + double pos_sum_events, + double neg_sum_events, + int pos_entries, + int neg_entries, + double pos_sum_entries, + double neg_sum_entries, + double pos_sum_squared, + double neg_sum_squared, + double pos_val_weight, + double neg_val_weight, + double pos_val2_weight, + double neg_val2_weight){ + + sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(id) + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; + + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting into Statistics: " + name); + + } + + void SQLiteDB::addData(string name, int id, string bin, double positive, double negative){ + sql = "INSERT INTO Data VALUES ('" + name + "'" + "," + "'" + to_string(id) + "'" + "," + "'" + bin + "'" + "," + "'" + to_string(positive) + "'" + "," + "'" + to_string(negative) + "')"; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting into Data: " + name + " " + to_string(id)); + + } + + //add cut to databse with primary keys region name and cut name + void SQLiteDB::addCut(string r_name, string c_name) { + + + sql = "INSERT INTO Cutflow VALUES ('" + r_name + "'" + "," + "'" + c_name + "')"; + //cout << sql << endl; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting cutflow: " + r_name + " " + c_name); + + } + + //add weight to database with primary keys region name, cut name, and weight id + void SQLiteDB::addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { + sql = "INSERT INTO Weights VALUES ('" + r_name + "'" + ",'" + c_name + "','" + to_string(id) + "','" + to_string(pos) + "','" + to_string(neg) + "','" + to_string(pos_sum) + "','" + to_string(neg_sum) \ + + "','" + to_string(pos_2sum) + "','" + to_string(neg_2sum) + "')"; + //cout << sql << endl; + rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); + //checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id)); + + } + + void SQLiteDB::closeDB() + + { + + // Close the SQL connection + sqlite3_close(db); + + } + + + diff --git a/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h b/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h index 483e05e0..c9a8acec 100644 --- a/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h +++ b/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h @@ -23,151 +23,24 @@ class SQLiteDB : public SQLiteBase { string sql; // Create a callback function - static int callback(void *NotUsed, int argc, char **argv, char **azColName) { + static int callback(void *NotUsed, int argc, char **argv, char **azColName); - // int argc: holds the number of results - // (array) azColName: holds each column returned - // (array) argv: holds each value + bool checkDBErrors(string msg); - for(int i = 0; i < argc; i++) { - - // Show column name, value, and newline - cout << azColName[i] << ": " << argv[i] << endl; - - } - - // Insert a newline - cout << endl; - - // Return successful - return 0; - } - - bool checkDBErrors(string msg) { - - if( rc ){ - // Show an error message - cout << "DB Error: " << sqlite3_errmsg(db) << " " << msg << endl; - return false; - - } - else { - cout << " success @ " << msg << endl; - return true; - } + public: - } + SQLiteDB(string path); + void createCutflowTables(); - public: + void createHistoTables(); - SQLiteDB(string path) : SQLiteBase(path){ - // Save the result of opening the file - rc = sqlite3_open(path.c_str(), &db); - // enable foreign key constraint - sqlite3_exec(db, "PRAGMA foreign_keys = ON;", 0 ,0 ,0); - bool open_success = checkDBErrors("opening DB"); - if(!open_success) { - sqlite3_close(db); - cout << "open DB failed!" << endl; - } - } - - void createCutflowTables() { - - // Save SQL to create a table - sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \ - "region_name TEXT NOT NULL,"\ - "cut_name TEXT NOT NULL," \ - "primary key (region_name, cut_name));"; - - // Run the SQL - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating cutflow table"); - - sql = "CREATE TABLE IF NOT EXISTS Weights(" \ - "r_name TEXT NOT NULL," \ - "c_name TEXT NOT NULL," \ - "id INTEGER NOT NULL," \ - "pos_entries INTEGER NOT NULL," \ - "neg_entries INTEGER NOT NULL," \ - "pos_sum DOUBLE NOT NULL," \ - "neg_sum DOUBLE NOT NULL," \ - "pos_squared_sum DOUBLE NOT NULL,"\ - "neg_squared_sum DOUBLE NOT NULL,"\ - "primary key (r_name, c_name, id)" \ - "foreign key (r_name, c_name) references Cutflow(region_name, cut_name) ON DELETE CASCADE);"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating weights table"); - } - - void createHistoTables(){ - sql = "CREATE TABLE IF NOT EXISTS HistoDescription("\ - "name TEXT NOT NULL,"\ - "num_of_bins INTEGER NOT NULL,"\ - "xmin DOUBLE NOT NULL,"\ - "xmax DOUBLE NOT NULL,"\ - "regions TEXT NOT NULL,"\ - "primary key(name) );"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating HistoDescription table"); - - sql = "CREATE TABLE IF NOT EXISTS Statistics("\ - "name TEXT NOT NULL,"\ - "id TEXT NOT NULL,"\ - "pos_num_events INTEGER NOT NULL,"\ - "neg_num_events INTEGER NOT NULL,"\ - "pos_sum_event_weights_over_events DOUBLE NOT NULL,"\ - "neg_sum_event_weights_over_events DOUBLE NOT NULL,"\ - "pos_entries INTEGER NOT NULL,"\ - "neg_entries INTEGER NOT NULL,"\ - "pos_sum_event_weights_over_entries DOUBLE NOT NULL,"\ - "neg_sum_event_weights_over_entries DOUBLE NOT NULL,"\ - "pos_sum_squared_weights DOUBLE NOT NULL,"\ - "neg_sum_squared_weights DOUBLE NOT NULL,"\ - "pos_value_times_weight DOUBLE NOT NULL,"\ - "neg_value_times_weight DOUBLE NOT NULL,"\ - "pos_value_squared_times_weight DOUBLE NOT NULL,"\ - "neg_value_squared_times_weight DOUBLE NOT NULL,"\ - "primary key(name, id) );"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating Statistics table"); - - sql = "CREATE TABLE IF NOT EXISTS Data("\ - "name TEXT NOT NULL,"\ - "id INTERGER NOT NULL,"\ - "bin TEXT NOT NULL,"\ - "positive DOUBLE NOT NULL,"\ - "negative DOUBLE NOT NULL,"\ - "primary key (name, id, bin) );"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating Data table"); - - } - - void createWeightNamesTable(){ - sql = "CREATE TABLE IF NOT EXISTS WeightDefinition("\ - "id INTERGER NOT NULL,"\ - "definition TEXT NOT NULL,"\ - "primary key (id) );"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating Weight Names table"); - } - - void addWeightDefinition(int id, string def){ - sql = "INSERT INTO WeightDefinition VALUES ('" + to_string(id) + "','" + def + "')"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - } - - void addHisto(string name, int bins, double xmin, double xmax, string regions){ - sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting into Histo: " + name); - } + void createWeightNamesTable(); + void addWeightDefinition(int id, string def); + + void addHisto(string name, int bins, double xmin, double xmax, string regions); + void addStatistic(string name, int id, int pos_events, @@ -183,52 +56,15 @@ class SQLiteDB : public SQLiteBase { double pos_val_weight, double neg_val_weight, double pos_val2_weight, - double neg_val2_weight){ + double neg_val2_weight); - sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(id) + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting into Statistics: " + name); + void addData(string name, int id, string bin, double positive, double negative); - } - - void addData(string name, int id, string bin, double positive, double negative){ - sql = "INSERT INTO Data VALUES ('" + name + "'" + "," + "'" + to_string(id) + "'" + "," + "'" + bin + "'" + "," + "'" + to_string(positive) + "'" + "," + "'" + to_string(negative) + "')"; - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting into Data: " + name + " " + to_string(id)); - - } - - //add cut to databse with primary keys region name and cut name - void addCut(string r_name, string c_name) { + void addCut(string r_name, string c_name); + void addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum); + + void closeDB(); - sql = "INSERT INTO Cutflow VALUES ('" + r_name + "'" + "," + "'" + c_name + "')"; - //cout << sql << endl; - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting cutflow: " + r_name + " " + c_name); - - } - - //add weight to database with primary keys region name, cut name, and weight id - void addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { - sql = "INSERT INTO Weights VALUES ('" + r_name + "'" + ",'" + c_name + "','" + to_string(id) + "','" + to_string(pos) + "','" + to_string(neg) + "','" + to_string(pos_sum) + "','" + to_string(neg_sum) \ - + "','" + to_string(pos_2sum) + "','" + to_string(neg_2sum) + "')"; - //cout << sql << endl; - rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); - //checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id)); - - } - - void closeDB() - - { - - // Close the SQL connection - sqlite3_close(db); - - } - - - }; + diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index cafe90f6..8bb2d84e 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -39,9 +39,8 @@ #include "SampleAnalyzer/Commons/Service/ExceptionService.h" #include "SampleAnalyzer/Commons/Service/Physics.h" -#ifdef SQLITE3_USE - #include "SampleAnalyzer/Interfaces/SQLite3/DatabaseManager.h" -#endif +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" + using namespace MA5; @@ -826,7 +825,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, } } -#ifdef SQLITE3_USE + // Create histo SQlite file for(MAuint32 i = 0; i < analyzers_.size(); ++i){ @@ -853,7 +852,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, } dbManager.closeDB(); } -#endif + //end of multi-weight cutflow code From 777deadfac1dc47cc01d48ef3726f719de21ccc4 Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 25 Nov 2022 07:41:14 -0700 Subject: [PATCH 36/53] changed SQLite interface to use Pointer to implementation design pattern to avoid using compiler flags --- madanalysis/build/makefile_writer.py | 3 +- .../Commons/Base/DatabaseManager.h | 72 ++---- tools/SampleAnalyzer/Commons/Base/SQLbase.h | 48 ---- .../{SQLiteDB.cpp => DatabaseManager.cpp} | 241 ++++++++++++------ .../Interfaces/sqlite/SQLiteDB.h | 70 ----- tools/SampleAnalyzer/Process/Plot/Histo.cpp | 2 +- 6 files changed, 188 insertions(+), 248 deletions(-) delete mode 100644 tools/SampleAnalyzer/Commons/Base/SQLbase.h rename tools/SampleAnalyzer/Interfaces/sqlite/{SQLiteDB.cpp => DatabaseManager.cpp} (51%) delete mode 100644 tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h diff --git a/madanalysis/build/makefile_writer.py b/madanalysis/build/makefile_writer.py index 2fdaa10b..208b3954 100644 --- a/madanalysis/build/makefile_writer.py +++ b/madanalysis/build/makefile_writer.py @@ -452,8 +452,7 @@ def Makefile( if options.has_sqlite: file.write('LIBFLAGS += -l sqlite3\n') - if options.has_sqlite_lib: - ##file.write('LIBFLAGS += -l sqlite3\n') + if options.has_sqlite_lib: file.write('LIBFLAGS += -l sqlite_for_ma5\n') diff --git a/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h b/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h index 7d02a5c1..b48076d3 100644 --- a/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h +++ b/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h @@ -2,48 +2,28 @@ #include -#include "SampleAnalyzer/Commons/Base/SQLbase.h" -#ifdef SQLITE3_USE - #include "SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h" -#endif +class SQLiteBase; class DatabaseManager { private: + SQLiteBase *manager; public: - DatabaseManager(std::string path) { - #ifdef SQLITE3_USE - manager = new SQLiteDB(path); - #else - manager = new SQLiteBase(path); - #endif - } - - ~DatabaseManager() { - delete manager; - } - - void createCutflowTables() { - manager->createCutflowTables(); - } - - void createHistoTables() { - manager->createHistoTables(); - } - - void createWeightNamesTable() { - manager->createWeightNamesTable(); - } - - void addWeightDefinition(int id, std::string def) { - manager->addWeightDefinition(id, def); - } - - void addHisto(std::string name, int bins, double xmin, double xmax, std::string regions) { - manager->addHisto(name, bins, xmin, xmax, regions); - } + DatabaseManager(std::string path); + + ~DatabaseManager(); + + void createCutflowTables(); + + void createHistoTables(); + + void createWeightNamesTable(); + + void addWeightDefinition(int id, std::string def); + + void addHisto(std::string name, int bins, double xmin, double xmax, std::string regions); void addStatistic(std::string name, int id, @@ -60,25 +40,13 @@ class DatabaseManager { double pos_val_weight, double neg_val_weight, double pos_val2_weight, - double neg_val2_weight) { - manager->addStatistic(name,id,pos_events, neg_events, pos_sum_events, neg_sum_events, pos_entries, neg_entries, pos_sum_entries, neg_sum_entries, pos_sum_squared, neg_sum_squared, pos_val_weight, neg_val_weight, pos_val2_weight, neg_val2_weight); - } - - void addData(std::string name, int id, std::string bin, double positive, double negative) { - manager->addData(name, id, bin, positive, negative); - } - - void addCut(std::string r_name, std::string c_name) { - manager->addCut(r_name, c_name); - } + double neg_val2_weight); - void addWeight(std::string r_name, std::string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { - manager->addWeight(r_name,c_name, id, pos, neg, pos_sum, neg_sum, pos_2sum, neg_2sum); - } + void addData(std::string name, int id, std::string bin, double positive, double negative); - void closeDB() { - manager->closeDB(); - } + void addCut(std::string r_name, std::string c_name); + void addWeight(std::string r_name, std::string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum); + void closeDB(); }; diff --git a/tools/SampleAnalyzer/Commons/Base/SQLbase.h b/tools/SampleAnalyzer/Commons/Base/SQLbase.h deleted file mode 100644 index 13674313..00000000 --- a/tools/SampleAnalyzer/Commons/Base/SQLbase.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include - - -class SQLiteBase { - public : - SQLiteBase(std::string path){} - - virtual ~SQLiteBase() {} - - virtual void createCutflowTables() {} - - virtual void createHistoTables() {} - - virtual void createWeightNamesTable() {} - - virtual void addWeightDefinition(int id, std::string def) {} - - virtual void addHisto(std::string name, int bins, double xmin, double xmax, std::string regions) {} - - virtual void addStatistic(std::string name, - int id, - int pos_events, - int neg_events, - double pos_sum_events, - double neg_sum_events, - int pos_entries, - int neg_entries, - double pos_sum_entries, - double neg_sum_entries, - double pos_sum_squared, - double neg_sum_squared, - double pos_val_weight, - double neg_val_weight, - double pos_val2_weight, - double neg_val2_weight) {} - - virtual void addData(std::string name, int id, std::string bin, double positive, double negative) {} - - virtual void addCut(std::string r_name, std::string c_name) {} - - virtual void addWeight(std::string r_name, std::string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) {} - - virtual void closeDB() {} - - -}; diff --git a/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.cpp b/tools/SampleAnalyzer/Interfaces/sqlite/DatabaseManager.cpp similarity index 51% rename from tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.cpp rename to tools/SampleAnalyzer/Interfaces/sqlite/DatabaseManager.cpp index 6345908f..4ecd5b8b 100644 --- a/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.cpp +++ b/tools/SampleAnalyzer/Interfaces/sqlite/DatabaseManager.cpp @@ -1,13 +1,32 @@ +// Implementation of Database Manager Interface using Pointer to Implementation Design Pattern, SQLiteBase class is only implemented if SQLite package is detected by compile scripts + + + #include #include #include -#include "SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h" +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" using namespace std; -int SQLiteDB::callback(void *NotUsed, int argc, char **argv, char **azColName) { +class SQLiteBase{ + private: + // Pointer to SQLite connection + sqlite3 *db; + + // Save any error messages + char *zErrMsg; + + // Save the result of opening the file + int rc; + + // Saved SQL + string sql; + + // Create a callback function + static int callback(void *NotUsed, int argc, char **argv, char **azColName){ // int argc: holds the number of results // (array) azColName: holds each column returned @@ -25,63 +44,68 @@ int SQLiteDB::callback(void *NotUsed, int argc, char **argv, char **azColName) { // Return successful return 0; -} -bool SQLiteDB::checkDBErrors(string msg){ - if( rc ){ - // Show an error message - cout << "DB Error: " << sqlite3_errmsg(db) << " " << msg << endl; - return false; - } - else { - cout << " success @ " << msg << endl; - return true; - } + } -} -SQLiteDB::SQLiteDB(string path) : SQLiteBase(path){ - // Save the result of opening the file - rc = sqlite3_open(path.c_str(), &db); - // enable foreign key constraint - sqlite3_exec(db, "PRAGMA foreign_keys = ON;", 0 ,0 ,0); - bool open_success = checkDBErrors("opening DB"); - if(!open_success) { - sqlite3_close(db); - cout << "open DB failed!" << endl; - } -} + bool checkDBErrors(string msg) { + if( rc ){ + // Show an error message + cout << "DB Error: " << sqlite3_errmsg(db) << " " << msg << endl; + return false; + } else { + cout << " success @ " << msg << endl; + return true; + } + + } -void SQLiteDB::createCutflowTables() { - // Save SQL to create a table - sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \ - "region_name TEXT NOT NULL,"\ - "cut_name TEXT NOT NULL," \ - "primary key (region_name, cut_name));"; + public: + + SQLiteBase(string path) { + // Save the result of opening the file + rc = sqlite3_open(path.c_str(), &db); + // enable foreign key constraint + sqlite3_exec(db, "PRAGMA foreign_keys = ON;", 0 ,0 ,0); + bool open_success = checkDBErrors("opening DB"); + if(!open_success) { + sqlite3_close(db); + cout << "open DB failed!" << endl; + } + } + + void createCutflowTables() { + + // Save SQL to create a table + sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \ + "region_name TEXT NOT NULL,"\ + "cut_name TEXT NOT NULL," \ + "primary key (region_name, cut_name));"; - // Run the SQL - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating cutflow table"); - - sql = "CREATE TABLE IF NOT EXISTS Weights(" \ - "r_name TEXT NOT NULL," \ - "c_name TEXT NOT NULL," \ - "id INTEGER NOT NULL," \ - "pos_entries INTEGER NOT NULL," \ - "neg_entries INTEGER NOT NULL," \ - "pos_sum DOUBLE NOT NULL," \ - "neg_sum DOUBLE NOT NULL," \ - "pos_squared_sum DOUBLE NOT NULL,"\ - "neg_squared_sum DOUBLE NOT NULL,"\ - "primary key (r_name, c_name, id)" \ - "foreign key (r_name, c_name) references Cutflow(region_name, cut_name) ON DELETE CASCADE);"; - - rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); - checkDBErrors("creating weights table"); -} + // Run the SQL + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating cutflow table"); + + sql = "CREATE TABLE IF NOT EXISTS Weights(" \ + "r_name TEXT NOT NULL," \ + "c_name TEXT NOT NULL," \ + "id INTEGER NOT NULL," \ + "pos_entries INTEGER NOT NULL," \ + "neg_entries INTEGER NOT NULL," \ + "pos_sum DOUBLE NOT NULL," \ + "neg_sum DOUBLE NOT NULL," \ + "pos_squared_sum DOUBLE NOT NULL,"\ + "neg_squared_sum DOUBLE NOT NULL,"\ + "primary key (r_name, c_name, id)" \ + "foreign key (r_name, c_name) references Cutflow(region_name, cut_name) ON DELETE CASCADE);"; -void SQLiteDB::createHistoTables(){ + rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); + checkDBErrors("creating weights table"); + } + + + void createHistoTables() { sql = "CREATE TABLE IF NOT EXISTS HistoDescription("\ "name TEXT NOT NULL,"\ "num_of_bins INTEGER NOT NULL,"\ @@ -127,27 +151,31 @@ void SQLiteDB::createHistoTables(){ } - void SQLiteDB::createWeightNamesTable(){ + + void createWeightNamesTable() { sql = "CREATE TABLE IF NOT EXISTS WeightDefinition("\ - "id INTERGER NOT NULL,"\ - "definition TEXT NOT NULL,"\ - "primary key (id) );"; + "id INTERGER NOT NULL,"\ + "definition TEXT NOT NULL,"\ + "primary key (id) );"; rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); checkDBErrors("creating Weight Names table"); } - void SQLiteDB::addWeightDefinition(int id, string def){ + + void addWeightDefinition(int id, string def) { sql = "INSERT INTO WeightDefinition VALUES ('" + to_string(id) + "','" + def + "')"; rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg); } - void SQLiteDB::addHisto(string name, int bins, double xmin, double xmax, string regions){ + + void addHisto(string name, int bins, double xmin, double xmax, string regions) { sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')"; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); //checkDBErrors("inserting into Histo: " + name); } - void SQLiteDB::addStatistic(string name, + + void addStatistic(string name, int id, int pos_events, int neg_events, @@ -162,7 +190,7 @@ void SQLiteDB::createHistoTables(){ double pos_val_weight, double neg_val_weight, double pos_val2_weight, - double neg_val2_weight){ + double neg_val2_weight) { sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(id) + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')"; @@ -171,42 +199,105 @@ void SQLiteDB::createHistoTables(){ } - void SQLiteDB::addData(string name, int id, string bin, double positive, double negative){ + + void addData(string name, int id, string bin, double positive, double negative) { sql = "INSERT INTO Data VALUES ('" + name + "'" + "," + "'" + to_string(id) + "'" + "," + "'" + bin + "'" + "," + "'" + to_string(positive) + "'" + "," + "'" + to_string(negative) + "')"; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); //checkDBErrors("inserting into Data: " + name + " " + to_string(id)); } - - //add cut to databse with primary keys region name and cut name - void SQLiteDB::addCut(string r_name, string c_name) { - + + void addCut(string r_name, string c_name) { + sql = "INSERT INTO Cutflow VALUES ('" + r_name + "'" + "," + "'" + c_name + "')"; //cout << sql << endl; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); //checkDBErrors("inserting cutflow: " + r_name + " " + c_name); - } - //add weight to database with primary keys region name, cut name, and weight id - void SQLiteDB::addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { + + void addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { sql = "INSERT INTO Weights VALUES ('" + r_name + "'" + ",'" + c_name + "','" + to_string(id) + "','" + to_string(pos) + "','" + to_string(neg) + "','" + to_string(pos_sum) + "','" + to_string(neg_sum) \ + "','" + to_string(pos_2sum) + "','" + to_string(neg_2sum) + "')"; //cout << sql << endl; rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg); //checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id)); - } - void SQLiteDB::closeDB() - - { - + + void closeDB() { // Close the SQL connection - sqlite3_close(db); - + sqlite3_close(db); } +}; + + + + +DatabaseManager::DatabaseManager(string path) { + manager = new SQLiteBase(path); +} + +DatabaseManager::~DatabaseManager() { + delete manager; +} + +void DatabaseManager::createCutflowTables() { + manager->createCutflowTables(); +} + +void DatabaseManager::createHistoTables() { + manager->createHistoTables(); +} + +void DatabaseManager::createWeightNamesTable() { + manager->createWeightNamesTable(); +} + +void DatabaseManager::addWeightDefinition(int id, std::string def) { + manager->addWeightDefinition(id, def); +} + +void DatabaseManager::addHisto(std::string name, int bins, double xmin, double xmax, std::string regions) { + manager->addHisto(name, bins, xmin, xmax, regions); +} + +void DatabaseManager::addStatistic(std::string name, + int id, + int pos_events, + int neg_events, + double pos_sum_events, + double neg_sum_events, + int pos_entries, + int neg_entries, + double pos_sum_entries, + double neg_sum_entries, + double pos_sum_squared, + double neg_sum_squared, + double pos_val_weight, + double neg_val_weight, + double pos_val2_weight, + double neg_val2_weight) { + manager->addStatistic(name,id,pos_events, neg_events, pos_sum_events, neg_sum_events, pos_entries, neg_entries, pos_sum_entries, neg_sum_entries, pos_sum_squared, neg_sum_squared, pos_val_weight, neg_val_weight, pos_val2_weight, neg_val2_weight); + } + +void DatabaseManager::addData(std::string name, int id, std::string bin, double positive, double negative) { + manager->addData(name, id, bin, positive, negative); +} + +void DatabaseManager::addCut(std::string r_name, std::string c_name) { + manager->addCut(r_name, c_name); +} + +void DatabaseManager::addWeight(std::string r_name, std::string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) { + manager->addWeight(r_name,c_name, id, pos, neg, pos_sum, neg_sum, pos_2sum, neg_2sum); +} + +void DatabaseManager::closeDB() { + manager->closeDB(); +} + diff --git a/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h b/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h deleted file mode 100644 index c9a8acec..00000000 --- a/tools/SampleAnalyzer/Interfaces/sqlite/SQLiteDB.h +++ /dev/null @@ -1,70 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "SampleAnalyzer/Commons/Base/SQLbase.h" - -using namespace std; - -class SQLiteDB : public SQLiteBase { - private: - // Pointer to SQLite connection - sqlite3 *db; - - // Save any error messages - char *zErrMsg; - - // Save the result of opening the file - int rc; - - // Saved SQL - string sql; - - // Create a callback function - static int callback(void *NotUsed, int argc, char **argv, char **azColName); - - bool checkDBErrors(string msg); - - public: - - SQLiteDB(string path); - - void createCutflowTables(); - - void createHistoTables(); - - void createWeightNamesTable(); - - void addWeightDefinition(int id, string def); - - void addHisto(string name, int bins, double xmin, double xmax, string regions); - - void addStatistic(string name, - int id, - int pos_events, - int neg_events, - double pos_sum_events, - double neg_sum_events, - int pos_entries, - int neg_entries, - double pos_sum_entries, - double neg_sum_entries, - double pos_sum_squared, - double neg_sum_squared, - double pos_val_weight, - double neg_val_weight, - double pos_val2_weight, - double neg_val2_weight); - - void addData(string name, int id, string bin, double positive, double negative); - - void addCut(string r_name, string c_name); - - void addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum); - - void closeDB(); - -}; - diff --git a/tools/SampleAnalyzer/Process/Plot/Histo.cpp b/tools/SampleAnalyzer/Process/Plot/Histo.cpp index 9cb51674..91c2d89f 100644 --- a/tools/SampleAnalyzer/Process/Plot/Histo.cpp +++ b/tools/SampleAnalyzer/Process/Plot/Histo.cpp @@ -75,7 +75,7 @@ void Histo::WriteSQL(DatabaseManager &db){ //for each weight histo,weight id pair: add bucket data to Data table db.addData(name_, weight_id.first, "underflow", weight_id.second.underflow_.first, weight_id.second.underflow_.second); for(int i = 0; i < nbins_; ++i){ - db.addData(name_, weight_id.first, to_string(i+1), weight_id.second.histo_[i].first, weight_id.second.histo_[i].second); + db.addData(name_, weight_id.first, std::to_string(i+1), weight_id.second.histo_[i].first, weight_id.second.histo_[i].second); } db.addData(name_, weight_id.first, "overflow", weight_id.second.overflow_.first, weight_id.second.overflow_.second); } From 1d4762d3be73dc361fd6352cca5a80010f0adbf6 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 19 Dec 2022 01:11:47 -0700 Subject: [PATCH 37/53] refactored database manager functionality to output manager, sample analyzer now passes in necessary data to output manager, further implementation only need to be added to the output manager --- .../Commons/Base/OutputManager.h | 65 +++++++++++++++++++ .../Process/Core/SampleAnalyzer.cpp | 16 ++++- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 tools/SampleAnalyzer/Commons/Base/OutputManager.h diff --git a/tools/SampleAnalyzer/Commons/Base/OutputManager.h b/tools/SampleAnalyzer/Commons/Base/OutputManager.h new file mode 100644 index 00000000..0269be52 --- /dev/null +++ b/tools/SampleAnalyzer/Commons/Base/OutputManager.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include + + +#include "SampleAnalyzer/Process/Analyzer/AnalyzerBase.h" +#include "SampleAnalyzer/Commons/DataFormat/SampleFormat.h" + +//include all output type interfaces here +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" + + +using namespace std; +using namespace MA5; + + + +//output manager object initializer takes in a AnalyzerBase pointer and a vector of arrays of size 3 +//each array constains [output type name, path to histogram file, path to cutflow file] +class OutputManager { + private: + //base pointer type OutputBase can take on any output interface, it's only purpose is to provide + //and execute function, new output formats require implementation of the output interface + vector > output_types_; + AnalyzerBase *analyzer_; + SampleFormat *samples_; + + public: + OutputManager(vector > output_types, AnalyzerBase *analyzer, SampleFormat *samples) : output_types_(output_types), analyzer_(analyzer), samples_(samples) {} + //for each output type, get output object from factory and call execute on it's interface + void Execute() { + + //implement each individual output type here + for(int i = 0; i < output_types_.size(); ++i){ + + //implementaton for SQLite + if(output_types_[i][0] == "sqlite"){ + DatabaseManager cutflow(output_types_[i][1]); + DatabaseManager histogram(output_types_[i][2]); + + histogram.createHistoTables(); + analyzer_->Manager()->GetPlotManager()->WriteSQL(histogram); + histogram.closeDB(); + + cutflow.createCutflowTables(); + bool addInitial = true; + samples_->mc()->WriteWeightNames(cutflow); + for(int i = 0; i < analyzer_->Manager()->Regions().size(); ++i){ + analyzer_->Manager()->Regions()[i]->WriteSQL(cutflow, addInitial); + + } + cutflow.closeDB(); + + } + } + } +}; + + + + + + diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp index 8bb2d84e..4a34699e 100644 --- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp +++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp @@ -39,7 +39,8 @@ #include "SampleAnalyzer/Commons/Service/ExceptionService.h" #include "SampleAnalyzer/Commons/Service/Physics.h" -#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" +//#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" +#include "SampleAnalyzer/Commons/Base/OutputManager.h" @@ -826,7 +827,19 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, } + for(MAuint32 i = 0; i < analyzers_.size(); ++i){ + + std::string histo_path = analyzers_[i]->Output() + "/Histograms/histo.db"; + std::string cutflow_path = analyzers_[i]->Output() + "/Cutflows/cutflows.db"; + vector > outputs; + outputs.push_back({"sqlite", cutflow_path, histo_path}); + OutputManager output_manager(outputs, analyzers_[i], &mySamples[0]); + output_manager.Execute(); + } + + + /* // Create histo SQlite file for(MAuint32 i = 0; i < analyzers_.size(); ++i){ std::string path = analyzers_[i]->Output() + "/Histograms/histo.db"; @@ -854,6 +867,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples, } //end of multi-weight cutflow code + */ // The user-defined stuff From f7198756331f1c857de292a7541c6a93f82a02e2 Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 3 Feb 2023 04:01:31 -0700 Subject: [PATCH 38/53] read sqlite db for histo data instead of SAF --- madanalysis/IOinterface/job_reader.py | 36 ++++++- madanalysis/IOinterface/sqlite_reader.py | 123 +++++++++++++++++++++++ 2 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 madanalysis/IOinterface/sqlite_reader.py diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py index 30bb01e3..1f34f7ed 100644 --- a/madanalysis/IOinterface/job_reader.py +++ b/madanalysis/IOinterface/job_reader.py @@ -30,6 +30,11 @@ from madanalysis.layout.histogram import Histogram from madanalysis.layout.histogram_logx import HistogramLogX from madanalysis.layout.histogram_frequency import HistogramFrequency +from madanalysis.IOinterface.sqlite_reader import getMeanAndStdev +from madanalysis.IOinterface.sqlite_reader import DBreader_debug + + + import glob import logging import shutil @@ -318,10 +323,18 @@ def ExtractHistos(self,dataset,plot,merging=False): while(os.path.isdir(self.safdir+"/"+name+"/MergingPlots_"+str(i))): i+=1 filename = self.safdir+"/"+name+"/MergingPlots_"+str(i-1)+"/Histograms/histos.saf" + ##file path for sqlite db + sqlite_db_filename = self.safdir+"/"+name+"/MergingPlots_"+str(i-1)+"/Histograms/histo.db" + + + else: while(os.path.isdir(self.safdir+"/"+name+"/MadAnalysis5job_"+str(i))): i+=1 filename = self.safdir+"/"+name+"/MadAnalysis5job_"+str(i-1)+"/Histograms/histos.saf" + ##file path for sqlite db + sqlite_db_filename = self.safdir+"/"+name+"/MadAnalysis5job_"+str(i-1)+"/Histograms/histo.db" + # Opening the file try: @@ -348,6 +361,11 @@ def ExtractHistos(self,dataset,plot,merging=False): data_negative = [] labels = [] + ## SqliteDB extractor + sqlite_output_dictionary = getMeanAndStdev(sqlite_db_filename) + #DBreader_debug(sqlite_output_dictionary) + + # Loop over the lines numline=0 for line in file: @@ -391,13 +409,25 @@ def ExtractHistos(self,dataset,plot,merging=False): elif words[0].lower()=='': histoTag.activate() elif words[0].lower()=='': + + histoTag.desactivate() plot.histos.append(copy.copy(histoinfo)) - plot.histos[-1].positive.array = data_positive[:] - plot.histos[-1].negative.array = data_negative[:] - histoinfo.Reset() + #plot.histos[-1].positive.array = data_positive[:] + #plot.histos[-1].negative.array = data_negative[:] + #histoinfo.Reset() data_positive = [] data_negative = [] + + for bin_index in sqlite_output_dictionary[histoinfo.name]: + data_positive.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) + data_negative.append(sqlite_output_dictionary[histoinfo.name][bin_index][2]) + histoinfo.Reset() + + plot.histos[-1].positive.array = data_positive[:] + plot.histos[-1].negative.array = data_negative[:] + + elif words[0].lower()=='': histoFreqTag.activate() elif words[0].lower()=='': diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py new file mode 100644 index 00000000..c7cc998e --- /dev/null +++ b/madanalysis/IOinterface/sqlite_reader.py @@ -0,0 +1,123 @@ +import sqlite3 +from matplotlib import pyplot as plt +import numpy as np +import math +import statistics + + +def getMeanAndStdev(path): + + con = sqlite3.connect(path) + cursor = con.cursor() + + bin_data = cursor.execute("select * from data;").fetchall() + + pos_bins = dict() + neg_bins = dict() + + ## bin_data has all data for the histogram, need to get mean and standard deviation for each bin + ## each row of the query is a tuple of 5 elements [histo name, weight id, bin, positive value, negative value] + ## sort them into +bin/-bin[name] -> bin # -> [mean, standard deviation] + + for row in bin_data: + ## if the histo name is not inside the bin dictionaries, create a new dictionary for each of +/- bin dictionary + ## append values to +/-bin[name][bin#] + + if row[0] not in pos_bins or row[0] not in neg_bins: + pos_bins[row[0]] = dict() + neg_bins[row[0]] = dict() + pos_bins[row[0]][row[2]] = [float(row[3])] + neg_bins[row[0]][row[2]] = [float(row[4])] + + else: + if row[2] in pos_bins[row[0]] or row[2] in neg_bins[row[0]]: + pos_bins[row[0]][row[2]].append(float(row[3])) + neg_bins[row[0]][row[2]].append(float(row[4])) + else : + pos_bins[row[0]][row[2]] = [float(row[3])] + neg_bins[row[0]][row[2]] = [float(row[4])] + + output = dict() + + for histo_name in pos_bins: + output[histo_name] = dict() + for bin_i in pos_bins[histo_name]: + output[histo_name][bin_i] = [statistics.mean(pos_bins[histo_name][bin_i]), statistics.stdev(pos_bins[histo_name][bin_i])] + + for histo_name in neg_bins: + for bin_i in neg_bins[histo_name]: + output[histo_name][bin_i].extend([statistics.mean(neg_bins[histo_name][bin_i]), statistics.stdev(neg_bins[histo_name][bin_i])]) + + return output + + + +## debug for printing out output dictionary +## structure is as follows: +## output[histogram_name][bin #] = [positive mean, positive stdev, negative mean, negative stddev] + + +def DBreader_debug(output): + + for name in output: + print(name) + for eachbin in output[name]: + print(eachbin) + for val in output[name][eachbin]: + print(val) + + + for histo in output: + num_of_keys = len(output[histo].keys()) + labels = [None] * num_of_keys + for i in range(1,num_of_keys): + labels[i] = i + labels[0] = 'underflow' + labels[num_of_keys-1] = 'overflow' + positives = [None] * num_of_keys + negatives = [None] * num_of_keys + for row in output[histo]: + if(row == 'underflow'): + positives[0] = output[histo][row][0] + negatives[0] = output[histo][row][2] + elif(row == 'overflow'): + positives[num_of_keys-1] = output[histo][row][0] + negatives[num_of_keys-1] = output[histo][row][2] + else: + positives[int(row)] = output[histo][row][0] + negatives[int(row)] = output[histo][row][2] + #for lable in lables: + # print(lable) + #for val in positives: + # print(val) + #for val in negatives: + # print(val) + x = np.arange(num_of_keys) + width = 0.5 + fig, ax = plt.subplots() + rects1 = ax.bar(x - width/3, positives, width, label="positives avg") + rects2 = ax.bar(x + width/3, negatives, width, label="negatives avg") + + ax.set_ylabel('Events Luminosity = ') + ax.set_title(histo) + ax.set_xticks(x, labels, rotation = 65) + ax.legend() + + #ax.bar_label(rects1, padding=3) + #ax.bar_label(rects2, padding=3) + + fig.tight_layout() + plt.show() + + + + + + + + + + + + + From 4aebe613f7a0d3e5777ed5856dc3109c64d2cd46 Mon Sep 17 00:00:00 2001 From: Kyle Date: Wed, 8 Feb 2023 03:06:50 -0700 Subject: [PATCH 39/53] added HistoRequency Fill method for multiweight --- .../Process/Plot/HistoFrequency.h | 34 +++++++++++++++++++ .../RegionSelectionManager.cpp | 14 ++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/tools/SampleAnalyzer/Process/Plot/HistoFrequency.h b/tools/SampleAnalyzer/Process/Plot/HistoFrequency.h index 6816d2ef..3f42c9c3 100644 --- a/tools/SampleAnalyzer/Process/Plot/HistoFrequency.h +++ b/tools/SampleAnalyzer/Process/Plot/HistoFrequency.h @@ -34,6 +34,15 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Process/Plot/PlotBase.h" +struct MultiWeightHistoFrequency { + + /// collection of observables for each weight id entry + std::map > stack_; + /// Sum of even-weights over entries for each id entry + std::pair sum_w_; + +}; + namespace MA5 { @@ -52,6 +61,7 @@ class HistoFrequency : public PlotBase /// Sum of event-weights over entries std::pair sum_w_; + std::map MultiWeightHistoFrequencyData; /// RegionSelections attached to the histo std::vector regions_; @@ -125,6 +135,30 @@ class HistoFrequency : public PlotBase } } + void Fill(const MAint32 &obs, std::map &weights){ + + for(auto &id_weight : weights) { + iterator it = MultiWeightHistoFrequencyData[id_weight.first].stack_.find(obs); + if(it == MultiWeightHistoFrequencyData[id_weight.first].stack_.end()){ + MultiWeightHistoFrequencyData[id_weight.first].stack_[obs] = std::make_pair(0.,0.); + } + else { + if(id_weight.second >= 0){ + multiweight_event_info[id_weight.first].nentries_.first++; + MultiWeightHistoFrequencyData[id_weight.first].sum_w_.first += id_weight.second; + MultiWeightHistoFrequencyData[id_weight.first].stack_[obs].first += id_weight.second; + + } else { + int absWeight = std::abs(id_weight.second); + multiweight_event_info[id_weight.first].nentries_.second++; + MultiWeightHistoFrequencyData[id_weight.first].sum_w_.second += absWeight; + MultiWeightHistoFrequencyData[id_weight.first].stack_[obs].second += absWeight; + } + } + } + + } + /// Write the plot in a ROOT file virtual void Write_TextFormat(std::ostream* output) { diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp index 77ac7d89..951b317b 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp @@ -113,8 +113,12 @@ void RegionSelectionManager::FillHisto(std::string const&histname, MAfloat64 val } catch (const std::exception& e) { MANAGE_EXCEPTION(e); } // Filling the histo - if (myhistof->FreshEvent()) myhistof->IncrementNEvents(weight_); + if (myhistof->FreshEvent()) { + myhistof->IncrementNEvents(weight_); + myhistof->IncrementNEvents(multiweight_); + } myhistof->Fill(val,weight_); + myhistof->Fill(val,multiweight_); } // LogX histo else if(dynamic_cast(plotmanager_.GetHistos()[i])!=0) @@ -128,7 +132,10 @@ void RegionSelectionManager::FillHisto(std::string const&histname, MAfloat64 val } catch (const std::exception& e) { MANAGE_EXCEPTION(e); } // Filling the histo - if (myhistoX->FreshEvent()) myhistoX->IncrementNEvents(weight_); + if (myhistoX->FreshEvent()) { + myhistoX->IncrementNEvents(weight_); + myhistoX->IncrementNEvents(multiweight_); + } myhistoX->Fill(val,weight_); } // Normal histo @@ -145,10 +152,11 @@ void RegionSelectionManager::FillHisto(std::string const&histname, MAfloat64 val // Filling the histo if (myhisto->FreshEvent()) { myhisto->IncrementNEvents(weight_); + myhisto->IncrementNEvents(multiweight_); } myhisto->Fill(val,weight_); - myhisto->IncrementNEvents(multiweight_); + //myhisto->IncrementNEvents(multiweight_); myhisto->Fill(val,multiweight_); } break; From 63b9f373e894ba90b3459da978efced088a1e073 Mon Sep 17 00:00:00 2001 From: Kyle Date: Wed, 8 Feb 2023 04:07:26 -0700 Subject: [PATCH 40/53] added HistoLogX Fill --- tools/SampleAnalyzer/Process/Plot/HistoLogX.h | 50 +++++++++++++++++++ .../RegionSelectionManager.cpp | 1 + 2 files changed, 51 insertions(+) diff --git a/tools/SampleAnalyzer/Process/Plot/HistoLogX.h b/tools/SampleAnalyzer/Process/Plot/HistoLogX.h index b9f170f3..2ef7c8c4 100644 --- a/tools/SampleAnalyzer/Process/Plot/HistoLogX.h +++ b/tools/SampleAnalyzer/Process/Plot/HistoLogX.h @@ -49,6 +49,7 @@ class HistoLogX : public Histo MAfloat64 log_xmin_; MAfloat64 log_xmax_; + // ------------------------------------------------------------- // method members // ------------------------------------------------------------- @@ -165,6 +166,55 @@ class HistoLogX : public Histo } } + void Fill(MAfloat64 value, std::map &weights){ + + try + { + if (std::isnan(value)) throw EXCEPTION_WARNING("Skipping a NaN (Not a Number) value in an histogram.","",0); + if (std::isinf(value)) throw EXCEPTION_WARNING("Skipping a Infinity value in an histogram.","",0); + } + catch (const std::exception& e) + { + MANAGE_EXCEPTION(e); + } + + for(auto &id_weight : weights){ + if(id_weight.second >= 0) { + + multiweight_event_info[id_weight.first].nentries_.first++; + MultiweightHistoData[id_weight.first].sum_w_.first +=id_weight.second; + MultiweightHistoData[id_weight.first].sum_ww_.first +=id_weight.second * id_weight.second; + MultiweightHistoData[id_weight.first].sum_xw_.first +=value * id_weight.second; + + //in log bountries map, access by weight id which gives a pair, first element is the min and second is the max + if( value < xmin_) { + MultiweightHistoData[id_weight.first].underflow_.first +=id_weight.second; + } else if (value >= xmax_) { + MultiweightHistoData[id_weight.first].overflow_.first +=id_weight.second; + } else { + MultiweightHistoData[id_weight.first].histo_[std::floor((std::log10(value)-log_xmin_)/step_)].first+=id_weight.second; + } + + } else { + multiweight_event_info[id_weight.first].nentries_.second++; + int absWeight = std::abs(id_weight.second); + MultiweightHistoData[id_weight.first].sum_w_.second +=absWeight; + MultiweightHistoData[id_weight.first].sum_ww_.second +=absWeight * absWeight; + MultiweightHistoData[id_weight.first].sum_xw_.second +=value * absWeight; + if(value < xmin_) { + MultiweightHistoData[id_weight.first].underflow_.second +=absWeight; + } else if (value >= xmax_) MultiweightHistoData[id_weight.first].overflow_.second+=absWeight; + else + { + MultiweightHistoData[id_weight.first].histo_[std::floor((std::log10(value)-log_xmin_)/step_)].second+=absWeight; + } + + + } + } + + } + /// Write the plot in a Text file virtual void Write_TextFormat(std::ostream* output); diff --git a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp index 951b317b..760ed885 100644 --- a/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp +++ b/tools/SampleAnalyzer/Process/RegionSelection/RegionSelectionManager.cpp @@ -137,6 +137,7 @@ void RegionSelectionManager::FillHisto(std::string const&histname, MAfloat64 val myhistoX->IncrementNEvents(multiweight_); } myhistoX->Fill(val,weight_); + myhistoX->Fill(val,multiweight_); } // Normal histo else if(dynamic_cast(plotmanager_.GetHistos()[i])!=0) From bc20d21985744579b5fe71609aad8247eb575cec Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 10 Feb 2023 03:36:44 -0700 Subject: [PATCH 41/53] append stdev array to positive and negative HistogramCore objects --- madanalysis/IOinterface/job_reader.py | 11 +++++++++++ madanalysis/layout/histogram_core.py | 1 + 2 files changed, 12 insertions(+) diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py index 1f34f7ed..49cdf5e0 100644 --- a/madanalysis/IOinterface/job_reader.py +++ b/madanalysis/IOinterface/job_reader.py @@ -417,15 +417,26 @@ def ExtractHistos(self,dataset,plot,merging=False): #plot.histos[-1].negative.array = data_negative[:] #histoinfo.Reset() data_positive = [] + data_positive_stdev = [] data_negative = [] + data_negative_stdev = [] + + # save bin mean and stdev into histogram_core for positive and negative values for bin_index in sqlite_output_dictionary[histoinfo.name]: data_positive.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) + data_positive_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1]) data_negative.append(sqlite_output_dictionary[histoinfo.name][bin_index][2]) + data_negative_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][3]) + histoinfo.Reset() plot.histos[-1].positive.array = data_positive[:] plot.histos[-1].negative.array = data_negative[:] + + plot.histos[-1].positive.stdev = data_positive_stdev[:] + plot.histos[-1].negative.stdev = data_negative_stdev[:] + elif words[0].lower()=='': diff --git a/madanalysis/layout/histogram_core.py b/madanalysis/layout/histogram_core.py index 0a3355a4..bb566599 100644 --- a/madanalysis/layout/histogram_core.py +++ b/madanalysis/layout/histogram_core.py @@ -50,6 +50,7 @@ def __init__(self): self.nan = 0. self.inf = 0. self.array = [] + self.stdev = [] def ComputeIntegral(self): From 5d326ee97864720f41f8e1b2791e47b99cc72b11 Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 10 Feb 2023 05:51:28 -0700 Subject: [PATCH 42/53] added error bar to plots, not sure if scale is correct --- madanalysis/layout/histogram.py | 8 ++++++++ madanalysis/layout/plotflow.py | 17 +++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/madanalysis/layout/histogram.py b/madanalysis/layout/histogram.py index 949eaf8b..a6ac745e 100644 --- a/madanalysis/layout/histogram.py +++ b/madanalysis/layout/histogram.py @@ -95,6 +95,14 @@ def FinalizeReading(self,main,dataset): data[-1]=0 self.summary.array = data[:] # [:] -> clone of data + #stdev + stdev_data = [] + for i in range(0, len(self.positive.array)): + stdev_data.append(max(0, self.positive.stdev[i]-self.negative.stdev[i])) + self.summary.stdev = stdev_data[:] + + + # Integral self.positive.ComputeIntegral() self.negative.ComputeIntegral() diff --git a/madanalysis/layout/plotflow.py b/madanalysis/layout/plotflow.py index 8fe99d39..1535b443 100644 --- a/madanalysis/layout/plotflow.py +++ b/madanalysis/layout/plotflow.py @@ -640,8 +640,17 @@ def DrawMATPLOTLIB(self,histos,scales,ref,irelhisto,filenamePy,outputnames): outputPy.write(str(histos[ind].summary.array[bin-1]*scales[ind])) outputPy.write('])\n\n') + #stdev + for ind in range(0, len(histos)): + outputPy.write(' # Creating array for stdev\n') + outputPy.write(' err = numpy.array([') + for bin in range(1, xnbin+1): + if bin !=1: + outputPy.write(',') + outputPy.write(str(histos[ind].summary.stdev[bin-1]*scales[ind])) + outputPy.write('])\n\n') - + # Canvas outputPy.write(' # Creating a new Canvas\n') dpi=80 @@ -795,7 +804,7 @@ def DrawMATPLOTLIB(self,histos,scales,ref,irelhisto,filenamePy,outputnames): mylinewidth = self.main.datasets[ind].linewidth mylinestyle = LineStyleType.convert2matplotlib(self.main.datasets[ind].linestyle) - outputPy.write(' pad.hist('+\ + outputPy.write(' n, _, _ = pad.hist('+\ 'x=xData, '+\ 'bins=xBinning, '+\ 'weights='+myweights+',\\\n'+\ @@ -824,6 +833,10 @@ def DrawMATPLOTLIB(self,histos,scales,ref,irelhisto,filenamePy,outputnames): ' orientation="vertical")\n\n') outputPy.write('\n') + + # error bar + outputPy.write(' plt.errorbar(xData, n, yerr = err, fmt=\'r.\')\n') + # Label outputPy.write(' # Axis\n') outputPy.write(" plt.rc('text',usetex=False)\n") From d71e17b6446f8515bc236e374cc4d7e3dff3974b Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 23 Feb 2023 07:18:55 -0700 Subject: [PATCH 43/53] changed histo mean and variation calculation in sqlite reader, there may be a bug with certain graphs, need to track it down --- madanalysis/IOinterface/job_reader.py | 24 +++++++-------- madanalysis/IOinterface/sqlite_reader.py | 38 ++++++++++++++++++++++-- madanalysis/layout/histogram.py | 4 +-- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py index 49cdf5e0..8a69f222 100644 --- a/madanalysis/IOinterface/job_reader.py +++ b/madanalysis/IOinterface/job_reader.py @@ -361,6 +361,8 @@ def ExtractHistos(self,dataset,plot,merging=False): data_negative = [] labels = [] + + ## SqliteDB extractor sqlite_output_dictionary = getMeanAndStdev(sqlite_db_filename) #DBreader_debug(sqlite_output_dictionary) @@ -416,26 +418,22 @@ def ExtractHistos(self,dataset,plot,merging=False): #plot.histos[-1].positive.array = data_positive[:] #plot.histos[-1].negative.array = data_negative[:] #histoinfo.Reset() - data_positive = [] - data_positive_stdev = [] - data_negative = [] - data_negative_stdev = [] + bin_means = [] + bin_stdev = [] # save bin mean and stdev into histogram_core for positive and negative values for bin_index in sqlite_output_dictionary[histoinfo.name]: - data_positive.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) - data_positive_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1]) - data_negative.append(sqlite_output_dictionary[histoinfo.name][bin_index][2]) - data_negative_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][3]) - + bin_means.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) + bin_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1]) + histoinfo.Reset() - plot.histos[-1].positive.array = data_positive[:] - plot.histos[-1].negative.array = data_negative[:] + plot.histos[-1].positive.array = bin_means[:] + ##plot.histos[-1].negative.array = data_negative[:] - plot.histos[-1].positive.stdev = data_positive_stdev[:] - plot.histos[-1].negative.stdev = data_negative_stdev[:] + plot.histos[-1].positive.stdev = bin_stdev[:] + ##plot.histos[-1].negative.stdev = data_negative_stdev[:] diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py index c7cc998e..72fed259 100644 --- a/madanalysis/IOinterface/sqlite_reader.py +++ b/madanalysis/IOinterface/sqlite_reader.py @@ -5,7 +5,7 @@ import statistics -def getMeanAndStdev(path): +def getMeanAndStdevOld(path): con = sqlite3.connect(path) cursor = con.cursor() @@ -16,7 +16,7 @@ def getMeanAndStdev(path): neg_bins = dict() ## bin_data has all data for the histogram, need to get mean and standard deviation for each bin - ## each row of the query is a tuple of 5 elements [histo name, weight id, bin, positive value, negative value] + ## each row of the query is a tuple of 5 elements [histo name, weight id, bin #, positive value, negative value] ## sort them into +bin/-bin[name] -> bin # -> [mean, standard deviation] for row in bin_data: @@ -50,6 +50,40 @@ def getMeanAndStdev(path): return output +def getMeanAndStdev(path): + + con = sqlite3.connect(path) + cursor = con.cursor() + bin_data = cursor.execute("select * from data;").fetchall() + + ## parse data in the form of parsed_data[histo_name][bin #][{positive value, negative value}] + parsed_data = dict() + for row in bin_data: + + histo_name = row[0] + weight_id = row[1] + bin_number = row[2] + value = float(row[3]) - abs(float(row[4])) + + if histo_name not in parsed_data: + ## if histo name is not in the parsed_data dictionary, then create a new bin dictionary for that histo, then for the bin, create a weigh id dictionary + parsed_data[histo_name] = dict() + parsed_data[histo_name][bin_number] = [] + + else: + ## since histo name is in the parsed_data dictionary, we need to check if the bin in the dictioary, if not then create a weight id dictionary for that bin + if bin_number not in parsed_data[histo_name]: + parsed_data[histo_name][bin_number] = [] + + parsed_data[histo_name][bin_number].append(value) + + output = dict() + for histo_name in parsed_data: + output[histo_name] = dict() + for bin_number in parsed_data[histo_name]: + output[histo_name][bin_number] = [statistics.mean(parsed_data[histo_name][bin_number]), statistics.stdev(parsed_data[histo_name][bin_number])] + + return output ## debug for printing out output dictionary diff --git a/madanalysis/layout/histogram.py b/madanalysis/layout/histogram.py index a6ac745e..0c4992b5 100644 --- a/madanalysis/layout/histogram.py +++ b/madanalysis/layout/histogram.py @@ -85,7 +85,7 @@ def FinalizeReading(self,main,dataset): # Data data = [] for i in range(0,len(self.positive.array)): - data.append(self.positive.array[i]-self.negative.array[i]) + data.append(self.positive.array[i]) if data[-1]<0: self.warnings.append(\ 'dataset='+dataset.name+\ @@ -98,7 +98,7 @@ def FinalizeReading(self,main,dataset): #stdev stdev_data = [] for i in range(0, len(self.positive.array)): - stdev_data.append(max(0, self.positive.stdev[i]-self.negative.stdev[i])) + stdev_data.append(max(0, self.positive.stdev[i])) self.summary.stdev = stdev_data[:] From 624a974ff7877aac5da268da423ba5320c73c6e3 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 13 Mar 2023 23:26:30 -0600 Subject: [PATCH 44/53] added weight statistics averages to sqlite loader and load from sqlite file only if it exists --- madanalysis/IOinterface/job_reader.py | 46 +++++++++++-------- madanalysis/IOinterface/sqlite_reader.py | 16 +++++++ .../layout/histogram_frequency_core.py | 1 + madanalysis/layout/histogram_logx.py | 2 + .../Process/Plot/HistoFrequency.h | 10 ++++ 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py index 8a69f222..5b0c73d0 100644 --- a/madanalysis/IOinterface/job_reader.py +++ b/madanalysis/IOinterface/job_reader.py @@ -364,7 +364,10 @@ def ExtractHistos(self,dataset,plot,merging=False): ## SqliteDB extractor - sqlite_output_dictionary = getMeanAndStdev(sqlite_db_filename) + sqlite_exists = os.path.isfile(sqlite_db_filename) + if sqlite_exists: + sqlite_output_dictionary = getMeanAndStdev(sqlite_db_filename) + histoStatistics = getHistoStatisticsAvg(sqlite_db_filename) #DBreader_debug(sqlite_output_dictionary) @@ -412,29 +415,32 @@ def ExtractHistos(self,dataset,plot,merging=False): histoTag.activate() elif words[0].lower()=='': - + histoTag.desactivate() plot.histos.append(copy.copy(histoinfo)) - #plot.histos[-1].positive.array = data_positive[:] - #plot.histos[-1].negative.array = data_negative[:] - #histoinfo.Reset() - bin_means = [] - bin_stdev = [] - - - # save bin mean and stdev into histogram_core for positive and negative values - for bin_index in sqlite_output_dictionary[histoinfo.name]: - bin_means.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) - bin_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1]) - - histoinfo.Reset() - - plot.histos[-1].positive.array = bin_means[:] - ##plot.histos[-1].negative.array = data_negative[:] - plot.histos[-1].positive.stdev = bin_stdev[:] - ##plot.histos[-1].negative.stdev = data_negative_stdev[:] + if not sqlite_exists: + plot.histos[-1].positive.array = data_positive[:] + plot.histos[-1].negative.array = data_negative[:] + histoinfo.Reset() + bin_means = [] + bin_stdev = [] + else: + bin_means = [] + bin_stdev = [] + + # save bin mean and stdev into histogram_core for positive and negative values + for bin_index in sqlite_output_dictionary[histoinfo.name]: + bin_means.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) + bin_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1]) + + histoinfo.Reset() + + plot.histos[-1].positive.array = bin_means[:] + ##plot.histos[-1].negative.array = bin_means[:] + plot.histos[-1].positive.stdev = bin_stdev[:] + ##plot.histos[-1].negative.stdev = bin_stdev[:] elif words[0].lower()=='': diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py index 72fed259..e3979f30 100644 --- a/madanalysis/IOinterface/sqlite_reader.py +++ b/madanalysis/IOinterface/sqlite_reader.py @@ -50,6 +50,8 @@ def getMeanAndStdevOld(path): return output + + def getMeanAndStdev(path): con = sqlite3.connect(path) @@ -85,6 +87,20 @@ def getMeanAndStdev(path): return output +def getHistoStatisticsAvg(path): + + con = sqlite3.connect(path) + cursor = con.cursor() + statistics = cursor.execute("select name, avg(pos_num_events), avg(neg_num_events), avg(pos_sum_event_weights_over_events), avg(neg_sum_event_weights_over_events), avg(pos_entries), avg(neg_entries), avg(pos_sum_event_weights_over_entries), avg(neg_sum_event_weights_over_entries), avg(pos_sum_squared_weights), avg(neg_sum_squared_weights), avg(pos_value_times_weight), avg(neg_value_times_weight), avg(pos_value_squared_times_weight), avg(neg_value_squared_times_weight) +from Statistics +group by name;").fetchall() + + + return statistics; + + + + ## debug for printing out output dictionary ## structure is as follows: diff --git a/madanalysis/layout/histogram_frequency_core.py b/madanalysis/layout/histogram_frequency_core.py index d797d74a..377fa93e 100644 --- a/madanalysis/layout/histogram_frequency_core.py +++ b/madanalysis/layout/histogram_frequency_core.py @@ -35,6 +35,7 @@ def __init__(self): self.overflow = 0. self.underflow = 0. self.array = [] + self.stdev = [] def ComputeIntegral(self): self.integral = 0 diff --git a/madanalysis/layout/histogram_logx.py b/madanalysis/layout/histogram_logx.py index bb16edbf..386be6f5 100644 --- a/madanalysis/layout/histogram_logx.py +++ b/madanalysis/layout/histogram_logx.py @@ -111,6 +111,8 @@ def FinalizeReading(self,main,dataset): data[-1]=0 self.summary.array = data[:] # [:] -> clone of data + + # Integral self.positive.ComputeIntegral() self.negative.ComputeIntegral() diff --git a/tools/SampleAnalyzer/Process/Plot/HistoFrequency.h b/tools/SampleAnalyzer/Process/Plot/HistoFrequency.h index 3f42c9c3..7835882c 100644 --- a/tools/SampleAnalyzer/Process/Plot/HistoFrequency.h +++ b/tools/SampleAnalyzer/Process/Plot/HistoFrequency.h @@ -33,6 +33,7 @@ // SampleAnalyzer headers #include "SampleAnalyzer/Process/Plot/PlotBase.h" +#include "SampleAnalyzer/Commons/Base/DatabaseManager.h" struct MultiWeightHistoFrequency { @@ -159,6 +160,15 @@ class HistoFrequency : public PlotBase } + + virtual void WriteSQL(DatabaseManager &db) { + + + + + + }; + /// Write the plot in a ROOT file virtual void Write_TextFormat(std::ostream* output) { From 4d0d1b7fc04e70b5c989a56cf27f987597e778ec Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 13 Mar 2023 23:47:14 -0600 Subject: [PATCH 45/53] fixed sqlite reader bug --- madanalysis/IOinterface/sqlite_reader.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py index e3979f30..fe359e32 100644 --- a/madanalysis/IOinterface/sqlite_reader.py +++ b/madanalysis/IOinterface/sqlite_reader.py @@ -91,11 +91,10 @@ def getHistoStatisticsAvg(path): con = sqlite3.connect(path) cursor = con.cursor() - statistics = cursor.execute("select name, avg(pos_num_events), avg(neg_num_events), avg(pos_sum_event_weights_over_events), avg(neg_sum_event_weights_over_events), avg(pos_entries), avg(neg_entries), avg(pos_sum_event_weights_over_entries), avg(neg_sum_event_weights_over_entries), avg(pos_sum_squared_weights), avg(neg_sum_squared_weights), avg(pos_value_times_weight), avg(neg_value_times_weight), avg(pos_value_squared_times_weight), avg(neg_value_squared_times_weight) -from Statistics -group by name;").fetchall() - - + + + statistics = cursor.execute("select name, avg(pos_num_events), avg(neg_num_events), avg(pos_sum_event_weights_over_events), avg(neg_sum_event_weights_over_events), avg(pos_entries), avg(neg_entries), avg(pos_sum_event_weights_over_entries), avg(neg_sum_event_weights_over_entries), avg(pos_sum_squared_weights), avg(neg_sum_squared_weights), avg(pos_value_times_weight), avg(neg_value_times_weight), avg(pos_value_squared_times_weight), avg(neg_value_squared_times_weight from Statistics group by name;").fetchall() + return statistics; From be33465bdf9c734989b6e56197278bd6d4281847 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 13 Mar 2023 23:56:31 -0600 Subject: [PATCH 46/53] fix bugs with sqlite reader query --- madanalysis/IOinterface/job_reader.py | 3 ++- madanalysis/IOinterface/sqlite_reader.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py index 5b0c73d0..af5538c7 100644 --- a/madanalysis/IOinterface/job_reader.py +++ b/madanalysis/IOinterface/job_reader.py @@ -31,7 +31,8 @@ from madanalysis.layout.histogram_logx import HistogramLogX from madanalysis.layout.histogram_frequency import HistogramFrequency from madanalysis.IOinterface.sqlite_reader import getMeanAndStdev -from madanalysis.IOinterface.sqlite_reader import DBreader_debug +from madanalysis.IOinterface.sqlite_reader import DBreader_debug +from madanalysis.IOinterface.sqlite_reader import getHistoStatisticsAvg diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py index fe359e32..9572910c 100644 --- a/madanalysis/IOinterface/sqlite_reader.py +++ b/madanalysis/IOinterface/sqlite_reader.py @@ -93,7 +93,7 @@ def getHistoStatisticsAvg(path): cursor = con.cursor() - statistics = cursor.execute("select name, avg(pos_num_events), avg(neg_num_events), avg(pos_sum_event_weights_over_events), avg(neg_sum_event_weights_over_events), avg(pos_entries), avg(neg_entries), avg(pos_sum_event_weights_over_entries), avg(neg_sum_event_weights_over_entries), avg(pos_sum_squared_weights), avg(neg_sum_squared_weights), avg(pos_value_times_weight), avg(neg_value_times_weight), avg(pos_value_squared_times_weight), avg(neg_value_squared_times_weight from Statistics group by name;").fetchall() + statistics = cursor.execute("select name, avg(pos_num_events), avg(neg_num_events), avg(pos_sum_event_weights_over_events), avg(neg_sum_event_weights_over_events), avg(pos_entries), avg(neg_entries), avg(pos_sum_event_weights_over_entries), avg(neg_sum_event_weights_over_entries), avg(pos_sum_squared_weights), avg(neg_sum_squared_weights), avg(pos_value_times_weight), avg(neg_value_times_weight), avg(pos_value_squared_times_weight), avg(neg_value_squared_times_weight) from Statistics group by name;").fetchall() return statistics; From a22ffc0bb8538f90188271ab285323c1a8ce094e Mon Sep 17 00:00:00 2001 From: Kyle Date: Wed, 15 Mar 2023 02:44:16 -0600 Subject: [PATCH 47/53] statistics table now uses averages of all weights --- madanalysis/IOinterface/job_reader.py | 76 +++++++++++++++++++----- madanalysis/IOinterface/sqlite_reader.py | 7 ++- 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py index af5538c7..4faf5f7b 100644 --- a/madanalysis/IOinterface/job_reader.py +++ b/madanalysis/IOinterface/job_reader.py @@ -362,7 +362,6 @@ def ExtractHistos(self,dataset,plot,merging=False): data_negative = [] labels = [] - ## SqliteDB extractor sqlite_exists = os.path.isfile(sqlite_db_filename) @@ -370,7 +369,8 @@ def ExtractHistos(self,dataset,plot,merging=False): sqlite_output_dictionary = getMeanAndStdev(sqlite_db_filename) histoStatistics = getHistoStatisticsAvg(sqlite_db_filename) #DBreader_debug(sqlite_output_dictionary) - + + # Loop over the lines numline=0 @@ -510,11 +510,19 @@ def ExtractHistos(self,dataset,plot,merging=False): # Looking from histogram statistics elif statisticsTag.activated and len(words)==2: + if statisticsTag.Nlines==0: results = self.ExtractStatisticsInt(words,numline,filename) + #print(results) if histoTag.activated: - histoinfo.positive.nevents=results[0] - histoinfo.negative.nevents=results[1] + + if sqlite_exists: + histoinfo.positive.nevents = int(histoStatistics[histoinfo.name][0]) + histoinfo.negative.nevents = int(histoStatistics[histoinfo.name][1]) + #print(histoinfo.positive.nevents, histoinfo.negative.nevents) + else : + histoinfo.positive.nevents=results[0] + histoinfo.negative.nevents=results[1] elif histoLogXTag.activated: histologxinfo.positive.nevents=results[0] histologxinfo.negative.nevents=results[1] @@ -524,9 +532,15 @@ def ExtractHistos(self,dataset,plot,merging=False): elif statisticsTag.Nlines==1: results = self.ExtractStatisticsFloat(words,numline,filename) + # print(results) if histoTag.activated: - histoinfo.positive.sumwentries=results[0] - histoinfo.negative.sumwentries=results[1] + if sqlite_exists: + histoinfo.positive.sumwentries = histoStatistics[histoinfo.name][2] + histoinfo.negative.sumwentries = histoStatistics[histoinfo.name][3] + #print(histoinfo.positive.sumwentries, histoinfo.negative.sumwentries) + else : + histoinfo.positive.sumwentries=results[0] + histoinfo.negative.sumwentries=results[1] elif histoLogXTag.activated: histologxinfo.positive.sumwentries=results[0] histologxinfo.negative.sumwentries=results[1] @@ -536,9 +550,15 @@ def ExtractHistos(self,dataset,plot,merging=False): elif statisticsTag.Nlines==2: results = self.ExtractStatisticsInt(words,numline,filename) + # print(results) if histoTag.activated: - histoinfo.positive.nentries=results[0] - histoinfo.negative.nentries=results[1] + if sqlite_exists: + histoinfo.positive.nentries = int(histoStatistics[histoinfo.name][4]) + histoinfo.negative.nentries = int(histoStatistics[histoinfo.name][5]) + # print(histoinfo.positive.nentries, histoinfo.negative.nentries) + else : + histoinfo.positive.nentries=results[0] + histoinfo.negative.nentries=results[1] elif histoLogXTag.activated: histologxinfo.positive.nentries=results[0] histologxinfo.negative.nentries=results[1] @@ -548,9 +568,15 @@ def ExtractHistos(self,dataset,plot,merging=False): elif statisticsTag.Nlines==3: results = self.ExtractStatisticsFloat(words,numline,filename) + # print(results) if histoTag.activated: - histoinfo.positive.sumw=results[0] - histoinfo.negative.sumw=results[1] + if sqlite_exists: + histoinfo.positive.sumw = histoStatistics[histoinfo.name][6] + histoinfo.negative.sumw = histoStatistics[histoinfo.name][7] + # print(histoinfo.positive.sumw, histoinfo.negative.sumw) + else : + histoinfo.positive.sumw=results[0] + histoinfo.negative.sumw=results[1] elif histoLogXTag.activated: histologxinfo.positive.sumw=results[0] histologxinfo.negative.sumw=results[1] @@ -560,27 +586,45 @@ def ExtractHistos(self,dataset,plot,merging=False): elif statisticsTag.Nlines==4 and not histoFreqTag.activated: results = self.ExtractStatisticsFloat(words,numline,filename) + # print(results) if histoTag.activated: - histoinfo.positive.sumw2=results[0] - histoinfo.negative.sumw2=results[1] + if sqlite_exists: + histoinfo.positive.sumw2 = histoStatistics[histoinfo.name][8] + histoinfo.negative.sumw2 = histoStatistics[histoinfo.name][9] + # print(histoinfo.positive.sumw2, histoinfo.negative.sumw2) + else : + histoinfo.positive.sumw2=results[0] + histoinfo.negative.sumw2=results[1] elif histoLogXTag.activated: histologxinfo.positive.sumw2=results[0] histologxinfo.negative.sumw2=results[1] elif statisticsTag.Nlines==5 and not histoFreqTag.activated: results = self.ExtractStatisticsFloat(words,numline,filename) + # print(results) if histoTag.activated: - histoinfo.positive.sumwx=results[0] - histoinfo.negative.sumwx=results[1] + if sqlite_exists: + histoinfo.positive.sumwx = histoStatistics[histoinfo.name][10] + histoinfo.negative.sumwx = histoStatistics[histoinfo.name][11] + # print(histoinfo.positive.sumwx, histoinfo.negative.sumwx) + else : + histoinfo.positive.sumwx=results[0] + histoinfo.negative.sumwx=results[1] elif histoLogXTag.activated: histologxinfo.positive.sumwx=results[0] histologxinfo.negative.sumwx=results[1] elif statisticsTag.Nlines==6 and not histoFreqTag.activated: results = self.ExtractStatisticsFloat(words,numline,filename) + # print(results) if histoTag.activated: - histoinfo.positive.sumw2x=results[0] - histoinfo.negative.sumw2x=results[1] + if sqlite_exists: + histoinfo.positive.sumw2x = histoStatistics[histoinfo.name][12] + histoinfo.negative.sumw2x = histoStatistics[histoinfo.name][13] + # print(histoinfo.positive.sumw2x, histoinfo.negative.sumw2x) + else : + histoinfo.positive.sumw2x=results[0] + histoinfo.negative.sumw2x=results[1] elif histoLogXTag.activated: histologxinfo.positive.sumw2x=results[0] histologxinfo.negative.sumw2x=results[1] diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py index 9572910c..0e9381a6 100644 --- a/madanalysis/IOinterface/sqlite_reader.py +++ b/madanalysis/IOinterface/sqlite_reader.py @@ -94,8 +94,13 @@ def getHistoStatisticsAvg(path): statistics = cursor.execute("select name, avg(pos_num_events), avg(neg_num_events), avg(pos_sum_event_weights_over_events), avg(neg_sum_event_weights_over_events), avg(pos_entries), avg(neg_entries), avg(pos_sum_event_weights_over_entries), avg(neg_sum_event_weights_over_entries), avg(pos_sum_squared_weights), avg(neg_sum_squared_weights), avg(pos_value_times_weight), avg(neg_value_times_weight), avg(pos_value_squared_times_weight), avg(neg_value_squared_times_weight) from Statistics group by name;").fetchall() + + statdict = dict() + for i in range(len(statistics)): + statdict[statistics[i][0]] = statistics[i][1:] - return statistics; + return statdict; + From 594af5de1a8a485559b0d85be13f0bae7c7ef4fa Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 20 Mar 2023 17:41:41 -0600 Subject: [PATCH 48/53] duplicated weight names to histo db file --- tools/SampleAnalyzer/Commons/Base/OutputManager.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/SampleAnalyzer/Commons/Base/OutputManager.h b/tools/SampleAnalyzer/Commons/Base/OutputManager.h index 0269be52..ce288f4e 100644 --- a/tools/SampleAnalyzer/Commons/Base/OutputManager.h +++ b/tools/SampleAnalyzer/Commons/Base/OutputManager.h @@ -42,6 +42,7 @@ class OutputManager { histogram.createHistoTables(); analyzer_->Manager()->GetPlotManager()->WriteSQL(histogram); + samples_->mc()->WriteWeightNames(histogram); histogram.closeDB(); cutflow.createCutflowTables(); From 7211feac90e99997a2511c5cde7db0299b5616e5 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 20 Mar 2023 19:15:35 -0600 Subject: [PATCH 49/53] regularized mean/stdev by sumw --- madanalysis/IOinterface/sqlite_reader.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py index 0e9381a6..7546e5b3 100644 --- a/madanalysis/IOinterface/sqlite_reader.py +++ b/madanalysis/IOinterface/sqlite_reader.py @@ -51,12 +51,24 @@ def getMeanAndStdevOld(path): return output +def getStatistics(stats): + histoname_dict = dict() + for entry in stats: + if entry[0] not in histoname_dict: + histoname_dict[entry[0]] = dict() + histoname_dict[entry[0]][entry[1]] = float(entry[2]) - float(entry[3]) + return histoname_dict + def getMeanAndStdev(path): con = sqlite3.connect(path) cursor = con.cursor() bin_data = cursor.execute("select * from data;").fetchall() + stats_data = cursor.execute("select name, id, pos_sum_event_weights_over_events, neg_sum_event_weights_over_events from Statistics").fetchall() + + statsdict = getStatistics(stats_data) + ## parse data in the form of parsed_data[histo_name][bin #][{positive value, negative value}] parsed_data = dict() @@ -65,8 +77,8 @@ def getMeanAndStdev(path): histo_name = row[0] weight_id = row[1] bin_number = row[2] - value = float(row[3]) - abs(float(row[4])) - + sumw = statsdict[histo_name][str(weight_id)] + value = (float(row[3]) - abs(float(row[4]))) / sumw if histo_name not in parsed_data: ## if histo name is not in the parsed_data dictionary, then create a new bin dictionary for that histo, then for the bin, create a weigh id dictionary parsed_data[histo_name] = dict() From faee6ffcd4e4530d41026d0a0e7b1a7ad1e059ea Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 20 Mar 2023 22:36:25 -0600 Subject: [PATCH 50/53] fixed underflow and overflow bins --- madanalysis/IOinterface/job_reader.py | 135 +++++++++++++++++--------- 1 file changed, 88 insertions(+), 47 deletions(-) diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py index 4faf5f7b..20fd6d35 100644 --- a/madanalysis/IOinterface/job_reader.py +++ b/madanalysis/IOinterface/job_reader.py @@ -430,19 +430,24 @@ def ExtractHistos(self,dataset,plot,merging=False): else: bin_means = [] bin_stdev = [] - + # save bin mean and stdev into histogram_core for positive and negative values for bin_index in sqlite_output_dictionary[histoinfo.name]: - bin_means.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) - bin_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1]) - + if bin_index not in ["underflow", "overflow"]: + bin_means.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) + bin_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1]) + elif bin_index == "underflow": + plot.histos[-1].positive.underflow = sqlite_output_dictionary[histoinfo.name][bin_index][0] + elif bin_index == "overflow": + plot.histos[-1].positive.overflow = sqlite_output_dictionary[histoinfo.name][bin_index][0] + histoinfo.Reset() plot.histos[-1].positive.array = bin_means[:] ##plot.histos[-1].negative.array = bin_means[:] plot.histos[-1].positive.stdev = bin_stdev[:] ##plot.histos[-1].negative.stdev = bin_stdev[:] - + elif words[0].lower()=='': histoFreqTag.activate() @@ -461,11 +466,28 @@ def ExtractHistos(self,dataset,plot,merging=False): elif words[0].lower()=='': histoLogXTag.desactivate() plot.histos.append(copy.copy(histologxinfo)) - plot.histos[-1].positive.array = data_positive[:] - plot.histos[-1].negative.array = data_negative[:] - histologxinfo.Reset() - data_positive = [] - data_negative = [] + + if not sqlite_exists: + + plot.histos[-1].positive.array = data_positive[:] + plot.histos[-1].negative.array = data_negative[:] + histologxinfo.Reset() + data_positive = [] + data_negative = [] + + else: + + for bin_index in sqlite_output_dictionary[histoinfo.name]: + bin_means.append(sqlite_output_dictionary[histologxinfo.name][bin_index][0]) + bin_stdev.append(sqlite_output_dictionary[histologxinfo.name][bin_index][1]) + + histoinfo.Reset() + + plot.histos[-1].positive.array = bin_means[:] + ##plot.histos[-1].negative.array = bin_means[:] + plot.histos[-1].positive.stdev = bin_stdev[:] + ##plot.histos[-1].negative.stdev = bin_stdev[:] + # Looking from histogram description elif descriptionTag.activated: @@ -513,121 +535,140 @@ def ExtractHistos(self,dataset,plot,merging=False): if statisticsTag.Nlines==0: results = self.ExtractStatisticsInt(words,numline,filename) - #print(results) + if histoTag.activated: - if sqlite_exists: histoinfo.positive.nevents = int(histoStatistics[histoinfo.name][0]) - histoinfo.negative.nevents = int(histoStatistics[histoinfo.name][1]) - #print(histoinfo.positive.nevents, histoinfo.negative.nevents) + histoinfo.negative.nevents = int(histoStatistics[histoinfo.name][1]) else : histoinfo.positive.nevents=results[0] histoinfo.negative.nevents=results[1] elif histoLogXTag.activated: - histologxinfo.positive.nevents=results[0] - histologxinfo.negative.nevents=results[1] + if sqlite_exists: + histologxinfo.positive.nevents = int(histoStatistics[histologxinfo.name][0]) + histologxinfo.negative.nevents = int(histoStatistics[histologxinfo.name][1]) + else : + histologxinfo.positive.nevents=results[0] + histologxinfo.negative.nevents=results[1] elif histoFreqTag.activated: histofreqinfo.positive.nevents=results[0] histofreqinfo.negative.nevents=results[1] elif statisticsTag.Nlines==1: results = self.ExtractStatisticsFloat(words,numline,filename) - # print(results) + if histoTag.activated: if sqlite_exists: histoinfo.positive.sumwentries = histoStatistics[histoinfo.name][2] - histoinfo.negative.sumwentries = histoStatistics[histoinfo.name][3] - #print(histoinfo.positive.sumwentries, histoinfo.negative.sumwentries) + histoinfo.negative.sumwentries = histoStatistics[histoinfo.name][3] else : histoinfo.positive.sumwentries=results[0] histoinfo.negative.sumwentries=results[1] elif histoLogXTag.activated: - histologxinfo.positive.sumwentries=results[0] - histologxinfo.negative.sumwentries=results[1] + if sqlite_exists: + histologxinfo.positive.sumwentries = histoStatistics[histologxinfo.name][2] + histologxinfo.negative.sumwentries = histoStatistics[histologxinfo.name][3] + else : + histologxinfo.positive.sumwentries=results[0] + histologxinfo.negative.sumwentries=results[1] elif histoFreqTag.activated: histofreqinfo.positive.sumwentries=results[0] histofreqinfo.negative.sumwentries=results[1] elif statisticsTag.Nlines==2: results = self.ExtractStatisticsInt(words,numline,filename) - # print(results) + if histoTag.activated: if sqlite_exists: histoinfo.positive.nentries = int(histoStatistics[histoinfo.name][4]) - histoinfo.negative.nentries = int(histoStatistics[histoinfo.name][5]) - # print(histoinfo.positive.nentries, histoinfo.negative.nentries) + histoinfo.negative.nentries = int(histoStatistics[histoinfo.name][5]) else : histoinfo.positive.nentries=results[0] histoinfo.negative.nentries=results[1] elif histoLogXTag.activated: - histologxinfo.positive.nentries=results[0] - histologxinfo.negative.nentries=results[1] + if sqlite_exists: + histologxinfo.positive.nentries = int(histoStatistics[histologxinfo.name][4]) + histologxinfo.negative.nentries = int(histoStatistics[histologxinfo.name][5]) + else : + histologxinfo.positive.nentries=results[0] + histologxinfo.negative.nentries=results[1] elif histoFreqTag.activated: histofreqinfo.positive.nentries=results[0] histofreqinfo.negative.nentries=results[1] elif statisticsTag.Nlines==3: results = self.ExtractStatisticsFloat(words,numline,filename) - # print(results) + if histoTag.activated: if sqlite_exists: histoinfo.positive.sumw = histoStatistics[histoinfo.name][6] - histoinfo.negative.sumw = histoStatistics[histoinfo.name][7] - # print(histoinfo.positive.sumw, histoinfo.negative.sumw) + histoinfo.negative.sumw = histoStatistics[histoinfo.name][7] else : histoinfo.positive.sumw=results[0] histoinfo.negative.sumw=results[1] elif histoLogXTag.activated: - histologxinfo.positive.sumw=results[0] - histologxinfo.negative.sumw=results[1] + if sqlite_exists: + histologxinfo.positive.sumw = histoStatistics[histologxinfo.name][6] + histologxinfo.negative.sumw = histoStatistics[histologxinfo.name][7] + else : + histologxinfo.positive.sumw=results[0] + histologxinfo.negative.sumw=results[1] elif histoFreqTag.activated: histofreqinfo.positive.sumw=results[0] histofreqinfo.negative.sumw=results[1] elif statisticsTag.Nlines==4 and not histoFreqTag.activated: results = self.ExtractStatisticsFloat(words,numline,filename) - # print(results) + if histoTag.activated: if sqlite_exists: histoinfo.positive.sumw2 = histoStatistics[histoinfo.name][8] - histoinfo.negative.sumw2 = histoStatistics[histoinfo.name][9] - # print(histoinfo.positive.sumw2, histoinfo.negative.sumw2) + histoinfo.negative.sumw2 = histoStatistics[histoinfo.name][9] else : histoinfo.positive.sumw2=results[0] histoinfo.negative.sumw2=results[1] elif histoLogXTag.activated: - histologxinfo.positive.sumw2=results[0] - histologxinfo.negative.sumw2=results[1] - + if sqlite_exists: + histologxinfo.positive.sumw2 = histoStatistics[histologxinfo.name][8] + histologxinfo.negative.sumw2 = histoStatistics[histologxinfo.name][9] + else : + histologxinfo.positive.sumw2=results[0] + histologxinfo.negative.sumw2=results[1] elif statisticsTag.Nlines==5 and not histoFreqTag.activated: results = self.ExtractStatisticsFloat(words,numline,filename) - # print(results) + if histoTag.activated: if sqlite_exists: histoinfo.positive.sumwx = histoStatistics[histoinfo.name][10] - histoinfo.negative.sumwx = histoStatistics[histoinfo.name][11] - # print(histoinfo.positive.sumwx, histoinfo.negative.sumwx) + histoinfo.negative.sumwx = histoStatistics[histoinfo.name][11] else : histoinfo.positive.sumwx=results[0] histoinfo.negative.sumwx=results[1] elif histoLogXTag.activated: - histologxinfo.positive.sumwx=results[0] - histologxinfo.negative.sumwx=results[1] + if sqlite_exists: + histologxinfo.positive.sumwx = histoStatistics[histologxinfo.name][10] + histologxinfo.negative.sumwx = histoStatistics[histologxinfo.name][11] + else : + histologxinfo.positive.sumwx=results[0] + histologxinfo.negative.sumwx=results[1] elif statisticsTag.Nlines==6 and not histoFreqTag.activated: results = self.ExtractStatisticsFloat(words,numline,filename) - # print(results) + if histoTag.activated: if sqlite_exists: histoinfo.positive.sumw2x = histoStatistics[histoinfo.name][12] - histoinfo.negative.sumw2x = histoStatistics[histoinfo.name][13] - # print(histoinfo.positive.sumw2x, histoinfo.negative.sumw2x) + histoinfo.negative.sumw2x = histoStatistics[histoinfo.name][13] else : histoinfo.positive.sumw2x=results[0] histoinfo.negative.sumw2x=results[1] elif histoLogXTag.activated: - histologxinfo.positive.sumw2x=results[0] - histologxinfo.negative.sumw2x=results[1] + if sqlite_exists: + histologxinfo.positive.sumw2x = histoStatistics[histologxinfo.name][12] + histologxinfo.negative.sumw2x = histoStatistics[histologxinfo.name][13] + else : + histologxinfo.positive.sumw2x=results[0] + histologxinfo.negative.sumw2x=results[1] else: logging.getLogger('MA5').warning('Extra line is found: '+line) From 984ec51285cedd43b1f89b182e323edd645c5f48 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 20 Mar 2023 22:47:39 -0600 Subject: [PATCH 51/53] fixed histologx underflow/overflow --- madanalysis/IOinterface/job_reader.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py index 20fd6d35..112e9f67 100644 --- a/madanalysis/IOinterface/job_reader.py +++ b/madanalysis/IOinterface/job_reader.py @@ -476,19 +476,26 @@ def ExtractHistos(self,dataset,plot,merging=False): data_negative = [] else: + bin_means = [] + bin_stdev = [] + + # save bin mean and stdev into histogram_core for positive and negative values + for bin_index in sqlite_output_dictionary[histologxinfo.name]: + if bin_index not in ["underflow", "overflow"]: + bin_means.append(sqlite_output_dictionary[histoinfo.name][bin_index][0]) + bin_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1]) + elif bin_index == "underflow": + plot.histos[-1].positive.underflow = sqlite_output_dictionary[histologxinfo.name][bin_index][0] + elif bin_index == "overflow": + plot.histos[-1].positive.overflow = sqlite_output_dictionary[histologxinfo.name][bin_index][0] - for bin_index in sqlite_output_dictionary[histoinfo.name]: - bin_means.append(sqlite_output_dictionary[histologxinfo.name][bin_index][0]) - bin_stdev.append(sqlite_output_dictionary[histologxinfo.name][bin_index][1]) - - histoinfo.Reset() + histologxinfo.Reset() plot.histos[-1].positive.array = bin_means[:] ##plot.histos[-1].negative.array = bin_means[:] plot.histos[-1].positive.stdev = bin_stdev[:] ##plot.histos[-1].negative.stdev = bin_stdev[:] - # Looking from histogram description elif descriptionTag.activated: if descriptionTag.Nlines==0: From c4619ce96a5bf38bd385cabdd2b5356fac7d3d22 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 27 Mar 2023 23:59:35 -0600 Subject: [PATCH 52/53] minor fixes --- madanalysis/IOinterface/sqlite_reader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py index 7546e5b3..60ff5bb4 100644 --- a/madanalysis/IOinterface/sqlite_reader.py +++ b/madanalysis/IOinterface/sqlite_reader.py @@ -56,7 +56,10 @@ def getStatistics(stats): for entry in stats: if entry[0] not in histoname_dict: histoname_dict[entry[0]] = dict() - histoname_dict[entry[0]][entry[1]] = float(entry[2]) - float(entry[3]) + sumw = float(entry[2]) - abs(float(entry[3])) + if sumw == 0: + raise Exception("sumw is 0 for histo " + entry[0] + " and weight id " + entry[1]) + histoname_dict[entry[0]][entry[1]] = sumw return histoname_dict @@ -78,7 +81,8 @@ def getMeanAndStdev(path): weight_id = row[1] bin_number = row[2] sumw = statsdict[histo_name][str(weight_id)] - value = (float(row[3]) - abs(float(row[4]))) / sumw + + value = (float(row[3]) - abs(float(row[4]))) /sumw if histo_name not in parsed_data: ## if histo name is not in the parsed_data dictionary, then create a new bin dictionary for that histo, then for the bin, create a weigh id dictionary parsed_data[histo_name] = dict() From 46c7292388f5496916771ff8790792f3c48fb8be Mon Sep 17 00:00:00 2001 From: Kyle Date: Tue, 28 Mar 2023 01:18:19 -0600 Subject: [PATCH 53/53] removed .DS_store from sqlite interface" git push " --- tools/SampleAnalyzer/Interfaces/sqlite/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tools/SampleAnalyzer/Interfaces/sqlite/.DS_Store diff --git a/tools/SampleAnalyzer/Interfaces/sqlite/.DS_Store b/tools/SampleAnalyzer/Interfaces/sqlite/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0