Proper Tutorial for Implementing Samples etc

CIValidations/FitterValidations.cpp

@@ -4,7 +4,6 @@
 #include "Utils/Comparison.h"
 #include "samplePDF/samplePDFTutorial.h"
 
-
 /// @todo add sample PDF object!
 ///
 ///
@@ -22,6 +21,7 @@ void FitVal(const std::string& Algo, bool MoreTests)
   MACH3LOG_INFO("Testing {}", Algo);
 
   covarianceXsec* xsec = MaCh3CovarianceFactory(FitManager, "Xsec");
+  covarianceOsc* osc = MaCh3CovarianceFactory<covarianceOsc>(FitManager, "Osc");
   std::unique_ptr<FitterBase> MaCh3Fitter = nullptr;
   if(Algo == "MCMC") {
     FitManager->OverrideSettings("General", "OutputFile", "MCMC_Test.root");
@@ -43,10 +43,6 @@ void FitVal(const std::string& Algo, bool MoreTests)
     throw MaCh3Exception(__FILE__ , __LINE__ );
   }
 
-  std::vector<std::string> OscCovMatrixFile = {"Inputs/Osc_Test.yaml"};
-  covarianceOsc* osc = new covarianceOsc(OscCovMatrixFile, "osc_cov");
-  osc->setParameters();
-
   std::string SampleConfig = {"Inputs/SamplePDF_Tutorial.yaml"};
   samplePDFTutorial *Sample = new samplePDFTutorial(SampleConfig, xsec);
   Sample->SetXsecCov(xsec);

Inputs/ManagerTest.yaml

@@ -1,6 +1,7 @@
 General:
   OutputFile: "Test.root"
   TutorialSamples: ["Inputs/SamplePDF_Tutorial.yaml"]
+  FittingAlgorithm: "MCMC"
 
   RealData: false
   Asimov: true
@@ -45,8 +46,10 @@ General:
     XsecPCAParams: [-999, -999]
     XsecStepScale: 1
 
+    OscCovName: "osc_cov"
     OscCovFile: ["Inputs/Osc_Test.yaml"]
     OscFix: ["baseline", "density", "Ye"]
+    OscStepScale: 1
 
   Predictive:
     Ntoy: 10
@@ -59,5 +62,3 @@ LikelihoodOptions:
   #False means you calculate W2 histogram only during first reweight, advisable for Barlow-Beeston
   UpdateW2: false
   TestStatistic: "Barlow-Beeston"
-  #TestStatistic: "Poisson"
-  #TestStatistic: "IceCube"

README.md

@@ -7,119 +7,49 @@ After this tutorial you should know how to run MCMC, implement systematic uncert
 
 ## How to Start?
 To compile simply
-```
+```bash
 mkdir build;
 cd build;
 cmake ../ -DPYTHON_ENABLED=ON [DPYTHON_ENABLED not mandatory]
 make -jN [set number of threads]
 make install
 ```
 then
-```
+```bash
 source bin/setup.MaCh3.sh
 source bin/setup.MaCh3Tutorial.sh
 ```
 alternatively you can use containers by
-```
+```bash
 docker pull ghcr.io/mach3-software/mach3tutorial:alma9latest
 ```
 To reed more how to use containers check our wiki [here](https://github.com/mach3-software/MaCh3/wiki/12.-Containers)
 
-## How to run LLH scan
-To run an LLH scan simply do
-```
-./bin/LLHScanTutorial Inputs/ManagerTest.yaml
-```
-These are very useful for validations and to compare with other fitters
-
 ## How to run MCMC
 To run MCMC simply
-```
+```bash
 ./bin/MCMCTutorial Inputs/ManagerTest.yaml
 ```
 Congratulations! 🎉
 You have just completed finished you first MCMC chain.
 
-## How to Plot?
-
-There are a number of apps included to make plots from the results of your fits, llh scans etc. You can find more details on them and how they work in the main MaCh3 wiki [here](https://github.com/mach3-software/MaCh3/wiki). There you will also find some instructions on how you can write yor own plotting scripts. 
-
-The plotting library is configured using yaml files. You can see some examples of such config files in the plotting directory, and a detailed explanation of them is given in [the wiki](https://github.com/mach3-software/MaCh3/wiki).
-
-Some examples on how to make some "standard" plots are given below.
-
 ### MCMC Chain
 You can produce simple plots with
-```
+```bash
 ./bin/ProcessMCMC bin/TutorialDiagConfig.yaml Test.root
 ```
 where Test.root is the output of running MCMCTutorial as described [here](#how-to-run-mcmc)
 
-You can then take the output of running ProcessMCMC which will be called something like <inputName>_Process.root, and make fancier error plots from it using the `GetPostfitParamPlots` app like 
+You can then take the output of running ProcessMCMC which will be called something like <inputName>_Process.root, and make fancier error plots from it using the `GetPostfitParamPlots` app like:
 
-```
+```bash
 GetPostfitParamPlots Test_Process.root
 ```
 
-### LLH Scans
-
-You can plot the results of an LLH scan using the aptly named PlotLLH app like so
-
-```
-PlotLLH LLH_Test.root
-```
-
-where LLH_Test.root is the result of running the LLH scan as described [here](#how-to-run-llh-scan).
-
-### Plotting with Python
-
-If you have installed the python interface for MaCh3 as described [here](https://github.com/mach3-software/MaCh3?tab=readme-ov-file#python) then you can also use the provided python plotting module. The details on how to write custom python scripts using this plotting module are detailed in [the wiki](https://github.com/mach3-software/MaCh3/wiki/15.-Plotting#custom-plotting-scripts). Here we will walk you through some example scripts.
-
-For the examples here, we will use matplotlib and numpy. These can be installed using the provided [requirements.txt](requirements.txt) by doing 
-
-```
-pip install -r requirements.txt
-```
-
-but note that these are just what is used for an example for the purpose of this tutorial. When making your own plots you are totally free to use whatever plotting libraries you like!
-
-You can use the example script [MCMCPlotting-tutorial.py](plotting/MCMC-plotting-tutorial.py) to plot the raw MCMC chain values that were obtained in the [how to run MCMC](#how-to-run-mcmc) section above by doing 
-
-```
-python plotting/MCMC-plotting-tutorial.py Test.root
-```
-
-This will give you some plots that look something like 
-
-<img width="350" alt="MCMC example" src="https://github.com/user-attachments/assets/bdb1792f-3d52-4ea3-8eb8-0e6cf7b9119a">
-
-After you have run this chain through the MCMC processor as described in the [How To Plot?](#how-to-plot) section, you can pass the processed file to [1DPosterior-tutorial.py](plotting/1DPosterior-tutorial.py) like
-
-```
-python plotting/1DPosterior-tutorial.py Test_Process.root
-```
-
-which will plot the projected one dimensional posteriors which should look something like 
-
-<img width="350" alt="1dPosterior example" src="https://github.com/user-attachments/assets/4ddf3abb-9794-4f21-ae9b-862718c2ff57">
-
-
-Similarly you can use [LLHScan-plotting-tutorial.py](plotting/LLHScan-plotting-tutorial.py) to plot the LLH scans you made in the [How to run LLH scan](#how-to-run-llh-scan) section like
-
-```
-python plotting/LLHScan-plotting-tutorial.py LLH_Test.root
-```
-
-which will give you some plots that look something like 
-
-<img width="350" alt="LLH scan example" src="https://github.com/user-attachments/assets/f16ad571-68da-42e3-ae6b-e984d03a58c3">
-
-
-
-## How to Setup your analysis
+## How to Develop Model of Systematic Uncertainties
 In the next step you gonna modify analysis setup and repeat steps.
 First let's better understand `Inputs/SystematicsTest.yaml`. This config controls what systematic uncertainties will be used in the analysis for example like this:
-```
+```yaml
 - Systematic:
     Names:
       FancyName: Norm_Param_0
@@ -148,7 +78,7 @@ Lastly we need to modify name of output file. This is governed by manager class
 `OutputFile: "Test_Modified.root"`.
 
 Now let's run MCMC again
-```
+```bash
 ./bin/MCMCTutorial Inputs/ManagerTest.yaml
 ```
 Congratulations! 🎉
@@ -157,17 +87,139 @@ Next step is to compare both chains.
 Warning: If you modified files in main folder not build you will have to call make install!
 ## How to Plot Comparisons?
 Now that you have two chains you can try comparing them using following.
-```
+```bash
 ./bin/ProcessMCMC bin/TutorialDiagConfig.yaml Test.root Default_Chain Test_Modified.root Modified_Chain
 ```
 This will produce pdf file with overlayed posteriors. Most should be similarly except modified parameter.
 
+## How to Develop New Samples
+First we gonna investigate how to modify sample, let's take a look at `Inputs/SamplePDF_Tutorial.yaml`. Each sample has set of cuts right now we only introduce cut on TrueNeutrinoEnergy.
+```yaml
+SelectionCuts:
+  - KinematicStr: "TrueNeutrinoEnergy"
+    Bounds: [ 0., 4 ]
+```
+We can introduce additional cut for example on Q2 by expanding config like this:
+```yaml
+SelectionCuts:
+  - KinematicStr: "TrueNeutrinoEnergy"
+    Bounds: [ 0., 4 ]
+  - KinematicStr: "TrueQ2"
+    Bounds: [ 0.6, 1 ]
+```
+
+You can also easily switch variable in which you bin sample.
+```yaml
+Binning:
+  XVarStr : "TrueNeutrinoEnergy"
+  XVarBins: [0.,  0.5,  1.,  1.25, 1.5, 1.75, 2., 2.25, 2.5, 2.75, 3., 3.25, 3.5, 3.75, 4., 5., 6., 10.]
+```
+You can try again to run MCMC and compare all 3 chains
+```bash
+./bin/ProcessMCMC bin/TutorialDiagConfig.yaml Test.root Default_Chain Test_Modified.root Modified_Chain Test_Modified_Sample.root ModifiedSameple_Chain
+```
+Up to this point we only modified sample but how to add new one? First make copy of sample config `Inputs/SamplePDF_Tutorial.yaml` and call it `Inputs/SamplePDF_User.yaml`. For the moment feel free to change name, binning etc but keep inputs the same. Go wild! Next go to `Inputs/ManagerTest.yaml`
+```yaml
+General:
+  TutorialSamples: ["Inputs/SamplePDF_Tutorial.yaml"]
+```
+To add you newly implemented sample you will have to expand config to for example:
+```yaml
+General:
+  TutorialSamples: ["Inputs/SamplePDF_Tutorial.yaml", "Inputs/SamplePDF_User.yaml"]
+```
 
 ## MCMC Diagnostic
 Crucial part of MCMC is diagnostic whether chain converged or not. You can read more on [here](https://github.com/mach3-software/MaCh3/wiki/11.-Step-size-tuning)
 
-```
+```bash
 ./bin/DiagMCMC Test.root bin/TutorialDiagConfig.yaml
 ```
 
-## More advanced development????
+### Useful Settings
+There are plenty of usefull settings
+**Fitting Algorithm**: Most likely you run MCMC but what if you want to use algorith like Minuit2?
+```yaml
+General:
+  FittingAlgorithm: "MCMC"
+```
+In `Inputs/ManagerTest.yaml` you should switch following setting to "Minuit2"
+
+**LLH Type**:
+By default we use Barlow-Beeston LLH, however several are implemented. For example by changing confign you can use Poisson or maybe IceCube.
+```yaml
+LikelihoodOptions:
+  TestStatistic: "Barlow-Beeston"
+```
+Read more [here](https://mach3-software.github.io/MaCh3/Structs_8h.html#a960da89e33ac45a56f7bbfac3068dc67)
+
+## How to Plot?
+
+There are a number of apps included to make plots from the results of your fits, llh scans etc. You can find more details on them and how they work in the main MaCh3 wiki [here](https://github.com/mach3-software/MaCh3/wiki). There you will also find some instructions on how you can write yor own plotting scripts.
+
+The plotting library is configured using yaml files. You can see some examples of such config files in the plotting directory, and a detailed explanation of them is given in [the wiki](https://github.com/mach3-software/MaCh3/wiki).
+
+Some examples on how to make some "standard" plots are given below.
+
+### How to run LLH scan
+You can run MCMC in very similar way as MCMC
+```bash
+./bin/LLHScanTutorial Inputs/ManagerTest.yaml
+```
+You can plot the results of an LLH scan using the aptly named PlotLLH app like so
+
+```bash
+PlotLLH LLH_Test.root
+```
+where LLH_Test.root is the result of running the LLH scan as described [here](#how-to-run-llh-scan).
+
+### Plotting with Python
+
+If you have installed the python interface for MaCh3 as described
+[here](https://github.com/mach3-software/MaCh3?tab=readme-ov-file#python)
+then you can also use the provided python plotting module. The details on how
+to write custom python scripts using this plotting module are detailed in
+[the wiki](https://github.com/mach3-software/MaCh3/wiki/15.-Plotting#custom-plotting-scripts).
+Here we will walk you through some example scripts.
+
+For the examples here, we will use matplotlib and numpy. These can be installed using the provided [requirements.txt](requirements.txt) by doing
+
+```bash
+pip install -r requirements.txt
+```
+
+but note that these are just what is used for an example for the purpose of this tutorial. When making your own plots you are totally free to use whatever plotting libraries you like!
+
+You can use the example script [MCMCPlotting-tutorial.py](plotting/MCMC-plotting-tutorial.py) to plot the raw MCMC chain values that were obtained in the [how to run MCMC](#how-to-run-mcmc) section above by doing
+
+```bash
+python plotting/MCMC-plotting-tutorial.py Test.root
+```
+
+This will give you some plots that look something like
+
+<img width="350" alt="MCMC example" src="https://github.com/user-attachments/assets/bdb1792f-3d52-4ea3-8eb8-0e6cf7b9119a">
+
+After you have run this chain through the MCMC processor as described in the [How To Plot?](#how-to-plot) section, you can pass the processed file to [1DPosterior-tutorial.py](plotting/1DPosterior-tutorial.py) like
+
+```bash
+python plotting/1DPosterior-tutorial.py Test_Process.root
+```
+
+which will plot the projected one dimensional posteriors which should look something like
+
+<img width="350" alt="1dPosterior example" src="https://github.com/user-attachments/assets/4ddf3abb-9794-4f21-ae9b-862718c2ff57">
+
+
+Similarly you can use [LLHScan-plotting-tutorial.py](plotting/LLHScan-plotting-tutorial.py) to plot the LLH scans you made in the [How to run LLH scan](#how-to-run-llh-scan) section like
+
+```bash
+python plotting/LLHScan-plotting-tutorial.py LLH_Test.root
+```
+
+which will give you some plots that look something like
+
+<img width="350" alt="LLH scan example" src="https://github.com/user-attachments/assets/f16ad571-68da-42e3-ae6b-e984d03a58c3">
+
+
+## More advanced development???

Tutorial/LLHScanTutorial.cpp

@@ -1,6 +1,5 @@
 // MaCh3 spline includes
 #include "mcmc/MaCh3Factory.h"
-#include "covariance/covarianceXsec.h"
 #include "samplePDF/samplePDFTutorial.h"
 
 int main(int argc, char *argv[]){
@@ -11,39 +10,26 @@ int main(int argc, char *argv[]){
 
   // Initialise covariance class reasonable for Systematics
   covarianceXsec* xsec = MaCh3CovarianceFactory(FitManager, "Xsec");
+  covarianceOsc*  osc  = MaCh3CovarianceFactory<covarianceOsc>(FitManager, "Osc");
 
-  auto OscMatrixFile = FitManager->raw()["General"]["Systematics"]["OscCovFile"].as<std::vector<std::string>>();
-  auto OscFixParams  = FitManager->raw()["General"]["Systematics"]["OscFix"].as<std::vector<std::string>>();
-  covarianceOsc* osc = new covarianceOsc(OscMatrixFile, "osc_cov");
-  // Fixed xsec parameters loop
-  if (OscFixParams.size() == 1 && OscFixParams.at(0) == "All") {
-    for (int j = 0; j < osc->GetNumParams(); j++) {
-      osc->toggleFixParameter(j);
-    }
-  } else {
-    for (unsigned int j = 0; j < OscFixParams.size(); j++) {
-      osc->toggleFixParameter(OscFixParams.at(j));
-    }
-  }
+  // Initialise samplePDF
+  auto SampleConfig = FitManager->raw()["General"]["TutorialSamples"].as<std::vector<std::string>>();
+  auto mySamples = MaCh3SamplePDFFactory<samplePDFTutorial>(SampleConfig, xsec, osc);
 
   // Create MCMC Class
-  std::unique_ptr<FitterBase> MaCh3Fitter = std::make_unique<mcmc>(FitManager);
+  std::unique_ptr<FitterBase> MaCh3Fitter = MaCh3FitterFactory(FitManager);
   // Add covariance to MCM
   MaCh3Fitter->addSystObj(xsec);
   MaCh3Fitter->addSystObj(osc);
-
-  auto SampleConfig = FitManager->raw()["General"]["TutorialSamples"].as<std::vector<std::string>>();
-  auto mySamples = MaCh3SamplePDFFactory<samplePDFTutorial>(SampleConfig, xsec, osc);
   for (size_t i = 0; i < SampleConfig.size(); ++i) {
     MaCh3Fitter->addSamplePDF(mySamples[i]);
   }
-
-  // Run MCMCM
+  // Run LLH scan
   MaCh3Fitter->RunLLHScan();
 
   delete FitManager;
   delete xsec;
-  MaCh3Fitter.reset();
+  delete osc;
   for (size_t i = 0; i < SampleConfig.size(); ++i) {
     delete mySamples[i];
   }