Merge pull request #979 from kcormi/tutorial_stat_routines

Add tutorial material
cms-analysis · Jun 24, 2024 · 5197481 · 5197481
2 parents 732b132 + 81bc683
commit 5197481
Show file tree

Hide file tree

Showing 16 changed files with 528 additions and 0 deletions.
diff --git a/data/tutorials/statistical_routines_tutorial/datacard.txt b/data/tutorials/statistical_routines_tutorial/datacard.txt
@@ -0,0 +1,17 @@
+imax 1                         # number of channels
+jmax 1                         # number of backgrounds
+kmax 3                         # number of nuisance parameters
+-------
+shapes * * FAKE
+------
+bin          bin1
+observation     6
+-----
+bin            bin1    bin1
+process         ggH      WW
+process           0       1
+rate            2.3     5.4
+-------
+lumi     lnN   1.01    1.01    # luminosity uncertainty
+xs_ggH   lnN   1.10       -    # gg->H cross section + signal efficiency + other minor ones
+xs_WW    lnN      -    1.16    # WW cross section + signal efficiency + other minor ones
diff --git a/data/tutorials/statistical_routines_tutorial/datacard_lowbackground.txt b/data/tutorials/statistical_routines_tutorial/datacard_lowbackground.txt
@@ -0,0 +1,17 @@
+imax 1                         # number of channels
+jmax 1                         # number of backgrounds
+kmax 3                         # number of nuisance parameters
+-------
+shapes * * FAKE
+------
+bin          bin1
+observation     0
+-----
+bin            bin1    bin1
+process         ggH      WW
+process           0       1
+rate            2.3       1
+-------
+lumi     lnN   1.01    1.01    # luminosity uncertainty
+xs_ggH   lnN   1.10       -    # gg->H cross section + signal efficiency + other minor ones
+xs_WW    lnN      -    1.16    # WW cross section + signal efficiency + other minor ones
diff --git a/data/tutorials/statistical_routines_tutorial/datacard_obs12.txt b/data/tutorials/statistical_routines_tutorial/datacard_obs12.txt
@@ -0,0 +1,17 @@
+imax 1                         # number of channels
+jmax 1                         # number of backgrounds
+kmax 3                         # number of nuisance parameters
+-------
+shapes * * FAKE
+------
+bin          bin1
+observation     12
+-----
+bin            bin1    bin1
+process         ggH      WW
+process           0       1
+rate            2.3     5.4
+-------
+lumi     lnN   1.01    1.01    # luminosity uncertainty
+xs_ggH   lnN   1.10       -    # gg->H cross section + signal efficiency + other minor ones
+xs_WW    lnN      -    1.16    # WW cross section + signal efficiency + other minor ones
diff --git a/data/tutorials/statistical_routines_tutorial/datacard_underfluctuation.txt b/data/tutorials/statistical_routines_tutorial/datacard_underfluctuation.txt
@@ -0,0 +1,17 @@
+imax 1                         # number of channels
+jmax 1                         # number of backgrounds
+kmax 3                         # number of nuisance parameters
+-------
+shapes * * FAKE
+------
+bin          bin1
+observation     13
+-----
+bin            bin1    bin1
+process         ggH      WW
+process           0       1
+rate            2.3     23.4
+-------
+lumi     lnN   1.01    1.01    # luminosity uncertainty
+xs_ggH   lnN   1.10       -    # gg->H cross section + signal efficiency + other minor ones
+xs_WW    lnN      -    1.16    # WW cross section + signal efficiency + other minor ones
diff --git a/data/tutorials/statistical_routines_tutorial/get_quantile.py b/data/tutorials/statistical_routines_tutorial/get_quantile.py
@@ -0,0 +1,38 @@
+from ROOT import TFile
+import numpy as np
+import scipy.stats as st
+
+from argparse import ArgumentParser
+
+parser = ArgumentParser()
+parser.add_argument("--input", default="higgsCombineTest.MultiDimFit.mH120.123456.root", help="input root file with toy results from MultiDimFit")
+parser.add_argument("--q0", action="store_true", help="use the q_0 test statistic rather than the profile likelihood ratio.")
+args = parser.parse_args()
+
+n_sigma = 1
+quantile_val = 2 * st.norm().cdf(-n_sigma)  # Get the quantile corresponding to the N sigma interval
+
+f = TFile(args.input, "READ")
+limit = f.Get("limit")
+n_entries = limit.GetEntries()
+
+m2nll_vals = []
+r_vals = []
+last_toy_num = -1
+for i in range(n_entries):
+    limit.GetEntry(i)
+    if limit.quantileExpected < 0:
+        if args.q0:
+            r_vals.append(limit.r)
+        continue
+
+    m2nll_vals.append(2 * limit.deltaNLL)
+
+
+test_stat_vals = m2nll_vals
+if args.q0:
+    test_stat_vals = np.where(np.array(r_vals) > 0, test_stat_vals, 0)
+
+test_stat_cutoff = np.quantile(test_stat_vals, 1 - quantile_val)
+t_stat_name = "q0" if args.q0 else "-2*deltaNLL"
+print(f"This point is rejected at the {n_sigma} sigma level if the test stat {t_stat_name}  > {test_stat_cutoff}")
diff --git a/docs/tutorial_stat_routines/limit_test_stat_distributions.png b/docs/tutorial_stat_routines/limit_test_stat_distributions.png
diff --git a/docs/tutorial_stat_routines/limit_test_stat_distributions_few_events.png b/docs/tutorial_stat_routines/limit_test_stat_distributions_few_events.png
diff --git a/docs/tutorial_stat_routines/llhood_scan.pdf b/docs/tutorial_stat_routines/llhood_scan.pdf
diff --git a/docs/tutorial_stat_routines/llhood_scan.png b/docs/tutorial_stat_routines/llhood_scan.png
diff --git a/docs/tutorial_stat_routines/llhood_scan_for_significance.png b/docs/tutorial_stat_routines/llhood_scan_for_significance.png
diff --git a/docs/tutorial_stat_routines/llhood_scan_obs10.png b/docs/tutorial_stat_routines/llhood_scan_obs10.png
diff --git a/docs/tutorial_stat_routines/llhood_scan_obs12.png b/docs/tutorial_stat_routines/llhood_scan_obs12.png
diff --git a/docs/tutorial_stat_routines/llhood_scan_obs2.png b/docs/tutorial_stat_routines/llhood_scan_obs2.png
diff --git a/docs/tutorial_stat_routines/r_scan.png b/docs/tutorial_stat_routines/r_scan.png
diff --git a/docs/tutorial_stat_routines/stat_routines.md b/docs/tutorial_stat_routines/stat_routines.md
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -32,6 +32,7 @@ nav:
             - "Solutions":                part5/longexerciseanswers.md
           - "Parametric Models":          tutorial2023/parametric_exercise.md
           - "Likelihood Based Unfolding": tutorial2023_unfolding/unfolding_exercise.md
+          - "Statitiscal Tests Basics": tutorial_stat_routines/stat_routines.md
         - Links & FAQ: part4/usefullinks.md
 
 theme: