From 045297d04b447a58efc391b1b2ef1b2f92a483a3 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Fri, 26 Jan 2024 12:59:43 -0500 Subject: [PATCH 1/3] optimise gradients * using vectorised `value_and_grad` instead of grad --- src/spey/backends/default_pdf/__init__.py | 8 ++------ src/spey/backends/default_pdf/simple_pdf.py | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/spey/backends/default_pdf/__init__.py b/src/spey/backends/default_pdf/__init__.py index 8dfdc92..a0ef7f4 100644 --- a/src/spey/backends/default_pdf/__init__.py +++ b/src/spey/backends/default_pdf/__init__.py @@ -2,7 +2,7 @@ from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Union -from autograd import grad, hessian, jacobian +from autograd import value_and_grad, hessian, jacobian from autograd import numpy as np from scipy.optimize import NonlinearConstraint @@ -251,11 +251,7 @@ def negative_loglikelihood(pars: np.ndarray) -> np.ndarray: ) - self.constraint_model.log_prob(pars) if do_grad: - grad_negative_loglikelihood = grad(negative_loglikelihood, argnum=0) - return lambda pars: ( - negative_loglikelihood(pars), - grad_negative_loglikelihood(pars), - ) + return value_and_grad(negative_loglikelihood, argnum=0) return negative_loglikelihood diff --git a/src/spey/backends/default_pdf/simple_pdf.py b/src/spey/backends/default_pdf/simple_pdf.py index e0cacb5..c1be7a1 100644 --- a/src/spey/backends/default_pdf/simple_pdf.py +++ b/src/spey/backends/default_pdf/simple_pdf.py @@ -2,7 +2,7 @@ from typing import Callable, List, Optional, Text, Tuple, Union -from autograd import grad, hessian +from autograd import value_and_grad, hessian from autograd import numpy as np from spey._version import __version__ @@ -162,11 +162,7 @@ def negative_loglikelihood(pars: np.ndarray) -> np.ndarray: return -self.main_model.log_prob(pars, data) if do_grad: - grad_negative_loglikelihood = grad(negative_loglikelihood, argnum=0) - return lambda pars: ( - negative_loglikelihood(pars), - grad_negative_loglikelihood(pars), - ) + return value_and_grad(negative_loglikelihood, argnum=0) return negative_loglikelihood From a46b31eb6ea7851181a58d21641943311889ef7b Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Sat, 27 Jan 2024 23:01:14 -0500 Subject: [PATCH 2/3] add callers for grad and hessian --- docs/api.rst | 11 ++++++ src/spey/__init__.py | 1 + src/spey/math.py | 86 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 src/spey/math.py diff --git a/docs/api.rst b/docs/api.rst index 376f4f5..2c8583e 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -82,6 +82,17 @@ Hypothesis testing asymptotic_calculator.compute_asymptotic_confidence_level toy_calculator.compute_toy_confidence_level +Gradient Tools +-------------- + +.. currentmodule:: spey.math + +.. autosummary:: + :toctree: _generated/ + + value_and_grad + hessian + Default Backends ---------------- diff --git a/src/spey/__init__.py b/src/spey/__init__.py index a00161c..1691a4b 100644 --- a/src/spey/__init__.py +++ b/src/spey/__init__.py @@ -25,6 +25,7 @@ "BackendBase", "ConverterBase", "about", + "math", ] diff --git a/src/spey/math.py b/src/spey/math.py new file mode 100644 index 0000000..04bb370 --- /dev/null +++ b/src/spey/math.py @@ -0,0 +1,86 @@ +from typing import Callable, Optional, Tuple + +from autograd import numpy + +from .interface.statistical_model import StatisticalModel +from .utils import ExpectationType + +# pylint: disable=E1101 + +__all__ = ["value_and_grad", "hessian"] + + +def __dir__(): + return __all__ + + +def value_and_grad( + statistical_model: StatisticalModel, + expected: ExpectationType = ExpectationType.observed, + data: Optional[numpy.ndarray] = None, +) -> Callable[[numpy.ndarray], Tuple[numpy.ndarray, numpy.ndarray]]: + """ + Retreive function to compute negative log-likelihood and its gradient. + + Args: + statistical_model (~spey.StatisticalModel): statistical model to be used. + expected (~spey.ExpectationType): Sets which values the fitting algorithm should focus and + p-values to be computed. + + * :obj:`~spey.ExpectationType.observed`: Computes the p-values with via post-fit + prescriotion which means that the experimental data will be assumed to be the truth + (default). + * :obj:`~spey.ExpectationType.aposteriori`: Computes the expected p-values with via + post-fit prescriotion which means that the experimental data will be assumed to be + the truth. + * :obj:`~spey.ExpectationType.apriori`: Computes the expected p-values with via pre-fit + prescription which means that the SM will be assumed to be the truth. + + data (``numpy.ndarray``, default ``None``): input data that to fit. If `None` observed + data will be used. + + Returns: + ``Callable[[numpy.ndarray], numpy.ndarray, numpy.ndarray]``: + negative log-likelihood and its gradient with respect to nuisance parameters + """ + val_and_grad = statistical_model.backend.get_objective_function( + expected=expected, data=data, do_grad=True + ) + return lambda pars: val_and_grad(numpy.array(pars)) + + +def hessian( + statistical_model: StatisticalModel, + expected: ExpectationType = ExpectationType.observed, + data: Optional[numpy.ndarray] = None, +) -> Callable[[numpy.ndarray], numpy.ndarray]: + r""" + Retreive the function to compute Hessian of negative log-likelihood + + .. math:: + + {\rm Hessian} = -\frac{\partial^2\mathcal{L}(\theta)}{\partial\theta_i\partial\theta_j} + + Args: + statistical_model (~spey.StatisticalModel): statistical model to be used. + expected (~spey.ExpectationType): Sets which values the fitting algorithm should focus and + p-values to be computed. + + * :obj:`~spey.ExpectationType.observed`: Computes the p-values with via post-fit + prescriotion which means that the experimental data will be assumed to be the truth + (default). + * :obj:`~spey.ExpectationType.aposteriori`: Computes the expected p-values with via + post-fit prescriotion which means that the experimental data will be assumed to be + the truth. + * :obj:`~spey.ExpectationType.apriori`: Computes the expected p-values with via pre-fit + prescription which means that the SM will be assumed to be the truth. + + data (``numpy.ndarray``, default ``None``): input data that to fit. If `None` observed + data will be used. + + Returns: + ``Callable[[numpy.ndarray], numpy.ndarray]``: + function to compute hessian of negative log-likelihood + """ + hess = statistical_model.backend.get_hessian_logpdf_func(expected=expected, data=data) + return lambda pars: -1.0 * hess(numpy.array(pars)) From fce776cb3b3865bf9065e054b692f9aa326d06e3 Mon Sep 17 00:00:00 2001 From: "Jack Y. Araz" Date: Sat, 27 Jan 2024 23:08:55 -0500 Subject: [PATCH 3/3] update changelog --- docs/releases/changelog-v0.1.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/releases/changelog-v0.1.md b/docs/releases/changelog-v0.1.md index d5142a4..d0cdc77 100644 --- a/docs/releases/changelog-v0.1.md +++ b/docs/releases/changelog-v0.1.md @@ -37,6 +37,12 @@ * Utilities to retreive bibtex information for third-party plug-ins. ([#32](https://github.com/SpeysideHEP/spey/pull/32)) +* Add math utilities for users to extract gradient and hessian of negative log-likelihood + ([#31](https://github.com/SpeysideHEP/spey/pull/31)) + +* Improve gradient execution for `default_pdf`. + ([#31](https://github.com/SpeysideHEP/spey/pull/31)) + ## Bug Fixes * In accordance to the latest updates ```UnCorrStatisticsCombiner``` has been updated with