"""
The delta-lambda PID: a Lagrangian generalization interpolating between the
delta-PID (Banerjee et al. 2018) and the BROJA/tilde-PID (Bertschinger et al. 2014).
Defines the generalized deficiency as:
delta_lambda(M : X \\ Y) := inf_{P(X'|MY)} E_M[D_KL(P(X|M) || P(X'|M))]
+ lambda * I(M; X' | Y)
As lambda -> infinity, this recovers the delta-PID (enforces Markov chain
M--Y--X', measures KL departure from copy).
As lambda -> 0, the KL term enforces the copy, P(X'|M) = P(X|M), and the
remaining term measures the departure from the Markov chain, so that
delta_lambda / lambda tends to the BROJA/tilde-PID unique information. The
deficiency itself vanishes in that limit, so the redundancy below tends to
min{I(M;X), I(M;Y)}, the MMI redundancy; Venkatesh et al. state the limit
informally as recovering the tilde-PID.
Redundancy is min-symmetrized:
RI_lambda(M : X; Y) = min{ I(M;X) - delta_lambda(M:X\\Y),
I(M;Y) - delta_lambda(M:Y\\X) }
References
----------
.. [1] P. Venkatesh, K. Gurushankar, G. Schamberg,
"Capturing and Interpreting Unique Information", arXiv:2302.11873, 2023.
Section III-G, Equation (26).
"""
import numpy as np
from ...algorithms.optimization import BaseAuxVarOptimizer, BaseConvexOptimizer, parallel_sweep
from ...exceptions import ditException
from ...multivariate import coinformation
from ..pid import BaseBivariatePID
__all__ = ("PID_DeltaLambda",)
class DeltaLambdaOptimizer(BaseConvexOptimizer, BaseAuxVarOptimizer):
"""
Optimizer for the delta-lambda generalized deficiency.
Minimizes E_M[D_KL(P(X|M) || P(X'|M))] + lambda * I(M; X' | Y)
over the channel P(X'|M,Y), where:
- source (X) is variable 0
- other (Y) is variable 1
- target (M) is variable 2
- auxiliary (X') is variable 3, conditioned on (M, Y) = {1, 2}
"""
construct_initial = BaseAuxVarOptimizer.construct_random_initial
def __init__(self, dist, source, other, target, lam=1.0, bound=None):
"""
Parameters
----------
dist : Distribution
The joint distribution.
source : iterable
The source variable (X).
other : iterable
The other source variable (Y).
target : iterable
The target variable (M).
lam : float
Lagrange multiplier. Large values enforce the Markov chain
(approaching delta-PID), small values enforce the copy
(the deficiency over lam approaching the BROJA unique information).
bound : int, None
Bound on |X'|. Defaults to |X|.
"""
dist = dist.coalesce([source, other, target])
super().__init__(dist, [[0], [1]], [2])
source_size = self._shape[0]
bound = min(bound, source_size) if bound is not None else source_size
# X' conditioned on (other=1, target=2) with alphabet size = |source|
self._construct_auxvars([({1, 2}, bound)])
self._lam = lam
self._source_rv = {0}
self._other_rv = {1}
self._target_rv = {2}
self._aux_rv = self._arvs
def _objective_gradient(self):
"""
Analytic gradient of the delta-lambda objective w.r.t. the joint.
The Lagrangian is ``kl + lam * I(X':M|Y)`` where
``kl = sum_{a,m} A[a,m] (log2 A[a,m] - log2 B[m,a])`` with
``A = p(X, M)`` (sum over Y, X') and ``B = p(X', M)`` (sum over X, Y).
Differentiating term by term:
* ``d kl / d A[a,m] = log2 A[a,m] - log2 B[m,a] + 1/ln2``
* ``d kl / d B[m,a] = -A[a,m] / (B[m,a] * ln2)``
and ``A`` / ``B`` are linear marginals of the joint, so each cell's
gradient is the appropriate broadcast of those two terms. The CMI term
reuses the standard conditional-MI gradient builder.
"""
cmi_g = self._conditional_mutual_information_grad(self._aux_rv, self._target_rv, self._other_rv)
lam = self._lam
ln2 = np.log(2)
def grad(pmf):
# A = p(X, M): sum over Y(1), X'(3) -> (|X|, |M|)
A = pmf.sum(axis=(1, 3))
# B = p(X', M): sum over X(0), Y(1) -> (|M|, |X'|)
B = pmf.sum(axis=(0, 1))
with np.errstate(divide="ignore"):
log_A = np.log2(np.maximum(A, 1e-300)) # (|X|, |M|)
log_B_aligned = np.log2(np.maximum(B, 1e-300)).T # (|X'|=|X|, |M|)
# contribution flowing through A[x, m], broadcast over Y and X'.
a_term = log_A - log_B_aligned + 1.0 / ln2 # (|X|, |M|)
g_a = a_term.reshape(A.shape[0], 1, A.shape[1], 1)
# contribution flowing through B[m, x'], broadcast over X and Y.
b_term = -A.T / (np.maximum(B, 1e-300) * ln2) # (|M|, |X'|)
g_b = b_term.reshape(1, 1, B.shape[0], B.shape[1])
kl_grad = np.broadcast_to(g_a, pmf.shape) + np.broadcast_to(g_b, pmf.shape)
return np.ascontiguousarray(kl_grad) + lam * cmi_g(pmf)
return grad
def _objective(self):
"""
Minimize E_M[D_KL(P(X|M) || P(X'|M))] + lambda * I(M; X' | Y).
"""
cmi_func = self._conditional_mutual_information(self._aux_rv, self._target_rv, self._other_rv)
lam = self._lam
source_axis = 0
other_axis = 1
aux_axis = 3
def objective(self, x):
pmf = self.construct_joint(x)
# P(X, M): marginalize out Y and X'
p_xm = pmf.sum(axis=(other_axis, aux_axis))
# P(X', M): marginalize out X and Y
p_xprime_m = pmf.sum(axis=(source_axis, other_axis))
# E_M[D_KL(P(X|M) || P(X'|M))]
# = sum_{a,m} P(X=a, M=m) * log2(P(X=a, M=m) / P(X'=a, M=m))
# p_xprime_m has shape (|M|, |X'|), transpose to (|X'|, |M|) = (|X|, |M|)
p_xprime_aligned = p_xprime_m.T
mask = p_xm > 0
log_ratio = np.where(
mask,
np.log2(p_xm + 1e-300) - np.log2(p_xprime_aligned + 1e-300),
0.0,
)
kl_term = np.sum(p_xm * log_ratio)
cmi_term = cmi_func(pmf)
return kl_term + lam * cmi_term
return objective
def _delta_lambda(d, source, other, target, lam=1.0, bound=None, niter=None, rng=None):
"""
Compute the generalized deficiency delta_lambda(M : source \\ other).
Parameters
----------
d : Distribution
The joint distribution.
source : iterable
The source with unique information.
other : iterable
The other source.
target : iterable
The target variable.
lam : float
Lagrange multiplier.
bound : int, None
Bound on |X'|. Defaults to |source|.
niter : int, None
Number of optimization restarts.
Returns
-------
delta : float
The generalized deficiency (non-negative).
"""
opt = DeltaLambdaOptimizer(d, source, other, target, lam=lam, bound=bound)
opt.optimize(niter=niter, rng=rng)
return max(opt.objective(opt._optima), 0.0)
[docs]
class PID_DeltaLambda(BaseBivariatePID):
r"""
The delta-lambda PID from Venkatesh, Gurushankar & Schamberg (2023).
A Lagrangian generalization that interpolates between the delta-PID
and the BROJA PID via a parameter lambda::
delta_lambda(M : X \ Y) = inf_{P(X'|MY)} E_M[D_KL(P(X|M) || P(X'|M))]
+ lambda * I(M; X' | Y)
Redundancy is symmetrized::
RI = min{ I(M;X) - delta_lambda(M:X\Y), I(M;Y) - delta_lambda(M:Y\X) }
The following parameters are passed as keyword arguments to the
constructor.
Parameters
----------
lam : float
Lagrange multiplier (default 1.0). Large values approach the
delta-PID. As lam tends to 0, the deficiency divided by lam tends to the
BROJA unique information, while the redundancy tends to that of the MMI.
bound : int, None
Bound on auxiliary variable cardinality.
niter : int, None
Number of optimization restarts.
"""
_name = "I_\u03b4\u03bb"
@staticmethod
def _measure(d, sources, target, lam=1.0, bound=None, niter=None, rng=0):
"""
Compute the delta-lambda PID redundancy for a pair of sources.
Parameters
----------
d : Distribution
The distribution to compute the PID for.
sources : iterable of iterables
The source variables (exactly two).
target : iterable
The target variable.
lam : float
Lagrange multiplier.
bound : int, None
Bound on auxiliary variable cardinality.
niter : int, None
Number of optimization restarts.
rng : np.random.Generator, int, None
The randomness of the optimizations (0 by default, so that results are
reproducible).
Returns
-------
ri : float
The delta-lambda PID redundancy.
"""
if len(sources) != 2:
msg = f"I_delta_lambda is only defined for two sources, {len(sources)} given."
raise ditException(msg)
source_a, source_b = sources
def _run(args, rng):
src, oth = args
return _delta_lambda(d, src, oth, target, lam=lam, bound=bound, niter=niter, rng=rng)
delta_ab, delta_ba = parallel_sweep(_run, [(source_a, source_b), (source_b, source_a)], base_seed=rng)
mi_a = coinformation(d, [source_a, target])
mi_b = coinformation(d, [source_b, target])
return min(mi_a - delta_ab, mi_b - delta_ba)