1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
use smallvec::smallvec;
use crate::{Element, Recordable, Shape, Tensor};
use super::{Cotangents, Operation, Reads, unary};
/// The log-sum-exp of a payload along one named axis:
/// `ln(sum(exp(x)))`, the softmax family's normalizer and a smooth
/// maximum; like `SumAlong`, the reduced axis is removed.
///
/// It is a fused primitive for the same reason as `LogSoftmax`: the
/// stable forward shifts by the axis maximum, which no composition of
/// recorded operations can express. The former composition
/// (`x - log_softmax(x)`, read from one arbitrary lane) returned `inf`
/// whenever that lane's log-probability underflowed to `-inf` for
/// finite extreme logits; the fused form is finite for every finite
/// operand. The gradient is the softmax, recovered from the operand
/// and the node's own output as `exp(operand - output)`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct LogSumExp {
pub(crate) axis: usize,
}
impl LogSumExp {
/// Returns the arity: one operand.
pub(crate) fn arity(&self) -> usize {
1
}
/// Returns the read set of the derivative rule below.
/// It reads its operand and its own output to recover the softmax.
pub(crate) fn reads(&self) -> Reads {
Reads {
operands: [true, false],
output: true,
}
}
/// Infers the shape of the result: the operand's shape with the
/// axis removed, checked against its rank.
pub(crate) fn infer_shape(&self, operands: &[Shape]) -> Shape {
let operand = unary(operands);
assert!(
self.axis < operand.rank(),
"axis {} is out of rank for {operand}",
self.axis
);
operand.without_axis(self.axis)
}
}
impl LogSumExp {
pub(crate) fn forward<E: Element>(&self, operands: &[&Tensor<E>]) -> Tensor<E> {
unary(operands).logsumexp(self.axis)
}
}
impl<Rule: Recordable> Operation<Rule> for LogSumExp {
fn backward(&self, operands: &[&Rule], output: &Rule, gradient: &Rule) -> Cotangents<Rule> {
let &operand = unary(operands);
// The derivative of log-sum-exp is the softmax; the shift
// cancels analytically, and `operand - output` reconstructs the
// stable log-probabilities directly.
let extent = operand.shape().axes()[self.axis];
let probabilities = (operand.clone() - output.broadcast_along(self.axis, extent)).exp();
smallvec![Some(
gradient.broadcast_along(self.axis, extent) * probabilities
)]
}
}
#[cfg(test)]
#[path = "tests/log_sum_exp_tests.rs"]
mod tests;