wyrm 0.9.1

A low-overhead, define-by-run autodifferentiation library.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160

//! Loss functions.
use std::cell::{Ref, RefCell};
use std::ops::Deref;
use std::rc::Rc;

use nodes::{BackwardAction, Bor, ForwardAction, IndexInputNode, LogSoftmaxNode, PassCounter};
use numerics;
use {Arr, Node, Variable};

/// Sparse categorical cross entropy loss.
///
/// Note that this performs a log-softmax operation
/// internally, so there is no need to perform a softmax
/// manually.
pub fn sparse_categorical_crossentropy<T>(
    x: &Variable<T>,
    y: &Variable<IndexInputNode>,
) -> Variable<SparseCategoricalCrossentropyNode<T>>
where
    T: Node<Value = Arr, InputGradient = Arr>,
{
    let node = SparseCategoricalCrossentropyNode::new(Rc::clone(&x.node), Rc::clone(&y.node));

    Variable::new(Rc::new(node), x.parameters.clone())
}

/// Sparse categorical cross-entropy loss node.
#[derive(Debug)]
pub struct SparseCategoricalCrossentropyNode<LHS> {
    operand: Rc<LHS>,
    log_softmax: LogSoftmaxNode<LHS>,
    y: Rc<IndexInputNode>,
    loss_value: RefCell<Arr>,
    gradient: RefCell<Arr>,
    needs_gradient: bool,
    counter: PassCounter,
}

impl<LHS> SparseCategoricalCrossentropyNode<LHS>
where
    LHS: Node<Value = Arr, InputGradient = Arr>,
{
    pub(crate) fn new(operand: Rc<LHS>, y: Rc<IndexInputNode>) -> Self {
        assert!(
            operand.value().rows() == 1,
            "Minibatches not supported: rows must be 1."
        );

        let log_softmax = LogSoftmaxNode::new(Rc::clone(&operand));
        let scalar_loss = {
            let log_softmax_value = log_softmax.value();

            let mut scalar_loss = 0.0;

            for &idx in y.value().iter() {
                scalar_loss += -log_softmax_value[(0, idx)];
            }

            scalar_loss
        };

        let mut loss_value = Arr::zeros((1, 1));
        loss_value.fill(scalar_loss);

        let gradient = operand.value().deref() * 0.0;
        let needs_gradient = operand.needs_gradient();

        SparseCategoricalCrossentropyNode {
            operand: operand,
            log_softmax: log_softmax,
            y: y,
            loss_value: RefCell::new(loss_value),
            gradient: RefCell::new(gradient),
            needs_gradient: needs_gradient,
            counter: PassCounter::default(),
        }
    }

    /// Return the predictions made by this layer.
    pub fn predictions(&self) -> Bor<Arr> {
        self.log_softmax.value()
    }
}

impl<LHS> Node for SparseCategoricalCrossentropyNode<LHS>
where
    LHS: Node<Value = Arr, InputGradient = Arr>,
{
    type Value = Arr;
    type InputGradient = Arr;

    fn forward(&self) {
        if self.counter.forward() == ForwardAction::Cached {
            return;
        }

        self.log_softmax.forward();
        self.y.forward();

        let softmax_value = self.log_softmax.value();
        debug_assert!(
            softmax_value.rows() == 1,
            "Minibatches not supported: rows must be 1."
        );
        let softmax_slice = softmax_value.deref().as_slice().unwrap();

        let mut loss_value = 0.0;

        for &idx in self.y.value().iter() {
            loss_value += -softmax_slice[idx];
        }

        self.loss_value.borrow_mut().fill(loss_value);
    }
    /// The backpropagation mechanics for this node are a little strange,
    /// because it uses the log-softmax node for the forward pass but not
    /// for the backward pass.
    fn backward(&self, _: &Ref<Self::InputGradient>) {
        // TODO: actually use the input gradient
        let beta = match self.counter.backward() {
            BackwardAction::Set => 0.0,
            BackwardAction::Increment => 1.0,
        };

        {
            let mut gradient = self.gradient.borrow_mut();
            let gradient_slice = gradient.as_slice_mut().unwrap();

            let value = self.log_softmax.value();
            let value_slice = value.as_slice().unwrap();

            for (grad, &val) in izip!(gradient_slice.iter_mut(), value_slice.iter()) {
                *grad = beta * *grad + numerics::exp(val);
            }

            for &idx in self.y.value().iter() {
                gradient_slice[idx] -= 1.0;
            }
        }

        if self.counter.recurse_backward() {
            self.log_softmax.zero_counter();
            self.operand.backward(&self.gradient.borrow());
        }
    }
    fn value(&self) -> Bor<Self::Value> {
        Bor::RefGuard(self.loss_value.borrow())
    }
    fn needs_gradient(&self) -> bool {
        self.needs_gradient
    }
    fn clear(&self) {
        if !self.counter.is_zero() {
            self.operand.clear();
            self.log_softmax.clear();
            self.y.clear();
            self.counter.clear();
        }
    }
}