1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

use super::ActivationFunction;
use crate::{Constant, KernelAdd, KernelError, KernelMul, Linear, PositiveDefiniteKernel};
use std::{
    fmt::Debug,
    ops::{Add, Mul},
};

/// https://arxiv.org/abs/1711.00165
#[derive(Clone, Debug)]
pub struct DeepNeuralNetwork<'a> {
    layers: Vec<&'a dyn ActivationFunction>,
}

impl<'a> DeepNeuralNetwork<'a> {
    pub fn new(layers: Vec<&'a dyn ActivationFunction>) -> Self {
        Self { layers }
    }
}
impl<'a> PositiveDefiniteKernel<Vec<f64>> for DeepNeuralNetwork<'a> {
    fn params_len(&self) -> usize {
        2 * (1 + self.layers.len())
    }

    fn value(&self, params: &[f64], x: &Vec<f64>, xprime: &Vec<f64>) -> Result<f64, KernelError> {
        if params.len() != self.params_len() {
            return Err(KernelError::ParametersLengthMismatch.into());
        }
        if x.len() != xprime.len() {
            return Err(KernelError::InvalidArgument.into());
        }

        let layer0 = Constant + Constant * Linear;
        let mut previous_layer_kernel = (
            layer0.value(&params[0..2], x, xprime)?,
            layer0.value(&params[0..2], x, x)?,
            layer0.value(&params[0..2], xprime, xprime)?,
        );
        let params = &params[2..];

        for (i, &layer) in self.layers.iter().enumerate() {
            let sigma_b = params[(i + 1) * 2];
            let sigma_w = params[(i + 1) * 2 + 1];
            let f = layer.f(previous_layer_kernel);
            let fxx = layer.f((
                previous_layer_kernel.1,
                previous_layer_kernel.1,
                previous_layer_kernel.1,
            ));
            let fxpxp = layer.f((
                previous_layer_kernel.2,
                previous_layer_kernel.2,
                previous_layer_kernel.2,
            ));

            previous_layer_kernel = (
                sigma_b + sigma_w * f,
                sigma_b + sigma_w * fxx,
                sigma_b + sigma_w * fxpxp,
            );
        }

        Ok(previous_layer_kernel.0)
    }
}

impl<'a, R> Add<R> for DeepNeuralNetwork<'a>
where
    R: PositiveDefiniteKernel<Vec<f64>>,
{
    type Output = KernelAdd<Self, R, Vec<f64>>;
    fn add(self, rhs: R) -> Self::Output {
        Self::Output::new(self, rhs)
    }
}

impl<'a, R> Mul<R> for DeepNeuralNetwork<'a>
where
    R: PositiveDefiniteKernel<Vec<f64>>,
{
    type Output = KernelMul<Self, R, Vec<f64>>;

    fn mul(self, rhs: R) -> Self::Output {
        Self::Output::new(self, rhs)
    }
}

#[cfg(test)]
mod tests {
    use crate::*;

    #[test]
    fn it_works() {
        let activfunc = ReLU;
        let kernel = DeepNeuralNetwork::new(vec![&activfunc]);

        let test_value = kernel.value(
            &[1.0, 1.0, 3.0, 4.0, 6.0],
            &vec![0.0, 0.0, 0.0],
            &vec![0.0, 0.0, 0.0],
        );

        match test_value {
            Err(KernelError::ParametersLengthMismatch) => (),
            _ => panic!(),
        };
    }
}