use nl::NeuralLayer;
use sample::Sample;
use matrix::Matrix;
use matrix::MatrixTrait;
use cost::CostFunction;
use cost::squared_error::SquaredError;
use utils::sample_input_to_matrix;
use utils::sample_output_to_matrix;
use rand::Rng;
use rand;
pub struct NeuralNetwork {
layers: Vec<NeuralLayer>,
cost_function: Box<CostFunction>,
shuffle_data: bool,
on_error_fn: Option<Box<Fn(f64)>>,
on_epoch_fn: Option<Box<Fn(&NeuralNetwork)>>,
}
impl NeuralNetwork {
pub fn new() -> NeuralNetwork {
NeuralNetwork {
layers: vec![],
cost_function: Box::new(SquaredError::new()),
shuffle_data: true,
on_error_fn: None,
on_epoch_fn: None,
}
}
pub fn set_shuffle_data(&mut self, enable: bool) {
self.shuffle_data = enable;
}
pub fn set_cost_function<T>(&mut self, cost_function: T)
where
T: 'static + CostFunction,
{
self.cost_function = Box::new(cost_function);
}
pub fn on_error<FN>(&mut self, callback_fn: FN)
where
FN: 'static + Fn(f64),
{
self.on_error_fn = Some(Box::new(callback_fn));
}
pub fn on_epoch<FN>(&mut self, callback_fn: FN)
where
FN: 'static + Fn(&NeuralNetwork),
{
self.on_epoch_fn = Some(Box::new(callback_fn));
}
fn emit_on_error(&self, err: f64) {
match self.on_error_fn {
Some(ref err_fn) => err_fn(err),
None => (),
}
}
fn emit_on_epoch(&self) {
match self.on_epoch_fn {
Some(ref epoch_fn) => epoch_fn(&self),
None => (),
}
}
pub fn add_layer(&mut self, layer: NeuralLayer) {
if self.layers.len() > 0 {
let prev_layer_neurons = self.layers[self.layers.len() - 1].neurons();
if prev_layer_neurons != layer.inputs() {
panic!(
"New layer should have enough inputs. \
Expected {}, got {}",
prev_layer_neurons,
layer.inputs()
);
}
}
self.layers.push(layer);
}
pub fn get_layers(&self) -> &Vec<NeuralLayer> {
&self.layers
}
pub fn forward(&self, sample: &Sample) -> Vec<Matrix> {
if self.layers.len() == 0 {
panic!("Neural network doesn't have any layers.");
}
let mut weights: Vec<Matrix> = vec![];
let mut prev_weight: Matrix = Matrix::zero(0, 0);
for (i, layer) in self.layers.iter().enumerate() {
let transposed_bias = layer.biases().transpose();
if i > 0 {
let mult: Matrix = prev_weight
.dot(&layer.weights().transpose())
.map(&|n, _, j| n + (1f64 * transposed_bias.get(0, j)))
.map_row(&|n| layer.activation.calc(n));
if i != self.layers.len() - 1 {
prev_weight = mult.clone();
}
weights.push(mult);
} else {
let samples_input: Matrix = sample_input_to_matrix(&sample);
let mult: Matrix = samples_input
.dot(&layer.weights().transpose())
.map(&|n, _, j| n + (1f64 * transposed_bias.get(0, j)))
.map_row(&|n| layer.activation.calc(n));
if self.layers.len() > 1 {
prev_weight = mult.clone();
}
weights.push(mult);
}
}
weights
}
pub fn evaluate(&self, sample: &Sample) -> Matrix {
let forward: Vec<Matrix> = self.forward(sample);
forward.last().unwrap().clone()
}
fn error(&self, prediction: &Matrix, target: &Matrix) -> f64 {
let err = self.cost_function.calc(prediction, target);
err
}
pub fn train(&mut self, mut samples: Vec<Sample>, epochs: i32, learning_rate: f64) {
for _ in 0..epochs {
let mut mut_samples = samples.as_mut_slice();
if self.shuffle_data {
rand::thread_rng().shuffle(&mut mut_samples);
}
let mut error_value = vec![];
for sample in mut_samples.iter() {
let mut output: Vec<Matrix> = self.forward(&sample);
output.reverse();
let mut delta: Matrix = Matrix::zero(0, 0);
for (i, layer) in output.iter().enumerate() {
let index: usize = self.layers.len() - 1 - i;
let error = if i == 0 {
let samples_outputs = sample_output_to_matrix(&sample);
let error = Matrix::generate(
samples_outputs.rows(),
samples_outputs.cols(),
&|m, n| samples_outputs.get(m, n) - layer.get(m, n),
);
error_value.push(self.error(&layer, &samples_outputs));
error
} else {
delta.dot(&self.layers[index + 1].weights().clone())
};
let forward_derivative: Matrix =
layer.map_row(&|n| self.layers[index].activation.derivative(n));
delta = Matrix::generate(layer.rows(), layer.cols(), &|m, n| {
error.get(m, n) * forward_derivative.get(m, n)
});
let biases = self.layers[index].biases().clone();
self.layers[index].set_biases(biases.map(&|n, i, j| {
n + (delta.get(j, i) * learning_rate)
}));
let mut prev_layer: Matrix = sample_input_to_matrix(&sample);
if i != output.len() - 1 {
prev_layer = output[i + 1].clone();
}
let syn: Matrix = delta.map(&|n, _, _| n * learning_rate).transpose().dot(
&prev_layer,
);
let this_layer_weights: &Matrix = &self.layers[index].weights().clone();
self.layers[index].set_weights(Matrix::generate(
this_layer_weights.rows(),
this_layer_weights.cols(),
&|m, n| syn.get(m, n) + this_layer_weights.get(m, n),
));
}
}
self.emit_on_error(
error_value.iter().fold(0f64, |n, sum| sum + n) / mut_samples.len() as f64,
);
self.emit_on_epoch();
}
}
}
#[cfg(test)]
mod tests {
use activation::Sigmoid;
use activation::SoftMax;
use activation::HyperbolicTangent;
use sample::Sample;
use nl::NeuralLayer;
use nn::NeuralNetwork;
use matrix::MatrixTrait;
use cost::cross_entropy::CrossEntropy;
#[test]
fn get_layers_test() {
let mut test = NeuralNetwork::new();
let layers = vec![NeuralLayer::new(1, 2, Sigmoid::new())];
for layer in layers {
test.add_layer(layer);
}
let get_layers = test.get_layers();
assert_eq!(get_layers.len(), 1);
}
#[test]
fn forward_test() {
let dataset = vec![Sample::new(vec![1f64, 0f64], vec![0f64])];
let mut test = NeuralNetwork::new();
let sig_activation = Sigmoid::new();
test.add_layer(NeuralLayer::new(1, 2, sig_activation));
let forward = test.forward(&dataset[0]);
assert_eq!(forward.len(), 1);
}
#[test]
fn forward_test_2layers() {
let dataset = vec![Sample::new(vec![1f64, 0f64], vec![0f64])];
let mut test = NeuralNetwork::new();
let sig_activation = Sigmoid::new();
test.add_layer(NeuralLayer::new(3, 2, sig_activation));
test.add_layer(NeuralLayer::new(1, 3, sig_activation));
let forward = test.forward(&dataset[0]);
assert_eq!(forward.len(), 2);
}
#[test]
fn train_test() {
let dataset = vec![Sample::new(vec![1f64, 0f64], vec![0f64])];
let mut test = NeuralNetwork::new();
let sig_activation = Sigmoid::new();
test.add_layer(NeuralLayer::new(1, 2, sig_activation));
test.train(dataset, 10, 0.1f64);
}
#[test]
fn train_test_2layers() {
let dataset = vec![
Sample::new(vec![1f64, 0f64], vec![0f64]),
Sample::new(vec![1f64, 1f64], vec![1f64]),
Sample::new(vec![1f64, 1f64], vec![1f64]),
];
let mut test = NeuralNetwork::new();
let sig_activation = Sigmoid::new();
test.add_layer(NeuralLayer::new(2, 2, sig_activation));
test.add_layer(NeuralLayer::new(1, 2, sig_activation));
let forward = test.forward(&dataset[1]);
test.train(dataset, 100, 0.1f64);
assert_eq!(forward.len(), 2);
}
#[test]
fn train_test_2layers_think() {
let dataset = vec![
Sample::new(vec![0f64, 0f64, 1f64], vec![0f64]),
Sample::new(vec![0f64, 1f64, 1f64], vec![0f64]),
Sample::new(vec![1f64, 0f64, 1f64], vec![1f64]),
Sample::new(vec![1f64, 1f64, 1f64], vec![1f64]),
];
let mut test = NeuralNetwork::new();
let sig_activation = Sigmoid::new();
test.add_layer(NeuralLayer::new(2, 3, sig_activation));
test.add_layer(NeuralLayer::new(1, 2, sig_activation));
test.train(dataset, 5, 0.1f64);
let think = test.evaluate(&Sample::predict(vec![1f64, 0f64, 1f64]));
assert_eq!(think.rows(), 1);
assert_eq!(think.cols(), 1);
}
#[test]
fn error_function_test() {
let dataset = vec![
Sample::new(vec![0f64, 0f64, 1f64], vec![0f64]),
Sample::new(vec![0f64, 1f64, 1f64], vec![0f64]),
Sample::new(vec![1f64, 0f64, 1f64], vec![1f64]),
Sample::new(vec![1f64, 1f64, 1f64], vec![1f64]),
];
let mut test = NeuralNetwork::new();
test.on_error(|err| {
assert!(err > 0f64);
});
let sig_activation = Sigmoid::new();
test.add_layer(NeuralLayer::new(2, 3, sig_activation));
test.add_layer(NeuralLayer::new(1, 2, sig_activation));
test.train(dataset, 5, 0.1f64);
}
#[test]
fn on_epoch_test() {
let dataset = vec![
Sample::new(vec![0f64, 0f64, 1f64], vec![0f64]),
Sample::new(vec![0f64, 1f64, 1f64], vec![0f64]),
Sample::new(vec![1f64, 0f64, 1f64], vec![1f64]),
Sample::new(vec![1f64, 1f64, 1f64], vec![1f64]),
];
let mut test = NeuralNetwork::new();
test.on_epoch(|this| {
assert_eq!(3, this.layers[0].weights().cols());
assert_eq!(2, this.layers[0].weights().rows());
assert_eq!(2, this.layers[1].weights().cols());
assert_eq!(1, this.layers[1].weights().rows());
});
let sig_activation = Sigmoid::new();
test.add_layer(NeuralLayer::new(2, 3, sig_activation));
test.add_layer(NeuralLayer::new(1, 2, sig_activation));
test.train(dataset, 5, 0.1f64);
}
#[test]
fn network_with_two_activations() {
let dataset = vec![
Sample::new(vec![0f64, 0f64, 1f64], vec![0f64]),
Sample::new(vec![0f64, 1f64, 1f64], vec![0f64]),
Sample::new(vec![1f64, 0f64, 1f64], vec![1f64]),
Sample::new(vec![1f64, 1f64, 1f64], vec![1f64]),
];
let mut test = NeuralNetwork::new();
test.add_layer(NeuralLayer::new(2, 3, Sigmoid::new()));
test.add_layer(NeuralLayer::new(1, 2, HyperbolicTangent::new()));
test.train(dataset, 5, 0.1f64);
let think = test.evaluate(&Sample::predict(vec![1f64, 0f64, 1f64]));
assert_eq!(think.rows(), 1);
assert_eq!(think.cols(), 1);
}
#[test]
fn two_hidden_layers() {
let dataset = vec![
Sample::new(vec![0f64, 0f64, 1f64], vec![0f64]),
Sample::new(vec![0f64, 1f64, 1f64], vec![0f64]),
Sample::new(vec![1f64, 0f64, 1f64], vec![1f64]),
Sample::new(vec![1f64, 1f64, 1f64], vec![1f64]),
];
let mut test = NeuralNetwork::new();
test.add_layer(NeuralLayer::new(2, 3, Sigmoid::new()));
test.add_layer(NeuralLayer::new(4, 2, Sigmoid::new()));
test.add_layer(NeuralLayer::new(1, 4, Sigmoid::new()));
test.train(dataset, 1, 0.1f64);
let think = test.evaluate(&Sample::predict(vec![1f64, 0f64, 1f64]));
assert_eq!(think.rows(), 1);
assert_eq!(think.cols(), 1);
}
#[test]
fn three_hidden_layers() {
let dataset = vec![
Sample::new(vec![0f64, 0f64, 1f64], vec![0f64]),
Sample::new(vec![0f64, 1f64, 1f64], vec![0f64]),
Sample::new(vec![1f64, 0f64, 1f64], vec![1f64]),
Sample::new(vec![1f64, 1f64, 1f64], vec![1f64]),
];
let mut test = NeuralNetwork::new();
test.add_layer(NeuralLayer::new(2, 3, Sigmoid::new()));
test.add_layer(NeuralLayer::new(1, 2, Sigmoid::new()));
test.set_cost_function(CrossEntropy);
test.train(dataset, 5, 0.1f64);
let think = test.evaluate(&Sample::predict(vec![1f64, 0f64, 1f64]));
assert_eq!(think.rows(), 1);
assert_eq!(think.cols(), 1);
}
#[test]
fn train_test_multiclass() {
let dataset = vec![
Sample::new(vec![1f64, 0f64, 2f64], vec![0f64, 1f64]),
Sample::new(vec![1f64, 1f64, 5f64], vec![1f64, 0f64]),
];
let mut test = NeuralNetwork::new();
let sig_activation = Sigmoid::new();
test.set_cost_function(CrossEntropy);
test.add_layer(NeuralLayer::new(3, 3, sig_activation));
test.add_layer(NeuralLayer::new(2, 3, SoftMax::new()));
test.train(dataset, 5, 0.01f64);
}
#[test]
fn shuffle_data() {
let dataset = vec![
Sample::new(vec![1f64, 0f64, 2f64], vec![0f64, 1f64]),
Sample::new(vec![1f64, 1f64, 5f64], vec![1f64, 0f64]),
];
let mut test = NeuralNetwork::new();
test.set_shuffle_data(true);
test.add_layer(NeuralLayer::new(3, 3, Sigmoid::new()));
test.add_layer(NeuralLayer::new(2, 3, SoftMax::new()));
test.train(dataset, 5, 0.01f64);
}
}