pub mod core {
use rand::{rngs::ThreadRng};
use rand::prelude::SliceRandom;
use std::time::{Duration,Instant};
use itertools::izip;
use scoped_threadpool::Pool;
use ndarray::{Array2,Array1,ArrayD,Axis,ArrayView2};
use ndarray_rand::{RandomExt,rand_distr::Uniform};
use ndarray_einsum_beta::*;
use std::io::{Read,Write, stdout};
use crossterm::{QueueableCommand, cursor};
use serde::{Serialize,Deserialize};
use std::mem::swap;
use std::fs::File;
use std::fs;
use std::path::Path;
const THREAD_COUNT:usize = 12usize;
use std::f32;
const DEFAULT_EVALUTATION_DATA:f32 = 0.1f32;
const DEFAULT_BATCH_SIZE:f32 = 0.002f32;
const DEFAULT_LEARNING_RATE:f32 = 0.1f32;
const DEFAULT_LAMBDA:f32 = 0.1f32;
const DEFAULT_EARLY_STOPPING:f32 = 400f32;
const DEFAULT_EVALUATION_MIN_CHANGE:f32 = 0.005f32;
const DEFAULT_LEARNING_RATE_DECAY:f32 = 0.5f32;
const DEFAULT_LEARNING_RATE_INTERVAL:f32 = 200f32;
pub enum EvaluationData<'b> {
Scaler(usize),
Percent(f32),
Actual(&'b Vec<(Vec<f32>,usize)>)
}
#[derive(Clone,Copy)]
pub enum MeasuredCondition {
Iteration(u32),
Duration(Duration)
}
#[derive(Clone,Copy)]
pub enum HaltCondition {
Iteration(u32),
Duration(Duration),
Accuracy(f32)
}
#[derive(Clone,Copy)]
pub enum Proportion {
Scaler(u32),
Percent(f32),
}
pub struct Trainer<'a> {
training_data: Vec<(Vec<f32>,usize)>,
k:usize,
evaluation_data: Vec<(Vec<f32>,usize)>,
halt_condition: Option<HaltCondition>,
log_interval: Option<MeasuredCondition>,
batch_size: usize,
learning_rate: f32,
lambda: f32,
early_stopping_condition: MeasuredCondition,
evaluation_min_change: Proportion,
learning_rate_decay: f32,
learning_rate_interval: MeasuredCondition,
checkpoint_interval: Option<MeasuredCondition>,
name: Option<&'a str>,
tracking: bool,
neural_network: &'a mut NeuralNetwork
}
impl<'a> Trainer<'a> {
pub fn evaluation_data(&mut self, evaluation_data:EvaluationData) -> &mut Trainer<'a> {
self.evaluation_data = match evaluation_data {
EvaluationData::Scaler(scaler) => { self.training_data.split_off(self.training_data.len() - scaler) }
EvaluationData::Percent(percent) => { self.training_data.split_off(self.training_data.len() - (self.training_data.len() as f32 * percent) as usize) }
EvaluationData::Actual(actual) => { actual.clone() }
};
return self;
}
pub fn halt_condition(&mut self, halt_condition:HaltCondition) -> &mut Trainer<'a> {
self.halt_condition = Some(halt_condition);
return self;
}
pub fn log_interval(&mut self, log_interval:MeasuredCondition) -> &mut Trainer<'a> {
self.log_interval = Some(log_interval);
return self;
}
pub fn batch_size(&mut self, batch_size:Proportion) -> &mut Trainer<'a> {
self.batch_size = match batch_size {
Proportion::Percent(percent) => { (self.training_data.len() as f32 * percent) as usize },
Proportion::Scaler(scaler) => { scaler as usize }
};
return self;
}
pub fn learning_rate(&mut self, learning_rate:f32) -> &mut Trainer<'a> {
self.learning_rate = learning_rate;
return self;
}
pub fn lambda(&mut self, lambda:f32) -> &mut Trainer<'a> {
self.lambda = lambda;
return self;
}
pub fn early_stopping_condition(&mut self, early_stopping_condition:MeasuredCondition) -> &mut Trainer<'a> {
self.early_stopping_condition = early_stopping_condition;
return self;
}
pub fn evaluation_min_change(&mut self, evaluation_min_change:Proportion) -> &mut Trainer<'a> {
self.evaluation_min_change = evaluation_min_change;
return self;
}
pub fn learning_rate_decay(&mut self, learning_rate_decay:f32) -> &mut Trainer<'a> {
self.learning_rate_decay = learning_rate_decay;
return self;
}
pub fn learning_rate_interval(&mut self, learning_rate_interval:MeasuredCondition) -> &mut Trainer<'a> {
self.learning_rate_interval = learning_rate_interval;
return self;
}
pub fn checkpoint_interval(&mut self, checkpoint_interval:MeasuredCondition) -> &mut Trainer<'a> {
self.checkpoint_interval = Some(checkpoint_interval);
return self;
}
pub fn name(&mut self, name:&'a str) -> &mut Trainer<'a> {
self.name=Some(name);
return self;
}
pub fn tracking(&mut self) -> &mut Trainer<'a> {
self.tracking = true;
return self;
}
pub fn go(&mut self) -> () {
self.neural_network.train_details(
&mut self.training_data,
self.k,
&self.evaluation_data,
self.halt_condition,
self.log_interval,
self.batch_size,
self.learning_rate,
self.lambda,
self.early_stopping_condition,
self.evaluation_min_change,
self.learning_rate_decay,
self.learning_rate_interval,
self.checkpoint_interval,
self.name,
self.tracking
);
}
}
#[derive(Clone,Copy,Serialize,Deserialize)]
pub enum Cost {
Quadratic,Crossentropy
}
impl Cost {
fn run(&self,y:&Array2<f32>,a:&Array2<f32>) -> f32 {
return match self {
Self::Quadratic => { quadratic(y,a) },
Self::Crossentropy => { cross_entropy(y,a) }
};
fn quadratic(y: &Array2<f32>, a: &Array2<f32>) -> f32 {
(y - a).mapv(|x| x.powi(2)).sum() / (2f32*a.nrows() as f32)
}
fn cross_entropy(y: &Array2<f32>, a: &Array2<f32>) -> f32 {
let part1 = a.mapv(f32::ln) * y;
let part2 = (1f32 - a).mapv(f32::ln) * (1f32 - y);
let mut cost:f32 = (part1 + part2).sum();
cost /= -(a.shape()[1] as f32);
return cost;
}
}
fn derivative(&self,y:&Array2<f32>,a:&Array2<f32>) -> Array2<f32> {
return match self {
Self::Quadratic => { a-y },
Self::Crossentropy => { -1f32 * ((1f32/a)*y) + (1f32-y) * (1f32/(1f32-a)) }
}
}
}
#[derive(Clone,Copy,Serialize,Deserialize)]
pub enum Activation {
Sigmoid,Softmax
}
impl Activation {
fn run(&self,y:&Array2<f32>) -> Array2<f32> {
return match self {
Self::Sigmoid => y.mapv(|x| Activation::sigmoid(x)),
Self::Softmax => Activation::softmax(y),
};
}
fn derivative(&self,z:&Array2<f32>) -> Array2<f32> {
return match self {
Self::Sigmoid => z.mapv(|x| -> f32 { sigmoid_derivative(x) }),
Self::Softmax => softmax_derivative(z),
};
fn sigmoid_derivative(z:f32) -> f32 {
let s:f32 = Activation::sigmoid(z);
return s*(1f32-s);
}
fn softmax_derivative(z:&Array2<f32>) -> Array2<f32> {
let mut derivatives:Array2<f32> = z.mapv(|x|x.exp());
let sums:Array1<f32> = derivatives.sum_axis(Axis(1));
let sqrd_sums:Array1<f32> = &sums * &sums;
for (mut row,sum,sqrd_sum) in izip!(
derivatives.axis_iter_mut(Axis(0)),
sums.iter(),
sqrd_sums.iter()
) {
row.mapv_inplace(|val| (val*(sum-val))/sqrd_sum);
}
return derivatives;
}
}
fn sigmoid(x: f32) -> f32 {
1f32 / (1f32 + (-x).exp())
}
fn softmax(y: &Array2<f32>) -> Array2<f32> {
let mut exp_matrix = y.clone();
let max_axis_vals = exp_matrix.fold_axis(Axis(1),0f32,|acc,x| (if acc > x { *acc } else { *x }));
for i in 0..exp_matrix.nrows() {
exp_matrix.row_mut(i).mapv_inplace(|x| x-max_axis_vals[i]);
}
exp_matrix.mapv_inplace(|x|x.exp());
let sum = exp_matrix.sum_axis(Axis(1));
for (sum,mut row) in izip!(sum.iter(),exp_matrix.axis_iter_mut(Axis(0))) {
row.mapv_inplace(|x| x / sum);
}
return exp_matrix;
}
}
pub struct Layer {
size: usize,
activation: Activation,
}
impl Layer {
pub fn new(size:usize,activation:Activation) -> Layer {
Layer {size,activation}
}
}
#[derive(Serialize,Deserialize,Clone)]
pub struct NeuralNetwork {
inputs: usize,
biases: Vec<Array2<f32>>,
connections: Vec<Array2<f32>>,
layers: Vec<Activation>,
cost: Cost,
}
impl NeuralNetwork {
pub fn new(inputs:usize,layers: &[Layer],cost:Option<Cost>) -> NeuralNetwork {
if layers.len() == 0 {
panic!("Requires >1 layers");
}
if inputs == 0 {
panic!("Input size must be >0");
}
for x in layers {
if x.size < 1usize {
panic!("All layer sizes must be >0");
}
}
let mut cost_function = Cost::Crossentropy;
if let Some(function) = cost {
cost_function = function;
}
let mut connections: Vec<Array2<f32>> = Vec::with_capacity(layers.len());
let mut biases: Vec<Array2<f32>> = Vec::with_capacity(layers.len());
let range = Uniform::new(-1f32, 1f32);
connections.push(Array2::random(
(layers[0].size,inputs),range) / (inputs as f32).sqrt()
);
biases.push(Array2::random((1,layers[0].size),range));
for i in 1..layers.len() {
connections.push(
Array2::random((layers[i].size,layers[i-1].size),range)
/ (layers[i-1].size as f32).sqrt()
);
biases.push(Array2::random((1,layers[i].size),range));
}
let layers:Vec<Activation> = layers.iter().map(|x|x.activation).collect();
NeuralNetwork{ inputs, biases, connections, layers, cost:cost_function}
}
pub fn activation(&mut self, index:usize, activation:Activation) {
if index >= self.layers.len() {
panic!("Layer {} does not exist. 0 <= given index < {}",index,self.layers.len());
}
self.layers[index] = activation;
}
pub fn run(&self, inputs:&Array2<f32>) -> Array2<f32> {
let mut activations:Array2<f32> = inputs.clone();
for i in 0..self.layers.len() {
let weighted_inputs:Array2<f32> = activations.dot(&self.connections[i].t());
let bias_matrix:Array2<f32> = Array2::ones((inputs.shape()[0],1)).dot(&self.biases[i]);
let inputs = weighted_inputs + bias_matrix;
activations = self.layers[i].run(&inputs);
}
return activations;
}
pub fn train(&mut self,training_data:&Vec<(Vec<f32>,usize)>,k:usize) -> Trainer {
for i in 0..training_data.len() {
let example = &training_data[i];
if example.0.len() != self.inputs {
panic!("Input size of example {} != size of input layer.",i);
}
else if k != self.biases[self.biases.len()-1].len() {
panic!("Output size of example {} != size of output layer.",i);
}
}
let mut rng = rand::thread_rng();
let mut temp_training_data = training_data.clone();
temp_training_data.shuffle(&mut rng);
let temp_evaluation_data = temp_training_data.split_off(training_data.len() - (training_data.len() as f32 * DEFAULT_EVALUTATION_DATA) as usize);
let multiplier:f32 = training_data[0].0.len() as f32 / training_data.len() as f32;
let early_stopping_condition:u32 = (DEFAULT_EARLY_STOPPING * multiplier).ceil() as u32;
let learning_rate_interval:u32 = (DEFAULT_LEARNING_RATE_INTERVAL * multiplier).ceil() as u32;
let batch_holder:f32 = DEFAULT_BATCH_SIZE * training_data.len() as f32;
let batch_size:usize = if batch_holder < 10f32 {
training_data.len()
}
else {
batch_holder.ceil() as usize
};
return Trainer {
training_data: temp_training_data,
k:k,
evaluation_data: temp_evaluation_data,
halt_condition: None,
log_interval: None,
batch_size: batch_size,
learning_rate: DEFAULT_LEARNING_RATE,
lambda: DEFAULT_LAMBDA,
early_stopping_condition: MeasuredCondition::Iteration(early_stopping_condition),
evaluation_min_change: Proportion::Percent(DEFAULT_EVALUATION_MIN_CHANGE),
learning_rate_decay: DEFAULT_LEARNING_RATE_DECAY,
learning_rate_interval: MeasuredCondition::Iteration(learning_rate_interval),
checkpoint_interval: None,
name: None,
tracking: false,
neural_network:self
};
}
fn train_details(&mut self,
training_data: &mut [(Vec<f32>,usize)],
k:usize,
evaluation_data: &[(Vec<f32>,usize)],
halt_condition: Option<HaltCondition>,
log_interval: Option<MeasuredCondition>,
batch_size: usize,
mut learning_rate: f32,
lambda: f32,
early_stopping_n: MeasuredCondition,
evaluation_min_change: Proportion,
learning_rate_decay: f32,
learning_rate_interval: MeasuredCondition,
checkpoint_interval: Option<MeasuredCondition>,
name: Option<&str>,
tracking:bool
) -> (){
if let Some(_) = checkpoint_interval {
if !Path::new("checkpoints").exists() {
fs::create_dir("checkpoints").unwrap();
}
if let Some(folder) = name {
let path = format!("checkpoints/{}",folder);
if Path::new(&path).exists() {
fs::remove_dir_all(&path).unwrap();
}
fs::create_dir(&path).unwrap();
}
}
let mut stdout = stdout();
let mut rng = rand::thread_rng();
let start_instant = Instant::now();
let mut iterations_elapsed = 0u32;
let mut best_accuracy_iteration = 0u32;
let mut best_accuracy_instant = Instant::now();
let mut best_accuracy = 0u32;
let starting_evaluation = self.evaluate(evaluation_data,k);
if let Some(_) = log_interval {
stdout.write(format!("Iteration: {}, Time: {}, Cost: {:.5}, Classified: {}/{} ({:.3}%), Learning rate: {}\n",
iterations_elapsed,
NeuralNetwork::time(start_instant),
starting_evaluation.0,
starting_evaluation.1,evaluation_data.len(),
(starting_evaluation.1 as f32)/(evaluation_data.len() as f32) * 100f32,
learning_rate
).as_bytes()).unwrap();
}
let mut last_checkpointed_instant = Instant::now();
let mut last_logged_instant = Instant::now();
training_data.shuffle(&mut rng);
let mut inner_training_data = NeuralNetwork::matrixify(training_data,k);
loop {
shuffle_matrix_data(&mut rng,&mut inner_training_data);
let batches = NeuralNetwork::batch_chunks(&inner_training_data,batch_size);
if tracking {
let mut percentage:f32 = 0f32;
stdout.queue(cursor::SavePosition).unwrap();
let backprop_start_instant = Instant::now();
let percent_change:f32 = 100f32 * batch_size as f32 / inner_training_data.0.nrows() as f32;
for batch in batches {
stdout.write(format!("Backpropagating: {:.2}%",percentage).as_bytes()).unwrap();
percentage += percent_change;
stdout.queue(cursor::RestorePosition).unwrap();
stdout.flush().unwrap();
let (new_connections,new_biases) = self.update_batch(&batch,learning_rate,lambda,training_data.len() as f32);
self.connections = new_connections;
self.biases = new_biases;
}
stdout.write(format!("Backpropagated: {}\n",NeuralNetwork::time(backprop_start_instant)).as_bytes()).unwrap();
}
else {
for batch in batches {
let (new_connections,new_biases) = self.update_batch(&batch,learning_rate,lambda,training_data.len() as f32);
self.connections = new_connections;
self.biases = new_biases;
}
}
iterations_elapsed += 1;
let evaluation = self.evaluate(evaluation_data,k);
match checkpoint_interval {
Some(MeasuredCondition::Iteration(iteration_interval)) => if iterations_elapsed % iteration_interval == 0 {
if let Some(folder) = name {
self.export(&format!("checkpoints/{}/{}",folder,iterations_elapsed));
}
else {
self.export(&format!("checkpoints/{}",iterations_elapsed));
}
},
Some(MeasuredCondition::Duration(duration_interval)) => if last_checkpointed_instant.elapsed() >= duration_interval {
if let Some(folder) = name {
self.export(&format!("checkpoints/{}/{}",folder,NeuralNetwork::time(start_instant)));
}
else {
self.export(&format!("checkpoints/{}",NeuralNetwork::time(start_instant)));
}
last_checkpointed_instant = Instant::now();
},
_ => {},
}
match log_interval {
Some(MeasuredCondition::Iteration(iteration_interval)) => if iterations_elapsed % iteration_interval == 0 {
log_fn(&mut stdout,iterations_elapsed,start_instant,learning_rate,evaluation,evaluation_data.len()
);
},
Some(MeasuredCondition::Duration(duration_interval)) => if last_logged_instant.elapsed() >= duration_interval {
log_fn(&mut stdout,iterations_elapsed,start_instant,learning_rate,evaluation,evaluation_data.len()
);
last_logged_instant = Instant::now();
},
_ => {},
}
if evaluation.1 as usize == evaluation_data.len() { break; }
match halt_condition {
Some(HaltCondition::Iteration(iteration)) => if iterations_elapsed == iteration { break; },
Some(HaltCondition::Duration(duration)) => if start_instant.elapsed() > duration { break; },
Some(HaltCondition::Accuracy(accuracy)) => if evaluation.1 >= (evaluation_data.len() as f32 * accuracy) as u32 { break; },
_ => {},
}
match evaluation_min_change {
Proportion::Percent(percent) => if (evaluation.1 as f32 / evaluation_data.len() as f32) > (best_accuracy as f32 / evaluation_data.len() as f32) + percent {
best_accuracy = evaluation.1;
best_accuracy_iteration = iterations_elapsed;
best_accuracy_instant = Instant::now();
}
Proportion::Scaler(scaler) => if evaluation.1 > best_accuracy + scaler {
best_accuracy = evaluation.1;
best_accuracy_iteration = iterations_elapsed;
best_accuracy_instant = Instant::now();
}
}
match early_stopping_n {
MeasuredCondition::Iteration(stopping_iteration) => if iterations_elapsed - best_accuracy_iteration == stopping_iteration { println!("---------------\nEarly stoppage!\n---------------"); break; },
MeasuredCondition::Duration(stopping_duration) => if best_accuracy_instant.elapsed() >= stopping_duration { println!("---------------\nEarly stoppage!\n---------------"); break; }
}
match learning_rate_interval {
MeasuredCondition::Iteration(interval_iteration) => if iterations_elapsed - best_accuracy_iteration == interval_iteration { learning_rate *= learning_rate_decay },
MeasuredCondition::Duration(interval_duration) => if best_accuracy_instant.elapsed() >= interval_duration { learning_rate *= learning_rate_decay }
}
}
let evaluation = self.evaluate(evaluation_data,k);
let new_percent = (evaluation.1 as f32)/(evaluation_data.len() as f32) * 100f32;
let starting_percent = (starting_evaluation.1 as f32)/(evaluation_data.len() as f32) * 100f32;
println!();
println!("Cost: {:.4} -> {:.4}",starting_evaluation.0,evaluation.0);
println!("Classified: {} ({:.2}%) -> {} ({:.2}%)",starting_evaluation.1,starting_percent,evaluation.1,new_percent);
println!("Cost: {:.4}",evaluation.0-starting_evaluation.0);
println!("Classified: +{} (+{:.3}%)",evaluation.1-starting_evaluation.1,new_percent - starting_percent);
println!("Time: {}",NeuralNetwork::time(start_instant));
println!();
fn log_fn(
stdout:&mut std::io::Stdout,
iterations_elapsed:u32,
start_instant:Instant,
learning_rate:f32,
evaluation: (f32,u32),
eval_len: usize
) -> () {
stdout.write(format!("Iteration: {}, Time: {}, Cost: {:.5}, Classified: {}/{} ({:.3}%), Learning rate: {}\n",
iterations_elapsed,
NeuralNetwork::time(start_instant),
evaluation.0,
evaluation.1,eval_len,
(evaluation.1 as f32)/(eval_len as f32) * 100f32,
learning_rate
).as_bytes()).unwrap();
}
fn shuffle_matrix_data(rng: &mut ThreadRng,example:&mut (Array2<f32>,Array2<f32>)) {
let length = example.0.nrows();
let mut indexs:Vec<usize> = (0usize..length).collect();
indexs.shuffle(rng);
for i in 0..length/2 {
if i <= indexs[i] {
swap(&mut example.0.row(i),&mut example.0.row(indexs[i]));
swap(&mut example.1.row(i),&mut example.1.row(indexs[i]));
}
}
}
}
fn update_batch(&self, batch: &(ArrayView2<f32>,ArrayView2<f32>), eta: f32, lambda:f32, n:f32) -> (Vec<Array2<f32>>,Vec<Array2<f32>>) {
let nabla_b_zeros:Vec<Array2<f32>> = self.biases.clone().iter().map(|x| x.map(|_| 0f32) ).collect();
let nabla_w_zeros:Vec<Array2<f32>> = self.connections.clone().iter().map(|x| x.map(|_| 0f32) ).collect();
let batch_len = batch.0.nrows();
let chunk_lengths:usize = if batch_len < THREAD_COUNT {
batch_len
} else {
(batch_len as f32 / THREAD_COUNT as f32).ceil() as usize
};
let input_chunks = batch.0.axis_chunks_iter(Axis(0),chunk_lengths);
let output_chunks = batch.1.axis_chunks_iter(Axis(0),chunk_lengths);
let chunks:Vec<(ArrayView2<f32>,ArrayView2<f32>)> = input_chunks.zip(output_chunks).collect();
let mut pool = Pool::new(chunks.len() as u32);
let mut out_nabla_b:Vec<Vec<Array2<f32>>> = vec!(nabla_b_zeros.clone();chunks.len());
let mut out_nabla_w:Vec<Vec<Array2<f32>>> = vec!(nabla_w_zeros.clone();chunks.len());
pool.scoped(|scope| {
for (chunk,nabla_w,nabla_b) in izip!(chunks,&mut out_nabla_w,&mut out_nabla_b) {
scope.execute(move || {
let (mut delta_nabla_b,mut delta_nabla_w):(Vec<Array2<f32>>,Vec<Array2<f32>>) = self.backpropagate(&chunk);
for layer in 0..self.biases.len() {
delta_nabla_w[layer] = &nabla_w[layer] + &delta_nabla_w[layer].t();
delta_nabla_b[layer] = &nabla_b[layer] + &delta_nabla_b[layer];
}
*nabla_w = delta_nabla_w;
*nabla_b = delta_nabla_b;
});
}
});
let mut nabla_b:Vec<Array2<f32>> = nabla_b_zeros.clone();
for example in &out_nabla_b {
for i in 0..example.len() {
nabla_b[i] += &example[i];
}
}
let mut nabla_w:Vec<Array2<f32>> = nabla_w_zeros.clone();
for example in &out_nabla_w {
for i in 0..example.len() {
nabla_w[i] += &example[i];
}
}
let return_connections:Vec<Array2<f32>> = self.connections.iter().zip(nabla_w).map(
| (w,nw) | (1f32-eta*(lambda/n))*w - ((eta / batch_len as f32)) * nw
).collect();
let mut return_biases:Vec<Array2<f32>> = self.biases.clone();
for i in 0..self.biases.len() {
return_biases[i] = &self.biases[i] - &((eta / batch_len as f32) * nabla_b[i].clone());
}
return (return_connections,return_biases);
}
fn backpropagate(&self, example:&(ArrayView2<f32>,ArrayView2<f32>)) -> (Vec<Array2<f32>>,Vec<Array2<f32>>) {
let number_of_examples = example.0.nrows();
let mut inputs:Vec<Array2<f32>> = Vec::with_capacity(self.biases.len());
let mut activations:Vec<Array2<f32>> = Vec::with_capacity(self.biases.len()+1);
activations.push(example.0.to_owned());
for i in 0..self.layers.len() {
let weighted_inputs = activations[i].dot(&self.connections[i].t());
let bias_matrix:Array2<f32> = Array2::ones((number_of_examples,1)).dot(&self.biases[i]);
inputs.push(weighted_inputs + bias_matrix);
activations.push(self.layers[i].run(&inputs[i]));
}
let target = example.1.clone();
let last_index = self.connections.len()-1;
let mut nabla_b:Vec<Array2<f32>> = Vec::with_capacity(self.biases.len());
let mut nabla_w:Vec<ArrayD<f32>> = Vec::with_capacity(self.connections.len());
let last_layer = self.layers[self.layers.len()-1];
let mut error:Array2<f32> = self.cost.derivative(&target.to_owned(),&activations[activations.len()-1]) * last_layer.derivative(&inputs[inputs.len()-1]);
nabla_b.insert(0,error.clone());
let weight_errors = einsum("ai,aj->aji", &[&error, &activations[last_index]]).unwrap();
nabla_w.insert(0,weight_errors);
for i in (1..self.layers.len()).rev() {
error = self.layers[i-1].derivative(&inputs[i-1]) *
error.dot(&self.connections[i]);
nabla_b.insert(0,error.clone());
let ein_sum = einsum("ai,aj->aji", &[&error, &activations[i-1]]).unwrap();
nabla_w.insert(0,ein_sum);
}
let nabla_b_sum = nabla_b.iter().map(|x| cast_1_to_2(x.sum_axis(Axis(0)))).collect();
let nabla_w_sum = nabla_w.iter().map(|x| cast_d_to_2(x.sum_axis(Axis(0)))).collect();
return (nabla_b_sum,nabla_w_sum);
fn cast_d_to_2(arrd:ArrayD<f32>) -> Array2<f32> {
let shape = (arrd.shape()[0],arrd.shape()[1]);
let mut arr2:Array2<f32> = Array2::zeros(shape);
for i in 0..shape.0 {
for t in 0..shape.1 {
arr2[[i,t]]=arrd[[i,t]];
}
}
return arr2;
}
fn cast_1_to_2(arr1:Array1<f32>) -> Array2<f32> {
let shape = (1,arr1.len());
let mut arr2:Array2<f32> = Array2::zeros(shape);
for i in 0..shape.1 {
arr2[[0,i]]=arr1[[i]];
}
return arr2;
}
}
fn batch_chunks(data:&(Array2<f32>,Array2<f32>),batch_size:usize) -> Vec<(ArrayView2<f32>,ArrayView2<f32>)>{
let input_chunks = data.0.axis_chunks_iter(Axis(0),batch_size);
let output_chunks = data.1.axis_chunks_iter(Axis(0),batch_size);
return input_chunks.zip(output_chunks).collect();
}
pub fn evaluate(&self, test_data:&[(Vec<f32>,usize)],k:usize) -> (f32,u32) {
let chunk_len:usize = if test_data.len() < THREAD_COUNT {
test_data.len()
} else {
(test_data.len() as f32 / THREAD_COUNT as f32).ceil() as usize
};
let chunks:Vec<_> = test_data.chunks(chunk_len).collect();
let mut pool = Pool::new(chunks.len() as u32);
let mut cost_vec = vec!(0f32;chunks.len());
let mut classified_vec = vec!(0u32;chunks.len());
pool.scoped(|scope| {
for (chunk,cost,classified) in izip!(chunks,&mut cost_vec,&mut classified_vec) {
scope.execute(move || {
let batch_tuple_matrix = NeuralNetwork::matrixify(&chunk,k);
let out = self.run(&batch_tuple_matrix.0);
let target = batch_tuple_matrix.1;
*cost = self.cost.run(&target,&out);
let output_class_indxs = max_output_indexs(&out);
let target_class_indxs:Vec<usize> = chunk.iter().map(|x|x.1).collect();
*classified = izip!(output_class_indxs.iter(),target_class_indxs.iter()).fold(0u32,|acc,(a,b)| {if a==b { acc+1u32 } else { acc }});
});
}
});
let cost:f32 = cost_vec.iter().sum();
let classified:u32 = classified_vec.iter().sum();
return (cost / chunk_len as f32, classified);
fn max_output_indexs(matrix:&Array2<f32>) -> Array1<usize> {
let examples = matrix.shape()[0];
let mut max_indexs:Array1<usize> = Array1::zeros(examples);
let mut max_values:Array1<f32> = Array1::zeros(examples);
for i in 0..examples {
for t in 0..matrix.row(i).len() {
if matrix[[i,t]] > max_values[i] {
max_indexs[i] = t;
max_values[i] = matrix[[i,t]];
}
}
}
return max_indexs;
}
}
pub fn evaluate_outputs(&self, test_data:&[(Vec<f32>,usize)],k:usize) -> (Array1<f32>,Array2<f32>) {
let chunks:Vec<Array2<f32>> = symbol_chunks(test_data,k);
let mut pool = Pool::new(chunks.len() as u32);
let mut classifications:Vec<Array1<f32>> = vec!(Array1::zeros(k);k);
pool.scoped(|scope| {
for (chunk,classification) in izip!(chunks,&mut classifications) {
scope.execute(move || {
let results = self.run(&chunk);
let classes:Array2<u32> = set_nonmax_zero(&results);
let class_sums:Array1<u32> = classes.sum_axis(Axis(0));
let number_of_examples = chunk.len_of(Axis(0));
*classification = class_sums.mapv(|val| (val as f32 / number_of_examples as f32));
});
}
});
let matrix:Array2<f32> = cast_array1s_to_array2(classifications,k);
let diagonal:Array1<f32> = matrix.clone().into_diag();
return (diagonal,matrix);
fn symbol_chunks(test_data:&[(Vec<f32>,usize)],k:usize) -> Vec<Array2<f32>> {
let mut chunks:Vec<Array2<f32>> = Vec::with_capacity(k);
let mut slice = (0usize,0usize);
loop {
slice.1+=1;
while test_data[slice.0].1 == test_data[slice.1+1].1 {
slice.1+=1;
if slice.1+1 == test_data.len() {
slice.1 += 1;
break;
}
}
let chunk_holder = NeuralNetwork::matrixify_inputs(&test_data[slice.0..slice.1]);
chunks.push(chunk_holder);
if chunks.len() == k { break };
slice.0 = slice.1;
}
if slice.1 != test_data.len() {
panic!("`evaluate outputs` requires data sorted by output.");
}
return chunks;
}
fn set_nonmax_zero(matrix:&Array2<f32>) -> Array2<u32> {
let mut max_indx = 0usize;
let shape = matrix.shape();
let mut zero_matrix = Array2::zeros((shape[0],shape[1]));
for i in 0..shape[0] {
for t in 1..shape[1] {
if matrix[[i,t]] > matrix[[i,max_indx]] {
max_indx=t;
}
}
zero_matrix[[i,max_indx]] = 1u32;
max_indx = 0usize;
}
return zero_matrix;
}
}
fn matrixify_inputs(examples:&[(Vec<f32>,usize)]) -> Array2<f32> {
let input_len = examples[0].0.len();
let example_len = examples.len();
let mut input_vec:Vec<f32> = Vec::with_capacity(example_len * input_len);
for example in examples {
input_vec.append(&mut example.0.clone());
}
let input_array:Array2<f32> = Array2::from_shape_vec((example_len,input_len),input_vec).unwrap();
return input_array;
}
fn matrixify(examples:&[(Vec<f32>,usize)],k:usize) -> (Array2<f32>,Array2<f32>) {
let input_len = examples[0].0.len();
let example_len = examples.len();
let mut input_vec:Vec<f32> = Vec::with_capacity(example_len * input_len);
let mut output_vec:Vec<f32> = Vec::with_capacity(example_len * k);
for example in examples {
input_vec.append(&mut example.0.clone());
let mut class_vec:Vec<f32> = vec!(0f32;k);
class_vec[example.1] = 1f32;
output_vec.append(&mut class_vec);
}
let input:Array2<f32> = Array2::from_shape_vec((example_len,input_len),input_vec).unwrap();
let output:Array2<f32> = Array2::from_shape_vec((example_len,k),output_vec).unwrap();
return (input,output);
}
fn time(instant:Instant) -> String {
let mut seconds = instant.elapsed().as_secs();
let hours = (seconds as f32 / 3600f32).floor();
seconds = seconds % 3600;
let minutes = (seconds as f32 / 60f32).floor();
seconds = seconds % 60;
let time = format!("{:#02}:{:#02}:{:#02}",hours,minutes,seconds);
return time;
}
pub fn print(&self) -> String {
let mut prt_string:String = String::new();
let max:usize = self.biases.iter().map(|x|x.shape()[1]).max().unwrap();
let width = self.connections.len();
for row in 0..max+2 {
for t in 0..width {
let diff = (max - self.biases[t].shape()[1]) / 2;
let spacing = 6*self.connections[t].shape()[1];
if row == diff {
prt_string.push_str(" ");
prt_string.push_str(&format!("┌ {: <1$}┐","",spacing));
prt_string.push_str(" ");
prt_string.push_str("┌ ┐");
prt_string.push_str(" ");
}
else if row == self.biases[t].shape()[1] + diff + 1 {
prt_string.push_str(" ");
prt_string.push_str(&format!("└ {: <1$}┘","",spacing));
prt_string.push_str(" ");
prt_string.push_str("└ ┘");
prt_string.push_str(" ");
}
else if row < diff || row > self.biases[t].shape()[1] + diff + 1 {
prt_string.push_str(" ");
prt_string.push_str(&format!(" {: <1$} ","",spacing));
prt_string.push_str(" ");
prt_string.push_str(" ");
prt_string.push_str(" ");
}
else {
let inner_row = row-diff-1;
if self.biases[t].shape()[1] / 2 == inner_row { prt_string.push_str("* "); }
else { prt_string.push_str(" "); }
prt_string.push_str("│ ");
for val in self.connections[t].row(inner_row) {
prt_string.push_str(&format!("{:+.2} ",val));
}
if inner_row == self.biases[t].shape()[1] / 2 {
prt_string.push_str("│ + │ ");
} else { prt_string.push_str("│ │ "); }
let val = self.biases[t][[0,inner_row]];
prt_string.push_str(&format!("{:+.2} ",val));
prt_string.push_str("│ ");
}
}
prt_string.push_str("\n");
}
prt_string.push_str("\n");
return prt_string;
}
pub fn export(&self,path:&str) -> () {
let file = File::create(format!("{}.json",path));
let serialized:String = serde_json::to_string(self).unwrap();
file.unwrap().write_all(serialized.as_bytes()).unwrap();
}
pub fn import(path:&str) -> NeuralNetwork {
let file = File::open(format!("{}.json",path));
let mut string_contents:String = String::new();
file.unwrap().read_to_string(&mut string_contents).unwrap();
let deserialized:NeuralNetwork = serde_json::from_str(&string_contents).unwrap();
return deserialized;
}
}
pub fn cast_array1s_to_array2<T:Default+Copy>(vec:Vec<Array1<T>>,k:usize) -> Array2<T> {
let mut arr2 = Array2::default((vec.len(),k));
let k = vec[0].len();
for i in 0..vec.len() {
if vec[i].len() != k { panic!("Cannot convert `Vec<Array1<T>>` to `Array2<T>`. vec[{}].len() ({}) does not equal k ({})",i,vec[i].len(),k); }
for t in 0..k {
arr2[[i,t]] = vec[i][t];
}
}
return arr2;
}
}
pub mod utilities {
extern crate ndarray;
use ndarray::{Array2,Array3};
use std::fmt::Display;
pub fn array2_prt<T:Display+Copy>(ndarray_param:&Array2<T>) -> String {
let mut prt_string:String = String::new();
let shape = ndarray_param.shape();
let spacing = 5*shape[1];
prt_string.push_str(&format!("┌ {: <1$}┐\n","",spacing));
for row in 0..shape[0] {
prt_string.push_str("│ ");
for val in ndarray_param.row(row) {
prt_string.push_str(&format!("{:+.1} ",val));
}
prt_string.push_str("│\n");
}
prt_string.push_str(&format!("└ {:<1$}┘\n","",spacing));
prt_string.push_str(&format!("{:<1$}","",(spacing/2)-1));
prt_string.push_str(&format!("[{},{}]\n",shape[0],shape[1]));
return prt_string;
}
pub fn array3_prt<T:Display+Copy>(ndarray_param:&Array3<T>) -> String {
let mut prt_string:String = String::new();
let shape = ndarray_param.shape();
let outer_spacing = (5*shape[0]*shape[2]) + (3*shape[0]) + 2;
prt_string.push_str(&format!("┌{: <1$}┐\n","",outer_spacing));
let inner_spacing = 5 * shape[2];
prt_string.push_str("│ ");
for _ in 0..shape[0] {
prt_string.push_str(&format!("┌ {: <1$}┐","",inner_spacing));
}
prt_string.push_str(" │\n");
for i in 0..shape[1] {
prt_string.push_str("│ ");
for t in 0..shape[0] {
prt_string.push_str("│ ");
for p in 0..shape[2] {
let val = ndarray_param[[t,i,p]];
prt_string.push_str(&format!("{:+.1} ",val));
}
prt_string.push_str("│");
}
prt_string.push_str(" │\n");
}
prt_string.push_str("│ ");
for _ in 0..shape[0] {
prt_string.push_str(&format!("└ {: <1$}┘","",inner_spacing));
}
prt_string.push_str(" │\n");
prt_string.push_str(&format!("└{:<1$}┘\n","",outer_spacing));
prt_string.push_str(&format!("{:<1$}","",(outer_spacing / 2) - 2));
prt_string.push_str(&format!("[{},{},{}]\n",shape[0],shape[1],shape[2]));
return prt_string;
}
pub fn counting_sort(data:&[(Vec<f32>,usize)],k:usize) -> Vec<(Vec<f32>,usize)> {
let mut count:Vec<usize> = vec!(0;k);
for i in 0..data.len() {
count[data[i].1] += 1;
}
for i in 1..count.len() {
count[i] += count[i-1];
}
let input_size = data[0].0.len();
let mut sorted_data:Vec<(Vec<f32>,usize)> = vec!((vec!(0f32;input_size),0usize);data.len());
for i in 0..data.len() {
sorted_data[count[data[i].1]-1] = data[i].clone();
count[data[i].1] -= 1;
}
return sorted_data;
}
}