esopt 0.2.0

General Evolution-Strategy-Optimizer implementation according to https://arxiv.org/abs/1703.03864 in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

//! Noisy polynomial example.
#![allow(clippy::missing_docs_in_private_items, clippy::print_stdout)]

use esopt::*;
use rand::prelude::*;

const DEGREE: usize = 3;
const REGULARIZE: Float = 0.025; //own L1 reg. factor

fn main() {
	//training data
	let points = vec![(0.0, 0.1), (1.0, 1.4), (2.0, 4.0), (3.0, 8.4), (4.0, 16.5)];
	/*let points = vec![    (0.0, 0.0),
		(1.0, 1.0),
		(2.0, 4.0),
		(3.0, 9.0),
		(4.0, 16.0)
	];*/

	//initialize required objects
	let mut opt = SGD::default(); //SGA optimizer (named SGD)
	opt.set_lr(0.1) //learning rate
		.set_beta(0.5) //momentum factor
		.set_lambda(0.01); //weight decay coefficient. we use our additional, own regularizer in the
				   // loss/objective function

	let eval = PolynomeEval::new(points); //target

	//create ES-Optimizer
	let mut es = ES::new(opt, eval); //Evolution-Strategy-Optimizer using optimizer and evaluator
	let params = gen_rnd_vec(DEGREE + 1, 0.1); //generate random initial parameters
	es.set_params(params); //initial parameters (important to specify the problem dimension, default is
					   // vec![0.0])

	//track the optimizer's results
	for i in 0..5 {
		let n = 200;
		let res = es.optimize_std_par(n); //optimize for n steps
		println!("After {:5} iterations:", (i + 1) * n);
		println!("-(MAE + L1): {:7.4}", res.0);
		println!("Gradnorm: {:7.5}", res.1);
		print_f(es.get_params());
		println!();
	}
}

fn calc_f(factors: &[Float], x: Float) -> Float {
	let mut result = factors[0];
	let mut current_pow = 1.0;
	for factor in factors.iter().skip(1) {
		current_pow *= x;
		result += factor * current_pow;
	}
	result
}

fn print_f(fct: &[Float]) {
	let mut str = String::new();
	for (i, factor) in fct.iter().enumerate() {
		str.push_str(&format!("{:7.4}x^{} + ", factor, i));
	}
	let tmp = str.len() - 3;
	str.truncate(tmp);
	println!("{}", str);
}

#[derive(Clone)]
struct PolynomeEval {
	target: Vec<(Float, Float)>, //points that should be matched by the polynomial function
}

impl PolynomeEval {
	pub fn new(points: Vec<(Float, Float)>) -> PolynomeEval {
		PolynomeEval { target: points }
	}
}

impl Evaluator for PolynomeEval {
	//evaluate as inverted mean absolute error to target (we want to minimize
	// instead of maximize)
	fn eval_train(&self, params: &[Float], _: usize) -> Float {
		let mut rng = thread_rng();
		//calculate mean absolute error of random batch of points -> seems to accept
		// outliers
		let mut mae = 0.0;
		let n = 4; //batch size
		for _ in 0..n {
			let i = rng.gen::<usize>() % self.target.len();
			let point = &self.target[i];
			let error = point.1 - calc_f(params, point.0);
			mae += error.abs();
		}
		mae /= n as Float;
		//regularize using L1
		let mut l1 = 0.0;
		for val in params.iter() {
			l1 += val.abs();
		}
		l1 /= params.len() as Float;
		l1 *= REGULARIZE;
		//return
		-(mae + l1)
	}

	//evaluate as inverted mean absolute error to target (we want to minimize
	// instead of maximize) here all data is used for deterministic progess
	// monitoring
	fn eval_test(&self, params: &[Float]) -> Float {
		//calculate mean absolute error
		let mut mae = 0.0;
		for i in 0..self.target.len() {
			let point = &self.target[i];
			let error = point.1 - calc_f(params, point.0);
			mae += error.abs();
		}
		mae /= self.target.len() as Float;
		//regularize using L1
		let mut l1 = 0.0;
		for val in params.iter() {
			l1 += val.abs();
		}
		l1 /= params.len() as Float;
		l1 *= REGULARIZE;
		//return
		-(mae + l1)
	}
}