forger 0.1.4

Forger is a library for reinforcement learning with Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

use peroxide::fuga::*;

pub trait Policy<A> {
    fn select_action(&mut self, action_rewards: &[(A, f64)]) -> Option<A>;
}

// ┌──────────────────────────────────────────────────────────┐
//  Epsilon Greedy (with Decay) Policy                                                            
// └──────────────────────────────────────────────────────────┘
pub struct EGreedyPolicy<A> {
    epsilon: f64,
    decay: f64,
    random: bool,
    _action_type: std::marker::PhantomData<A>,
}

impl<A: Clone> EGreedyPolicy<A> {
    pub fn new(epsilon: f64, decay: f64) -> Self {
        Self {
            epsilon,
            decay,
            random: true,
            _action_type: std::marker::PhantomData,
        }
    }

    pub fn decay_epsilon(&mut self) {
        self.epsilon *= self.decay;
    }

    pub fn eval(&mut self) {
        self.random = false;
    }
}

impl<A: Clone> Policy<A> for EGreedyPolicy<A> {
    fn select_action(&mut self, action_rewards: &[(A, f64)]) -> Option<A> {
        if action_rewards.is_empty() {
            return None;
        }

        let epsilon = self.epsilon;

        let u = Uniform(0f64, 1f64);
        let sample = u.sample(1)[0];

        if sample < epsilon && self.random {
            let mut rng = thread_rng();
            Some(action_rewards.choose(&mut rng).unwrap().0.clone())
        } else {
            let mut max_reward = action_rewards[0].1;
            let mut max_actions = vec![];

            for (a, r) in action_rewards.iter() {
                if *r > max_reward {
                    max_reward = *r;
                    max_actions = vec![a.clone()];
                } else if *r == max_reward {
                    max_actions.push(a.clone());
                }
            }

            let mut rng = thread_rng();
            Some(max_actions.choose(&mut rng).unwrap().clone())
        }
    }
}

pub struct EGreedyPolicyMin<A> {
    epsilon: f64,
    decay: f64,
    random: bool,
    _action_type: std::marker::PhantomData<A>,
}

impl<A: Clone> EGreedyPolicyMin<A> {
    pub fn new(epsilon: f64, decay: f64) -> Self {
        Self {
            epsilon,
            decay,
            random: true,
            _action_type: std::marker::PhantomData,
        }
    }

    pub fn decay_epsilon(&mut self) {
        self.epsilon *= self.decay;
    }

    pub fn eval(&mut self) {
        self.random = false;
    }
}

impl<A: Clone> Policy<A> for EGreedyPolicyMin<A> {
    fn select_action(&mut self, action_rewards: &[(A, f64)]) -> Option<A> {
        if action_rewards.is_empty() {
            return None;
        }

        let epsilon = self.epsilon;

        let u = Uniform(0f64, 1f64);
        let sample = u.sample(1)[0];

        if sample < epsilon && self.random {
            let mut rng = thread_rng();
            Some(action_rewards.choose(&mut rng).unwrap().0.clone())
        } else {
            let mut min_reward = action_rewards[0].1;
            let mut min_actions = vec![];

            for (a, r) in action_rewards.iter() {
                if *r < min_reward {
                    min_reward = *r;
                    min_actions = vec![a.clone()];
                } else if *r == min_reward {
                    min_actions.push(a.clone());
                }
            }

            let mut rng = thread_rng();
            Some(min_actions.choose(&mut rng).unwrap().clone())
        }
    }
}