gymnasia 3.0.5

OpenAI Gymnasium environments in pure Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

use crate::core::{Env, StepResult};

use super::Wrapper;

/// Normalizes rewards using a running mean and variance estimate.
///
/// Uses the Welford online algorithm for numerically stable computation.
pub struct NormalizeReward<E: Env> {
    env: E,
    gamma: f64,
    epsilon: f64,
    running_mean: f64,
    running_var: f64,
    count: f64,
    returns: f64,
}

impl<E: Env> NormalizeReward<E> {
    /// Wrap `env` with reward normalization.
    pub fn new(env: E) -> Self {
        Self::with_params(env, 0.99, 1e-8)
    }

    /// Wrap with custom discount factor and epsilon.
    pub fn with_params(env: E, gamma: f64, epsilon: f64) -> Self {
        Self {
            env,
            gamma,
            epsilon,
            running_mean: 0.0,
            running_var: 1.0,
            count: 0.0,
            returns: 0.0,
        }
    }

    /// Current running mean.
    pub fn running_mean(&self) -> f64 {
        self.running_mean
    }

    /// Current running variance.
    pub fn running_var(&self) -> f64 {
        self.running_var
    }

    fn update_and_normalize(&mut self, reward: f64, terminated: bool) -> f64 {
        self.returns = self.returns * self.gamma + reward;

        // Welford online update
        self.count += 1.0;
        let delta = self.returns - self.running_mean;
        self.running_mean += delta / self.count;
        let delta2 = self.returns - self.running_mean;
        self.running_var =
            (self.running_var + (delta * delta2 - self.running_var) / self.count).max(0.0);

        let normalized = reward / (self.running_var.sqrt() + self.epsilon);

        if terminated {
            self.returns = 0.0;
        }

        normalized
    }
}

impl<E: Env> Env for NormalizeReward<E> {
    type Action = E::Action;
    type Observation = E::Observation;
    type ActionSpace = E::ActionSpace;
    type ObservationSpace = E::ObservationSpace;
    type ResetOptions = E::ResetOptions;

    fn step(&mut self, action: Self::Action) -> StepResult<Self::Observation> {
        let mut result = self.env.step(action);
        result.reward = self.update_and_normalize(result.reward, result.terminated);
        result
    }

    fn reset(&mut self, seed: Option<u64>, options: Self::ResetOptions) -> Self::Observation {
        self.returns = 0.0;
        self.env.reset(seed, options)
    }

    fn action_space(&self) -> &Self::ActionSpace {
        self.env.action_space()
    }
    fn observation_space(&self) -> &Self::ObservationSpace {
        self.env.observation_space()
    }
    fn close(&mut self) {
        self.env.close();
    }
}

impl<E: Env> Wrapper for NormalizeReward<E> {
    type Inner = E;
    fn inner(&self) -> &E {
        &self.env
    }
    fn inner_mut(&mut self) -> &mut E {
        &mut self.env
    }
    fn into_inner(self) -> E {
        self.env
    }
}