rl 0.4.0

A reinforcement learning library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

use burn::prelude::*;
use gym_rs::core::{ActionReward, Env};
use gym_rs::envs::classical_control::cartpole::{CartPoleEnv, CartPoleObservation};
use gym_rs::utils::renderer::RenderMode;
use rand::seq::IteratorRandom;
use rand::thread_rng;
use strum::{EnumIter, FromRepr, IntoEnumIterator, VariantArray};

use crate::env::{DiscreteActionSpace, Environment, Report, ToTensor};

fn obs2arr(observation: CartPoleObservation) -> [f32; 4] {
    Vec::from(observation)
        .into_iter()
        .map(|x| x as f32)
        .collect::<Vec<_>>()
        .try_into()
        .expect("vec is length 4")
}

/// Actions for the [`CartPole`] environment, representing applying a left or right force to the cart
#[derive(FromRepr, EnumIter, VariantArray, Clone, Copy, Debug)]
pub enum CPAction {
    Left = 0,
    Right = 1,
}

impl From<usize> for CPAction {
    fn from(value: usize) -> Self {
        Self::from_repr(value).expect("CPAction::from is only called with valid values [0, 1]")
    }
}

impl<B: Backend<IntElem = i32>> ToTensor<B, 2, Int> for Vec<CPAction> {
    fn to_tensor(self, device: &B::Device) -> Tensor<B, 2, Int> {
        let len = self.len();
        let data = Data::new(
            self.into_iter().map(|x| x as i32).collect::<Vec<_>>(),
            [len].into(),
        );
        Tensor::from_data(data, device).unsqueeze_dim(1)
    }
}

impl<B: Backend<FloatElem = f32>> ToTensor<B, 2, Float> for Vec<[f32; 4]> {
    fn to_tensor(self, device: &B::Device) -> Tensor<B, 2, Float> {
        let len = self.len();
        let data = Data::new(
            self.into_iter().flatten().collect::<Vec<_>>(),
            [len * 4].into(),
        );
        Tensor::from_data(data, device).reshape([-1, 4])
    }
}

/// The classic CartPole reinforcement learning environment
///
/// This implementation is a thin wrapper around [gym_rs](https://github.com/MathisWellmann/gym-rs)
#[derive(Debug)]
pub struct CartPole {
    gym_env: CartPoleEnv,
    pub report: Report,
}

impl CartPole {
    pub fn new(render_mode: RenderMode) -> Self {
        Self {
            gym_env: CartPoleEnv::new(render_mode),
            report: Report::new(vec!["reward", "loss"]),
        }
    }
}

impl Environment for CartPole {
    type State = [f32; 4];
    type Action = CPAction;

    fn random_action(&self) -> Self::Action {
        CPAction::iter().choose(&mut thread_rng()).unwrap()
    }

    fn step(&mut self, action: Self::Action) -> (Option<Self::State>, f32) {
        let ActionReward {
            observation,
            reward,
            done,
            ..
        } = self.gym_env.step(action as usize);

        let next_state = if done {
            None
        } else {
            Some(obs2arr(observation))
        };

        self.report.entry("reward").and_modify(|x| *x += *reward);

        (next_state, *reward as f32)
    }

    fn reset(&mut self) -> Self::State {
        obs2arr(self.gym_env.reset(None, false, None).0)
    }
}

impl DiscreteActionSpace for CartPole {
    fn actions(&self) -> Vec<Self::Action> {
        CPAction::VARIANTS.to_vec()
    }
}