border-core 0.0.5

Reinforcement learning library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

//! A generic implementation of [StepProcessorBase](crate::StepProcessorBase).
use std::{default::Default, marker::PhantomData};
use crate::{Obs, Env, StepProcessorBase};
use super::{Batch, SubBatch};

/// Configuration of [SimpleStepProcessor].
#[derive(Clone, Debug)]
pub struct SimpleStepProcessorConfig {
}

impl Default for SimpleStepProcessorConfig {
    fn default() -> Self {
        Self {}
    }
}

/// A generic implementation of [StepProcessorBase](crate::StepProcessorBase).
///
/// It supports 1-step TD backup for non-vectorized environment:
/// `E::Obs.len()` must be 1.
pub struct SimpleStepProcessor<E, O, A> {
    prev_obs: Option<O>,
    phantom: PhantomData<(E, A)>
}

impl<E, O, A> StepProcessorBase<E> for SimpleStepProcessor<E, O, A>
where
    E: Env,
    O: SubBatch + From<E::Obs>,
    A: SubBatch + From<E::Act>,
{
    type Config = SimpleStepProcessorConfig;
    type Output = Batch<O, A>;

    fn build(_config: &Self::Config) -> Self {
        Self {
            prev_obs: None,
            phantom: PhantomData
        }
    }

    fn reset(&mut self, init_obs: E::Obs) {
        self.prev_obs = Some(init_obs.into());
    }

    fn process(&mut self, step: crate::Step<E>) -> Self::Output {
        assert_eq!(step.obs.len(), 1);

        let batch = if self.prev_obs.is_none() {
            panic!("prev_obs is not set. Forgot to call reset()?");
        } else {
            let next_obs = step.obs.clone().into();
            let obs = self.prev_obs.replace(step.obs.into()).unwrap();
            let act = step.act.into();
            let reward = step.reward;
            let is_done = step.is_done;
            let ix_sample = None;
            let weight = None;

            if is_done[0] == 1 {
                self.prev_obs.replace(step.init_obs.into());
            }

            Batch {obs, act, next_obs, reward, is_done, ix_sample, weight}
        };

        batch
    }
}