use candle_core::Tensor;
use crate::gym::{Gym, ResetInfo, StepInfo};
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct EpisodeStatistics {
pub episode_return: f32,
pub episode_length: usize,
}
#[derive(Clone, Debug)]
pub struct EpisodeStatisticsInfo<I> {
pub inner: I,
pub completed_episode: Option<EpisodeStatistics>,
}
pub struct RecordEpisodeStatisticsGym<G> {
gym: G,
episode_return: f32,
episode_length: usize,
}
impl<G> RecordEpisodeStatisticsGym<G> {
pub fn new(gym: G) -> Self {
Self {
gym,
episode_return: 0.0,
episode_length: 0,
}
}
}
impl<G, I> Gym<EpisodeStatisticsInfo<I>> for RecordEpisodeStatisticsGym<G>
where
G: Gym<I>,
{
type Error = G::Error;
type SpaceError = G::SpaceError;
fn reset(&mut self) -> Result<ResetInfo<EpisodeStatisticsInfo<I>>, Self::Error> {
self.episode_return = 0.0;
self.episode_length = 0;
let reset = self.gym.reset()?;
Ok(ResetInfo {
state: reset.state,
info: EpisodeStatisticsInfo {
inner: reset.info,
completed_episode: None,
},
})
}
fn step(&mut self, action: Tensor) -> Result<StepInfo<EpisodeStatisticsInfo<I>>, Self::Error> {
let step = self.gym.step(action)?;
self.episode_return += step.reward;
self.episode_length += 1;
let completed_episode = if step.done || step.truncated {
let statistics = EpisodeStatistics {
episode_return: self.episode_return,
episode_length: self.episode_length,
};
self.episode_return = 0.0;
self.episode_length = 0;
Some(statistics)
} else {
None
};
Ok(StepInfo {
state: step.state,
reward: step.reward,
done: step.done,
truncated: step.truncated,
info: EpisodeStatisticsInfo {
inner: step.info,
completed_episode,
},
})
}
fn action_space(&self) -> Box<dyn crate::spaces::Space<Error = Self::SpaceError>> {
self.gym.action_space()
}
fn observation_space(&self) -> Box<dyn crate::spaces::Space<Error = Self::SpaceError>> {
self.gym.observation_space()
}
}
#[derive(Debug, Clone)]
pub struct RawRewardInfo<I> {
pub inner: I,
pub raw_reward: Option<f32>,
}
pub struct RecordRawRewardGym<G> {
gym: G,
}
impl<G> RecordRawRewardGym<G> {
pub fn new(gym: G) -> Self {
Self { gym }
}
}
impl<G, I> Gym<RawRewardInfo<I>> for RecordRawRewardGym<G>
where
G: Gym<I>,
{
type Error = <G as Gym<I>>::Error;
type SpaceError = <G as Gym<I>>::SpaceError;
fn reset(&mut self) -> Result<ResetInfo<RawRewardInfo<I>>, Self::Error> {
let reset = self.gym.reset()?;
Ok(ResetInfo {
state: reset.state,
info: RawRewardInfo {
inner: reset.info,
raw_reward: None,
},
})
}
fn step(&mut self, action: Tensor) -> Result<StepInfo<RawRewardInfo<I>>, Self::Error> {
let step = self.gym.step(action)?;
Ok(StepInfo {
state: step.state,
reward: step.reward,
done: step.done,
truncated: step.truncated,
info: RawRewardInfo {
inner: step.info,
raw_reward: Some(step.reward),
},
})
}
fn action_space(&self) -> Box<dyn crate::spaces::Space<Error = Self::SpaceError>> {
self.gym.action_space()
}
fn observation_space(&self) -> Box<dyn crate::spaces::Space<Error = Self::SpaceError>> {
self.gym.observation_space()
}
}
#[cfg(test)]
mod tests {
use crate::{
gym::Gym,
wrappers::test_support::{TestGym, action},
};
use super::{EpisodeStatistics, RecordEpisodeStatisticsGym, RecordRawRewardGym};
#[test]
fn records_statistics_at_the_wrapped_episode_boundary() {
let gym = TestGym::new([
TestGym::step(1.0, 2.5, false, false, 1),
TestGym::step(2.0, -0.5, true, false, 2),
]);
let mut wrapper = RecordEpisodeStatisticsGym::new(gym);
assert!(wrapper.reset().unwrap().info.completed_episode.is_none());
assert!(
wrapper
.step(action())
.unwrap()
.info
.completed_episode
.is_none()
);
assert_eq!(
wrapper.step(action()).unwrap().info.completed_episode,
Some(EpisodeStatistics {
episode_return: 2.0,
episode_length: 2,
})
);
}
#[test]
fn adds_raw_reward_without_changing_the_transition() {
let gym = TestGym::new([TestGym::step(1.0, 2.5, false, false, 7)]);
let mut wrapper = RecordRawRewardGym::new(gym);
let reset = wrapper.reset().unwrap();
assert_eq!(reset.info.raw_reward, None);
let step = wrapper.step(action()).unwrap();
assert_eq!(step.reward, 2.5);
assert_eq!(step.info.raw_reward, Some(2.5));
assert_eq!(step.info.inner.sequence, 7);
}
}