modurl 0.1.0

Modular reinforcement learning algorithms and utilities built on Candle
Documentation
use candle_core::{D, Device, Tensor};

use crate::sampling::sample_u32_inclusive;

pub trait Space {
    type Error;

    /// Returns one unbatched environment action. Concrete spaces document its
    /// exact shape.
    fn sample(&self, device: &Device) -> Result<Tensor, Self::Error>;
    /// Returns true if `x` has the concrete space's unbatched environment
    /// action shape and is within the space.
    fn contains(&self, x: &Tensor) -> bool;
    /// Returns the shape of one latent policy output consumed by
    /// [`Space::tensor_from_neurons`].
    fn shape(&self) -> Vec<usize>;
    /// Converts latent policy output shaped `[batch_size, ...self.shape()]`
    /// into batched environment actions. Concrete spaces document the output
    /// shape.
    ///
    /// A continuous [`BoxSpace`] clamps each component to its bounds. PPO keeps
    /// its original latent action separately for probability calculations.
    fn tensor_from_neurons(&self, neurons: &Tensor) -> Result<Tensor, Self::Error>;
}

#[derive(Clone)]
pub struct Discrete {
    possible_values: usize,
}

impl Space for Discrete {
    type Error = candle_core::Error;

    /// Samples one scalar environment action shaped `[]`.
    fn sample(&self, device: &Device) -> Result<Tensor, Self::Error> {
        let value = sample_u32_inclusive(0, self.possible_values as u32 - 1, device)?;
        Tensor::from_vec(vec![value], vec![], device)
    }

    /// Tests one scalar environment action `x` shaped `[]`.
    fn contains(&self, x: &Tensor) -> bool {
        if x.dims() != Vec::<usize>::new() {
            return false;
        }
        let value = x.to_vec0::<u32>().expect("Failed to convert to u32.");
        if value < self.possible_values as u32 {
            return true;
        }
        false
    }

    /// Returns latent policy shape `[action_count]` (or `[]` for a one-action
    /// space).
    fn shape(&self) -> Vec<usize> {
        if self.possible_values == 1 {
            vec![]
        } else {
            vec![self.possible_values]
        }
    }

    /// Converts logits `neurons` shaped `[batch_size, action_count]` into
    /// scalar action indices shaped `[batch_size]`.
    fn tensor_from_neurons(&self, neurons: &Tensor) -> Result<Tensor, Self::Error> {
        neurons.argmax(D::Minus1)
    }
}

impl Discrete {
    pub fn new(possible_values: usize) -> Self {
        Self { possible_values }
    }

    pub fn get_possible_values(&self) -> usize {
        self.possible_values
    }
}

/// A bounded, n-dimensional continuous space.
///
/// The `low` and `high` tensors define the inclusive component bounds.
/// [`Space::tensor_from_neurons`] clamps policy outputs to these bounds before
/// they are sent to an environment.
#[derive(Clone)]
pub struct BoxSpace {
    low: Tensor,
    high: Tensor,
}

impl Space for BoxSpace {
    type Error = candle_core::Error;

    /// Samples one environment action shaped `self.shape()`.
    fn sample(&self, device: &Device) -> Result<Tensor, Self::Error> {
        let mut values = vec![];
        let dtype = self.low.dtype();

        // Perform host-side sampling in F64 so F64-backed spaces retain their
        // precision. The final sample still uses the space's declared dtype.
        let low = self
            .low
            .to_dtype(candle_core::DType::F64)?
            .flatten_all()
            .expect("Failed to flatten tensor.");
        let high = self
            .high
            .to_dtype(candle_core::DType::F64)?
            .flatten_all()
            .expect("Failed to flatten tensor.");
        let rng_bases = Tensor::rand(0.0_f64, 1.0_f64, low.shape(), device)?;
        for i in 0..low.shape().dim(0).expect("Failed to get dim.") {
            let mut low = low
                .get(i)
                .expect("Failed to get value.")
                .to_vec0::<f64>()
                .expect("Failed to convert to f64.");
            let mut high = high
                .get(i)
                .expect("Failed to get value.")
                .to_vec0::<f64>()
                .expect("Failed to convert to f64.");
            low = finitize(low, dtype);
            high = finitize(high, dtype);

            let rng_base = rng_bases
                .get(i)
                .expect("Failed to get value.")
                .to_vec0::<f64>()
                .expect("Failed to convert to f64.");
            let adjusted_range = high - low;
            let rand_num = low + adjusted_range * rng_base;

            values.push(rand_num);
        }
        Tensor::from_vec(values, low.shape(), device)?.to_dtype(dtype)
    }

    /// Tests one environment action `x` shaped `self.shape()`.
    fn contains(&self, x: &Tensor) -> bool {
        // This is kinda weird, because if the shape is not equal,
        // Should we just say false, or should we return an error?
        if *x.shape() != *self.low.shape() {
            return false;
        }
        let low = self
            .low
            .to_dtype(candle_core::DType::F64)
            .and_then(|tensor| tensor.flatten_all())
            .expect("Failed to flatten tensor.");
        let high = self
            .high
            .to_dtype(candle_core::DType::F64)
            .and_then(|tensor| tensor.flatten_all())
            .expect("Failed to flatten tensor.");
        let x = x
            .to_dtype(candle_core::DType::F64)
            .and_then(|tensor| tensor.flatten_all())
            .expect("Failed to flatten tensor.");
        // So inefficient :*(
        for i in 0..low.shape().dim(0).expect("Failed to get dim.") {
            let low = low
                .get(i)
                .expect("Failed to get value.")
                .to_vec0::<f64>()
                .expect("Failed to convert to f64.");
            let high = high
                .get(i)
                .expect("Failed to get value.")
                .to_vec0::<f64>()
                .expect("Failed to convert to f64.");
            let value = x
                .get(i)
                .expect("Failed to get value.")
                .to_vec0::<f64>()
                .expect("Failed to convert to f64.");
            if value < low || value > high {
                return false;
            }
        }
        true
    }

    fn shape(&self) -> Vec<usize> {
        self.low.shape().clone().into_dims()
    }

    /// Clips `neurons` shaped `[batch_size, ...self.shape()]`, preserving that
    /// shape.
    fn tensor_from_neurons(&self, neurons: &Tensor) -> Result<Tensor, Self::Error> {
        // Continuous policies (for example, an unsquashed Gaussian) may
        // produce values outside the environment's bounds. Keep the latent
        // action unchanged for probability calculations, but clip the action
        // that is actually sent to the environment.
        let low = self
            .low
            .to_device(neurons.device())?
            .to_dtype(neurons.dtype())?;
        let high = self
            .high
            .to_device(neurons.device())?
            .to_dtype(neurons.dtype())?;
        neurons
            .broadcast_maximum(&low)
            .and_then(|values| values.broadcast_minimum(&high))
    }
}

impl BoxSpace {
    /// Creates a box whose `low` and `high` bounds have the same
    /// `action_shape`, which becomes the shape of one unbatched value.
    pub fn new(low: Tensor, high: Tensor) -> Self {
        assert!(low.shape() == high.shape());
        Self { low, high }
    }

    pub fn new_with_universal_bounds(
        shape: Vec<usize>,
        low: f32,
        high: f32,
        device: &Device,
    ) -> Self {
        let mut lows = vec![];
        let mut highs = vec![];
        for _ in 0..shape.iter().product::<usize>() {
            lows.push(low);
            highs.push(high);
        }
        let lows = Tensor::from_vec(lows, shape.clone(), device).expect("Failed to create tensor.");
        let highs =
            Tensor::from_vec(highs, shape.clone(), device).expect("Failed to create tensor.");
        Self::new(lows, highs)
    }

    pub fn new_unbounded(shape: Vec<usize>, device: &Device) -> Self {
        Self::new_with_universal_bounds(shape, f32::NEG_INFINITY, f32::INFINITY, device)
    }
}

// A helper function to finitize the value.
// This way random_range can work with f32::INFINITY and f32::NEG_INFINITY.
// We just return f32::MAX / 2.0 and f32::MIN / 2.0 respectively.
// It's pretty dumb, but it lies and says min and max aren't finite otherwise.
fn finitize(value: f64, dtype: candle_core::DType) -> f64 {
    let finite_limit = if dtype == candle_core::DType::F64 {
        f64::MAX / 2.0
    } else {
        f64::from(f32::MAX / 2.0)
    };
    if value == f64::INFINITY {
        return finite_limit;
    }
    if value == f64::NEG_INFINITY {
        return -finite_limit;
    }
    value
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn box_space_clips_batched_policy_outputs_to_bounds() {
        let space = BoxSpace::new_with_universal_bounds(vec![3], -1.0, 1.0, &Device::Cpu);
        let neurons = Tensor::from_vec(
            vec![-2.0_f32, -0.5, 0.25, 0.75, 1.5, 3.0],
            (2, 3),
            &Device::Cpu,
        )
        .unwrap();

        let actions = space.tensor_from_neurons(&neurons).unwrap();

        assert_eq!(
            actions.to_vec2::<f32>().unwrap(),
            vec![vec![-1.0, -0.5, 0.25], vec![0.75, 1.0, 1.0]]
        );
    }

    #[test]
    fn box_space_clips_using_policy_output_dtype() {
        let space = BoxSpace::new_with_universal_bounds(vec![3], -1.0, 1.0, &Device::Cpu);
        let neurons = Tensor::from_vec(
            vec![-2.0_f64, -0.5, 0.25, 0.75, 1.5, 3.0],
            (2, 3),
            &Device::Cpu,
        )
        .unwrap();

        let actions = space.tensor_from_neurons(&neurons).unwrap();

        assert_eq!(actions.dtype(), candle_core::DType::F64);
        assert_eq!(
            actions.to_vec2::<f64>().unwrap(),
            vec![vec![-1.0, -0.5, 0.25], vec![0.75, 1.0, 1.0]]
        );
    }

    #[test]
    fn box_space_samples_are_f32() {
        let space = BoxSpace::new_with_universal_bounds(vec![3], -1.0, 1.0, &Device::Cpu);
        let sample = space.sample(&Device::Cpu).unwrap();

        assert_eq!(sample.dtype(), candle_core::DType::F32);
        assert!(space.contains(&sample));
    }

    #[test]
    fn box_space_preserves_f64_sampling_and_boundary_precision() {
        let low = Tensor::new(&[0.0_f64], &Device::Cpu).unwrap();
        let high = Tensor::new(&[1.0_f64], &Device::Cpu).unwrap();
        let space = BoxSpace::new(low, high);

        let sample = space.sample(&Device::Cpu).unwrap();
        let just_outside = Tensor::new(&[1.0_f64 + 1e-12], &Device::Cpu).unwrap();

        assert_eq!(sample.dtype(), candle_core::DType::F64);
        assert!(space.contains(&sample));
        assert!(!space.contains(&just_outside));
    }
}