use candle_core::{D, Device, Tensor};
use crate::sampling::sample_u32_inclusive;
pub trait Space {
type Error;
fn sample(&self, device: &Device) -> Result<Tensor, Self::Error>;
fn contains(&self, x: &Tensor) -> bool;
fn shape(&self) -> Vec<usize>;
fn tensor_from_neurons(&self, neurons: &Tensor) -> Result<Tensor, Self::Error>;
}
#[derive(Clone)]
pub struct Discrete {
possible_values: usize,
}
impl Space for Discrete {
type Error = candle_core::Error;
fn sample(&self, device: &Device) -> Result<Tensor, Self::Error> {
let value = sample_u32_inclusive(0, self.possible_values as u32 - 1, device)?;
Tensor::from_vec(vec![value], vec![], device)
}
fn contains(&self, x: &Tensor) -> bool {
if x.dims() != Vec::<usize>::new() {
return false;
}
let value = x.to_vec0::<u32>().expect("Failed to convert to u32.");
if value < self.possible_values as u32 {
return true;
}
false
}
fn shape(&self) -> Vec<usize> {
if self.possible_values == 1 {
vec![]
} else {
vec![self.possible_values]
}
}
fn tensor_from_neurons(&self, neurons: &Tensor) -> Result<Tensor, Self::Error> {
neurons.argmax(D::Minus1)
}
}
impl Discrete {
pub fn new(possible_values: usize) -> Self {
Self { possible_values }
}
pub fn get_possible_values(&self) -> usize {
self.possible_values
}
}
#[derive(Clone)]
pub struct BoxSpace {
low: Tensor,
high: Tensor,
}
impl Space for BoxSpace {
type Error = candle_core::Error;
fn sample(&self, device: &Device) -> Result<Tensor, Self::Error> {
let mut values = vec![];
let dtype = self.low.dtype();
let low = self
.low
.to_dtype(candle_core::DType::F64)?
.flatten_all()
.expect("Failed to flatten tensor.");
let high = self
.high
.to_dtype(candle_core::DType::F64)?
.flatten_all()
.expect("Failed to flatten tensor.");
let rng_bases = Tensor::rand(0.0_f64, 1.0_f64, low.shape(), device)?;
for i in 0..low.shape().dim(0).expect("Failed to get dim.") {
let mut low = low
.get(i)
.expect("Failed to get value.")
.to_vec0::<f64>()
.expect("Failed to convert to f64.");
let mut high = high
.get(i)
.expect("Failed to get value.")
.to_vec0::<f64>()
.expect("Failed to convert to f64.");
low = finitize(low, dtype);
high = finitize(high, dtype);
let rng_base = rng_bases
.get(i)
.expect("Failed to get value.")
.to_vec0::<f64>()
.expect("Failed to convert to f64.");
let adjusted_range = high - low;
let rand_num = low + adjusted_range * rng_base;
values.push(rand_num);
}
Tensor::from_vec(values, low.shape(), device)?.to_dtype(dtype)
}
fn contains(&self, x: &Tensor) -> bool {
if *x.shape() != *self.low.shape() {
return false;
}
let low = self
.low
.to_dtype(candle_core::DType::F64)
.and_then(|tensor| tensor.flatten_all())
.expect("Failed to flatten tensor.");
let high = self
.high
.to_dtype(candle_core::DType::F64)
.and_then(|tensor| tensor.flatten_all())
.expect("Failed to flatten tensor.");
let x = x
.to_dtype(candle_core::DType::F64)
.and_then(|tensor| tensor.flatten_all())
.expect("Failed to flatten tensor.");
for i in 0..low.shape().dim(0).expect("Failed to get dim.") {
let low = low
.get(i)
.expect("Failed to get value.")
.to_vec0::<f64>()
.expect("Failed to convert to f64.");
let high = high
.get(i)
.expect("Failed to get value.")
.to_vec0::<f64>()
.expect("Failed to convert to f64.");
let value = x
.get(i)
.expect("Failed to get value.")
.to_vec0::<f64>()
.expect("Failed to convert to f64.");
if value < low || value > high {
return false;
}
}
true
}
fn shape(&self) -> Vec<usize> {
self.low.shape().clone().into_dims()
}
fn tensor_from_neurons(&self, neurons: &Tensor) -> Result<Tensor, Self::Error> {
let low = self
.low
.to_device(neurons.device())?
.to_dtype(neurons.dtype())?;
let high = self
.high
.to_device(neurons.device())?
.to_dtype(neurons.dtype())?;
neurons
.broadcast_maximum(&low)
.and_then(|values| values.broadcast_minimum(&high))
}
}
impl BoxSpace {
pub fn new(low: Tensor, high: Tensor) -> Self {
assert!(low.shape() == high.shape());
Self { low, high }
}
pub fn new_with_universal_bounds(
shape: Vec<usize>,
low: f32,
high: f32,
device: &Device,
) -> Self {
let mut lows = vec![];
let mut highs = vec![];
for _ in 0..shape.iter().product::<usize>() {
lows.push(low);
highs.push(high);
}
let lows = Tensor::from_vec(lows, shape.clone(), device).expect("Failed to create tensor.");
let highs =
Tensor::from_vec(highs, shape.clone(), device).expect("Failed to create tensor.");
Self::new(lows, highs)
}
pub fn new_unbounded(shape: Vec<usize>, device: &Device) -> Self {
Self::new_with_universal_bounds(shape, f32::NEG_INFINITY, f32::INFINITY, device)
}
}
fn finitize(value: f64, dtype: candle_core::DType) -> f64 {
let finite_limit = if dtype == candle_core::DType::F64 {
f64::MAX / 2.0
} else {
f64::from(f32::MAX / 2.0)
};
if value == f64::INFINITY {
return finite_limit;
}
if value == f64::NEG_INFINITY {
return -finite_limit;
}
value
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn box_space_clips_batched_policy_outputs_to_bounds() {
let space = BoxSpace::new_with_universal_bounds(vec![3], -1.0, 1.0, &Device::Cpu);
let neurons = Tensor::from_vec(
vec![-2.0_f32, -0.5, 0.25, 0.75, 1.5, 3.0],
(2, 3),
&Device::Cpu,
)
.unwrap();
let actions = space.tensor_from_neurons(&neurons).unwrap();
assert_eq!(
actions.to_vec2::<f32>().unwrap(),
vec![vec![-1.0, -0.5, 0.25], vec![0.75, 1.0, 1.0]]
);
}
#[test]
fn box_space_clips_using_policy_output_dtype() {
let space = BoxSpace::new_with_universal_bounds(vec![3], -1.0, 1.0, &Device::Cpu);
let neurons = Tensor::from_vec(
vec![-2.0_f64, -0.5, 0.25, 0.75, 1.5, 3.0],
(2, 3),
&Device::Cpu,
)
.unwrap();
let actions = space.tensor_from_neurons(&neurons).unwrap();
assert_eq!(actions.dtype(), candle_core::DType::F64);
assert_eq!(
actions.to_vec2::<f64>().unwrap(),
vec![vec![-1.0, -0.5, 0.25], vec![0.75, 1.0, 1.0]]
);
}
#[test]
fn box_space_samples_are_f32() {
let space = BoxSpace::new_with_universal_bounds(vec![3], -1.0, 1.0, &Device::Cpu);
let sample = space.sample(&Device::Cpu).unwrap();
assert_eq!(sample.dtype(), candle_core::DType::F32);
assert!(space.contains(&sample));
}
#[test]
fn box_space_preserves_f64_sampling_and_boundary_precision() {
let low = Tensor::new(&[0.0_f64], &Device::Cpu).unwrap();
let high = Tensor::new(&[1.0_f64], &Device::Cpu).unwrap();
let space = BoxSpace::new(low, high);
let sample = space.sample(&Device::Cpu).unwrap();
let just_outside = Tensor::new(&[1.0_f64 + 1e-12], &Device::Cpu).unwrap();
assert_eq!(sample.dtype(), candle_core::DType::F64);
assert!(space.contains(&sample));
assert!(!space.contains(&just_outside));
}
}