use std::num::NonZeroUsize;
use candle_core::{D, Device, Tensor};
use candle_nn::ops::softmax;
use crate::distributions::{DifferentiableExpectation, DistEval, Distribution, ExpectationTerms};
#[derive(Clone, Copy, Debug, Default)]
pub struct CategoricalDistribution;
#[derive(Debug, thiserror::Error)]
pub enum CategoricalDistributionError {
#[error("categorical tensor operation failed: {0}")]
TensorError(#[source] candle_core::Error),
#[error("a categorical distribution requires at least one category")]
NoCategories,
}
impl From<candle_core::Error> for CategoricalDistributionError {
fn from(error: candle_core::Error) -> Self {
Self::TensorError(error)
}
}
fn log_softmax(tensor: &Tensor, dim: D) -> Result<Tensor, candle_core::Error> {
let max = tensor.max_keepdim(dim)?.expand(tensor.dims())?;
let exps = (tensor - &max)?.exp()?;
let sum_exps = exps.sum_keepdim(dim)?.expand(tensor.dims())?;
let log_sum_exps = sum_exps.log()?;
tensor - max - log_sum_exps
}
impl CategoricalDistribution {
fn validate(outputs: &Tensor) -> Result<(usize, usize), CategoricalDistributionError> {
let shape = outputs.dims2()?;
if shape.1 == 0 {
return Err(CategoricalDistributionError::NoCategories);
}
Ok(shape)
}
fn probs(&self, outputs: &Tensor) -> Result<Tensor, CategoricalDistributionError> {
Self::validate(outputs)?;
Ok(softmax(outputs, D::Minus1)?)
}
fn log_probs(&self, outputs: &Tensor) -> Result<Tensor, CategoricalDistributionError> {
Self::validate(outputs)?;
Ok(log_softmax(outputs, D::Minus1)?)
}
fn gumbel_noise(
shape: &[usize],
dtype: candle_core::DType,
device: &Device,
) -> Result<Tensor, candle_core::Error> {
let u = Tensor::rand(0f32, 1f32, shape, device)?.to_dtype(dtype)?;
let gumbel = (-1.0 * (&u.log()?))?.log()?;
gumbel.neg() }
}
impl Distribution for CategoricalDistribution {
type Error = CategoricalDistributionError;
fn sample(&self, outputs: &Tensor) -> Result<Tensor, Self::Error> {
Self::validate(outputs)?;
let shape = outputs.dims();
let device = outputs.device();
let noise = Self::gumbel_noise(shape, outputs.dtype(), device)?;
Ok((outputs + noise)?)
}
fn mode(&self, outputs: &Tensor) -> Result<Tensor, Self::Error> {
Self::validate(outputs)?;
Ok(outputs.clone())
}
fn dist_eval(&self, outputs: &Tensor, actions: &Tensor) -> Result<DistEval, Self::Error> {
let log_probs = self.log_probs(outputs)?;
let actions_argmax = actions.argmax(D::Minus1)?; let log_prob = log_probs
.gather(&actions_argmax.unsqueeze(1)?, D::Minus1)?
.squeeze(1)?;
let probs = self.probs(outputs)?;
let entropy = (probs.clone() * log_probs)?.sum(D::Minus1)?.neg()?;
Ok(DistEval::new(log_prob, entropy))
}
}
impl DifferentiableExpectation for CategoricalDistribution {
fn expectation(
&self,
outputs: &Tensor,
_samples: NonZeroUsize,
) -> Result<ExpectationTerms, Self::Error> {
let (batch_size, categories) = Self::validate(outputs)?;
let actions = Tensor::arange(0u32, categories as u32, outputs.device())?
.broadcast_left(batch_size)?;
let log_probabilities = self.log_probs(outputs)?;
let weights = self.probs(outputs)?;
Ok(ExpectationTerms::new(actions, log_probabilities, weights)?)
}
fn default_target_entropy(&self, outputs: &Tensor) -> Result<f64, Self::Error> {
Ok(0.98 * (Self::validate(outputs)?.1 as f64).ln())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_log_softmax() {
let device = Device::Cpu;
let x = Tensor::from_vec(vec![1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0], &[2, 3], &device).unwrap();
let result = log_softmax(&x, D::Minus1).unwrap();
let result_vec = result.to_vec2::<f32>().unwrap();
let expected = [
vec![-2.407606f32, -1.4076059, -0.40760595],
vec![-2.407606f32, -1.4076059, -0.40760595],
];
for i in 0..2 {
for j in 0..3 {
let diff = (result_vec[i][j] - expected[i][j]).abs();
assert!(
diff < 1e-5,
"Mismatch at [{}, {}]: got {}, expected {}, diff {}",
i,
j,
result_vec[i][j],
expected[i][j],
diff
);
}
}
}
#[test]
fn expectation_enumerates_actions_with_normalized_weights() {
let logits = Tensor::from_vec(vec![0.0f32, 1.0, 2.0], (1, 3), &Device::Cpu).unwrap();
let distribution = CategoricalDistribution;
let terms = distribution
.expectation(&logits, NonZeroUsize::MIN)
.unwrap();
assert_eq!(terms.actions().dims(), &[1, 3]);
assert_eq!(
terms.actions().to_vec2::<u32>().unwrap(),
vec![vec![0, 1, 2]]
);
let weight_sum = terms
.weights()
.sum_all()
.unwrap()
.to_scalar::<f32>()
.unwrap();
assert!((weight_sum - 1.0).abs() < 1e-6);
assert!(
(distribution.default_target_entropy(&logits).unwrap() - 0.98 * 3.0f64.ln()).abs()
< 1e-12
);
}
#[test]
fn categorical_distribution_requires_a_category() {
let outputs = Tensor::zeros((2, 0), candle_core::DType::F32, &Device::Cpu).unwrap();
assert!(matches!(
CategoricalDistribution.sample(&outputs),
Err(CategoricalDistributionError::NoCategories)
));
}
}