use std::num::NonZeroUsize;
use candle_core::Tensor;
use candle_nn::Module;
use crate::distributions::{DifferentiableExpectation, Distribution, ExpectationTerms};
pub trait ProbabilisticPolicy {
type Error;
fn sample(&self, state: &Tensor) -> Result<Tensor, Self::Error>;
fn mode(&self, state: &Tensor) -> Result<Tensor, Self::Error>;
fn log_prob_and_entropy(
&self,
state: &Tensor,
action: &Tensor,
) -> Result<(Tensor, Tensor), Self::Error>;
}
pub trait ExpectationPolicy: ProbabilisticPolicy {
fn expectation(
&self,
state: &Tensor,
samples: NonZeroUsize,
) -> Result<ExpectationTerms, Self::Error>;
fn default_target_entropy(&self, state: &Tensor) -> Result<f64, Self::Error>;
}
pub struct ProbabilisticPolicyModel<D>
where
D: Distribution,
{
module: Box<dyn Module>,
distribution: D,
}
impl<D> ProbabilisticPolicyModel<D>
where
D: Distribution + Default,
<D as Distribution>::Error: std::fmt::Debug,
{
pub fn new(module: impl Module + 'static) -> Self {
Self {
module: Box::new(module),
distribution: D::default(),
}
}
}
impl<D> ProbabilisticPolicyModel<D>
where
D: Distribution,
<D as Distribution>::Error: std::fmt::Debug,
{
pub fn with_distribution(module: impl Module + 'static, distribution: D) -> Self {
Self {
module: Box::new(module),
distribution,
}
}
pub fn distribution(&self) -> &D {
&self.distribution
}
fn outputs(&self, state: &Tensor) -> candle_core::Result<Tensor> {
self.module.forward(state)
}
}
#[derive(Debug, thiserror::Error)]
pub enum ProbabilisticPolicyModelError<DE>
where
DE: std::fmt::Debug,
{
#[error("policy module failed: {0}")]
ModuleError(#[source] candle_core::Error),
#[error("policy distribution failed: {0}")]
DistError(#[source] DE),
}
impl<DE> From<candle_core::Error> for ProbabilisticPolicyModelError<DE>
where
DE: std::fmt::Debug,
{
fn from(error: candle_core::Error) -> Self {
ProbabilisticPolicyModelError::ModuleError(error)
}
}
impl<D> ProbabilisticPolicy for ProbabilisticPolicyModel<D>
where
D: Distribution,
<D as Distribution>::Error: std::fmt::Debug,
{
type Error = ProbabilisticPolicyModelError<<D as Distribution>::Error>;
fn sample(&self, state: &Tensor) -> Result<Tensor, Self::Error> {
let outputs = self.outputs(state)?;
self.distribution
.sample(&outputs)
.map_err(ProbabilisticPolicyModelError::DistError)
}
fn mode(&self, state: &Tensor) -> Result<Tensor, Self::Error> {
let outputs = self.outputs(state)?;
self.distribution
.mode(&outputs)
.map_err(ProbabilisticPolicyModelError::DistError)
}
fn log_prob_and_entropy(
&self,
state: &Tensor,
action: &Tensor,
) -> Result<(Tensor, Tensor), Self::Error> {
let outputs = self.outputs(state)?;
let dist_eval = self
.distribution
.dist_eval(&outputs, action)
.map_err(ProbabilisticPolicyModelError::DistError)?;
let log_prob = dist_eval.log_prob().clone();
let entropy = dist_eval.entropy().clone();
Ok((log_prob, entropy))
}
}
impl<D> ExpectationPolicy for ProbabilisticPolicyModel<D>
where
D: DifferentiableExpectation,
<D as Distribution>::Error: std::fmt::Debug,
{
fn expectation(
&self,
state: &Tensor,
samples: NonZeroUsize,
) -> Result<ExpectationTerms, Self::Error> {
let outputs = self.outputs(state)?;
self.distribution
.expectation(&outputs, samples)
.map_err(ProbabilisticPolicyModelError::DistError)
}
fn default_target_entropy(&self, state: &Tensor) -> Result<f64, Self::Error> {
let outputs = self.outputs(state)?.detach();
self.distribution
.default_target_entropy(&outputs)
.map_err(ProbabilisticPolicyModelError::DistError)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{distributions::GaussianDistribution, models::MLP};
use candle_core::{DType, Device, Tensor};
use candle_nn::{Activation, VarBuilder, VarMap};
fn create_test_policy(
input_size: usize,
action_size: usize,
hidden_sizes: Vec<usize>,
) -> Result<ProbabilisticPolicyModel<GaussianDistribution>, candle_core::Error> {
let device = Device::Cpu;
let varmap = VarMap::new();
let vb = VarBuilder::from_varmap(&varmap, DType::F64, &device);
let mlp = MLP::builder()
.input_size(input_size)
.output_size(action_size * 2)
.vb(vb)
.hidden_layer_sizes(hidden_sizes)
.activation(Activation::Relu)
.build()?;
Ok(ProbabilisticPolicyModel::new(mlp))
}
#[test]
fn configured_distribution_is_stored_by_the_policy() {
use crate::distributions::{TanhTransform, TransformedDistribution};
let device = Device::Cpu;
let varmap = VarMap::new();
let mlp = MLP::builder()
.input_size(2)
.output_size(2)
.vb(VarBuilder::from_varmap(&varmap, DType::F32, &device))
.hidden_layer_sizes(vec![4])
.build()
.unwrap();
let policy = ProbabilisticPolicyModel::with_distribution(
mlp,
TransformedDistribution::new(GaussianDistribution::default(), TanhTransform),
);
let actions = policy
.sample(&Tensor::zeros((3, 2), DType::F32, &device).unwrap())
.unwrap();
assert_eq!(actions.dims(), &[3, 1]);
assert!(
actions
.abs()
.unwrap()
.max_all()
.unwrap()
.to_scalar::<f32>()
.unwrap()
<= 1.0
);
}
fn create_test_state(
batch_size: usize,
state_dim: usize,
) -> Result<Tensor, candle_core::Error> {
Tensor::randn(0.0, 1.0, (batch_size, state_dim), &Device::Cpu)
}
#[test]
fn test_policy_creation() {
let policy = create_test_policy(4, 2, vec![32, 32]);
assert!(policy.is_ok(), "Failed to create probabilistic policy");
}
#[test]
fn test_sample_action_shape() {
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let state = create_test_state(1, 4).unwrap();
let action = policy.sample(&state).unwrap();
assert_eq!(action.dims(), &[1, 2]);
}
#[test]
fn test_sample_batch_consistency() {
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let batch_sizes = vec![1, 5, 10];
for batch_size in batch_sizes {
let state = create_test_state(batch_size, 4).unwrap();
let action = policy.sample(&state).unwrap();
assert_eq!(
action.dims(),
&[batch_size, 2],
"Action shape incorrect for batch size {}",
batch_size
);
}
}
#[test]
fn test_log_prob_and_entropy_shape() {
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let batch_size = 5;
let state = create_test_state(batch_size, 4).unwrap();
let action = create_test_state(batch_size, 2).unwrap();
let (log_prob, entropy) = policy.log_prob_and_entropy(&state, &action).unwrap();
assert_eq!(log_prob.dims(), &[batch_size]);
assert_eq!(entropy.dims(), &[batch_size]);
}
#[test]
fn test_entropy_is_positive() {
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let state = create_test_state(1, 4).unwrap();
let action = create_test_state(1, 2).unwrap();
let (_, entropy) = policy.log_prob_and_entropy(&state, &action).unwrap();
let entropy_val = entropy.to_vec1::<f64>().unwrap()[0];
assert!(
entropy_val.is_finite(),
"Entropy is not finite: {}",
entropy_val
);
assert!(
entropy_val > -100.0,
"Entropy is not reasonable: {}",
entropy_val
);
assert!(entropy_val < 100.0, "Entropy is too large: {}", entropy_val);
}
#[test]
fn test_log_prob_is_finite() {
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let state = create_test_state(3, 4).unwrap();
let action = create_test_state(3, 2).unwrap();
let (log_prob, _) = policy.log_prob_and_entropy(&state, &action).unwrap();
let log_prob_vals = log_prob.to_vec1::<f64>().unwrap();
for (i, val) in log_prob_vals.iter().enumerate() {
assert!(
val.is_finite(),
"Log probability {} is not finite: {}",
i,
val
);
}
}
#[test]
fn test_deterministic_with_same_state() {
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let state = Tensor::zeros((1, 4), candle_core::DType::F64, &Device::Cpu).unwrap();
let action = Tensor::zeros((1, 2), candle_core::DType::F64, &Device::Cpu).unwrap();
let (log_prob1, entropy1) = policy.log_prob_and_entropy(&state, &action).unwrap();
let (log_prob2, entropy2) = policy.log_prob_and_entropy(&state, &action).unwrap();
let log_prob1_val = log_prob1.to_vec1::<f64>().unwrap()[0];
let log_prob2_val = log_prob2.to_vec1::<f64>().unwrap()[0];
let entropy1_val = entropy1.to_vec1::<f64>().unwrap()[0];
let entropy2_val = entropy2.to_vec1::<f64>().unwrap()[0];
assert!(
(log_prob1_val - log_prob2_val).abs() < 1e-6,
"Log probabilities should be deterministic"
);
assert!(
(entropy1_val - entropy2_val).abs() < 1e-6,
"Entropies should be deterministic"
);
}
#[test]
fn test_different_action_sizes() {
let action_sizes = vec![1, 2, 4, 8];
for action_size in action_sizes {
let policy = create_test_policy(4, action_size, vec![32, 32]).unwrap();
let state = create_test_state(1, 4).unwrap();
let action = policy.sample(&state).unwrap();
assert_eq!(
action.dims(),
&[1, action_size],
"Action size incorrect for action_size {}",
action_size
);
}
}
#[test]
fn configured_gaussian_policy_preserves_multidimensional_actions() {
let device = Device::Cpu;
let varmap = VarMap::new();
let actor = MLP::builder()
.input_size(4)
.output_size(12)
.vb(VarBuilder::from_varmap(&varmap, DType::F64, &device))
.hidden_layer_sizes(vec![8])
.build()
.unwrap();
let policy = ProbabilisticPolicyModel::with_distribution(
actor,
GaussianDistribution::new(vec![2, 3]).unwrap(),
);
let states = create_test_state(5, 4).unwrap();
let actions = policy.sample(&states).unwrap();
assert_eq!(actions.dims(), &[5, 2, 3]);
let (log_probabilities, entropies) =
policy.log_prob_and_entropy(&states, &actions).unwrap();
assert_eq!(log_probabilities.dims(), &[5]);
assert_eq!(entropies.dims(), &[5]);
let terms = policy
.expectation(&states, NonZeroUsize::new(2).unwrap())
.unwrap();
assert_eq!(terms.actions().dims(), &[5, 2, 2, 3]);
assert_eq!(terms.log_probabilities().dims(), &[5, 2]);
}
#[test]
fn test_log_prob_and_entropy_are_finite_for_default_network() {
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let state = create_test_state(1, 4).unwrap();
let action = create_test_state(1, 2).unwrap();
let (log_prob, entropy) = policy.log_prob_and_entropy(&state, &action).unwrap();
assert!(log_prob.to_vec1::<f64>().unwrap()[0].is_finite());
assert!(entropy.to_vec1::<f64>().unwrap()[0].is_finite());
}
#[test]
fn test_gradient_flow() {
let device = Device::Cpu;
let varmap = VarMap::new();
let vb = VarBuilder::from_varmap(&varmap, DType::F64, &device);
let mlp = MLP::builder()
.input_size(4)
.output_size(4)
.vb(vb)
.hidden_layer_sizes(vec![16])
.activation(Activation::Relu)
.build()
.unwrap();
let policy: ProbabilisticPolicyModel<GaussianDistribution> =
ProbabilisticPolicyModel::new(mlp);
let state = create_test_state(2, 4).unwrap();
let action = policy.sample(&state).unwrap();
let (log_prob, entropy) = policy.log_prob_and_entropy(&state, &action).unwrap();
let _ = (log_prob.mean(0).unwrap(), entropy.mean(0).unwrap());
}
#[test]
fn fuzz_log_prob_and_entropy() {
const FUZZ_ITERATIONS: usize = 30;
for i in 0..FUZZ_ITERATIONS {
let state_dim = 2 + (i % 8); let action_dim = 1 + (i % 5); let batch_size = 1 + (i % 15);
let policy = create_test_policy(state_dim, action_dim, vec![32, 32]).unwrap();
let state = create_test_state(batch_size, state_dim).unwrap();
let action = create_test_state(batch_size, action_dim).unwrap();
let (log_prob, entropy) = policy.log_prob_and_entropy(&state, &action).unwrap();
assert_eq!(
log_prob.dims(),
&[batch_size],
"Iteration {}: Log prob shape incorrect",
i
);
assert_eq!(
entropy.dims(),
&[batch_size],
"Iteration {}: Entropy shape incorrect",
i
);
let log_prob_vals = log_prob.to_vec1::<f64>().unwrap();
let entropy_vals = entropy.to_vec1::<f64>().unwrap();
for (j, val) in log_prob_vals.iter().enumerate() {
assert!(
val.is_finite(),
"Iteration {}, sample {}: Log prob is not finite: {}",
i,
j,
val
);
}
for (j, val) in entropy_vals.iter().enumerate() {
assert!(
val.is_finite(),
"Iteration {}, sample {}: Entropy is not finite: {}",
i,
j,
val
);
assert!(
*val > -100.0,
"Iteration {}, sample {}: Entropy too negative: {}",
i,
j,
val
);
}
}
}
#[test]
fn fuzz_different_architectures() {
const FUZZ_ITERATIONS: usize = 25;
for i in 0..FUZZ_ITERATIONS {
let state_dim = 2 + (i % 6); let action_dim = 1 + (i % 4);
let hidden_layers = match i % 4 {
0 => vec![16],
1 => vec![32, 16],
2 => vec![64, 32, 16],
_ => vec![128, 64],
};
let policy = create_test_policy(state_dim, action_dim, hidden_layers).unwrap();
let state = create_test_state(5, state_dim).unwrap();
let action = policy.sample(&state).unwrap();
assert_eq!(
action.dims(),
&[5, action_dim],
"Iteration {}: Sampling failed",
i
);
let result = policy.log_prob_and_entropy(&state, &action);
assert!(
result.is_ok(),
"Iteration {}: Log prob computation failed: {:?}",
i,
result.err()
);
}
}
#[test]
fn fuzz_numerical_stability() {
const FUZZ_ITERATIONS: usize = 40;
for i in 0..FUZZ_ITERATIONS {
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let scale = match i % 4 {
0 => 0.01,
1 => 1.0,
2 => 5.0,
_ => 10.0,
};
let mut state = create_test_state(3, 4).unwrap();
state = (state * scale).unwrap();
let mut action = create_test_state(3, 2).unwrap();
action = (action * (scale * 0.1)).unwrap();
let result = policy.log_prob_and_entropy(&state, &action);
assert!(
result.is_ok(),
"Iteration {} (scale={}): Computation failed: {:?}",
i,
scale,
result.err()
);
if let Ok((log_prob, entropy)) = result {
let log_prob_vals = log_prob.to_vec1::<f64>().unwrap();
let entropy_vals = entropy.to_vec1::<f64>().unwrap();
for val in log_prob_vals {
assert!(
val.is_finite(),
"Iteration {} (scale={}): Log prob not finite: {}",
i,
scale,
val
);
}
for val in entropy_vals {
assert!(
val.is_finite(),
"Iteration {} (scale={}): Entropy not finite: {}",
i,
scale,
val
);
}
}
}
}
#[test]
fn fuzz_batch_size_stress_test() {
const FUZZ_ITERATIONS: usize = 20;
for i in 0..FUZZ_ITERATIONS {
let batch_size = match i % 6 {
0 => 1,
1 => 2,
2 => 5,
3 => 10,
4 => 50,
_ => 100,
};
let policy = create_test_policy(4, 2, vec![32, 32]).unwrap();
let state = create_test_state(batch_size, 4).unwrap();
let action = policy.sample(&state).unwrap();
assert_eq!(
action.dims(),
&[batch_size, 2],
"Iteration {}: Batch size {} failed",
i,
batch_size
);
let (log_prob, entropy) = policy.log_prob_and_entropy(&state, &action).unwrap();
assert_eq!(
log_prob.dims(),
&[batch_size],
"Iteration {}: Log prob batch size incorrect",
i
);
assert_eq!(
entropy.dims(),
&[batch_size],
"Iteration {}: Entropy batch size incorrect",
i
);
let log_prob_vals = log_prob.to_vec1::<f64>().unwrap();
let entropy_vals = entropy.to_vec1::<f64>().unwrap();
assert!(
log_prob_vals.iter().all(|v| v.is_finite()),
"Iteration {}: Some log prob values not finite",
i
);
assert!(
entropy_vals.iter().all(|v| v.is_finite()),
"Iteration {}: Some entropy values not finite",
i
);
}
}
}