use std::fmt::Debug;
use bon::bon;
use candle_core::Tensor;
use candle_nn::{Optimizer, VarMap};
use super::{
CriticCountRequirement, DeterministicActorCriticAgent,
DeterministicActorCriticCollectionLogEntry, DeterministicActorCriticError,
DeterministicActorCriticLogEntry, DeterministicActorCriticLogger,
DeterministicActorCriticResult, DeterministicActorCriticStrategy, DeterministicCritic,
};
use crate::{
agents::{
Agent, ReplayStorageConfig,
sac::{SACCriticAggregationMode, SACCriticError, aggregate_critic_values},
},
gym::MultiGym,
spaces::{BoxSpace, Space},
};
pub trait TD3Logger<I = ()> {
fn log(&mut self, entry: &DeterministicActorCriticLogEntry);
fn log_collection(&mut self, _entry: &DeterministicActorCriticCollectionLogEntry<I>) {}
}
struct TD3LoggingInfo<'a, I> {
logger: &'a mut dyn TD3Logger<I>,
}
impl<I> DeterministicActorCriticLogger<I> for Option<TD3LoggingInfo<'_, I>> {
fn log_update(&mut self, entry: &DeterministicActorCriticLogEntry) {
if let Some(info) = self {
info.logger.log(entry);
}
}
fn log_collection(&mut self, entry: &DeterministicActorCriticCollectionLogEntry<I>) {
if let Some(info) = self {
info.logger.log_collection(entry);
}
}
}
pub(crate) struct TD3Strategy {
target_policy_noise: f64,
target_noise_clip: f64,
actor_update_interval: usize,
target_aggregation_mode: SACCriticAggregationMode,
actor_aggregation_mode: Option<SACCriticAggregationMode>,
}
impl DeterministicActorCriticStrategy for TD3Strategy {
fn critic_count_requirement(&self) -> CriticCountRequirement {
CriticCountRequirement::NonEmpty
}
fn actor_update_interval(&self) -> usize {
self.actor_update_interval
}
fn target_policy_noise(&self) -> f64 {
self.target_policy_noise
}
fn target_noise_clip(&self) -> f64 {
self.target_noise_clip
}
fn aggregate_target_values(&self, values: Vec<Tensor>) -> Result<Tensor, SACCriticError> {
aggregate_critic_values(&values, self.target_aggregation_mode)
}
fn aggregate_actor_values(&self, values: Vec<Tensor>) -> Result<Tensor, SACCriticError> {
match self.actor_aggregation_mode {
Some(mode) => aggregate_critic_values(&values, mode),
None => values
.into_iter()
.next()
.ok_or(SACCriticError::NoCriticValues),
}
}
}
pub struct TD3Agent<'a, AO, CO, GE, SE, I = ()>
where
AO: Optimizer,
CO: Optimizer,
GE: Debug,
SE: Debug,
{
inner: DeterministicActorCriticAgent<'a, AO, CO, GE, SE, TD3Strategy>,
logging_info: Option<TD3LoggingInfo<'a, I>>,
}
#[bon]
impl<'a, AO, CO, GE, SE, I> TD3Agent<'a, AO, CO, GE, SE, I>
where
AO: Optimizer,
CO: Optimizer,
GE: Debug,
SE: Debug,
{
#[builder]
pub fn new(
#[builder(with = |actor: impl candle_core::Module + 'static| Box::new(actor))]
online_actor: Box<dyn candle_core::Module>,
#[builder(with = |actor: impl candle_core::Module + 'static| Box::new(actor))]
target_actor: Box<dyn candle_core::Module>,
online_actor_vars: &'a VarMap,
target_actor_vars: &'a mut VarMap,
actor_optimizer: AO,
critics: Vec<DeterministicCritic<'a, CO>>,
action_space: BoxSpace,
observation_space: Box<dyn Space<Error = SE>>,
replay_storage_config: ReplayStorageConfig,
logger: Option<&'a mut dyn TD3Logger<I>>,
#[builder(default = 0.99)]
gamma: f64,
#[builder(default = 0.005)]
tau: f64,
#[builder(default = 0.1)]
exploration_noise: f64,
#[builder(default = 0.2)]
target_policy_noise: f64,
#[builder(default = 0.5)]
target_noise_clip: f64,
#[builder(default = 2)]
actor_update_interval: usize,
#[builder(default = SACCriticAggregationMode::Min)]
target_aggregation_mode: SACCriticAggregationMode,
actor_aggregation_mode: Option<SACCriticAggregationMode>,
#[builder(default = 1_000_000)]
replay_capacity: usize,
#[builder(default = 256)]
batch_size: usize,
#[builder(default = 1)]
update_frequency: usize,
#[builder(default = 1_000)]
training_start: usize,
training_horizon: usize,
#[builder(default = candle_core::DType::F32)] dtype: candle_core::DType,
) -> DeterministicActorCriticResult<Self, GE, SE> {
let strategy = TD3Strategy {
target_policy_noise,
target_noise_clip,
actor_update_interval,
target_aggregation_mode,
actor_aggregation_mode,
};
let inner = DeterministicActorCriticAgent::builder()
.online_actor(online_actor)
.target_actor(target_actor)
.online_actor_vars(online_actor_vars)
.target_actor_vars(target_actor_vars)
.actor_optimizer(actor_optimizer)
.critics(critics)
.strategy(strategy)
.action_space(action_space)
.observation_space(observation_space)
.replay_storage_config(replay_storage_config)
.gamma(gamma)
.tau(tau)
.exploration_noise(exploration_noise)
.replay_capacity(replay_capacity)
.batch_size(batch_size)
.update_frequency(update_frequency)
.training_start(training_start)
.training_horizon(training_horizon)
.dtype(dtype)
.build()?;
Ok(Self {
inner,
logging_info: logger.map(|logger| TD3LoggingInfo { logger }),
})
}
pub fn get_action_space(&self) -> &BoxSpace {
self.inner.get_action_space()
}
pub fn get_observation_space(&self) -> &dyn Space<Error = SE> {
self.inner.get_observation_space()
}
pub fn act_deterministic(
&self,
observation: &Tensor,
) -> Result<Tensor, DeterministicActorCriticError<GE, SE>> {
self.inner.act_deterministic(observation)
}
}
impl<'a, AO, CO, GE, SE, I> Agent<I> for TD3Agent<'a, AO, CO, GE, SE, I>
where
AO: Optimizer,
CO: Optimizer,
GE: Debug,
SE: Debug,
{
type Error = DeterministicActorCriticError<GE, SE>;
type GymError = GE;
type SpaceError = SE;
fn act(&mut self, observation: &Tensor) -> Result<Tensor, Self::Error> {
self.inner.act(observation)
}
#[cfg_attr(
feature = "tracing",
tracing::instrument(
name = "td3.learn",
target = "modurl::performance",
skip_all,
fields(num_timesteps)
)
)]
fn learn(
&mut self,
env: &mut dyn MultiGym<I, Error = GE, SpaceError = SE>,
num_timesteps: usize,
) -> Result<(), Self::Error> {
self.inner.learn(env, num_timesteps, &mut self.logging_info)
}
}
#[cfg(test)]
mod tests {
use super::{TD3Agent, TD3Logger, TD3Strategy};
use crate::{
agents::{
Agent, ReplayDeviceStrategy, ReplayStorageConfig,
deterministic_actor_critic::{
DeterministicActorCriticLogEntry, DeterministicActorCriticStrategy,
},
sac::{SACCritic, SACCriticAggregationMode, ScalarStateActionCritic},
test_support::{CountingOptimizer, FixedContinuousEnv},
},
gym::{MultiGym, VectorizedGymWrapper},
models::MLP,
spaces::BoxSpace,
};
use candle_core::{DType, Device, Tensor};
use candle_nn::{VarBuilder, VarMap};
#[test]
fn strategy_uses_the_configured_critic_aggregation() {
let strategy = TD3Strategy {
target_policy_noise: 0.2,
target_noise_clip: 0.5,
actor_update_interval: 2,
target_aggregation_mode: SACCriticAggregationMode::Mean,
actor_aggregation_mode: Some(SACCriticAggregationMode::Mean),
};
let values = || {
vec![
Tensor::new(&[1.0f32, 3.0], &Device::Cpu).unwrap(),
Tensor::new(&[3.0f32, 1.0], &Device::Cpu).unwrap(),
Tensor::new(&[2.0f32, 2.0], &Device::Cpu).unwrap(),
]
};
assert_eq!(
strategy
.aggregate_target_values(values())
.unwrap()
.to_vec1::<f32>()
.unwrap(),
vec![2.0, 2.0]
);
assert_eq!(
strategy
.aggregate_actor_values(values())
.unwrap()
.to_vec1::<f32>()
.unwrap(),
vec![2.0, 2.0]
);
}
#[derive(Default)]
struct RecordingLogger {
actor_updates: Vec<bool>,
}
impl TD3Logger for RecordingLogger {
fn log(&mut self, entry: &DeterministicActorCriticLogEntry) {
self.actor_updates.push(entry.actor_updated);
assert_eq!(entry.critic_losses.len(), 3);
assert_eq!(entry.critic_q_values.len(), 3);
assert!(
entry
.critic_q_values
.iter()
.all(|values| values.dims() == [1])
);
assert_eq!(entry.bellman_targets.dims(), &[1]);
assert_eq!(entry.replay_actions.dims(), &[1, 1]);
assert_eq!(entry.critic_learning_rates, vec![1e-3, 1e-3, 1e-3]);
assert_eq!(entry.actor_loss.is_some(), entry.actor_updated);
assert_eq!(entry.policy_q_values.is_some(), entry.actor_updated);
assert_eq!(entry.policy_actions.is_some(), entry.actor_updated);
}
}
fn actor(vars: &VarMap, device: &Device) -> MLP {
MLP::builder()
.input_size(4)
.output_size(1)
.hidden_layer_sizes(vec![4])
.output_activation(Tensor::tanh)
.vb(VarBuilder::from_varmap(vars, DType::F32, device))
.build()
.unwrap()
}
fn critic<'a>(
online_vars: &'a VarMap,
target_vars: &'a mut VarMap,
device: &Device,
) -> SACCritic<'a, CountingOptimizer> {
let network = |vars| {
MLP::builder()
.input_size(5)
.output_size(1)
.hidden_layer_sizes(vec![4])
.vb(VarBuilder::from_varmap(vars, DType::F32, device))
.build()
.unwrap()
};
SACCritic::builder()
.online_network(ScalarStateActionCritic::new(network(online_vars)))
.target_network(ScalarStateActionCritic::new(network(target_vars)))
.online_vars(online_vars)
.target_vars(target_vars)
.optimizer(CountingOptimizer::with_learning_rate(1e-3))
.build()
.unwrap()
}
#[test]
fn public_agent_supports_generalized_critic_ensembles_and_delays_actor_updates() {
let device = Device::Cpu;
let mut env: VectorizedGymWrapper<FixedContinuousEnv> = vec![
FixedContinuousEnv::new(device.clone()),
FixedContinuousEnv::new(device.clone()),
]
.into();
let online_actor_vars = VarMap::new();
let mut target_actor_vars = VarMap::new();
let online_critic_vars_1 = VarMap::new();
let mut target_critic_vars_1 = VarMap::new();
let online_critic_vars_2 = VarMap::new();
let mut target_critic_vars_2 = VarMap::new();
let online_critic_vars_3 = VarMap::new();
let mut target_critic_vars_3 = VarMap::new();
let critic_1 = critic(&online_critic_vars_1, &mut target_critic_vars_1, &device);
let critic_2 = critic(&online_critic_vars_2, &mut target_critic_vars_2, &device);
let critic_3 = critic(&online_critic_vars_3, &mut target_critic_vars_3, &device);
let mut logger = RecordingLogger::default();
let mut agent = TD3Agent::builder()
.online_actor(actor(&online_actor_vars, &device))
.target_actor(actor(&target_actor_vars, &device))
.online_actor_vars(&online_actor_vars)
.target_actor_vars(&mut target_actor_vars)
.actor_optimizer(CountingOptimizer::with_learning_rate(1e-3))
.critics(vec![critic_1, critic_2, critic_3])
.action_space(BoxSpace::new_with_universal_bounds(
vec![1],
-1.0,
1.0,
&device,
))
.observation_space(env.observation_space())
.replay_storage_config(ReplayStorageConfig::new(ReplayDeviceStrategy::OneDevice(
device,
)))
.exploration_noise(0.0)
.target_policy_noise(0.0)
.target_aggregation_mode(SACCriticAggregationMode::Mean)
.actor_aggregation_mode(SACCriticAggregationMode::Median)
.replay_capacity(4)
.batch_size(1)
.training_start(1)
.training_horizon(2)
.logger(&mut logger)
.build()
.unwrap();
agent.learn(&mut env, 2).unwrap();
assert_eq!(agent.inner.actor_optimizer.steps, 1);
assert_eq!(agent.inner.critics[0].optimizer().steps, 2);
assert_eq!(agent.inner.critics[1].optimizer().steps, 2);
assert_eq!(agent.inner.critics[2].optimizer().steps, 2);
assert_eq!(logger.actor_updates, vec![false, true]);
}
}