use super::Tensor;
use super::buffer::BatchData;
use super::loss::{double_bellman_targets, weighted_td_huber_loss};
use super::market_sim::Action;
use super::modulation::{GainCache, InteroceptiveCore, add_column_sums, temperature};
use super::state::AccountBody;
use super::trunk::{CausalConfig, CausalTrunk, MarketTrunk, Trunk, TrunkState};
use crate::matrix::Matrix;
use crate::optimizers::{Optimizer, step_clipped};
use crate::param::{Linear, Param};
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
#[derive(Clone, Debug, PartialEq)]
pub struct AgentConfig {
pub obs_dim: usize,
pub hidden: usize,
pub gamma: f32,
pub optimizer: Optimizer,
pub max_grad_norm: f32,
pub tau0: f32,
pub beta: f32,
pub target_sync: usize,
pub seed: u64,
pub causal: Option<CausalConfig>,
}
impl AgentConfig {
pub fn new(obs_dim: usize) -> Self {
Self {
obs_dim,
hidden: 128,
gamma: 0.99,
optimizer: Optimizer::adam(3e-4),
max_grad_norm: 10.0,
tau0: 1.0,
beta: 5.0,
target_sync: 250,
seed: 0,
causal: None,
}
}
}
#[derive(Clone, Debug)]
pub struct QNetwork {
pub trunk: Trunk,
pub core: InteroceptiveCore,
pub head: Linear,
pub head_bias: Param,
}
#[derive(Clone, Debug)]
pub struct ForwardCache {
pub trunk: TrunkState,
pub gain: GainCache,
pub h_mod: Matrix,
pub q: Matrix,
}
impl QNetwork {
pub fn new(obs_dim: usize, hidden: usize, rng: &mut StdRng) -> Self {
Self::with_trunk(
Trunk::Mlp(Box::new(MarketTrunk::new(obs_dim, hidden, rng))),
hidden,
rng,
)
}
pub fn causal(obs_dim: usize, hidden: usize, config: &CausalConfig, rng: &mut StdRng) -> Self {
let trunk = CausalTrunk::new(obs_dim, hidden, config, rng);
Self::with_trunk(Trunk::Causal(Box::new(trunk)), hidden, rng)
}
fn with_trunk(trunk: Trunk, hidden: usize, rng: &mut StdRng) -> Self {
Self {
trunk,
core: InteroceptiveCore::new(hidden, Action::COUNT, rng),
head: Linear::new(hidden, Action::COUNT, rng),
head_bias: Param::zeros(1, Action::COUNT),
}
}
pub fn forward(&self, obs: &Matrix, body: &Matrix) -> ForwardCache {
let trunk = self.trunk.forward(obs);
let (h_mod, gain) = self.core.modulate(trunk.h(), body);
let mut q = self.head.forward(&h_mod);
for r in 0..q.rows {
for (v, b) in q.row_mut(r).iter_mut().zip(&self.head_bias.value.data) {
*v += b;
}
}
self.core.add_bias(body, &mut q);
ForwardCache {
trunk,
gain,
h_mod,
q,
}
}
pub fn backward(&mut self, obs: &Matrix, body: &Matrix, cache: &ForwardCache, grad_q: &Matrix) {
add_column_sums(grad_q, &mut self.head_bias);
self.core.add_bias_backward(body, grad_q);
let grad_h_mod = self.head.backward(&cache.h_mod, grad_q);
let grad_h = self
.core
.modulate_backward(cache.trunk.h(), body, &cache.gain, &grad_h_mod);
self.trunk.backward(obs, &cache.trunk, &grad_h);
}
pub fn params_mut(&mut self) -> Vec<&mut Param> {
let mut params = self.trunk.params_mut();
params.extend(self.core.params_mut());
params.extend(self.head.params_mut());
params.push(&mut self.head_bias);
params
}
}
pub struct InteroceptiveAgent {
pub config: AgentConfig,
pub online: QNetwork,
pub target: QNetwork,
updates: usize,
td_errors: Vec<f32>,
}
impl InteroceptiveAgent {
pub fn new(config: AgentConfig) -> Self {
let mut rng = StdRng::seed_from_u64(config.seed);
let online = match &config.causal {
Some(causal) => QNetwork::causal(config.obs_dim, config.hidden, causal, &mut rng),
None => QNetwork::new(config.obs_dim, config.hidden, &mut rng),
};
Self {
target: online.clone(),
online,
config,
updates: 0,
td_errors: Vec::new(),
}
}
pub fn forward(&self, obs: &Tensor, body: &AccountBody) -> (Tensor, Tensor) {
let cache = self.online.forward(obs, &body_rows(body, obs.rows));
(cache.q, cache.h_mod)
}
pub fn q_values(&self, obs: &Matrix, bodies: &Matrix) -> Matrix {
self.online.forward(obs, bodies).q
}
pub fn select_action(&self, obs: &Tensor, body: &AccountBody, epsilon: f32) -> Action {
let (q, _) = self.forward(obs, body);
self.choose(q.row(0), body, epsilon, &mut rand::thread_rng())
}
pub fn choose(
&self,
q: &[f32],
body: &AccountBody,
epsilon: f32,
rng: &mut impl Rng,
) -> Action {
let best = argmax(q);
if rng.gen_range(0.0..1.0f32) >= epsilon {
return Action::from_index(best);
}
let tau = temperature(self.config.tau0, self.config.beta, body.drawdown_velocity);
if tau < 1e-3 {
return Action::from_index(best);
}
let mut weights = [0.0f32; Action::COUNT];
for (w, v) in weights.iter_mut().zip(q) {
*w = ((v - q[best]) / tau).exp();
}
let mut draw = rng.gen_range(0.0..weights.iter().sum::<f32>());
for (i, w) in weights.iter().enumerate() {
if draw < *w {
return Action::from_index(i);
}
draw -= w;
}
Action::from_index(best)
}
pub fn train_step(&mut self, batch: &BatchData) -> f32 {
self.train_step_weighted(batch, &[])
}
pub fn train_step_weighted(&mut self, batch: &BatchData, weights: &[f32]) -> f32 {
let next_online = self.online.forward(&batch.next_obs, &batch.next_body).q;
let next_q = self.target.forward(&batch.next_obs, &batch.next_body).q;
let mut targets = vec![0.0; batch.len()];
double_bellman_targets(
&batch.rewards,
&batch.dones,
&next_online,
&next_q,
self.config.gamma,
&mut targets,
);
let cache = self.online.forward(&batch.obs, &batch.body);
let mut grad = Matrix::new(batch.len(), Action::COUNT);
self.td_errors.resize(batch.len(), 0.0);
let loss = weighted_td_huber_loss(
&cache.q,
&batch.actions,
&targets,
weights,
&mut grad,
&mut self.td_errors,
);
self.online.backward(&batch.obs, &batch.body, &cache, &grad);
self.updates += 1;
step_clipped(
&mut self.online.params_mut(),
&self.config.optimizer,
self.updates,
1.0,
self.config.max_grad_norm,
)
.expect("host parameters cannot fail a step");
if self.updates.is_multiple_of(self.config.target_sync) {
self.target = self.online.clone();
}
loss
}
pub fn last_td_errors(&self) -> &[f32] {
&self.td_errors
}
pub fn updates(&self) -> usize {
self.updates
}
}
pub fn body_rows(body: &AccountBody, n: usize) -> Matrix {
Matrix::from_vec(n, 4, body.to_array().repeat(n))
}
fn argmax(values: &[f32]) -> usize {
let mut best = 0;
for (i, v) in values.iter().enumerate() {
if *v > values[best] {
best = i;
}
}
best
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn margin_stress_raises_the_flatten_logit_reflexively() {
let agent = InteroceptiveAgent::new(AgentConfig::new(12));
let mut rng = StdRng::seed_from_u64(7);
let obs = Matrix::from_vec(1, 12, (0..12).map(|_| rng.gen_range(-1.0..1.0)).collect());
let calm = AccountBody {
margin_stress: 0.05,
..AccountBody::default()
};
let stressed = AccountBody {
margin_stress: 0.95,
..calm
};
let flatten = Action::EmergencyFlatten as usize;
let (q_calm, _) = agent.forward(&obs, &calm);
let (q_stress, _) = agent.forward(&obs, &stressed);
let shift = q_stress.row(0)[flatten] - q_calm.row(0)[flatten];
assert!(shift > 3.0, "flatten logit moved by only {shift}");
assert_eq!(argmax(q_stress.row(0)), flatten);
}
#[test]
fn panic_makes_exploration_greedy() {
let agent = InteroceptiveAgent::new(AgentConfig {
beta: 20.0,
..AgentConfig::new(4)
});
let panic = AccountBody::from_array([0.1, 1.0, 0.05, 0.0]);
let q = [0.0, 0.1, 0.0, 0.0, 0.0];
let mut rng = StdRng::seed_from_u64(0);
for _ in 0..100 {
assert_eq!(agent.choose(&q, &panic, 1.0, &mut rng), Action::BuySmall);
}
}
#[test]
fn backward_matches_finite_differences() {
let mut rng = StdRng::seed_from_u64(3);
let net = QNetwork::new(3, 6, &mut rng);
gradients_match_finite_differences(net, 3, &mut rng);
}
#[test]
fn a_causal_trunk_backward_matches_finite_differences() {
let mut rng = StdRng::seed_from_u64(4);
let config = CausalConfig {
lookback: 3,
d_model: 8,
layers: 2,
heads: 2,
};
let net = QNetwork::causal(17, 6, &config, &mut rng);
gradients_match_finite_differences(net, 17, &mut rng);
}
fn gradients_match_finite_differences(mut net: QNetwork, obs_dim: usize, rng: &mut StdRng) {
let mut random = |rows, cols| {
Matrix::from_vec(
rows,
cols,
(0..rows * cols).map(|_| rng.gen_range(-1.0..1.0)).collect(),
)
};
let obs = random(4, obs_dim);
let body = Matrix::from_vec(4, 4, random(4, 4).data.iter().map(|v| v.abs()).collect());
let weights = random(4, Action::COUNT);
let loss = |net: &QNetwork| -> f32 {
let q = net.forward(&obs, &body).q;
q.data.iter().zip(&weights.data).map(|(a, b)| a * b).sum()
};
let cache = net.forward(&obs, &body);
net.backward(&obs, &body, &cache, &weights);
let analytic: Vec<Vec<f32>> = net
.params_mut()
.iter()
.map(|p| p.grad.data.clone())
.collect();
let eps = 1e-3;
for (p, grads) in analytic.iter().enumerate() {
for (i, &g) in grads.iter().enumerate() {
let mut plus = net.clone();
plus.params_mut()[p].value.data[i] += eps;
let mut minus = net.clone();
minus.params_mut()[p].value.data[i] -= eps;
let numeric = (loss(&plus) - loss(&minus)) / (2.0 * eps);
assert!(
(numeric - g).abs() < 1e-2 * (1.0 + g.abs()),
"param {p}[{i}]: analytic {g}, numeric {numeric}"
);
}
}
}
}