rusting_brain 5.0.0

A Rust deep-learning library for training and running transformer language models, dense and mixture-of-experts, on CPU or CUDA.
Documentation
//! Trunk, core and Q head together, and the DQN agent around them.

use super::Tensor;
use super::buffer::BatchData;
use super::loss::{double_bellman_targets, weighted_td_huber_loss};
use super::market_sim::Action;
use super::modulation::{GainCache, InteroceptiveCore, add_column_sums, temperature};
use super::state::AccountBody;
use super::trunk::{CausalConfig, CausalTrunk, MarketTrunk, Trunk, TrunkState};
use crate::matrix::Matrix;
use crate::optimizers::{Optimizer, step_clipped};
use crate::param::{Linear, Param};
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};

#[derive(Clone, Debug, PartialEq)]
pub struct AgentConfig {
    pub obs_dim: usize,
    pub hidden: usize,
    pub gamma: f32,
    pub optimizer: Optimizer,
    pub max_grad_norm: f32,
    /// Exploration temperature for a calm body.
    pub tau0: f32,
    /// How sharply drawdown velocity cools exploration.
    pub beta: f32,
    /// Updates between copies of the online network into the target network.
    pub target_sync: usize,
    pub seed: u64,
    /// `Some` encodes the market with a [`CausalTrunk`] over the window's
    /// candles instead of the two-layer MLP.
    pub causal: Option<CausalConfig>,
}

impl AgentConfig {
    pub fn new(obs_dim: usize) -> Self {
        Self {
            obs_dim,
            hidden: 128,
            gamma: 0.99,
            optimizer: Optimizer::adam(3e-4),
            max_grad_norm: 10.0,
            tau0: 1.0,
            beta: 5.0,
            target_sync: 250,
            seed: 0,
            causal: None,
        }
    }
}

/// `Q(s, b) = W_head (h * (1 + tanh(W_gain b + c))) + c_head + W_bias b`,
/// with `h` the trunk's encoding of `s`.
#[derive(Clone, Debug)]
pub struct QNetwork {
    pub trunk: Trunk,
    pub core: InteroceptiveCore,
    pub head: Linear,
    pub head_bias: Param,
}

/// Everything [`QNetwork::backward`] needs from the forward pass.
#[derive(Clone, Debug)]
pub struct ForwardCache {
    pub trunk: TrunkState,
    pub gain: GainCache,
    /// Modulated hidden state `h'`, `[n, hidden]`.
    pub h_mod: Matrix,
    /// `[n, actions]`.
    pub q: Matrix,
}

impl QNetwork {
    pub fn new(obs_dim: usize, hidden: usize, rng: &mut StdRng) -> Self {
        Self::with_trunk(
            Trunk::Mlp(Box::new(MarketTrunk::new(obs_dim, hidden, rng))),
            hidden,
            rng,
        )
    }

    /// A [`CausalTrunk`] in place of the MLP.
    pub fn causal(obs_dim: usize, hidden: usize, config: &CausalConfig, rng: &mut StdRng) -> Self {
        let trunk = CausalTrunk::new(obs_dim, hidden, config, rng);
        Self::with_trunk(Trunk::Causal(Box::new(trunk)), hidden, rng)
    }

    fn with_trunk(trunk: Trunk, hidden: usize, rng: &mut StdRng) -> Self {
        Self {
            trunk,
            core: InteroceptiveCore::new(hidden, Action::COUNT, rng),
            head: Linear::new(hidden, Action::COUNT, rng),
            head_bias: Param::zeros(1, Action::COUNT),
        }
    }

    /// `obs` is `[n, obs_dim]`, `body` is `[n, 4]`.
    pub fn forward(&self, obs: &Matrix, body: &Matrix) -> ForwardCache {
        let trunk = self.trunk.forward(obs);
        let (h_mod, gain) = self.core.modulate(trunk.h(), body);
        let mut q = self.head.forward(&h_mod);
        for r in 0..q.rows {
            for (v, b) in q.row_mut(r).iter_mut().zip(&self.head_bias.value.data) {
                *v += b;
            }
        }
        self.core.add_bias(body, &mut q);
        ForwardCache {
            trunk,
            gain,
            h_mod,
            q,
        }
    }

    /// Accumulates `dL/dparam` for every parameter from `dL/dQ`.
    pub fn backward(&mut self, obs: &Matrix, body: &Matrix, cache: &ForwardCache, grad_q: &Matrix) {
        add_column_sums(grad_q, &mut self.head_bias);
        self.core.add_bias_backward(body, grad_q);
        let grad_h_mod = self.head.backward(&cache.h_mod, grad_q);
        let grad_h = self
            .core
            .modulate_backward(cache.trunk.h(), body, &cache.gain, &grad_h_mod);
        self.trunk.backward(obs, &cache.trunk, &grad_h);
    }

    pub fn params_mut(&mut self) -> Vec<&mut Param> {
        let mut params = self.trunk.params_mut();
        params.extend(self.core.params_mut());
        params.extend(self.head.params_mut());
        params.push(&mut self.head_bias);
        params
    }
}

/// A Double DQN agent with a target network.
pub struct InteroceptiveAgent {
    pub config: AgentConfig,
    pub online: QNetwork,
    pub target: QNetwork,
    updates: usize,
    td_errors: Vec<f32>,
}

impl InteroceptiveAgent {
    pub fn new(config: AgentConfig) -> Self {
        let mut rng = StdRng::seed_from_u64(config.seed);
        let online = match &config.causal {
            Some(causal) => QNetwork::causal(config.obs_dim, config.hidden, causal, &mut rng),
            None => QNetwork::new(config.obs_dim, config.hidden, &mut rng),
        };
        Self {
            target: online.clone(),
            online,
            config,
            updates: 0,
            td_errors: Vec::new(),
        }
    }

    /// Q-values and modulated hidden state for every row of `obs`, all under
    /// the same body.
    pub fn forward(&self, obs: &Tensor, body: &AccountBody) -> (Tensor, Tensor) {
        let cache = self.online.forward(obs, &body_rows(body, obs.rows));
        (cache.q, cache.h_mod)
    }

    /// Q-values with one body per row, `bodies` being `[n, 4]`.
    pub fn q_values(&self, obs: &Matrix, bodies: &Matrix) -> Matrix {
        self.online.forward(obs, bodies).q
    }

    /// Epsilon-greedy over the first row of `obs`, where the exploratory
    /// draw is Boltzmann at the body's temperature.
    pub fn select_action(&self, obs: &Tensor, body: &AccountBody, epsilon: f32) -> Action {
        let (q, _) = self.forward(obs, body);
        self.choose(q.row(0), body, epsilon, &mut rand::thread_rng())
    }

    /// The policy on one row of Q-values. With probability `epsilon`, sample
    /// `softmax(q / tau)` with `tau` from [`temperature`]; otherwise argmax.
    /// A panicking body has `tau` near zero, so even exploration turns greedy.
    pub fn choose(
        &self,
        q: &[f32],
        body: &AccountBody,
        epsilon: f32,
        rng: &mut impl Rng,
    ) -> Action {
        let best = argmax(q);
        if rng.gen_range(0.0..1.0f32) >= epsilon {
            return Action::from_index(best);
        }
        let tau = temperature(self.config.tau0, self.config.beta, body.drawdown_velocity);
        if tau < 1e-3 {
            return Action::from_index(best);
        }
        let mut weights = [0.0f32; Action::COUNT];
        for (w, v) in weights.iter_mut().zip(q) {
            *w = ((v - q[best]) / tau).exp();
        }
        let mut draw = rng.gen_range(0.0..weights.iter().sum::<f32>());
        for (i, w) in weights.iter().enumerate() {
            if draw < *w {
                return Action::from_index(i);
            }
            draw -= w;
        }
        Action::from_index(best)
    }

    /// One Double DQN update: the online network picks the next action and
    /// the target network scores it. Returns the Huber loss.
    pub fn train_step(&mut self, batch: &BatchData) -> f32 {
        self.train_step_weighted(batch, &[])
    }

    /// [`InteroceptiveAgent::train_step`] with each row's loss scaled by
    /// `weights`, as [`ReplayBuffer::weights`](super::ReplayBuffer::weights)
    /// reports them; empty means all ones.
    pub fn train_step_weighted(&mut self, batch: &BatchData, weights: &[f32]) -> f32 {
        let next_online = self.online.forward(&batch.next_obs, &batch.next_body).q;
        let next_q = self.target.forward(&batch.next_obs, &batch.next_body).q;
        let mut targets = vec![0.0; batch.len()];
        double_bellman_targets(
            &batch.rewards,
            &batch.dones,
            &next_online,
            &next_q,
            self.config.gamma,
            &mut targets,
        );

        let cache = self.online.forward(&batch.obs, &batch.body);
        let mut grad = Matrix::new(batch.len(), Action::COUNT);
        self.td_errors.resize(batch.len(), 0.0);
        let loss = weighted_td_huber_loss(
            &cache.q,
            &batch.actions,
            &targets,
            weights,
            &mut grad,
            &mut self.td_errors,
        );
        self.online.backward(&batch.obs, &batch.body, &cache, &grad);

        self.updates += 1;
        step_clipped(
            &mut self.online.params_mut(),
            &self.config.optimizer,
            self.updates,
            1.0,
            self.config.max_grad_norm,
        )
        .expect("host parameters cannot fail a step");
        if self.updates.is_multiple_of(self.config.target_sync) {
            self.target = self.online.clone();
        }
        loss
    }

    /// The TD error of each row of the last update's batch, for
    /// [`ReplayBuffer::update_priorities`](super::ReplayBuffer::update_priorities).
    pub fn last_td_errors(&self) -> &[f32] {
        &self.td_errors
    }

    pub fn updates(&self) -> usize {
        self.updates
    }
}

/// `n` copies of `body` as a `[n, 4]` matrix.
pub fn body_rows(body: &AccountBody, n: usize) -> Matrix {
    Matrix::from_vec(n, 4, body.to_array().repeat(n))
}

fn argmax(values: &[f32]) -> usize {
    let mut best = 0;
    for (i, v) in values.iter().enumerate() {
        if *v > values[best] {
            best = i;
        }
    }
    best
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn margin_stress_raises_the_flatten_logit_reflexively() {
        let agent = InteroceptiveAgent::new(AgentConfig::new(12));
        let mut rng = StdRng::seed_from_u64(7);
        let obs = Matrix::from_vec(1, 12, (0..12).map(|_| rng.gen_range(-1.0..1.0)).collect());
        let calm = AccountBody {
            margin_stress: 0.05,
            ..AccountBody::default()
        };
        let stressed = AccountBody {
            margin_stress: 0.95,
            ..calm
        };

        let flatten = Action::EmergencyFlatten as usize;
        let (q_calm, _) = agent.forward(&obs, &calm);
        let (q_stress, _) = agent.forward(&obs, &stressed);
        let shift = q_stress.row(0)[flatten] - q_calm.row(0)[flatten];
        assert!(shift > 3.0, "flatten logit moved by only {shift}");
        assert_eq!(argmax(q_stress.row(0)), flatten);
    }

    #[test]
    fn panic_makes_exploration_greedy() {
        let agent = InteroceptiveAgent::new(AgentConfig {
            beta: 20.0,
            ..AgentConfig::new(4)
        });
        let panic = AccountBody::from_array([0.1, 1.0, 0.05, 0.0]);
        let q = [0.0, 0.1, 0.0, 0.0, 0.0];
        let mut rng = StdRng::seed_from_u64(0);
        for _ in 0..100 {
            assert_eq!(agent.choose(&q, &panic, 1.0, &mut rng), Action::BuySmall);
        }
    }

    #[test]
    fn backward_matches_finite_differences() {
        let mut rng = StdRng::seed_from_u64(3);
        let net = QNetwork::new(3, 6, &mut rng);
        gradients_match_finite_differences(net, 3, &mut rng);
    }

    #[test]
    fn a_causal_trunk_backward_matches_finite_differences() {
        let mut rng = StdRng::seed_from_u64(4);
        let config = CausalConfig {
            lookback: 3,
            d_model: 8,
            layers: 2,
            heads: 2,
        };
        // Three candles of five features, then two position features.
        let net = QNetwork::causal(17, 6, &config, &mut rng);
        gradients_match_finite_differences(net, 17, &mut rng);
    }

    fn gradients_match_finite_differences(mut net: QNetwork, obs_dim: usize, rng: &mut StdRng) {
        let mut random = |rows, cols| {
            Matrix::from_vec(
                rows,
                cols,
                (0..rows * cols).map(|_| rng.gen_range(-1.0..1.0)).collect(),
            )
        };
        let obs = random(4, obs_dim);
        let body = Matrix::from_vec(4, 4, random(4, 4).data.iter().map(|v| v.abs()).collect());
        let weights = random(4, Action::COUNT);
        let loss = |net: &QNetwork| -> f32 {
            let q = net.forward(&obs, &body).q;
            q.data.iter().zip(&weights.data).map(|(a, b)| a * b).sum()
        };

        let cache = net.forward(&obs, &body);
        net.backward(&obs, &body, &cache, &weights);
        let analytic: Vec<Vec<f32>> = net
            .params_mut()
            .iter()
            .map(|p| p.grad.data.clone())
            .collect();

        let eps = 1e-3;
        for (p, grads) in analytic.iter().enumerate() {
            for (i, &g) in grads.iter().enumerate() {
                let mut plus = net.clone();
                plus.params_mut()[p].value.data[i] += eps;
                let mut minus = net.clone();
                minus.params_mut()[p].value.data[i] -= eps;
                let numeric = (loss(&plus) - loss(&minus)) / (2.0 * eps);
                assert!(
                    (numeric - g).abs() < 1e-2 * (1.0 + g.abs()),
                    "param {p}[{i}]: analytic {g}, numeric {numeric}"
                );
            }
        }
    }
}