use crate::graph::network::Network;
use crate::trainer::core::{RlContext, RlStep, RlStepReport, StepEnv, StepTrainer};
pub struct DecisionLagTrainer<T: RlStep> {
inner: T,
lag: usize,
grace: usize,
pub steps_since_promotion: usize,
relay: RelayStats,
}
#[derive(Default)]
struct RelayStats {
step: Option<usize>,
faces: usize,
promoted: usize,
stalled: usize,
}
impl<T: RlStep> DecisionLagTrainer<T> {
pub fn new(inner: T) -> Self {
DecisionLagTrainer {
inner,
lag: 0,
grace: 5,
steps_since_promotion: 0,
relay: RelayStats::default(),
}
}
pub fn with_lag(mut self, k: usize) -> Self {
self.lag = k;
self
}
pub fn lag(&self) -> usize {
self.lag
}
pub fn with_grace(mut self, n: usize) -> Self {
self.grace = n;
self
}
pub fn grace(&self) -> usize {
self.grace
}
}
impl<T: RlStep> StepTrainer for DecisionLagTrainer<T> {
fn make_optimizer(&self, net: &Network) -> Box<dyn flodl::nn::optim::Optimizer> {
self.inner.make_optimizer(net)
}
fn describe(&self) -> Option<serde_json::Value> {
let base = self.inner.describe();
Some(match base {
Some(mut v) => {
if let Some(obj) = v.as_object_mut() {
obj.insert("decision_lag".into(), serde_json::json!(self.lag > 0));
obj.insert("decision_lag_steps".into(), serde_json::json!(self.lag));
obj.insert("grace_periods".into(), serde_json::json!(self.grace));
}
v
}
None => serde_json::json!({
"decision_lag": self.lag > 0,
"decision_lag_steps": self.lag,
"grace_periods": self.grace
}),
})
}
}
impl<T: RlStep> RlStep for DecisionLagTrainer<T> {
fn train_step(
&mut self,
net: &mut Network,
optimizer: &mut dyn flodl::nn::optim::Optimizer,
step: usize,
ctx: &RlContext<'_>,
) -> flodl::tensor::Result<RlStepReport> {
if self.lag == 0 {
return self.inner.train_step(net, optimizer, step, ctx);
}
if step < self.grace {
tracing::debug!(
"decision-lag grace step {step}/{}: plain per-net training (relay engages at step {})",
self.grace,
self.grace
);
return self.inner.train_step(net, optimizer, step, ctx);
}
if self.relay.step != Some(step) {
if let Some(prev) = self.relay.step {
tracing::debug!(
"decision-lag step {} (lag {}): {} face decision(s) on frozen weights │ {} shadow promotion(s) │ {} stalled",
prev,
self.lag,
self.relay.faces,
self.relay.promoted,
self.relay.stalled,
);
} else {
let grace_note = if self.grace > 0 {
format!(", after a {}-step grace period", self.grace)
} else {
String::new()
};
tracing::info!(
"decision-lag relay active (lag {} engine step(s){}): each net DECIDES and ranks on held (frozen) weights while a shadow forked from them takes its optimizer step(s) and is promoted at the cycle boundary — ≈2× env per step",
self.lag,
grace_note
);
}
self.relay = RelayStats {
step: Some(step),
..RelayStats::default()
};
}
self.relay.faces += 1;
let face_weights_before = net.export_weights()?;
let face_report = self.inner.train_step(net, optimizer, step, ctx)?;
net.import_weights(&face_weights_before)?;
let lag = self.lag;
let rel = step - self.grace;
let boundary = (rel + 1) % lag == 0;
let cycle_start = self.grace + (rel + 1).saturating_sub(lag);
let mut promoted: Option<Vec<f32>> = None;
if boundary {
let blueprint = net.topology_blueprint();
let mut shadow_net = Network::build(&blueprint, net_device(net))?;
shadow_net.import_weights(&face_weights_before)?;
let mut shadow_opt = self.inner.make_optimizer(&shadow_net);
for i in 0..lag {
let s = cycle_start + i;
let shadow_ctx = RlContext {
fitness: ctx.fitness,
metrics: ctx.metrics,
env: StepEnv { step: s, ..ctx.env },
net_hash: ctx.net_hash,
challenged: false,
net_seed: ctx
.net_seed
.wrapping_add(0x9E37_79B9_7F4A_7C15)
.wrapping_add(i as u64),
};
self.inner
.train_step(&mut shadow_net, shadow_opt.as_mut(), s, &shadow_ctx)?;
}
promoted = Some(shadow_net.export_weights()?);
}
match promoted {
Some(promoted) => {
net.import_weights(&promoted)?;
self.steps_since_promotion = 0;
if promoted == face_weights_before {
self.relay.stalled += 1;
tracing::warn!(
"decision-lag [{}] step {step}: shadow promotion changed no weights — the relay is NOT learning (lr 0? saturated net?); the face will not evolve",
ctx.net_hash,
);
} else {
self.relay.promoted += 1;
tracing::debug!(
"decision-lag [{}] step {step}: face fit {:.4} (deciding pass on held weights) → shadow took {lag} step(s) & was promoted",
ctx.net_hash,
face_report.fitness,
);
}
}
None => {
self.steps_since_promotion += 1;
tracing::debug!(
"decision-lag [{}] step {step}: face decided on held weights; shadow trains at the cycle boundary (lag {lag})",
ctx.net_hash,
);
}
}
Ok(face_report)
}
}
fn net_device(net: &Network) -> flodl::Device {
net.layers
.first()
.map(|l| l.weight.variable.data().device())
.unwrap_or(flodl::Device::CPU)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::engine::fitness::{Direction, Fitness};
use crate::trainer::core::{RlContext, StepEnv};
use flodl::nn::Module;
struct ProbeTrainer {
lr: f64,
}
impl StepTrainer for ProbeTrainer {
fn make_optimizer(&self, net: &Network) -> Box<dyn flodl::nn::optim::Optimizer> {
use flodl::nn::Module;
Box::new(flodl::nn::Adam::new(&net.parameters(), self.lr))
}
}
impl RlStep for ProbeTrainer {
fn train_step(
&mut self,
net: &mut Network,
optimizer: &mut dyn flodl::nn::optim::Optimizer,
_step: usize,
_ctx: &RlContext<'_>,
) -> flodl::tensor::Result<RlStepReport> {
use flodl::nn::Module;
let x = flodl::Variable::new(
flodl::Tensor::from_f32(&[0.5; 4], &[1, 4], flodl::Device::CPU)?,
true,
);
let out = net.forward(&x)?;
let loss = out.mean()?;
loss.set_requires_grad(true)?;
optimizer.zero_grad();
loss.backward()?;
optimizer.step()?;
let fit = out.mean()?.item().unwrap_or(0.0) as f32;
Ok(RlStepReport {
train_loss: 0.0,
fitness: fit,
informative: vec![],
challenged_turns: 0,
rl: None,
})
}
}
fn ctx(step: usize) -> RlContext<'static> {
let fitness: &'static Fitness =
Box::leak(Box::new(Fitness::reported(Direction::Maximize, "reward")));
let metrics: &'static [crate::engine::fitness::Metric] = &[];
RlContext {
fitness,
metrics,
challenged: false,
env: StepEnv {
step,
run_seed: 42,
pop_size: 2,
live_count: 2,
checkpoint_every: 100,
smoothing_window: 10,
max_steps: Some(100),
},
net_hash: "probe",
net_seed: 7,
}
}
fn probe_net() -> Network {
let mut topo = crate::graph::topology::Topology::new(0, None);
topo.options.input_dim = Some(4);
topo.options.output_dim = Some(2);
topo.nodes.push(crate::graph::node::Node::new_input(0, 1));
topo.nodes
.push(crate::graph::node::Node::new_output(1, 1, 1));
topo.finalize();
Network::build(&topo, flodl::Device::CPU).unwrap()
}
#[test]
fn relay_face_never_moves_and_shadow_promotes() {
let mut lagged = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 })
.with_lag(1)
.with_grace(0);
let mut face = probe_net();
let face_start = face.export_weights().unwrap();
let mut face_opt = flodl::nn::Adam::new(&face.parameters(), 1e-2);
let report = lagged
.train_step(&mut face, &mut face_opt, 0, &ctx(0))
.unwrap();
let face_after = face.export_weights().unwrap();
assert_ne!(
face_start, face_after,
"promotion must install learned weights into the face"
);
assert!(report.fitness.is_finite());
assert_eq!(
lagged.steps_since_promotion, 0,
"promotion resets the clock"
);
}
#[test]
fn relay_report_describes_the_deciding_face_not_the_learner() {
let mut lagged = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 })
.with_lag(1)
.with_grace(0);
let mut face = probe_net();
let mut face_opt = flodl::nn::Adam::new(&face.parameters(), 1e-2);
let _ = lagged
.train_step(&mut face, &mut face_opt, 0, &ctx(0))
.unwrap();
let face_before_step1 = face.export_weights().unwrap();
let report1 = lagged
.train_step(&mut face, &mut face_opt, 1, &ctx(1))
.unwrap();
let mut replay = probe_net();
replay.import_weights(&face_before_step1).unwrap();
let x = flodl::Variable::new(
flodl::Tensor::from_f32(&[0.5; 4], &[1, 4], flodl::Device::CPU).unwrap(),
false,
);
let out = replay.forward(&x).unwrap();
let expected = out.mean().unwrap().item().unwrap() as f32;
assert!(
(report1.fitness - expected).abs() < 1e-5,
"report {} must describe the deciding face ({expected})",
report1.fitness
);
}
#[test]
fn lag_k_holds_the_face_for_k_steps_then_promotes() {
let mut lagged = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 })
.with_lag(3)
.with_grace(0);
let mut face = probe_net();
let mut opt = flodl::nn::Adam::new(&face.parameters(), 1e-2);
let held = face.export_weights().unwrap();
lagged.train_step(&mut face, &mut opt, 0, &ctx(0)).unwrap();
assert_eq!(
face.export_weights().unwrap(),
held,
"step 0 holds the face"
);
assert_eq!(lagged.steps_since_promotion, 1);
lagged.train_step(&mut face, &mut opt, 1, &ctx(1)).unwrap();
assert_eq!(
face.export_weights().unwrap(),
held,
"step 1 holds the face"
);
assert_eq!(lagged.steps_since_promotion, 2);
lagged.train_step(&mut face, &mut opt, 2, &ctx(2)).unwrap();
assert_ne!(
face.export_weights().unwrap(),
held,
"step 2 is the cycle boundary and promotes the shadow"
);
assert_eq!(
lagged.steps_since_promotion, 0,
"promotion resets the clock"
);
}
#[test]
fn with_lag_zero_disables_the_relay() {
let mut lagged = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 })
.with_lag(0)
.with_grace(0);
assert_eq!(lagged.lag(), 0, "0 = relay off");
let mut face = probe_net();
let start = face.export_weights().unwrap();
let mut opt = flodl::nn::Adam::new(&face.parameters(), 1e-2);
lagged.train_step(&mut face, &mut opt, 0, &ctx(0)).unwrap();
assert_ne!(
start,
face.export_weights().unwrap(),
"lag 0 trains plainly — the face moves"
);
assert_eq!(
lagged.steps_since_promotion, 0,
"no promotion ever happened"
);
}
#[test]
fn conservative_defaults_lag_off_grace_five() {
let d = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 });
assert_eq!(d.lag(), 0, "unset lag = relay off (conservative)");
assert_eq!(d.grace(), 5, "unset grace = 5 plain steps");
let blob = d.describe().unwrap();
assert_eq!(blob["decision_lag"], serde_json::json!(false));
assert_eq!(blob["decision_lag_steps"], serde_json::json!(0));
assert_eq!(blob["grace_periods"], serde_json::json!(5));
}
#[test]
fn grace_defers_relay_engagement_then_runs_a_full_cycle() {
let mut lagged = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 })
.with_lag(3)
.with_grace(2);
let mut face = probe_net();
let mut opt = flodl::nn::Adam::new(&face.parameters(), 1e-2);
let s0 = face.export_weights().unwrap();
lagged.train_step(&mut face, &mut opt, 0, &ctx(0)).unwrap();
let s1 = face.export_weights().unwrap();
assert_ne!(s0, s1, "grace step 0 trains plainly");
lagged.train_step(&mut face, &mut opt, 1, &ctx(1)).unwrap();
let s2 = face.export_weights().unwrap();
assert_ne!(s1, s2, "grace step 1 trains plainly");
assert_eq!(lagged.steps_since_promotion, 0, "no promotion during grace");
lagged.train_step(&mut face, &mut opt, 2, &ctx(2)).unwrap();
assert_eq!(face.export_weights().unwrap(), s2, "rel 0 holds the face");
assert_eq!(lagged.steps_since_promotion, 1);
lagged.train_step(&mut face, &mut opt, 3, &ctx(3)).unwrap();
assert_eq!(face.export_weights().unwrap(), s2, "rel 1 holds the face");
assert_eq!(lagged.steps_since_promotion, 2);
lagged.train_step(&mut face, &mut opt, 4, &ctx(4)).unwrap();
assert_ne!(
face.export_weights().unwrap(),
s2,
"boundary promotes the first full cycle"
);
assert_eq!(
lagged.steps_since_promotion, 0,
"promotion resets the clock"
);
}
#[test]
fn describe_declares_the_relay_and_its_lag() {
let lagged = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 }).with_lag(4);
let blob = lagged.describe().unwrap();
assert_eq!(blob["decision_lag"], serde_json::json!(true));
assert_eq!(blob["decision_lag_steps"], serde_json::json!(4));
assert_eq!(
blob["grace_periods"],
serde_json::json!(5),
"unset grace rides the blob at its default"
);
}
#[test]
fn relay_telemetry_counts_faces_per_step_and_rolls_at_the_boundary() {
let mut lagged = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 })
.with_lag(1)
.with_grace(0);
let mut a = probe_net();
let mut b = probe_net();
let mut oa = flodl::nn::Adam::new(&a.parameters(), 1e-2);
let mut ob = flodl::nn::Adam::new(&b.parameters(), 1e-2);
lagged.train_step(&mut a, &mut oa, 0, &ctx(0)).unwrap();
lagged.train_step(&mut b, &mut ob, 0, &ctx(0)).unwrap();
assert_eq!(lagged.relay.faces, 2, "both nets of step 0 counted");
assert_eq!(
lagged.relay.promoted + lagged.relay.stalled,
2,
"each net's promotion is classified exactly once"
);
lagged.train_step(&mut a, &mut oa, 1, &ctx(1)).unwrap();
assert_eq!(lagged.relay.step, Some(1));
assert_eq!(lagged.relay.faces, 1);
}
#[test]
fn relay_promotion_actually_learns_face_moves_over_steps() {
let mut lagged = DecisionLagTrainer::new(ProbeTrainer { lr: 1e-2 })
.with_lag(1)
.with_grace(0);
let mut face = probe_net();
let mut face_opt = flodl::nn::Adam::new(&face.parameters(), 1e-2);
let start = face.export_weights().unwrap();
for step in 0..5 {
let _ = lagged
.train_step(&mut face, &mut face_opt, step, &ctx(step))
.unwrap();
}
let end = face.export_weights().unwrap();
assert_ne!(start, end, "five relay steps must move the face");
}
}