use crate::graph::network::Network;
use crate::trainer::challenge::{
ChallengeDecay, ChallengePolicy, ForcedAction, challenge_fires, forced_action,
};
use crate::trainer::core::{RlContext, RlStep, RlStepMeta, RlStepReport};
use crate::trainer::env::Env;
use crate::utils::fitness::Metric;
use crate::utils::race_steps::train_one_step_pred_only;
use flodl::nn::Module;
use flodl::nn::optim::Optimizer;
use flodl::tensor::Result;
use flodl::{Device, Tensor, Variable};
use std::cmp::Ordering;
const DEFAULT_GRAD_CLIP: f32 = 2.0;
const DEFAULT_MATCHES_PER_STEP: usize = 1;
const DEFAULT_EVAL_MATCHES_PER_STEP: usize = 5;
const DEFAULT_CHALLENGE_PROB: f32 = 0.1;
const DEFAULT_CHALLENGE_DECAY: ChallengeDecay = ChallengeDecay::Custom { exponent: 0.5 };
const EVAL_SEED_OFFSET: u64 = 1000;
#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Signal {
Turns,
Return {
discount: f32,
},
}
impl Signal {
pub fn name(self) -> &'static str {
match self {
Signal::Turns => "turns",
Signal::Return { .. } => "return",
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ObsMemory {
None,
Delta,
Ema { beta: f32 },
}
impl ObsMemory {
pub fn width_factor(self) -> usize {
match self {
ObsMemory::None => 1,
ObsMemory::Delta | ObsMemory::Ema { .. } => 2,
}
}
}
struct MemoryState {
memory: ObsMemory,
prev: Vec<f32>,
ema: Vec<f32>,
}
impl MemoryState {
fn new(memory: ObsMemory, first: &[f32]) -> Self {
Self {
memory,
prev: first.to_vec(),
ema: first.to_vec(),
}
}
fn row(&mut self, obs: &[f32]) -> Vec<f32> {
let mut fed = obs.to_vec();
match self.memory {
ObsMemory::None => {}
ObsMemory::Delta => {
fed.extend(obs.iter().zip(&self.prev).map(|(o, p)| o - p));
self.prev = obs.to_vec();
}
ObsMemory::Ema { beta } => {
for (e, o) in self.ema.iter_mut().zip(obs) {
*e = beta * *e + (1.0 - beta) * o;
}
fed.extend(&self.ema);
}
}
fed
}
}
pub struct Reinforce<E: Env> {
env: E,
device: Device,
learning_rate: f32,
grad_clip: f32,
matches_per_step: usize,
eval_matches_per_step: usize,
per_clock_exam: bool,
signal: Signal,
obs_memory: ObsMemory,
challenge_prob: f32,
challenge_policy: ChallengePolicy,
challenge_decay: ChallengeDecay,
lr_schedule: Option<crate::trainer::LrSchedule>,
optimizer: crate::trainer::OptimizerKind,
action_rule: ActionRule,
eval_action_rule: ActionRule,
softmax_temperature: f32,
}
impl<E: Env> Reinforce<E> {
pub fn new(env: E, device: Device) -> Self {
Self {
env,
device,
learning_rate: crate::utils::training::DEFAULT_LR,
grad_clip: DEFAULT_GRAD_CLIP,
matches_per_step: DEFAULT_MATCHES_PER_STEP,
eval_matches_per_step: DEFAULT_EVAL_MATCHES_PER_STEP,
per_clock_exam: true,
signal: Signal::Turns,
obs_memory: ObsMemory::None,
challenge_prob: DEFAULT_CHALLENGE_PROB,
challenge_policy: ChallengePolicy::LeastAction,
challenge_decay: DEFAULT_CHALLENGE_DECAY,
lr_schedule: None,
optimizer: crate::trainer::OptimizerKind::default(),
action_rule: ACTION_RULE,
eval_action_rule: EVAL_ACTION_RULE,
softmax_temperature: SOFTMAX_TEMPERATURE,
}
}
pub fn with_optimizer(mut self, optimizer: crate::trainer::OptimizerKind) -> Self {
self.optimizer = optimizer;
self
}
pub fn with_named_lr_schedule(mut self, schedule: crate::trainer::LrSchedule) -> Self {
self.lr_schedule = Some(schedule);
self
}
pub fn with_matches_per_step(mut self, n: usize) -> Self {
self.matches_per_step = n;
self
}
pub fn with_eval_matches_per_step(mut self, n: usize) -> Self {
self.eval_matches_per_step = n;
self
}
pub fn with_per_clock_exam(mut self, on: bool) -> Self {
self.per_clock_exam = on;
self
}
pub fn with_signal(mut self, signal: Signal) -> Self {
self.signal = signal;
self
}
pub fn with_action_rule(mut self, rule: ActionRule, temperature: f32) -> Self {
self.action_rule = rule;
self.softmax_temperature = if temperature.is_finite() && temperature > 0.0 {
temperature
} else {
1.0
};
self
}
pub fn action_rule(&self) -> (ActionRule, f32) {
(self.action_rule, self.softmax_temperature)
}
pub fn with_eval_action_rule(mut self, rule: ActionRule) -> Self {
self.eval_action_rule = rule;
self
}
pub fn eval_action_rule(&self) -> ActionRule {
self.eval_action_rule
}
pub fn with_obs_memory(mut self, memory: ObsMemory) -> Self {
self.obs_memory = match memory {
ObsMemory::Ema { beta } if beta.is_finite() => ObsMemory::Ema {
beta: beta.clamp(0.0, 1.0),
},
ObsMemory::Ema { .. } => ObsMemory::Ema { beta: 0.9 },
other => other,
};
self
}
pub fn with_learning_rate(mut self, lr: f32) -> Self {
self.learning_rate = lr;
self
}
pub fn with_grad_clip(mut self, clip: f32) -> Self {
self.grad_clip = clip;
self
}
pub fn with_challenge(
mut self,
prob: f32,
policy: ChallengePolicy,
decay: ChallengeDecay,
) -> Self {
self.challenge_prob = prob;
self.challenge_policy = policy;
self.challenge_decay = decay;
self
}
pub fn env(&self) -> &E {
&self.env
}
pub fn env_mut(&mut self) -> &mut E {
&mut self.env
}
pub fn signal(&self) -> Signal {
self.signal
}
fn play_episode(
&mut self,
net: &mut Network,
start_seed: u64,
forced: Option<ForcedAction>,
) -> Result<Episode> {
self.play_episode_with_rule(net, start_seed, forced, self.action_rule)
}
fn play_episode_with_rule(
&mut self,
net: &mut Network,
start_seed: u64,
forced: Option<ForcedAction>,
rule: ActionRule,
) -> Result<Episode> {
net.eval();
let mut rng = fastrand::Rng::with_seed(start_seed ^ ACTION_RULE_SEED_SALT);
let max_turns = self.env.max_turns();
let mut episode = Episode::default();
let mut obs = self.env.reset(start_seed)?;
let mut memory = MemoryState::new(self.obs_memory, &obs);
for _ in 0..max_turns {
let fed = memory.row(&obs);
let row = Tensor::from_f32(&fed, &[1, fed.len() as i64], self.device)?;
let values = net
.forward(&Variable::new(row, false))?
.data()
.to_f32_vec()?;
let action = match forced {
Some(forced) => forced.resolve(&values),
None => choose_action(rule, self.softmax_temperature, &values, &mut rng),
};
let (next, reward, done) = self.env.step(action)?;
episode.turns.push((fed, action));
episode.rewards.push(reward);
obs = next;
if done {
break;
}
}
Ok(episode)
}
fn reinforce_update(
&self,
net: &mut Network,
optimizer: &mut dyn Optimizer,
episodes: &[Episode],
input_dim: usize,
) -> Result<f32> {
let actions = self.env.action_count();
let total: usize = episodes.iter().map(Episode::len).sum();
let mut flat = Vec::with_capacity(total * input_dim);
let mut mask = vec![0.0f32; total * actions];
let mut returns = Vec::with_capacity(total);
let mut row = 0usize;
for episode in episodes {
let g = episode.returns(self.signal);
for (t, (obs, action)) in episode.turns.iter().enumerate() {
flat.extend_from_slice(obs);
if *action >= actions {
return Err(flodl::tensor::TensorError::new(&format!(
"reinforce: the policy chose action {action}, but the environment has \
{actions} — the topology's output_dim must equal Env::action_count"
)));
}
mask[row * actions + action] = 1.0;
returns.push(g[t]);
row += 1;
}
}
let baseline = returns.iter().sum::<f32>() / returns.len() as f32;
let advantages: Vec<f32> = returns.iter().map(|g| g - baseline).collect();
let adv_std =
(advantages.iter().map(|a| a * a).sum::<f32>() / advantages.len() as f32).sqrt();
let scale = 1.0 / adv_std.max(1e-6);
net.train();
let x = Variable::new(
Tensor::from_f32(&flat, &[total as i64, input_dim as i64], self.device)?,
true,
);
let pred = net.forward(&x)?;
let logp = pred.data().log_softmax(1)?;
let m = Tensor::from_f32(&mask, &[total as i64, actions as i64], self.device)?;
let chosen = logp.mul(&m)?;
let adv = Tensor::from_f32(&advantages, &[total as i64, 1], self.device)?;
let weighted = chosen
.sum_dims(&[1], false)?
.reshape(&[total as i64, 1])?
.mul(&adv)?;
let s = Tensor::from_f32(
&[loss_scale(scale, total, LOSS_MEAN_OVER_TURNS)],
&[1],
self.device,
)?;
let loss = Variable::new(weighted.sum()?.mul(&s)?, false);
net.eval();
let dummy = Tensor::from_f32(&vec![0.0; input_dim], &[1, input_dim as i64], self.device)?;
train_one_step_pred_only(
net,
optimizer,
&|_: &Variable| Ok(loss.clone()),
&dummy,
self.grad_clip,
)
}
fn exam_axis(&self, step: usize, era_len: usize) -> usize {
if self.per_clock_exam {
step
} else {
eval_era_of(step, era_len)
}
}
fn score_metrics(&self, metrics: &[Metric], outcomes: &[f32]) -> Result<Vec<f32>> {
if metrics.is_empty() {
return Ok(Vec::new());
}
let t = Tensor::from_f32(outcomes, &[outcomes.len() as i64], self.device)?;
let (pred, target) = (Variable::new(t.clone(), false), Variable::new(t, false));
metrics.iter().map(|m| m.score(&pred, &target)).collect()
}
}
impl<E: Env> RlStep for Reinforce<E> {
fn restore_optimizer(
&self,
net: &Network,
path: &std::path::Path,
) -> Option<Box<dyn Optimizer>> {
self.optimizer.load(net, self.learning_rate, path)
}
fn make_optimizer(&self, net: &Network) -> Box<dyn Optimizer> {
self.optimizer.make(net, self.learning_rate)
}
fn describe(&self) -> Option<serde_json::Value> {
let mut blob = serde_json::json!({
"trainer": "reinforce",
"update": "REINFORCE (log_softmax autodiff; per-turn returns, batch-mean baseline, advantage normalization)",
"env": self.env.describe(),
"actions": self.env.action_count(),
"max_turns": self.env.max_turns(),
"signal": self.signal,
"matches_per_step": self.matches_per_step,
"eval_matches_per_step": self.eval_matches_per_step,
"exam": if self.per_clock_exam { "per_clock" } else { "era" },
"learning_rate": self.learning_rate,
"grad_clip": self.grad_clip,
"challenge_prob": self.challenge_prob,
"challenge_policy": self.challenge_policy.to_string(),
"challenge_decay": self.challenge_decay.to_string(),
});
if self.obs_memory != ObsMemory::None {
blob["obs_memory"] =
serde_json::to_value(self.obs_memory).unwrap_or(serde_json::Value::Null);
}
if let Some(named) = &self.lr_schedule {
blob["lr_schedule"] = serde_json::Value::String(named.label());
}
if self.optimizer != crate::trainer::OptimizerKind::default() {
blob["optimizer"] = serde_json::Value::String(self.optimizer.label().to_string());
}
if self.action_rule != ActionRule::Argmax {
blob["action_rule"] = serde_json::Value::String(self.action_rule.as_str().to_string());
if self.softmax_temperature != SOFTMAX_TEMPERATURE {
blob["softmax_temperature"] = serde_json::json!(self.softmax_temperature);
}
}
if self.eval_action_rule != EVAL_ACTION_RULE {
blob["eval_action_rule"] =
serde_json::Value::String(self.eval_action_rule.as_str().to_string());
}
Some(blob)
}
fn train_step(
&mut self,
net: &mut Network,
optimizer: &mut dyn Optimizer,
step: usize,
ctx: &RlContext<'_>,
) -> Result<RlStepReport> {
let actions = self.env.action_count();
let challenged = challenge_fires(
ctx.env.run_seed,
ctx.net_seed,
step,
self.challenge_prob,
ctx.env.max_steps,
self.challenge_decay,
);
let forced =
challenged.then(|| forced_action(self.challenge_policy, ctx.net_seed, step, actions));
if let Some(named) = &self.lr_schedule {
optimizer.set_lr(named.lr(self.learning_rate, step, ctx.env.max_steps) as f64);
}
let train_base = ctx.net_seed ^ ctx.env.run_seed;
let attempt_block = ctx.env.attempt.wrapping_mul(self.matches_per_step as u64);
let mut episodes: Vec<Episode> = Vec::with_capacity(self.matches_per_step);
for match_i in 0..self.matches_per_step {
let seed = episode_start_seed(train_base, step, attempt_block + match_i as u64);
episodes.push(self.play_episode(net, seed, forced)?);
}
let total: usize = episodes.iter().map(Episode::len).sum();
let train_loss = if total == 0 {
0.0
} else {
let input_dim = episodes
.iter()
.find_map(|e| e.turns.first().map(|(obs, _)| obs.len()))
.expect("a played episode has at least one turn");
self.reinforce_update(net, optimizer, &episodes, input_dim)?
};
let eval_base = eval_seed_base(ctx.env.run_seed);
let exam = self.exam_axis(step, ctx.env.checkpoint_every);
let mut outcomes: Vec<f32> = Vec::with_capacity(self.eval_matches_per_step);
let mut eval_turns = 0usize;
for match_i in 0..self.eval_matches_per_step {
let seed = episode_start_seed(eval_base, exam, match_i as u64);
let episode = self.play_episode_with_rule(net, seed, None, self.eval_action_rule)?;
eval_turns += episode.len();
outcomes.push(episode.outcome(self.signal));
}
Ok(RlStepReport {
train_loss,
fitness: mean(&outcomes),
informative: self.score_metrics(ctx.metrics, &outcomes)?,
rl: Some(RlStepMeta {
matches: self.matches_per_step + self.eval_matches_per_step,
train_turns: total,
eval_turns,
}),
})
}
}
pub fn episode_start_seed(seed_base: u64, step: usize, match_i: u64) -> u64 {
seed_base
^ (step as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15)
^ match_i.wrapping_mul(0xBF58_476D_1CE4_E5B9)
}
fn eval_seed_base(run_seed: u64) -> u64 {
run_seed.wrapping_add(EVAL_SEED_OFFSET)
}
fn eval_era_of(step: usize, era_len: usize) -> usize {
if era_len == 0 { step } else { step / era_len }
}
fn greedy_action(values: &[f32]) -> usize {
let mut best = 0usize;
for (i, v) in values.iter().enumerate() {
let better = if v.is_nan() {
false } else if values[best].is_nan() {
true } else {
matches!(v.partial_cmp(&values[best]), Some(Ordering::Greater))
};
if better {
best = i;
}
}
best
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ActionRule {
Argmax,
Softmax,
}
impl ActionRule {
pub fn as_str(self) -> &'static str {
match self {
ActionRule::Argmax => "argmax",
ActionRule::Softmax => "softmax",
}
}
}
pub const ACTION_RULE: ActionRule = ActionRule::Argmax;
pub const EVAL_ACTION_RULE: ActionRule = ActionRule::Argmax;
pub const SOFTMAX_TEMPERATURE: f32 = 1.0;
pub const LOSS_MEAN_OVER_TURNS: bool = true;
const ACTION_RULE_SEED_SALT: u64 = 0xA5A5_5A5A_C3C3_3C3C;
fn choose_action(
rule: ActionRule,
temperature: f32,
values: &[f32],
rng: &mut fastrand::Rng,
) -> usize {
match rule {
ActionRule::Argmax => greedy_action(values),
ActionRule::Softmax => softmax_action(values, temperature, rng),
}
}
fn softmax_action(values: &[f32], temperature: f32, rng: &mut fastrand::Rng) -> usize {
let t = if temperature.is_finite() && temperature > 0.0 {
temperature
} else {
1.0
};
let max = values
.iter()
.copied()
.filter(|v| !v.is_nan())
.fold(f32::NEG_INFINITY, f32::max);
let max = if max.is_finite() { max } else { 0.0 };
let mass: Vec<f32> = values
.iter()
.map(|v| {
if v.is_nan() {
0.0
} else {
((v - max) / t).exp()
}
})
.collect();
let total: f32 = mass.iter().sum();
if !(total > 0.0) {
return greedy_action(values);
}
let draw = rng.f32() * total;
let mut acc = 0.0;
for (i, w) in mass.iter().enumerate() {
acc += w;
if draw < acc {
return i;
}
}
mass.iter().rposition(|w| *w > 0.0).unwrap_or(0)
}
fn loss_scale(scale: f32, turns: usize, mean_over_turns: bool) -> f32 {
if mean_over_turns {
scale / turns.max(1) as f32
} else {
scale
}
}
fn mean(values: &[f32]) -> f32 {
if values.is_empty() {
0.0
} else {
values.iter().sum::<f32>() / values.len() as f32
}
}
#[derive(Default)]
struct Episode {
turns: Vec<(Vec<f32>, usize)>,
rewards: Vec<f32>,
}
impl Episode {
fn len(&self) -> usize {
self.turns.len()
}
fn returns(&self, signal: Signal) -> Vec<f32> {
let n = self.len();
let mut out = vec![0.0f32; n];
match signal {
Signal::Turns => {
for (t, g) in out.iter_mut().enumerate() {
*g = (n - t) as f32;
}
}
Signal::Return { discount } => {
let mut running = 0.0f32;
for t in (0..n).rev() {
running = self.rewards[t] + discount * running;
out[t] = running;
}
}
}
out
}
fn outcome(&self, signal: Signal) -> f32 {
match signal {
Signal::Turns => self.len() as f32,
Signal::Return { .. } => self.rewards.iter().sum(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::trainer::env::toy::ToyEnv;
use crate::utils::fitness::{Direction, Fitness};
#[test]
fn loss_is_the_turn_mean_not_the_sum() {
let scale = 2.0;
assert_eq!(loss_scale(scale, 1, true), scale);
assert_eq!(loss_scale(scale, 500, true), scale / 500.0);
assert_eq!(loss_scale(scale, 500, false), scale);
assert_eq!(loss_scale(scale, 0, true), scale);
}
fn tiny_net(output_dim: usize, dropout_prob: f32) -> Network {
use crate::graph::node::Node;
use crate::graph::topology::{Connection, Port, Topology, TopologyOptions};
let mut topo = Topology::new(
3,
Some(TopologyOptions {
input_dim: Some(2),
output_dim: Some(output_dim),
dropout_prob,
..Default::default()
}),
);
topo.nodes.push(Node::new_input(0, 2));
topo.nodes.push(Node::new_hidden(1, 1, 1));
topo.nodes.push(Node::new_output(2, 1, 1));
topo.connections.push(Connection {
from: Port { node: 0, index: 0 },
to: Port { node: 1, index: 0 },
});
topo.connections.push(Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 0 },
});
topo.finalize();
Network::build(&topo, Device::CPU).unwrap()
}
fn ctx<'a>(fitness: &'a Fitness, net_seed: u64, run_seed: u64, step: usize) -> RlContext<'a> {
RlContext {
fitness,
metrics: &[],
env: crate::trainer::StepEnv {
step,
run_seed,
pop_size: 1,
live_count: 1,
checkpoint_every: 0,
smoothing_window: 1,
max_steps: Some(10),
attempt: 0,
},
net_hash: "test-net",
net_seed,
}
}
struct LadderEnv {
max_turns: usize,
turns: usize,
seed: u64,
}
impl LadderEnv {
fn new(max_turns: usize) -> Self {
Self {
max_turns,
turns: 0,
seed: 0,
}
}
}
impl Env for LadderEnv {
fn reset(&mut self, seed: u64) -> Result<Vec<f32>> {
self.seed = seed;
self.turns = 0;
Ok(vec![0.5, -0.5])
}
fn step(&mut self, _action: usize) -> Result<(Vec<f32>, f32, bool)> {
self.turns += 1;
let reward = self.turns as f32;
Ok((vec![reward, 0.0], reward, self.turns >= self.max_turns))
}
fn action_count(&self) -> usize {
2
}
fn max_turns(&self) -> usize {
self.max_turns
}
fn describe(&self) -> serde_json::Value {
serde_json::json!({ "env": "ladder", "max_turns": self.max_turns })
}
}
#[test]
fn reinforce_is_an_rl_step() {
fn assert_rl_step<T: RlStep>() {}
assert_rl_step::<Reinforce<ToyEnv>>();
}
#[test]
fn episode_seeds_are_pure_and_domain_separated() {
assert_eq!(
episode_start_seed(11, 3, 2),
episode_start_seed(11, 3, 2),
"pure: same inputs, same seed"
);
let base = episode_start_seed(11, 3, 2);
assert_ne!(
base,
episode_start_seed(12, 3, 2),
"a different net is a different run"
);
assert_ne!(
base,
episode_start_seed(11, 4, 2),
"a different clock is a different run"
);
assert_ne!(
base,
episode_start_seed(11, 3, 3),
"a different match is a different run"
);
assert_ne!(
episode_start_seed(11, 3, 2),
episode_start_seed(11, 2, 3),
"step and match must not be interchangeable"
);
assert_eq!(episode_start_seed(42, 0, 0), 42);
assert_eq!(episode_start_seed(0, 1, 0), 0x9E37_79B9_7F4A_7C15);
}
#[test]
fn the_exam_base_is_shared_and_off_the_train_draw() {
assert_ne!(
eval_seed_base(42),
42,
"the exam must not reuse the train base"
);
assert_eq!(eval_seed_base(42), 1042, "a plain offset: golden value");
for net_seed in [1u64, 99] {
assert_eq!(
episode_start_seed(eval_seed_base(7), 2, 1),
episode_start_seed(eval_seed_base(7), 2, 1),
"net {net_seed} sits its own copy of the same exam"
);
}
}
#[test]
fn greedy_action_is_argmax_with_ties_to_the_lowest_index() {
assert_eq!(greedy_action(&[0.1, 0.9]), 1);
assert_eq!(greedy_action(&[0.9, 0.1]), 0);
assert_eq!(greedy_action(&[1.0, 1.0]), 0, "ties go to the lowest index");
assert_eq!(greedy_action(&[0.0, f32::NAN]), 0);
assert_eq!(greedy_action(&[f32::NAN, 0.0]), 1);
assert_eq!(greedy_action(&[f32::NAN, f32::NAN]), 0);
assert_eq!(greedy_action(&[f32::INFINITY, f32::NEG_INFINITY]), 0);
assert_eq!(greedy_action(&[3.0]), 0);
assert_eq!(greedy_action(&[]), 0);
}
#[test]
fn softmax_sampling_replays_from_its_seed() {
let values = [0.0, 0.0, 0.0];
let draw = |seed: u64| {
let mut rng = fastrand::Rng::with_seed(seed);
(0..64)
.map(|_| softmax_action(&values, 1.0, &mut rng))
.collect::<Vec<_>>()
};
assert_eq!(draw(7), draw(7), "the same seed replays the same moves");
assert_ne!(draw(7), draw(8), "a different seed is a different draw");
let moves = draw(7);
assert!(
moves.iter().any(|&a| a != moves[0]),
"a flat policy must explore: {moves:?}"
);
}
#[test]
fn the_acting_rule_selects_argmax_or_softmax() {
let sharp = [0.0, 40.0, 0.0];
for seed in 0..32u64 {
let mut rng = fastrand::Rng::with_seed(seed);
assert_eq!(softmax_action(&sharp, 1.0, &mut rng), 1, "seed {seed}");
let mut rng = fastrand::Rng::with_seed(seed);
assert_eq!(choose_action(ActionRule::Argmax, 1.0, &sharp, &mut rng), 1);
}
let mut rng = fastrand::Rng::with_seed(3);
assert_eq!(softmax_action(&[f32::NAN, 1.0], 1.0, &mut rng), 1);
let mut rng = fastrand::Rng::with_seed(3);
assert_eq!(softmax_action(&[f32::NAN, f32::NAN], 1.0, &mut rng), 0);
let mut rng = fastrand::Rng::with_seed(5);
assert_eq!(softmax_action(&sharp, 0.0, &mut rng), 1);
assert_eq!(ACTION_RULE, ActionRule::Argmax);
assert_eq!(ACTION_RULE.as_str(), "argmax");
assert_eq!(ActionRule::Softmax.as_str(), "softmax");
}
struct PickyEnv {
turns: usize,
}
impl Env for PickyEnv {
fn reset(&mut self, _seed: u64) -> Result<Vec<f32>> {
self.turns = 0;
Ok(vec![0.5, -0.5])
}
fn step(&mut self, action: usize) -> Result<(Vec<f32>, f32, bool)> {
self.turns += 1;
let dead = action == 0;
Ok((vec![0.0, 0.0], 1.0, dead || self.turns >= 2))
}
fn action_count(&self) -> usize {
2
}
fn max_turns(&self) -> usize {
2
}
fn describe(&self) -> serde_json::Value {
serde_json::json!({ "env": "picky", "max_turns": 2 })
}
}
#[test]
fn the_exam_plays_under_its_own_rule_not_the_training_one() {
assert_eq!(EVAL_ACTION_RULE, ActionRule::Argmax);
let fitness = Fitness::reported(Direction::Maximize, "turns");
let read = |train: ActionRule, eval: ActionRule| {
let mut trainer = Reinforce::new(PickyEnv { turns: 0 }, Device::CPU)
.with_matches_per_step(0)
.with_eval_matches_per_step(8)
.with_action_rule(train, 1e6)
.with_eval_action_rule(eval);
assert_eq!(trainer.eval_action_rule(), eval);
let mut net = tiny_net(2, 0.0);
let mut optimizer = trainer.make_optimizer(&net);
trainer
.train_step(&mut net, &mut *optimizer, 1, &ctx(&fitness, 42, 7, 1))
.unwrap()
.fitness
};
let greedy = read(ActionRule::Softmax, ActionRule::Argmax);
assert_eq!(
greedy,
read(ActionRule::Argmax, ActionRule::Argmax),
"the training rule cannot move the exam"
);
assert_ne!(
greedy,
read(ActionRule::Argmax, ActionRule::Softmax),
"a sampled exam at a flat temperature grades a different policy"
);
}
#[test]
fn describe_omits_the_acting_rule_while_it_is_the_default() {
let trainer = Reinforce::new(ToyEnv::new(2, 5), Device::CPU);
let blob = trainer.describe().expect("reinforce records a blob");
assert_eq!(blob["trainer"], serde_json::json!("reinforce"));
assert!(
blob.get("action_rule").is_none(),
"the reference recipe adds no key, so old runs still resume: {blob}"
);
assert!(
blob.get("eval_action_rule").is_none(),
"and neither does the default exam: {blob}"
);
let sampled = trainer.with_eval_action_rule(ActionRule::Softmax);
let blob = sampled.describe().expect("reinforce records a blob");
assert_eq!(blob["eval_action_rule"], serde_json::json!("softmax"));
assert_eq!(
blob["action_rule"],
serde_json::Value::Null,
"the TRAIN arm stays the reference recipe"
);
}
#[test]
fn returns_follow_the_signal() {
let episode = Episode {
turns: vec![(vec![0.0; 2], 0), (vec![0.0; 2], 1), (vec![0.0; 2], 0)],
rewards: vec![1.0, 2.0, 3.0],
};
assert_eq!(episode.returns(Signal::Turns), vec![3.0, 2.0, 1.0]);
assert_eq!(episode.outcome(Signal::Turns), 3.0);
let undiscounted = Signal::Return { discount: 1.0 };
assert_eq!(episode.returns(undiscounted), vec![6.0, 5.0, 3.0]);
assert_eq!(
episode.outcome(undiscounted),
6.0,
"the outcome is the episode total"
);
let discounted = Signal::Return { discount: 0.5 };
assert_eq!(episode.returns(discounted), vec![2.75, 3.5, 3.0]);
assert_eq!(
episode.outcome(discounted),
6.0,
"the discount shapes credit, not the total"
);
let empty = Episode::default();
assert!(empty.returns(Signal::Turns).is_empty());
assert_eq!(empty.outcome(Signal::Turns), 0.0);
assert_eq!(empty.outcome(Signal::Return { discount: 0.9 }), 0.0);
}
#[test]
fn a_step_seeds_train_episodes_per_net() {
let fitness = Fitness::reported(Direction::Maximize, "turns");
let step = |net_seed: u64| {
let mut trainer =
Reinforce::new(ToyEnv::new(2, 5), Device::CPU).with_eval_matches_per_step(0);
let mut net = tiny_net(2, 0.0);
let mut optimizer = trainer.make_optimizer(&net);
trainer
.train_step(&mut net, &mut *optimizer, 3, &ctx(&fitness, net_seed, 7, 3))
.unwrap();
trainer.env().last_seed()
};
assert_eq!(step(100), episode_start_seed(100 ^ 7, 3, 0));
assert_ne!(
step(100),
step(101),
"a different net explores a different world"
);
assert_eq!(
step(100),
episode_start_seed(100 ^ 7, 3, 0),
"and the seed is a pure function of the recorded inputs"
);
}
#[test]
fn the_exam_seeds_are_shared_across_nets() {
let fitness = Fitness::reported(Direction::Maximize, "turns");
let last_exam_seed = |net_seed: u64, step: usize| {
let mut trainer =
Reinforce::new(ToyEnv::new(2, 5), Device::CPU).with_eval_matches_per_step(2);
let mut net = tiny_net(2, 0.0);
let mut optimizer = trainer.make_optimizer(&net);
trainer
.train_step(
&mut net,
&mut *optimizer,
step,
&ctx(&fitness, net_seed, 7, step),
)
.unwrap();
trainer.env().last_seed()
};
assert_eq!(
last_exam_seed(100, 3),
episode_start_seed(eval_seed_base(7), 3, 1)
);
assert_eq!(
last_exam_seed(100, 3),
last_exam_seed(999, 3),
"the exam is the same exam for every net"
);
assert_ne!(
last_exam_seed(100, 3),
last_exam_seed(100, 4),
"a fresh clock draws fresh games — the default per-clock exam"
);
}
#[test]
fn every_match_inside_one_clock_draws_its_own_game() {
let step = 3usize;
for (kind, base) in [("train", 100 ^ 7), ("eval", eval_seed_base(7))] {
let seeds: Vec<u64> = (0..5).map(|i| episode_start_seed(base, step, i)).collect();
let unique: std::collections::HashSet<u64> = seeds.iter().copied().collect();
assert_eq!(
unique.len(),
5,
"{kind}: five matches, five games: {seeds:?}"
);
}
let fitness = Fitness::reported(Direction::Maximize, "turns");
let mut trainer = Reinforce::new(ToyEnv::new(2, 5), Device::CPU)
.with_matches_per_step(3)
.with_eval_matches_per_step(0);
let mut net = tiny_net(2, 0.0);
let mut optimizer = trainer.make_optimizer(&net);
trainer
.train_step(
&mut net,
&mut *optimizer,
step,
&ctx(&fitness, 100, 7, step),
)
.unwrap();
assert_eq!(
trainer.env().last_seed(),
episode_start_seed(100 ^ 7, step, 2),
"the third train match of the clock played its own seed"
);
}
#[test]
fn the_default_exam_draws_fresh_games_every_clock() {
let fitness = Fitness::reported(Direction::Maximize, "turns");
let last_exam_seed = |net_seed: u64, step: usize| {
let mut trainer =
Reinforce::new(ToyEnv::new(2, 5), Device::CPU).with_eval_matches_per_step(1);
let mut net = tiny_net(2, 0.0);
let mut optimizer = trainer.make_optimizer(&net);
let mut c = ctx(&fitness, net_seed, 7, step);
c.env.checkpoint_every = 4;
trainer
.train_step(&mut net, &mut *optimizer, step, &c)
.unwrap();
trainer.env().last_seed()
};
assert_eq!(
last_exam_seed(100, 5),
episode_start_seed(eval_seed_base(7), 5, 0),
"the exam's step axis is the CLOCK"
);
assert_ne!(
last_exam_seed(100, 5),
last_exam_seed(100, 7),
"two clocks inside one era are two exams"
);
assert_eq!(
last_exam_seed(100, 5),
last_exam_seed(999, 5),
"and it is still one exam for every net"
);
}
#[test]
fn the_era_exam_is_stable_within_an_era_and_rotates_between_them() {
assert_eq!(eval_era_of(0, 20), 0);
assert_eq!(eval_era_of(19, 20), 0, "the era's last clock");
assert_eq!(eval_era_of(20, 20), 1, "the next era's first clock");
assert_eq!(eval_era_of(141, 20), 7);
assert_eq!(eval_era_of(7, 0), 7, "no tick ⇒ the per-clock exam");
assert_eq!(eval_era_of(7, 1), 7, "an era of one clock ⇒ per-clock too");
let fitness = Fitness::reported(Direction::Maximize, "turns");
let last_exam_seed = |step: usize| {
let mut trainer = Reinforce::new(ToyEnv::new(2, 5), Device::CPU)
.with_eval_matches_per_step(1)
.with_per_clock_exam(false);
let mut net = tiny_net(2, 0.0);
let mut optimizer = trainer.make_optimizer(&net);
let mut c = ctx(&fitness, 7, 7, step);
c.env.checkpoint_every = 4;
trainer
.train_step(&mut net, &mut *optimizer, step, &c)
.unwrap();
trainer.env().last_seed()
};
assert_eq!(last_exam_seed(5), last_exam_seed(7), "one era, one exam");
assert_ne!(
last_exam_seed(5),
last_exam_seed(9),
"a new era is a new exam"
);
assert_eq!(
last_exam_seed(5),
episode_start_seed(eval_seed_base(7), eval_era_of(5, 4), 0),
"the exam seed is a pure function of (base, era, match)"
);
}
#[test]
fn a_step_reports_exact_volume_and_is_a_pure_function_of_its_inputs() {
let fitness = Fitness::reported(Direction::Maximize, "turns");
let step = || {
let mut trainer = Reinforce::new(LadderEnv::new(4), Device::CPU);
let mut net = tiny_net(2, 0.0);
let mut optimizer = trainer.make_optimizer(&net);
let before = net.export_weights().unwrap();
let report = trainer
.train_step(&mut net, &mut *optimizer, 2, &ctx(&fitness, 42, 7, 2))
.unwrap();
(report, before, net.export_weights().unwrap())
};
let (report, before, after) = step();
assert_eq!(report.fitness, 4.0, "every exam episode is exactly 4 turns");
let rl = report
.rl
.expect("an RL step reports its environment volume");
assert_eq!(rl.matches, 6, "1 train + 5 eval episodes by default");
assert_eq!(rl.train_turns, 4, "one train episode, capped at max_turns");
assert_eq!(rl.eval_turns, 20, "five exam episodes, capped at max_turns");
assert_eq!(rl.turns(), 24);
assert!(
report.train_loss.is_finite(),
"a real update reports a finite loss"
);
assert_ne!(
before, after,
"the update moved the weights, so the step is not a no-op"
);
let (again, _, _) = step();
assert_eq!(report.train_loss, again.train_loss);
assert_eq!(report.fitness, again.fitness);
assert_eq!(report.rl, again.rl);
}
#[test]
fn obs_memory_rows_are_pure_and_cold_start_free() {
let a = vec![1.0f32, 2.0];
let b = vec![3.0f32, 4.0];
let mut none = MemoryState::new(ObsMemory::None, &a);
assert_eq!(none.row(&a), a, "no channel: the row IS the observation");
assert_eq!(ObsMemory::None.width_factor(), 1);
let mut delta = MemoryState::new(ObsMemory::Delta, &a);
assert_eq!(
delta.row(&a),
vec![1.0, 2.0, 0.0, 0.0],
"turn 0 has no past to subtract"
);
assert_eq!(
delta.row(&b),
vec![3.0, 4.0, 2.0, 2.0],
"then the discrete derivative"
);
assert_eq!(ObsMemory::Delta.width_factor(), 2);
let mut ema = MemoryState::new(ObsMemory::Ema { beta: 0.5 }, &a);
assert_eq!(
ema.row(&a),
vec![1.0, 2.0, 1.0, 2.0],
"seeded AT the first observation, so no spike"
);
assert_eq!(
ema.row(&b),
vec![3.0, 4.0, 2.0, 3.0],
"ema = 0.5·prev + 0.5·obs"
);
assert_eq!(ObsMemory::Ema { beta: 0.5 }.width_factor(), 2);
let mut again = MemoryState::new(ObsMemory::Ema { beta: 0.5 }, &a);
assert_eq!(again.row(&a), vec![1.0, 2.0, 1.0, 2.0]);
assert_eq!(again.row(&b), vec![3.0, 4.0, 2.0, 3.0]);
}
#[test]
fn the_signal_choice_changes_the_objective_and_the_unit() {
let fitness = Fitness::reported(Direction::Maximize, "score");
let run = |signal: Signal| {
let mut trainer = Reinforce::new(LadderEnv::new(4), Device::CPU).with_signal(signal);
let mut net = tiny_net(2, 0.0);
let mut optimizer = trainer.make_optimizer(&net);
trainer
.train_step(&mut net, &mut *optimizer, 0, &ctx(&fitness, 42, 7, 0))
.unwrap()
};
let turns = run(Signal::Turns);
let ret = run(Signal::Return { discount: 1.0 });
assert_eq!(turns.fitness, 4.0, "turns survived");
assert_eq!(ret.fitness, 10.0, "1 + 2 + 3 + 4 of reward");
assert!(turns.train_loss.is_finite() && ret.train_loss.is_finite());
assert_ne!(
turns.train_loss, ret.train_loss,
"the signal changes the objective"
);
}
#[test]
fn describe_records_the_world_the_caps_and_the_knobs() {
let trainer = Reinforce::new(ToyEnv::new(3, 12), Device::CPU);
let blob = trainer.describe().unwrap();
assert_eq!(blob["trainer"], "reinforce");
assert_eq!(blob["env"]["env"], "toy", "the world describes itself");
assert_eq!(blob["actions"], 3);
assert_eq!(blob["max_turns"], 12);
assert_eq!(blob["signal"]["kind"], "turns");
assert_eq!(blob["challenge_policy"], "least_action");
assert_eq!(blob["challenge_decay"], "custom:0.5");
assert_eq!(blob["matches_per_step"], 1);
assert_eq!(blob["eval_matches_per_step"], 5);
assert_eq!(blob["exam"], "per_clock");
let changed = trainer
.with_signal(Signal::Return { discount: 0.5 })
.with_matches_per_step(3)
.with_per_clock_exam(false)
.with_learning_rate(2e-3)
.with_challenge(0.0, ChallengePolicy::Random, ChallengeDecay::Linear)
.describe()
.unwrap();
assert_ne!(
blob, changed,
"a changed knob must change the recorded blob"
);
assert_eq!(changed["signal"]["kind"], "return");
assert_eq!(changed["signal"]["discount"], 0.5);
assert_eq!(changed["challenge_policy"], "random");
assert_eq!(changed["challenge_decay"], "linear");
assert_eq!(changed["exam"], "era", "the axis is recorded by name");
}
}