1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
//! Entrant generation: exactly one child per culled slot (Iter 5 contract).
//!
//! `generate_child` rolls `crossover_prob` → roulette-select parents →
//! crossover (≤3 attempts) → clone-fittest fallback → random fallback;
//! then rolls `mutate_prob` for a yes/no mutation. The child always gets
//! its own `derive_seed(run_seed, clock)` weight seed and a `created_from`
//! lineage string. Catch-up replays the shared stream solo so the child
//! rejoin the group at the current clock.
use flodl::nn::Optimizer;
use flodl::tensor::Result;
use log::{debug, info};
/// Parent-pairing attempts before falling back to a random topology.
/// Distinct from `crossover_rolls` (rolls per step) and `crossover_retries`
/// (gate-rejection retries) — this bounds only the draw of fresh parent
/// PAIRS when no compatible pairing is found.
const MAX_PARENT_PAIRINGS: usize = 3;
use crate::graph::network::Network;
use crate::graph::node::NodeKind;
use crate::graph::topology::Topology;
use crate::state::{NetMetrics, NetState, write_net_state};
use crate::utils::seed::derive_seed;
use super::smoothing::rolling_mean;
use super::race_engine::RaceEngine;
/// A freshly-built child net being caught up solo before it rejoins the group.
///
/// Holds the child's state (written to `nets/<hash>.json`), its live `Network`
/// (coefficients live here, mutate in place during catch-up), and its
/// `Optimizer` (Adam state carries forward across catch-up steps).
pub struct RaceChild {
pub state: NetState,
pub net: Network,
pub optimizer: Box<dyn Optimizer>,
}
impl RaceEngine {
// ── Child generation (Iter 5: roulette → crossover → mutate?) ───────────
/// Generate exactly one child for a culled slot (Iter 5 contract).
///
/// The decision is deterministic: the evolve/random switch is rolled from
/// `derive_seed(run_seed, clock)`, so two identical-seed runs make the
/// same call at the same step.
///
/// - `crossover_prob` fires → roulette-select 1–2 parents from the live
/// ranking (by smoothed fitness), crossover them (up to 3 attempts,
/// then fall back to the fittest parent's clone; if even that is
/// impossible, a fresh random topology), then mutate with the configured
/// per-type probability.
/// - otherwise → a fresh random child from the run's resolved pools.
///
/// The child always gets its own weight seed `derive_seed(run_seed, clock)`
/// and `created_from` lineage recording what actually happened.
pub(crate) fn generate_child(&mut self, clock: usize, child_idx: usize) -> Result<RaceChild> {
// Same-clock siblings must differ: mix the child ordinal into the
// derivation so two culled slots at one step make distinct children
// (the pop-shrink bug: identical roll seeds ⇒ identical children ⇒
// dedupe collapsed the pop).
let roll_seed = derive_seed(self.header.run_seed, clock * 1024 + child_idx);
let mut rng = fastrand::Rng::with_seed(roll_seed);
// Crossover attempt: independent of mutation. When ``crossover_prob``
// does not fire we skip straight to a random child (the not-evolved
// branch). When it fires we try roulette → crossover up to 3 times;
// a hard failure (no live parents, etc.) falls back to random so the
// race never stalls.
let try_crossover = rng.f32() < self.config.crossover_prob;
if try_crossover {
match self.evolved_child(clock, child_idx, &mut rng) {
Ok(child) => Ok(child),
Err(e) => {
// Crossover machinery failed hard (not just a no-op):
// fall back to a random child so the race never stalls.
if self.verbose_detail() {
info!(
"step {} │ evolved child failed ({:?}) → random fallback",
clock, e,
);
}
self.random_child(clock, child_idx)
}
}
} else {
self.random_child(clock, child_idx)
}
}
/// The evolved branch: roulette parents → crossover → mutate.
/// The evolved branch: roulette parents → crossover → maybe mutate.
/// Called only when ``crossover_prob`` fires; returns an error when there
/// are no live parents to select from (which is treated as "no crossover
/// this step" by ``generate_child``, which falls back to random).
fn evolved_child(
&mut self,
clock: usize,
child_idx: usize,
rng: &mut fastrand::Rng,
) -> Result<RaceChild> {
// 1. Ranking: live hashes ordered best-first by smoothed fitness.
let hashes = self.state.live_hashes();
if hashes.is_empty() {
return Err(flodl::tensor::TensorError::new(
"race: no live nets to select parents from",
));
}
let direction = self.fitness.direction();
let mut ranked: Vec<(String, f32)> = hashes
.iter()
.map(|h| {
(
h.clone(),
rolling_mean(self.rolling_fitness.get(h).unwrap()),
)
})
.collect();
ranked.sort_by(|a, b| direction.cmp(b.1, a.1));
let scores: Vec<f32> = ranked.iter().map(|(_, s)| *s).collect();
// 3. Crossover: up to 3 attempts; a no-op (incompatible dims) retries
// with FRESH roulette-drawn parents (the draw lives inside the
// attempt loop — see below).
let mut child_topo = None;
let mut cx_note = String::new();
// Operator pool: resolved once per child (not per attempt) so a
// single child's retries vary the PARENTS, not the op. Draw is
// uniform over the pool; empty config pool ⇒ both ops (engine
// convention). Uniform uses its own swap_prob — no more magic 0.5.
let ops = self
.config
.resolved_crossover_ops()
.map_err(|e| flodl::tensor::TensorError::new(&e))?;
let op = ops[rng.usize(..ops.len())];
// Parent count is a property of the OPERATOR, not a config knob:
// every CrossoverOp is binary today ⇒ n_parents = op.required_parents().
// The old set_crossover_parents knob is gone — redundant surface.
let n_parents = op.required_parents();
// Up to 3 attempts. The FULL attempt is retried — new roulette parent
// draws each pass, not the same pair with new op randomness. A pair
// with no compatible pivot (one_point) or mismatched hidden counts
// (uniform) can never succeed, so retrying only the operator would
// burn all 3 attempts on a doomed pairing. Parents persist past the
// loop only if the LAST attempt used them (for lineage metadata).
let mut parent_hashes: Vec<String> = Vec::new();
for _ in 0..MAX_PARENT_PAIRINGS {
// 2. Roulette-select parents — INSIDE the attempt loop: a failed
// pairing gets fresh parents, per the design contract.
let parent_positions: Vec<usize> = (0..n_parents)
.filter_map(|_| {
crate::evolution::selection::SelectionMethod::Roulette
.apply(&scores, direction, rng, 0)
.into_iter()
.next()
})
.collect();
let attempt_parents: Vec<String> = parent_positions
.iter()
.map(|&p| ranked[p].0.clone())
.collect();
let mut pa = self.load_parent_topology(&attempt_parents[0])?;
if n_parents == 2 {
let mut pb = self.load_parent_topology(&attempt_parents[1])?;
let cx = match op {
crate::engine::config::CrossoverOp::OnePoint => {
Topology::cx_one_point(&mut pa, &mut pb, rng)
}
crate::engine::config::CrossoverOp::Uniform => {
const UNIFORM_SWAP_PROB: f32 = 0.5;
Topology::cx_uniform(&mut pa, &mut pb, UNIFORM_SWAP_PROB, rng)
}
};
if cx {
child_topo = Some(pa);
cx_note = match op {
crate::engine::config::CrossoverOp::OnePoint => "crossover-one-point",
crate::engine::config::CrossoverOp::Uniform => "crossover-uniform",
}
.to_string();
parent_hashes = attempt_parents;
break;
}
// No-op — next loop pass draws a fresh parent pair.
} else {
// Single-parent mode: the child is the parent with a new seed
// (asexual reproduction; crossover needs two).
child_topo = Some(pa);
cx_note = "clone".to_string();
parent_hashes = attempt_parents;
break;
}
}
let mut child_topo = match child_topo {
Some(t) => t,
None => {
// MAX_PARENT_PAIRINGS exhausted → fresh random topology, NOT a
// clone of the fittest: cloning reinforces the leader and
// starves diversity; a random entrant brings new blood. The
// race never stalls either way (pop size stays constant).
if self.verbose_detail() {
info!(
"step {} │ crossover produced no child after {} parent-pairing attempts (no compatible pivot/dims) → random topology fallback",
clock,
MAX_PARENT_PAIRINGS,
);
}
return self.random_child(clock, child_idx);
}
};
// 4. Mutation: yes/no rolled per child from ``mutate_prob``; the op
// type is drawn from the three built-in mutation kinds (activation /
// combine / standardize).
let mutated = if rng.f32() < self.config.mutate_prob {
let kinds = [
crate::evolution::mutation::MutationMethod::Activation { prob: 1.0 },
crate::evolution::mutation::MutationMethod::CombineOp { prob: 1.0 },
crate::evolution::mutation::MutationMethod::Standardize { prob: 1.0 },
];
let kind = &kinds[rng.usize(0..kinds.len())];
self.apply_mutation(&mut child_topo, kind, rng);
cx_note.push_str("+mut");
true
} else {
false
};
self.finalize_child(
child_topo,
clock,
child_idx,
&cx_note,
&parent_hashes,
mutated,
)
}
/// Load a live net's topology by hash (parents must still be live —
/// they were just stepped this loop iteration, so they are).
fn load_parent_topology(&self, hash: &str) -> Result<Topology> {
match self.state.net(hash) {
Some(state) => state.topology(),
None => Err(flodl::tensor::TensorError::new(&format!(
"race: parent {hash} not live"
))),
}
}
/// Apply one mutation op to the child topology. `prob: 1.0` inside the
/// method means "definitely mutate" (the yes/no was already rolled).
fn apply_mutation(
&self,
topo: &mut Topology,
kind: &crate::evolution::mutation::MutationMethod,
rng: &mut fastrand::Rng,
) {
use crate::evolution::mutation::MutationMethod;
match kind {
MutationMethod::Activation { .. } => {
let pool = self
.config
.resolved_activation_pool()
.unwrap_or_else(|_| crate::evolution::pools::all_activations());
if let Some(node) = topo.nodes.iter_mut().find(|n| n.kind == NodeKind::Hidden) {
node.activation = pool[rng.usize(0..pool.len())];
}
}
MutationMethod::CombineOp { .. } => {
let pool = self
.config
.resolved_combine_pool()
.unwrap_or_else(|_| crate::evolution::pools::all_combine_ops());
if let Some(node) = topo.nodes.iter_mut().find(|n| n.kind == NodeKind::Hidden) {
node.combine_op = Some(pool[rng.usize(0..pool.len())]);
}
}
MutationMethod::Standardize { .. } => {
let pool = self
.config
.resolved_standardize_pool()
.unwrap_or_else(|_| crate::evolution::pools::all_standardize_ops());
if let Some(node) = topo.nodes.iter_mut().find(|n| n.kind == NodeKind::Hidden) {
node.standardize = Some(pool[rng.usize(0..pool.len())]);
}
}
}
topo.refresh_labels();
topo.finalize();
}
/// A fresh random child from the run's resolved pools (the crossover_prob
/// "random" branch, and the last-resort fallback for a failed evolved
/// path). Mirrors the generational `create_individual` shape.
fn random_child(&mut self, clock: usize, child_idx: usize) -> Result<RaceChild> {
let roll_seed = derive_seed(self.header.run_seed, clock * 1024 + child_idx);
let mut rng = fastrand::Rng::with_seed(roll_seed);
let topo_opts = self.config.topology_options;
let n_hidden = rng.usize(topo_opts.min_hidden_num_nodes..=topo_opts.max_hidden_num_nodes);
// Stamp the run's dataset dims + hidden-dim pool so the child trains on
// the same shared stream the rest of the population uses.
// match the actual data).
let mut opts = topo_opts;
opts.input_dim = Some(self.header.input_dim);
opts.output_dim = Some(self.header.output_dim);
// The topology's internal rng must also derive from the run seed —
// seeding it with the template's default would make every random
// child with the same n_hidden structurally identical.
let topo_seed = derive_seed(self.header.run_seed, clock * 1024 + child_idx) as usize;
let mut graph = Topology::new(topo_seed, Some(opts));
graph.create_random_hidden_nodes(n_hidden);
// Children draw from the config's resolved pools — an empty config
// pool falls back to the full known-op set (empty ⇒ all).
let activation = self
.config
.resolved_activation_pool()
.unwrap_or_else(|_| crate::evolution::pools::all_activations());
let combine = self
.config
.resolved_combine_pool()
.unwrap_or_else(|_| crate::evolution::pools::all_combine_ops());
let standardize = self
.config
.resolved_standardize_pool()
.unwrap_or_else(|_| crate::evolution::pools::all_standardize_ops());
let hidden_dim_pool = self.config.hidden_dim_pool.clone().unwrap_or(4..=8);
let stride = self.config.hidden_dim_stride.max(1);
for node in &mut graph.nodes {
if node.kind == NodeKind::Hidden {
let n = ((hidden_dim_pool.end() - hidden_dim_pool.start()) / stride) + 1;
node.hidden_dim = Some(hidden_dim_pool.start() + rng.usize(0..n) * stride);
node.activation = activation[rng.usize(0..activation.len())];
node.combine_op = Some(combine[rng.usize(0..combine.len())]);
node.standardize = Some(standardize[rng.usize(0..standardize.len())]);
}
}
graph.refresh_labels();
graph.finalize();
// The random branch rolls mutate_prob too — the yes/no is per child,
// not per branch (Iter-5 contract: every child gets the mutation roll).
let mutated = rng.f32() < self.config.mutate_prob;
if mutated {
let kinds = [
crate::evolution::mutation::MutationMethod::Activation { prob: 1.0 },
crate::evolution::mutation::MutationMethod::CombineOp { prob: 1.0 },
crate::evolution::mutation::MutationMethod::Standardize { prob: 1.0 },
];
let kind = &kinds[rng.usize(0..kinds.len())];
self.apply_mutation(&mut graph, kind, &mut rng);
}
self.finalize_child(graph, clock, child_idx, "random", &[], mutated)
}
/// Shared child finalization: fresh weight seed from `run_seed + clock`,
/// lineage string, `NetState` starting at step 0, built `Network` +
/// `Optimizer` ready for catch-up.
fn finalize_child(
&self,
mut topo: Topology,
clock: usize,
child_idx: usize,
origin: &str,
parents: &[String],
mutated: bool,
) -> Result<RaceChild> {
let new_seed = derive_seed(self.header.run_seed, clock * 1024 + child_idx) as usize;
topo.options.topology_seed = new_seed;
let lineage = if parents.is_empty() {
origin.to_string()
} else {
format!("{origin}:parents={}", parents.join(","))
};
let lineage = if mutated {
format!("{lineage}+mut")
} else {
lineage
};
// The child's step counter starts at **0** (it has trained zero steps
// so far); `entered_at_step` is patched to the run clock right after,
// so catch-up advances it 0 → clock and it rejoins in lockstep.
let mut state = NetState::new(&topo, 0, Some(lineage))?;
state.entered_at_step = clock;
let _net_device = self.config.device();
let net = Network::build(&topo, self.config.device())?;
state.stamp_meta(&net, &self.meta_ctx);
let optimizer = self.trainer.make_optimizer(&net);
Ok(RaceChild {
state,
net,
optimizer,
})
}
// ── Catch-up ────────────────────────────────────────────────────────────
/// Catch-up a newcomer by replaying the shared stream solo for steps
/// `0..clock`. Each step calls the same `train_one_step` and
/// `eval_one_step` plus `seed_step_randomness` as the group loop, so the
/// child reproduces exactly what a group net would have seen at each
/// step — that's the replay contract.
///
/// The child's network is rebuilt from its topology + seed at insertion
/// time (in `clone_fittest_parent`), so catch-up starts from fresh weights
/// (the same starting point the group nets had at step 0). This is correct:
/// a group net also started fresh at step 0 and saw the same batch sequence.
///
/// Writes the child's state file **once at the end** (with the final step
/// and last metrics). During catch-up the child is not live in the population,
/// so no one reads its file per-step; for Item-6 resume, the recorded
/// `step` field is what matters, not intermediate writes. The asymmetry
/// with the group loop (which writes every step) is noted — Item 6 may
/// make catch-up write per-step too for full parity, but it's not required
/// for Item 4's determinism exit criterion.
pub(crate) fn catch_up(&mut self, child: &mut RaceChild, clock: usize) -> Result<()> {
self.catch_up_range(child, 0, clock)
}
/// Incremental catch-up: replay steps `from..to` (half-open). Used by the
/// checkpoint-gated evolution path to evaluate the child at each gate
/// without replaying the whole history — `from` is where the child's
/// replay currently stands, `to` the next gate (or the clock).
pub(crate) fn catch_up_range(
&mut self,
child: &mut RaceChild,
from: usize,
to: usize,
) -> Result<()> {
for step in from..to {
// The child replays through the caller's training scheme — the
// same contract group-step nets use, so the recipe is identical.
let run_data = crate::trainer::RunData {
dataset: &self.dataset,
stream: &self.stream,
};
let ctx = crate::trainer::StepContext {
data: Some(&run_data),
fitness: Some(&self.fitness),
metrics: &self.metrics,
env: crate::trainer::StepEnv {
step,
run_seed: self.header.run_seed,
pop_size: self.config.pop_size,
live_count: self.state.live_count(),
checkpoint_every: self.config.checkpoint_every,
smoothing_window: crate::engine::smoothing::SMOOTHING_WINDOW,
},
net_hash: &child.state.hash,
net_seed: child.state.net_seed as u64,
};
let report =
self.trainer
.train_step(&mut child.net, &mut *child.optimizer, step, &ctx)?;
let metrics = NetMetrics {
step,
train_loss: report.train_loss,
eval_loss: report.eval_loss,
fitness: report.fitness,
informative: report.informative,
};
child.state.record_metrics(metrics.clone());
child.state.advance_step();
// Keep the rolling buffer in sync (catch-up steps count toward the
// child's smoothed fitness once it rejoins).
if let Some(buf) = self.rolling_fitness.get_mut(&child.state.hash) {
buf.push(metrics.fitness);
}
// Per-step catch-up lines are debug-only — the user-facing log
// is one "catch-up: N steps … done" message from the caller.
log::debug!(
"catch-up step {}: net {} train_loss↓ {:.4} eval_loss↓ {:?} fitness{} {:.4}",
step,
child.state.hash,
metrics.train_loss,
metrics.eval_loss,
self.fitness.direction().arrow(),
metrics.fitness,
);
}
// Write the child's final caught-up state once. (Per-step writes during
// catch-up are a later nicety — see the function doc.)
write_net_state(&self.run_dir, &child.state)?;
Ok(())
}
/// Iter-6 Tier B: rebuild a net from its persisted `NetState` and replay
/// steps `0..step`, asserting **metric parity** with the recorded
/// `last_metrics`.
///
/// Resume = replay: coefficients are never persisted, so a reconstructed
/// net must re-derive its entire training history from the deterministic
/// stream. If the replayed metrics differ from what was recorded, the
/// reconstruction is broken (seed drift, primitive change, config
/// mismatch) — fail loudly rather than silently diverge.
///
/// Returns a `RaceChild` ready for insertion into the live maps (same
/// shape `generate_child` + `catch_up` produce for newborns).
pub(crate) fn replay_loaded_net(&mut self, state: NetState) -> Result<RaceChild> {
let clock = state.step;
let recorded = state.last_metrics.clone();
let mut child = self.build_child_from_state(state)?;
self.catch_up(&mut child, clock)?;
if let (Some(recorded), Some(replayed)) = (recorded, child.state.last_metrics.clone()) {
if recorded != replayed {
return Err(flodl::tensor::TensorError::new(&format!(
"race resume parity failure: net {} at step {} — recorded \
train_loss={:?} eval_loss={:?} fitness={:.6}, replayed \
train_loss={:?} eval_loss={:?} fitness={:.6} — \
reconstruction is not bit-identical (seed/primitive/config drift)",
child.state.hash,
clock,
recorded.train_loss,
recorded.eval_loss,
recorded.fitness,
replayed.train_loss,
replayed.eval_loss,
replayed.fitness,
)));
}
debug!(
"parity ok: net {} replayed to step {} bit-identical",
child.state.hash, clock,
);
}
Ok(child)
}
/// Rebuild a `RaceChild` from a persisted `NetState`: same topology, same
/// recorded weight seed and lineage — but step reset to 0 so `catch_up`
/// can replay the full history. Unlike `finalize_child`, the seed is
/// **preserved**, never re-derived.
fn build_child_from_state(&self, state: NetState) -> Result<RaceChild> {
let topo = state.topology()?;
let net = Network::build(&topo, self.config.device())?;
let optimizer = self.trainer.make_optimizer(&net);
let rebuilt = NetState {
step: 0,
is_alive: true,
last_metrics: None,
// A revived net is alive again: clear any cull markers.
culled_at_step: None,
cull_reason: None,
final_smoothed_fitness: None,
// Keep the rest (hash, topology, net_seed, lineage, entry step, meta).
..state
};
Ok(RaceChild {
state: rebuilt,
net,
optimizer,
})
}
}