use super::*;
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct KnobDrift {
pub(crate) label: &'static str,
pub(crate) recorded: String,
pub(crate) current: String,
pub(crate) undo: String,
}
impl CoreEngine {
pub(crate) fn behavioural_drift(&self) -> Vec<KnobDrift> {
let recorded = &self.header.config;
let current = &self.config;
let mut drift: Vec<KnobDrift> = Vec::new();
let mut push = |label: &'static str, rec: String, cur: String, undo: String| {
if rec != cur {
drift.push(KnobDrift {
label,
recorded: rec,
current: cur,
undo,
});
}
};
push(
"smoothing_window (K)",
recorded.smoothing_window.to_string(),
current.smoothing_window.to_string(),
format!(
"set_fitness_smoothing_window({})",
recorded.smoothing_window
),
);
push(
"policy_scoped_fitness",
recorded.policy_scoped_fitness.to_string(),
current.policy_scoped_fitness.to_string(),
format!(
"set_fitness_policy_scoped({})",
recorded.policy_scoped_fitness
),
);
let rec_mean = if recorded.mean_type.is_empty() {
crate::engine::config::DEFAULT_MEAN_TYPE.to_string()
} else {
recorded.mean_type.clone()
};
push(
"mean_type",
rec_mean.clone(),
current.mean_type.to_string(),
format!("set_fitness_mean_type({rec_mean})",),
);
let rec_selection = if recorded.crossover_selection.is_empty() {
SelectionMethod::Roulette.to_string()
} else {
recorded.crossover_selection.clone()
};
push(
"crossover_selection",
rec_selection.clone(),
current.crossover_selection.to_string(),
format!("set_crossover_selection({rec_selection})"),
);
push(
"train_merit_p_min (p_min)",
fmt_cfg(recorded.train_merit_p_min),
fmt_cfg(current.train_merit_p_min),
format!("set_ticket_p_min({})", fmt_cfg(recorded.train_merit_p_min)),
);
push(
"train_merit_p_max (p_max)",
fmt_cfg(recorded.train_merit_p_max),
fmt_cfg(current.train_merit_p_max),
format!("set_ticket_p_max({})", fmt_cfg(recorded.train_merit_p_max)),
);
let rec_legs = crate::engine::config::TrainTicket::parse(&recorded.train_ticket)
.unwrap_or(crate::engine::config::DEFAULT_TRAIN_TICKET);
push(
"train_ticket",
rec_legs.to_string(),
current.train_ticket.to_string(),
format!("set_train_ticket(\"{rec_legs}\")"),
);
push(
"train_merit_learn_only",
recorded.train_merit_learn_only.to_string(),
current.train_merit_learn_only.to_string(),
format!("set_ticket_learn_only({})", recorded.train_merit_learn_only),
);
push(
"crossover_fitter_base",
recorded.crossover_fitter_base.to_string(),
current.crossover_fitter_base.to_string(),
format!(
"set_crossover_fitter_base({})",
recorded.crossover_fitter_base
),
);
let rec_pool = if recorded.feature_split_pool.is_empty() {
vec![
recorded
.feature_split
.clone()
.unwrap_or_else(|| "none".to_string()),
]
} else {
recorded.feature_split_pool.clone()
};
push(
"drift_prob",
fmt_cfg(recorded.drift_prob),
fmt_cfg(current.drift_prob),
format!("set_topology_drift({})", fmt_cfg(recorded.drift_prob)),
);
push(
"mutation_novelty_min",
fmt_cfg(recorded.mutation_novelty_min),
fmt_cfg(current.mutation_novelty_min),
format!(
"set_mutation_novelty_min({})",
fmt_cfg(recorded.mutation_novelty_min)
),
);
push(
"mutation_novelty_draws",
recorded.mutation_novelty_draws.to_string(),
current.mutation_novelty_draws.to_string(),
format!(
"set_mutation_novelty_draws({})",
recorded.mutation_novelty_draws
),
);
push(
"feature_split_pool",
rec_pool.join(","),
current.feature_split_pool_names().join(","),
format!(
"set_topology_feature_split_pool([{}])",
rec_pool
.iter()
.map(|p| format!("\"{p}\""))
.collect::<Vec<_>>()
.join(", ")
),
);
for (label, setter, rec, cur) in [
(
"crossover_cull_policy",
"set_crossover_cull_policy",
recorded.crossover_cull_policy.clone(),
current.crossover_cull_policy.to_string(),
),
(
"mutation_cull_policy",
"set_mutation_cull_policy",
recorded.mutation_cull_policy.clone(),
current.mutation_cull_policy.to_string(),
),
] {
let rec = if rec.is_empty() {
"lowest_fitness".to_string()
} else {
rec
};
push(label, rec.clone(), cur, format!("{setter}({rec})"));
}
for (label, setter, rec, cur) in [
(
"mutation_probation_steps",
"set_mutation_probation_steps",
recorded.mutation_probation_steps,
current.mutation_probation_steps,
),
(
"crossover_probation_steps",
"set_crossover_probation_steps",
recorded.crossover_probation_steps,
current.crossover_probation_steps,
),
(
"init_probation_steps",
"set_pop_init_probation_steps",
recorded.init_probation_steps,
current.init_probation_steps,
),
(
"recovery_retries",
"set_pop_recovery_retries",
recorded.recovery_retries,
current.recovery_retries,
),
] {
push(
label,
rec.to_string(),
cur.to_string(),
format!("{setter}({rec})"),
);
}
for (label, setter, rec, cur) in [
(
"crossover_share",
"set_crossover_share",
recorded.crossover_share,
current.crossover_share,
),
(
"mutation_share",
"set_mutation_share",
recorded.mutation_share,
current.mutation_share,
),
] {
push(
label,
fmt_cfg(rec),
fmt_cfg(cur),
format!("{setter}({})", fmt_cfg(rec)),
);
}
for (label, setter, rec, cur) in [(
"recovery_same_seed",
"set_pop_recovery_same_seed",
recorded.recovery_same_seed,
current.recovery_same_seed,
)] {
push(
label,
rec.to_string(),
cur.to_string(),
format!("{setter}({rec})"),
);
}
for (label, setter, rec, cur) in [(
"recovery_tolerance",
"set_pop_recovery_tol",
recorded.recovery_tolerance,
current.recovery_tolerance,
)] {
push(
label,
rec.to_string(),
cur.to_string(),
format!("{setter}({rec})"),
);
}
drift
}
pub(crate) fn load_live_frontier(&mut self) -> Result<()> {
for drift in self.behavioural_drift() {
tracing::warn!(
"race resume: {} differs from the record — the run recorded {}, this resume sets {} \
— the search from here is NOT the recorded one; pass {} to replay the recorded race",
drift.label,
drift.recorded,
drift.current,
drift.undo
);
}
let run_dir = self.run_dir.clone();
self.novelty.rebuild_from_nets(&run_dir);
let nets_dir = run_dir.join("nets");
let mut loaded = 0usize;
if let Some(manifest) = crate::state::load_frontier_manifest(&run_dir)? {
let history = Self::load_history_windows(&run_dir, self.config.smoothing_window);
let mut replayed: Vec<Vec<String>> = Vec::new();
let loaded = self.replay_manifest_frontier(&manifest, &history, &mut replayed)?;
self.report_resume(loaded, &replayed, &nets_dir, &run_dir)?;
return Ok(());
}
let entries = std::fs::read_dir(&nets_dir).map_err(|source| {
crate::utils::error::EngineError::Io {
path: nets_dir.display().to_string(),
source,
}
})?;
let mut replayed: Vec<Vec<String>> = Vec::new();
let history = Self::load_history_windows(&run_dir, self.config.smoothing_window);
for entry in entries {
let path = entry
.map_err(|source| crate::utils::error::EngineError::Io {
path: nets_dir.display().to_string(),
source,
})?
.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let raw = std::fs::read_to_string(&path).map_err(|source| {
crate::utils::error::EngineError::Io {
path: path.display().to_string(),
source,
}
})?;
let state = NetState::from_json(&raw)?;
if !state.is_alive {
tracing::debug!(
"race resume: skipping tombstone net {} (culled)",
state.hash
);
continue;
}
let entered = state.entered_at_step;
if entered > 0 {
if let Some(lm) = state.last_metrics.as_ref() {
if entered > lm.step + 1 {
tracing::warn!(
"race resume: skipping GHOST net {} (entered_at_step {} but last trained clock {} — a rejected child that leaked into nets/)",
state.hash,
entered,
lm.step
);
continue;
}
}
}
let clock = state
.last_metrics
.as_ref()
.map(|m| m.step)
.unwrap_or(state.step);
if let Some(trainer_state) = state.trainer_state.clone() {
self.trainer
.restore_trainer_state(&state.hash, &trainer_state);
}
let child = self.load_child_from_weights(state, clock)?;
let (hash, step) = self.insert_replayed(child, &history);
loaded += 1;
replayed.push(vec![hash, step.to_string()]);
}
self.report_resume(loaded, &replayed, &nets_dir, &run_dir)?;
Ok(())
}
pub(crate) fn replay_manifest_frontier(
&mut self,
manifest: &crate::state::FrontierManifest,
history: &HashMap<String, crate::engine::smoothing::HistoryReplay>,
replayed: &mut Vec<Vec<String>>,
) -> Result<usize> {
if manifest.run_seed != self.header.run_seed {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: ckpt.json belongs to run seed {} but engine.json records {} — the run \
dir mixes two runs",
manifest.run_seed, self.header.run_seed
))
.into());
}
let nets_dir = self.run_dir.join("nets");
if manifest.nets.len() != self.config.pop_size {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: expected {} live nets in population (per config.pop_size), but found \
{} in ckpt.json — pass the pop size this run used, or start a fresh race",
self.config.pop_size,
manifest.nets.len()
))
.into());
}
let mut loaded = 0usize;
for entry in &manifest.nets {
let path = nets_dir.join(format!("{}.json", entry.hash));
let raw = std::fs::read_to_string(&path).map_err(|source| {
crate::utils::error::EngineError::InvalidOptions(format!(
"resume: ckpt.json lists net {} at clock {} but {} is unreadable ({source}) — \
the frontier snapshot is incomplete",
entry.hash,
entry.step,
path.display()
))
})?;
let mut state = NetState::from_json(&raw)?;
if state.hash != entry.hash {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: {} holds net {} but ckpt.json lists it as {} — the files and the \
manifest disagree",
path.display(),
state.hash,
entry.hash
))
.into());
}
let file_clock = state
.last_metrics
.as_ref()
.map(|m| m.step)
.unwrap_or(state.step);
if file_clock < entry.step {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: net {} is recorded at clock {file_clock} but ckpt.json says the \
frontier was at {} — the net files predate the manifest (a checkpoint \
write failed?); this run cannot be resumed exactly",
entry.hash, entry.step
))
.into());
}
if file_clock > entry.step {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: net {} on disk is at clock {file_clock}, past the checkpoint's \
{} — its weights are newer than the snapshot and cannot be rewound; \
resume from a later checkpoint (the stop snapshot) instead",
entry.hash, entry.step
))
.into());
}
state.is_alive = true;
if let Some(trainer_state) = state.trainer_state.clone() {
self.trainer
.restore_trainer_state(&state.hash, &trainer_state);
}
let child = self.load_child_from_weights(state, entry.step)?;
let (hash, step) = self.insert_replayed(child, history);
replayed.push(vec![hash, format!("{step} (manifest {})", entry.step)]);
loaded += 1;
}
Ok(loaded)
}
pub(crate) fn load_child_from_weights(
&self,
state: NetState,
_clock: usize,
) -> Result<crate::engine::child::RaceChild> {
let topo = state.topology()?;
let mut net = crate::graph::network::Network::build(&topo, self.config.device())?;
let nets = self.run_dir.join("nets");
let weights = nets.join(format!("{}.safetensors", state.hash));
crate::utils::safetensors::load_safetensors(&mut net, &weights).map_err(|e| {
crate::utils::error::EngineError::InvalidOptions(format!(
"resume: net {} has no loadable weights ({}): {e} — the checkpoint is \
incomplete (a failed tick write?)",
state.hash,
weights.display()
))
})?;
let optim = nets.join(format!("{}.optim", state.hash));
let optimizer = if self.config.save_optim && optim.exists() {
match self.trainer.restore_optimizer(&net, &optim) {
Some(o) => o,
None => {
tracing::warn!(
"resume: {} exists but this trainer cannot restore optimizer state \
— the moments restart (option A)",
optim.display()
);
self.trainer.make_optimizer(&net)
}
}
} else {
self.trainer.make_optimizer(&net)
};
Ok(crate::engine::child::RaceChild {
state,
net,
optimizer,
cx: None,
})
}
pub(crate) fn insert_replayed(
&mut self,
child: crate::engine::child::RaceChild,
history: &HashMap<String, crate::engine::smoothing::HistoryReplay>,
) -> (String, usize) {
let hash = child.state.hash.clone();
let step = child
.state
.last_metrics
.as_ref()
.map(|m| m.step)
.unwrap_or(child.state.step);
let state = child.state.clone();
self.state.insert(state, None);
self.networks.insert(hash.clone(), child.net);
self.optimizers.insert(hash.clone(), child.optimizer);
let mut buf = RollingBuffer::new(self.config.smoothing_window);
let mut train_buf = RollingBuffer::new(self.config.smoothing_window);
let mut eval_buf = RollingBuffer::new(self.config.smoothing_window);
let mut trend = crate::engine::smoothing::ReadingTrend::default();
let mut metric_bufs = self.metric_buffers();
match history.get(&hash) {
Some(replay) if !replay.window.is_empty() => {
for (fitness, train_loss, eval_loss) in &replay.window {
buf.push(*fitness);
train_buf.push(*train_loss);
if let Some(eval) = eval_loss {
eval_buf.push(*eval);
}
}
trend = replay.trend;
for (label, series) in &replay.metric_windows {
if let Some(i) = self.metrics.iter().position(|m| m.label == *label) {
for v in series {
metric_bufs[i].push(*v);
}
}
}
}
_ => {
if let Some(m) = self.state.net(&hash).and_then(|s| s.last_metrics.as_ref()) {
buf.push(m.fitness);
train_buf.push(m.train_loss);
if let Some(e) = m.eval_loss {
eval_buf.push(e);
}
trend.push(m.fitness);
for (i, v) in m.informative.iter().enumerate() {
if let Some(buf) = metric_bufs.get_mut(i) {
buf.push(*v);
}
}
}
}
}
let epoch = buf.len();
self.rolling_fitness.insert(hash.clone(), buf);
self.rolling_train.insert(hash.clone(), train_buf);
self.rolling_eval.insert(hash.clone(), eval_buf);
self.rolling_metrics.insert(hash.clone(), metric_bufs);
self.rolling_epoch.insert(hash.clone(), epoch);
self.rolling_trend.insert(hash.clone(), trend);
(hash, step)
}
pub(crate) fn report_resume(
&self,
loaded: usize,
replayed: &[Vec<String>],
_nets_dir: &std::path::Path,
run_dir: &std::path::Path,
) -> Result<()> {
if loaded != self.config.pop_size {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: expected {} live nets in population (per config.pop_size), but found \
{} live nets in {}",
self.config.pop_size,
loaded,
run_dir.display()
))
.into());
}
crate::utils::cfy::emit(&crate::utils::cfy::card(
"Race resume",
&crate::utils::cfy::table(&["hash", "replayed step"], replayed),
Some(&format!("loaded {loaded} nets from {}", run_dir.display())),
));
Ok(())
}
pub(crate) fn load_history_windows(
run_dir: &std::path::Path,
window: usize,
) -> HashMap<String, crate::engine::smoothing::HistoryReplay> {
let Ok(raw) = std::fs::read_to_string(run_dir.join("history.csv")) else {
return HashMap::new();
};
let mut lines = raw.lines();
let header = lines.next().unwrap_or_default();
let cols = crate::engine::format::csv_split_line(header);
let find = |name: &str| cols.iter().position(|c| *c == name);
let (Some(hash_i), Some(train_i), Some(eval_i), Some(fit_i)) = (
find("hash"),
find("train_loss"),
find("eval_loss"),
find("fitness"),
) else {
return HashMap::new();
};
const FIXED: [&str; 10] = [
"step",
"hash",
"net_seed",
"origin",
"entered_at_step",
"train_loss",
"eval_loss",
"fitness",
"fitness_smooth",
"fitness_smooth_std",
];
let metric_cols: Vec<(usize, String)> = cols
.iter()
.enumerate()
.filter(|(_, c)| !FIXED.iter().any(|f| **c == *f))
.map(|(i, c)| (i, c.trim_start_matches("m:").to_string()))
.collect();
let mut out: HashMap<String, crate::engine::smoothing::HistoryReplay> = HashMap::new();
for line in lines {
let fields = crate::engine::format::csv_split_line(line);
let needed = hash_i.max(train_i).max(eval_i).max(fit_i);
if fields.len() <= needed {
continue;
}
let Ok(train_loss) = fields[train_i].parse::<f32>() else {
continue;
};
let Ok(fitness) = fields[fit_i].parse::<f32>() else {
continue;
};
let eval_loss = fields[eval_i].parse::<f32>().ok();
let replay = out.entry(fields[hash_i].to_string()).or_default();
replay.trend.push(fitness);
replay.window.push((fitness, train_loss, eval_loss));
if replay.window.len() > window {
replay.window.remove(0);
}
for (col, label) in &metric_cols {
let Some(value) = fields.get(*col).and_then(|f| f.parse::<f32>().ok()) else {
continue;
};
if !replay.metric_windows.iter().any(|(l, _)| l == label) {
replay.metric_windows.push((label.clone(), Vec::new()));
}
if let Some((_, series)) =
replay.metric_windows.iter_mut().find(|(l, _)| l == label)
{
series.push(value);
if series.len() > window {
series.remove(0);
}
}
}
}
out
}
}