use super::config::RaceConfig;
use super::core::{CoreEngine, CoreEngineParts, assert_trainer_blob_matches};
use crate::state::RunHeader;
use crate::trainer::stream::{BatchStream, PoolSplit};
use crate::utils::fitness::Fitness;
use flodl::tensor::Result;
pub struct TabularEngine(pub(crate) CoreEngine);
impl std::ops::Deref for TabularEngine {
type Target = CoreEngine;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for TabularEngine {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl TabularEngine {
pub fn from_spec(spec: crate::engine::run_spec::RunSpec) -> Result<Self> {
Ok(Self(CoreEngine::from_spec(spec)?))
}
pub fn from_tabular_spec<T: crate::trainer::TabularStep + 'static>(
spec: crate::engine::run_spec::TabularSpec<T>,
) -> Result<Self> {
let crate::engine::run_spec::TabularSpec {
data_dir,
config,
fitness,
trainer,
seed,
run_dir,
} = spec;
Ok(Self(CoreEngine::from_spec(
crate::engine::run_spec::RunSpec::tabular(
data_dir, config, fitness, trainer, seed, run_dir,
),
)?))
}
pub fn resume_from(
run_dir: impl Into<std::path::PathBuf>,
fitness: Fitness,
trainer: impl crate::trainer::TabularStep + 'static,
) -> Result<Self> {
let run_dir = run_dir.into();
let header = crate::state::load_engine_json(&run_dir)?;
let Some(data_dir) = header.data_dir.clone() else {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume_from: {} records no data_dir — it was written before the header kept one, so pass it explicitly with TabularEngine::resume(dir, data_dir, …)",
run_dir.display()
))
.into());
};
let mut config = RaceConfig::from_snapshot(&header.config);
CoreEngine::apply_header_search_space(&mut config, &header);
Self::resume(
run_dir,
std::path::PathBuf::from(data_dir),
config,
fitness,
trainer,
)
}
pub fn resume(
run_dir: std::path::PathBuf,
data_dir: std::path::PathBuf,
mut config: RaceConfig,
fitness: Fitness,
trainer: impl crate::trainer::TabularStep + 'static,
) -> Result<Self> {
let trainer = crate::trainer::AnyTrainer::tabular(trainer);
let header = crate::state::load_engine_json(&run_dir)?;
let persisted_counters = RunHeader {
culls: header.culls,
run_elapsed_secs: header.run_elapsed_secs,
children_born_at_clock: header.children_born_at_clock.clone(),
..header.clone()
};
match header.engine_mode.as_deref() {
Some("tabular") | None => {}
Some(other) => {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: {run_dir_display} was recorded as mode \"{other}\", not \"tabular\" — use RlEngine::resume for RL runs",
run_dir_display = run_dir.display(),
))
.into())
}
}
assert_trainer_blob_matches(&header.trainer, &trainer, "resume")?;
config
.validate_search_pools()
.map_err(crate::utils::error::EngineError::InvalidOptions)?;
CoreEngine::rewind_csvs_to_manifest(&run_dir)?;
let explicit_test_rows = header.explicit_test_rows;
let recorded_eval_split = header
.eval_split
.clone()
.or_else(|| explicit_test_rows.map(|_| "test".to_string()));
let roles = super::core::roles::resolve(&data_dir, config.device())?;
if roles.eval_split != recorded_eval_split {
let name = |s: &Option<String>| {
s.clone()
.unwrap_or_else(|| "none (one pool, split by eval_share)".to_string())
};
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: the run ranked on `{}` but {} now supplies `{}` — the eval pool is the row set the nets were selected against, so a re-laid-out data dir cannot continue the recorded race",
name(&recorded_eval_split),
data_dir.display(),
name(&roles.eval_split),
))
.into());
}
let exam = roles.exam;
match (exam.as_ref(), header.exam_rows) {
(Some(ds), Some(recorded)) if recorded != ds.len() => {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: header records exam_rows = {recorded} but {} holds {} rows — the exam set changed, so its reading would not be comparable to the recorded run's",
data_dir.display(),
ds.len()
))
.into())
}
(None, Some(recorded)) => {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: the run recorded an exam ({recorded} rows) but {} no longer supplies one",
data_dir.display(),
))
.into())
}
_ => {}
}
let Some(dataset) = roles.dataset else {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: {} supplied no dataset",
data_dir.display()
))
.into());
};
let mut recorded_metrics = Vec::with_capacity(header.informative_metrics.len());
for name in &header.informative_metrics {
match crate::trainer::menu::metric_for(name) {
Some(metric) => recorded_metrics.push(metric),
None => {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: engine.json records metric {name:?}, which this build no longer provides — the run wrote that column and cannot be continued without it"
))
.into())
}
}
}
let requested: Vec<String> = config
.metrics
.iter()
.map(|m| m.label().to_string())
.collect();
let recorded: Vec<String> = recorded_metrics
.iter()
.map(|m| m.label().to_string())
.collect();
if !requested.is_empty() && requested != recorded {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: metrics changed — the run recorded [{}] but this call asks for [{}]; the metric set is part of the record (it is history.csv's header), so pass the recorded set or start a new run",
recorded.join(", "),
requested.join(", ")
))
.into());
}
config.metrics = recorded_metrics.clone();
let metrics = recorded_metrics;
let run_seed = header.run_seed;
let header_stream_ratio = header.train_eval_split_ratio.unwrap_or(0.2); config.eval_share = header_stream_ratio;
let (batch_size, eval_batch_size) = match trainer.stream_shape() {
Some(shape) => (shape.batch_size, shape.eval_batch_size),
None => (16, 16), };
let split = match explicit_test_rows {
Some(test_len) => {
if test_len == 0 || test_len >= dataset.len() {
return Err(crate::utils::error::EngineError::InvalidOptions(format!(
"resume: header records explicit_test_rows = {test_len} but the dataset at {} holds {} rows — the run dir and data dir disagree",
data_dir.display(),
dataset.len(),
))
.into());
}
PoolSplit::explicit(dataset.len() - test_len, test_len, run_seed)
}
None => PoolSplit::of(&dataset, header_stream_ratio, run_seed),
};
let eval_era_every = header.eval_era_every.unwrap_or(config.checkpoint_every);
if header
.eval_era_every
.is_some_and(|recorded| recorded != config.checkpoint_every)
{
tracing::info!(
"resume: eval-permutation era from the record = {eval_era_every} clock(s); \
this call's checkpoint_every is {} (the tick cadence, not the era)",
config.checkpoint_every
);
}
let stream = Some(
BatchStream::new(run_seed, batch_size, split)
.with_eval_batch_size(eval_batch_size)
.with_checkpoint_every(eval_era_every),
);
let dataset = Some(dataset);
let log_level = config.log_level;
let meta_ctx = crate::engine::config::RunMetaCtx {
input_dim: header.input_dim,
output_dim: header.output_dim,
batch_size: Some(batch_size),
dropout_prob: header.topology_options.dropout_prob,
fitness_label: header.fitness_label.0.clone(),
direction: format!("{:?}", header.fitness_direction).to_lowercase(),
pop_size: config.pop_size,
run_seed,
};
let mut engine = CoreEngine::assemble(CoreEngineParts {
run_dir: run_dir.clone(),
header,
config,
meta_ctx,
stream,
dataset,
explicit_test_rows,
exam,
fitness,
metrics,
trainer,
log_level,
resumed: true,
});
engine.load_live_frontier()?;
engine.culls = persisted_counters.culls;
engine.children_born_at_clock = persisted_counters.children_born_at_clock;
engine.elapsed_base_secs = persisted_counters.run_elapsed_secs;
Ok(Self(engine))
}
}