1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
//! The reference [`TabularStep`]: the classic tabular recipe the race engine used to run inline,
//! extracted verbatim so `TabularTrainer` is a drop-in — identical RNG discipline, identical
use crate::graph::network::Network;
use crate::trainer::{TabularContext, TabularStep};
use crate::utils::race_steps::{deterministic_train_step, eval_one_step};
use flodl::nn::optim::Optimizer;
use flodl::tensor::Result;
use flodl::{Tensor, Variable};
/// Classic tabular scheme: one train step + one eval step on the step's shared
/// batches. Also owns the loss, LR (via `make_optimizer`), and grad clip.
pub struct TabularTrainer {
/// The loss this scheme trains against (supervised paradigm).
pub loss_fn: crate::trainer::BoxedLossFn,
/// Adam learning rate for every optimizer this scheme creates.
pub learning_rate: f32,
/// Gradient-norm clip applied per optimizer step (0 = off).
pub grad_clip: f32,
/// Custom training batch size requested from the engine stream.
pub batch_size: usize,
/// Custom evaluation batch size requested from the engine stream.
pub eval_batch_size: usize,
/// Optional LR schedule: pure function of the step clock (`None` = fixed LR);
/// MUST be deterministic in `step` alone — see `TabularStep::scheduled_lr`.
pub lr_schedule: Option<Box<dyn Fn(usize) -> f64 + Send + Sync>>,
/// A NAMED schedule (`cosine`, `onecycle`, …), which wins over the closure above when set.
pub lr_schedule_name: Option<crate::trainer::LrSchedule>,
/// Which update the optimizer makes (`adam` by default).
pub optimizer: crate::trainer::OptimizerKind,
/// What objective `loss_fn` optimizes, for the run record. The engine can't
/// see inside the closure, so replay tools refuse an unlabeled scheme.
pub loss_label: Option<String>,
}
impl TabularTrainer {
/// A tabular scheme is defined by its loss — constructor takes
pub fn new(
loss_fn: impl Fn(&Variable, &Variable) -> Result<Variable> + Send + Sync + 'static,
) -> Self {
Self {
loss_fn: Box::new(loss_fn),
..Self::default()
}
}
/// Name the loss this scheme trains against, for `engine.json` — a plain
/// name lets replay tools rebuild it; a custom one records the truth.
pub fn with_loss_label(mut self, label: impl Into<String>) -> Self {
self.loss_label = Some(label.into());
self
}
/// Set the Adam learning rate (default: the crate's historical 1e-3).
pub fn with_learning_rate(mut self, lr: f32) -> Self {
self.learning_rate = lr;
self
}
/// Set the gradient-norm clip (default 1.0; 0 disables clipping).
pub fn with_grad_clip(mut self, clip: f32) -> Self {
self.grad_clip = clip;
self
}
/// Set the training batch size (default 16).
pub fn with_batch_size(mut self, size: usize) -> Self {
self.batch_size = size;
self
}
/// Set the evaluation batch size (default 16).
pub fn with_eval_batch_size(mut self, size: usize) -> Self {
self.eval_batch_size = size;
self
}
/// Set an LR schedule: a pure function of the step clock returning the rate
/// for that step; the closure must be deterministic in `step` alone.
pub fn with_lr_schedule(
mut self,
schedule: impl Fn(usize) -> f64 + Send + Sync + 'static,
) -> Self {
self.lr_schedule = Some(Box::new(schedule));
self
}
/// Set a NAMED schedule — the one a config file, a CLI flag or a Python keyword can spell.
pub fn with_named_lr_schedule(mut self, schedule: crate::trainer::LrSchedule) -> Self {
self.lr_schedule_name = Some(schedule);
self
}
/// Choose the update by name (`adam` is the default).
pub fn with_optimizer(mut self, optimizer: crate::trainer::OptimizerKind) -> Self {
self.optimizer = optimizer;
self
}
}
impl Default for TabularTrainer {
fn default() -> Self {
Self {
// Fallback loss (plain cross-entropy) so `Default` stays
// available; real runs construct via `TabularTrainer::new(loss)`.
loss_fn: Box::new(crate::utils::score::cross_entropy_onehot_loss),
learning_rate: crate::utils::training::DEFAULT_LR,
grad_clip: 1.0,
batch_size: 16,
eval_batch_size: 16,
// Unlabeled by default: the engine cannot see inside `loss_fn`,
// silence is the honest record. `with_loss_label` names
loss_label: None,
lr_schedule: None,
lr_schedule_name: None,
optimizer: crate::trainer::OptimizerKind::default(),
}
}
}
impl TabularStep for TabularTrainer {
/// Reload `nets/<hash>.optim` into a fresh optimizer of the KIND this scheme trains
fn restore_optimizer(
&self,
net: &Network,
path: &std::path::Path,
) -> Option<Box<dyn Optimizer>> {
self.optimizer.load(net, self.learning_rate, path)
}
fn make_optimizer(&self, net: &Network) -> Box<dyn Optimizer> {
self.optimizer.make(net, self.learning_rate)
}
/// Record this scheme's training recipe in `engine.json` (see
/// [`TabularStep::describe`]).
fn describe(&self) -> Option<serde_json::Value> {
let mut blob = serde_json::json!({
"trainer": "tabular",
"optimizer": self.optimizer.label(),
"loss": self.loss_label,
"learning_rate": self.learning_rate,
"grad_clip": self.grad_clip,
"batch_size": self.batch_size,
"eval_batch_size": self.eval_batch_size,
// The BOOLEAN stays for compatibility: runs recorded before named schedules compare
// this key on resume, so changing it to a string would refuse every one of them.
"lr_schedule": self.lr_schedule.is_some() || self.lr_schedule_name.is_some(),
});
// Only a NAMED schedule gets a label, and only when one is set — so a run
// with no schedule writes exactly the blob it wrote before this existed.
if let Some(named) = &self.lr_schedule_name {
blob["lr_schedule_name"] = serde_json::Value::String(named.label());
}
Some(blob)
}
fn loss(&self) -> crate::trainer::LossFn<'_> {
&*self.loss_fn
}
/// A NAMED schedule scales the base rate; an anonymous closure is taken
/// word. Named wins, because it is the one that can be recorded and checked.
fn scheduled_lr(&self, step: usize, max_steps: Option<usize>) -> Option<f64> {
match (&self.lr_schedule_name, &self.lr_schedule) {
(Some(named), _) => Some(named.lr(self.learning_rate, step, max_steps) as f64),
(None, Some(anonymous)) => Some(anonymous(step)),
(None, None) => None,
}
}
fn stream_shape(&self) -> Option<crate::trainer::StreamShape> {
Some(crate::trainer::StreamShape {
batch_size: self.batch_size,
eval_batch_size: self.eval_batch_size,
})
}
fn train_step(
&mut self,
net: &mut Network,
optimizer: &mut dyn Optimizer,
step: usize,
ctx: &TabularContext<'_>,
) -> Result<crate::trainer::TabularStepReport> {
// The tabular recipe uses every engine handle: shared stream
// step's batches, loss + fitness + metrics for scoring.
let fitness = ctx.fitness;
let loss_fn: crate::trainer::LossFn<'_> = &*self.loss_fn; // our own loss
// call_index = hash of the net's name, so each net sees distinct dropout at one step (live nets vs catching-up children too).
let h: u64 = ctx
.net_hash
.bytes()
.fold(0u64, |a, b| a.wrapping_add(b as u64))
.wrapping_add(ctx.env.attempt);
let batch: (Tensor, Tensor) = ctx.data.train_batch_at(step as u64, ctx.env.attempt)?;
// LR schedule first (pure function of step — replay-safe),
// deterministic train step (seeds the RNG, then forward/backward).
if let Some(lr) = self.scheduled_lr(step, ctx.env.max_steps) {
optimizer.set_lr(lr);
}
let train_loss = deterministic_train_step(
ctx.net_seed,
step as u64,
h,
net,
optimizer,
loss_fn,
&batch,
self.grad_clip,
)?;
// Eval on the step's shared eval batch (held-out stream), never
// train batch — this must mirror replay exactly.
let eval_batch: (Tensor, Tensor) = ctx.data.eval_batch(step as u64)?;
let report = eval_one_step(net, loss_fn, fitness, ctx.metrics, &eval_batch)?;
Ok(crate::trainer::TabularStepReport {
train_loss,
eval_loss: report.eval_loss,
fitness: report.fitness,
informative: report.metrics,
})
}
}