gras 2.0.1

A Genetic Programming framework for Neural Architecture Search in Rust
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
//! The deterministic shared batch stream for the step-race engine.

use flodl::Tensor;
use flodl::tensor::Result;

use crate::utils::seed::derive_seed;
use crate::utils::tabular_data::Dataset;

/// Per-step eval batches derive from `run_seed + EVAL_STREAM_OFFSET`, never
/// `run_seed` itself — train and eval randomness must not share a seed.
pub const EVAL_STREAM_OFFSET: u64 = 0xFFFF;

/// Deterministic seeded shuffle split: `train_ratio` of the rows train, the rest
/// evaluate. An empty eval side falls back to the whole pool (train/eval overlap).
fn seeded_split(len: usize, train_ratio: f32, eval_ratio: f32, seed: u64) -> (Vec<i64>, Vec<i64>) {
    assert!(
        (train_ratio + eval_ratio - 1.0).abs() < 1e-6,
        "ratios must sum to 1.0"
    );
    let mut indices: Vec<i64> = (0..len as i64).collect();
    let mut rng = fastrand::Rng::with_seed(seed);
    for i in (1..indices.len()).rev() {
        let j = rng.usize(0..=i);
        indices.swap(i, j);
    }
    let split = (len as f32 * train_ratio).round() as usize;
    let train: Vec<i64> = indices[..split].to_vec();
    let eval: Vec<i64> = indices[split..].to_vec();
    if eval.is_empty() {
        static WARNED: std::sync::Once = std::sync::Once::new();
        WARNED.call_once(|| {
            tracing::warn!(
                "eval set is empty (train_ratio={train_ratio}) — falling back to using ALL data for eval (train/eval overlap)"
            );
        });
        (train, indices.clone())
    } else {
        (train, eval)
    }
}

// ── PoolSplit (D12) ─────────────────────────────────────────────────────────

/// The static train/eval pool split, fixed once at run start: every net trains
/// the train pool and is evaluated on the eval pool — identical conditions.
#[derive(Clone, Debug)]
pub struct PoolSplit {
    /// Row indices nets train on (the stream draws from here).
    pub train: Vec<i64>,
    /// Row indices nets are evaluated on (eval batches draw from here) —
    /// held-out rows, so the reading uses every test row the run was given.
    pub eval: Vec<i64>,
}

impl PoolSplit {
    /// Split `dataset.len()` rows into train/eval pools with a seeded, deterministic shuffle:
    /// `train_eval_split_ratio` of the rows train, the rest are the eval pool.
    pub fn new(len: usize, train_eval_split_ratio: f32, seed: u64) -> Self {
        let (train, eval) = seeded_split(
            len,
            1.0 - train_eval_split_ratio,
            train_eval_split_ratio,
            seed,
        );
        PoolSplit { train, eval }
    }

    /// Split from an existing dataset — convenience over [`PoolSplit::new`].
    pub fn of(dataset: &Dataset, train_eval_split_ratio: f32, seed: u64) -> Self {
        Self::new(dataset.len(), train_eval_split_ratio, seed)
    }

    /// Split from an EXPLICIT train/test layout: `train` = all train rows;
    /// `eval` = a seeded shuffle of all test rows.
    pub fn explicit(train_len: usize, test_len: usize, seed: u64) -> Self {
        assert!(test_len > 0, "explicit split needs a non-empty test pool");
        let train: Vec<i64> = (0..train_len as i64).collect();
        let mut eval: Vec<i64> = (train_len as i64..(train_len + test_len) as i64).collect();
        // Seeded shuffle of the TEST rows only (the eval permutation re-shuffles per
        // era anyway, but this keeps the raw pool order seed-dependent).
        let mut rng = fastrand::Rng::with_seed(seed);
        for i in (1..eval.len()).rev() {
            let j = rng.usize(0..=i);
            eval.swap(i, j);
        }
        PoolSplit { train, eval }
    }
}

// ── BatchStream ─────────────────────────────────────────────────────────────

/// The shared, deterministic, infinite batch stream: batch for step *s*
/// `f(run_seed, s)` — no cursor, no state — so resume replays exactly.
#[derive(Clone, Debug)]
pub struct BatchStream {
    run_seed: u64,
    batch_size: usize,
    eval_batch_size: usize,
    checkpoint_every: usize,
    train_pool: Vec<i64>,
    eval_pool: Vec<i64>,
}

impl BatchStream {
    /// Build a stream over a run's pools. Holds only indices — rows are gathered
    /// at materialization time, so the stream is cheap to clone per net.
    pub fn new(run_seed: u64, batch_size: usize, split: PoolSplit) -> Self {
        BatchStream {
            run_seed,
            batch_size,
            eval_batch_size: batch_size,
            checkpoint_every: 100, // default, can be overridden via with_checkpoint_every
            train_pool: split.train,
            eval_pool: split.eval,
        }
    }

    /// Override the eval batch size (trainer's `stream_shape` hook). Train and eval
    /// walk separate permutations, so the two sizes are independent.
    pub fn with_eval_batch_size(mut self, eval_batch_size: usize) -> Self {
        self.eval_batch_size = eval_batch_size.max(1);
        self
    }

    /// Set the checkpoint cadence — the eval permutation rotates per era, so it must agree
    /// engine's `checkpoint_every`.
    pub fn with_checkpoint_every(mut self, n: usize) -> Self {
        self.checkpoint_every = n;
        self
    }

    /// In-place variant of [`Self::with_checkpoint_every`], called by the engine at run start /
    /// resume so a config edit can't desync the eval rotation.
    pub fn set_run_checkpoint_every(&mut self, n: usize) {
        self.checkpoint_every = n;
    }

    /// Train batch size the stream materializes.
    pub fn batch_size(&self) -> usize {
        self.batch_size
    }

    /// The checkpoint cadence this stream was built with — the eval
    /// permutation rotates per era = `step / checkpoint_every` (`0` = never rotate).
    pub fn checkpoint_every(&self) -> usize {
        self.checkpoint_every
    }

    /// Eval batch size the stream materializes.
    pub fn eval_batch_size(&self) -> usize {
        self.eval_batch_size
    }

    /// The pool index vectors this stream draws from (train, eval) — read-only
    /// view; never mutated after construction.
    pub fn pools(&self) -> (&[i64], &[i64]) {
        (&self.train_pool, &self.eval_pool)
    }

    /// The share of the rows this stream TRAINS on (`train ÷ (train + eval)`), i.e.
    pub fn train_share(&self) -> f32 {
        self.train_pool.len() as f32 / (self.train_pool.len() + self.eval_pool.len()) as f32
    }

    /// Gather `indices` into one `(inputs, targets)` batch tensor pair.
    fn gather(dataset: &Dataset, indices: &[i64]) -> Result<(Tensor, Tensor)> {
        let idx = Tensor::from_i64(indices, &[indices.len() as i64], dataset.inputs.device())?;
        let x = dataset.inputs.index_select(0, &idx)?;
        let y = dataset.targets.index_select(0, &idx)?;
        Ok((x, y))
    }

    /// A contiguous window of `batch_size` pool rows starting at `offset`
    /// (wrapping): one permutation consumed round-robin.
    fn window(pool: &[i64], batch_size: usize, offset: usize) -> Vec<i64> {
        let n = pool.len();
        (0..batch_size).map(|i| pool[(offset + i) % n]).collect()
    }

    /// The **train** batch for `step` — the clock's own pass; see [`Self::train_batch_at`].
    pub fn train_batch(&self, dataset: &Dataset, step: u64) -> Result<(Tensor, Tensor)> {
        self.train_batch_at(dataset, step, 0)
    }

    /// The **train** batch for `step`'s `attempt`: `0` is the clock's own pass, `1..` a recovery retry on `set_pop_recovery_same_seed(false)`, each one batch further along the…
    pub fn train_batch_at(
        &self,
        dataset: &Dataset,
        step: u64,
        attempt: u64,
    ) -> Result<(Tensor, Tensor)> {
        // One seeded permutation of the train pool, derived from the run
        // seed only — steps walk it round-robin via a step-derived offset.
        let perm_seed = derive_seed(self.run_seed, 0);
        let mut perm = self.train_pool.clone();
        let mut rng = fastrand::Rng::with_seed(perm_seed);
        for i in (1..perm.len()).rev() {
            let j = rng.usize(0..=i);
            perm.swap(i, j);
        }
        // A retry takes the NEXT window, so it sees rows the clock's own pass did not.
        let offset = (step as usize)
            .wrapping_mul(self.batch_size)
            .wrapping_add((attempt as usize).wrapping_mul(self.batch_size));
        Self::gather(dataset, &Self::window(&perm, self.batch_size, offset))
    }

    /// The eval permutation for `era` — re-shuffled per checkpoint; pure function
    /// of `(run_seed, era)`; era 0 matches the legacy fixed permutation.
    fn eval_permutation(&self, era: u64) -> Vec<i64> {
        let perm_seed = derive_seed(self.run_seed.wrapping_add(EVAL_STREAM_OFFSET), era as usize);
        let mut perm = self.eval_pool.clone();
        let mut rng = fastrand::Rng::with_seed(perm_seed);
        for i in (1..perm.len()).rev() {
            let j = rng.usize(0..=i);
            perm.swap(i, j);
        }
        perm
    }

    /// The era a step belongs to (steps in the same era share one eval permutation — the rotation
    /// boundary is the engine's checkpoint).
    fn era_of(&self, step: u64) -> u64 {
        if self.checkpoint_every == 0 {
            return 0;
        }
        (step as usize / self.checkpoint_every) as u64
    }

    /// The **eval** batch for `step` — the held-out batch every live net is scored
    /// on. Rotated per checkpoint era; pure in `(run_seed, step)`.
    pub fn eval_batch(&self, dataset: &Dataset, step: u64) -> Result<(Tensor, Tensor)> {
        let perm = self.eval_permutation(self.era_of(step));
        let offset = (step as usize).wrapping_mul(self.eval_batch_size);
        Self::gather(dataset, &Self::window(&perm, self.eval_batch_size, offset))
    }
}

// ── Tests ───────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {
    use super::*;
    use flodl::Device;

    fn small_dataset() -> Dataset {
        // 64 rows, 2 features, 2 classes — deterministic, tiny.
        crate::utils::tabular_data::synthetic_classification(64, 2, 2, 7, Device::CPU).unwrap()
    }

    #[test]
    fn pool_split_is_disjoint_and_covering() {
        let ds = small_dataset();
        let split = PoolSplit::of(&ds, 0.25, 42);
        assert_eq!(split.train.len() + split.eval.len(), ds.len());
        let mut seen = split.train.clone();
        seen.extend(split.eval.clone());
        seen.sort();
        seen.dedup();
        assert_eq!(seen.len(), ds.len(), "pools must not overlap");
        // Every held-out row is evaluated — no row is set aside from the reading.
        assert_eq!(split.eval.len(), ds.len() - split.train.len());
    }

    #[test]
    fn pool_split_is_deterministic() {
        let a = PoolSplit::new(100, 0.2, 123);
        let b = PoolSplit::new(100, 0.2, 123);
        let c = PoolSplit::new(100, 0.2, 124);
        assert_eq!(a.train, b.train);
        assert_eq!(a.eval, b.eval);
        assert_ne!(a.train, c.train, "different seed ⇒ different split");
    }

    #[test]
    fn explicit_split_separates_layouts_by_construction() {
        let split = PoolSplit::explicit(80, 20, 42);
        // Train pool = EXACTLY the train rows, in order, unshuffled.
        assert_eq!(
            split.train,
            (0..80).collect::<Vec<i64>>(),
            "every train row is a training row"
        );
        // Eval = EXACTLY the test rows — disjoint from train by construction,
        // covering all of them.
        let mut held: Vec<i64> = split.eval.clone();
        held.sort();
        assert_eq!(held, (80..100).collect::<Vec<i64>>());
        // Seeded: same (train_len, test_len, seed) ⇒ same split; a different
        // seed re-shuffles the test side
        let again = PoolSplit::explicit(80, 20, 42);
        assert_eq!(split.eval, again.eval);
        let other = PoolSplit::explicit(80, 20, 43);
        assert_ne!(split.eval, other.eval, "seed re-shuffles the test pool");
        // The shuffled side still covers exactly the test rows.
        let mut other_held: Vec<i64> = other.eval.clone();
        other_held.sort();
        assert_eq!(other_held, (80..100).collect::<Vec<i64>>());
    }

    #[test]
    #[should_panic(expected = "non-empty test pool")]
    fn explicit_split_rejects_empty_test_pool() {
        let _ = PoolSplit::explicit(80, 0, 42);
    }

    #[test]
    fn stream_batch_is_pure_function_of_seed_and_step() {
        let ds = small_dataset();
        let mk = |seed| BatchStream::new(seed, 8, PoolSplit::of(&ds, 0.25, seed));
        let s = mk(99);
        // Same (seed, step) ⇒ identical batch, however many times it's asked.
        let (x1, y1) = s.train_batch(&ds, 5).unwrap();
        let (x2, y2) = s.train_batch(&ds, 5).unwrap();
        assert_eq!(x1.to_f32_vec().unwrap(), x2.to_f32_vec().unwrap());
        assert_eq!(y1.to_f32_vec().unwrap(), y2.to_f32_vec().unwrap());
        // A second stream with the same seed materializes the same batch.
        let (x3, _) = mk(99).train_batch(&ds, 5).unwrap();
        assert_eq!(x1.to_f32_vec().unwrap(), x3.to_f32_vec().unwrap());
        // Different step ⇒ different batch (stream advances).
        let (x4, _) = s.train_batch(&ds, 6).unwrap();
        assert_ne!(x1.to_f32_vec().unwrap(), x4.to_f32_vec().unwrap());
        // Different run_seed ⇒ different batch (runs are isolated).
        let (x5, _) = mk(100).train_batch(&ds, 5).unwrap();
        assert_ne!(x1.to_f32_vec().unwrap(), x5.to_f32_vec().unwrap());
    }

    #[test]
    fn eval_stream_is_independent_of_train_stream() {
        let ds = small_dataset();
        let s = BatchStream::new(99, 8, PoolSplit::of(&ds, 0.25, 99));
        let (tx, _) = s.train_batch(&ds, 0).unwrap();
        let (ex, _) = s.eval_batch(&ds, 0).unwrap();
        // Eval comes from the eval pool — a train batch row set must never
        // appear as the eval batch at the same step.
        assert_ne!(tx.to_f32_vec().unwrap(), ex.to_f32_vec().unwrap());
    }

    /// A recovery retry walks ONE batch further: attempt 0 is the clock's own draw, so the
    /// default streak re-trains the same rows bit for bit.
    #[test]
    fn retry_attempt_walks_the_next_batch() {
        let ds = small_dataset();
        let s = BatchStream::new(99, 8, PoolSplit::of(&ds, 0.25, 99));
        let (own, _) = s.train_batch(&ds, 5).unwrap();
        let (zero, _) = s.train_batch_at(&ds, 5, 0).unwrap();
        assert_eq!(
            own.to_f32_vec().unwrap(),
            zero.to_f32_vec().unwrap(),
            "attempt 0 IS the clock's own draw"
        );
        let (first, _) = s.train_batch_at(&ds, 5, 1).unwrap();
        assert_ne!(
            own.to_f32_vec().unwrap(),
            first.to_f32_vec().unwrap(),
            "a retry on a fresh draw trains different rows"
        );
        assert_eq!(
            first.to_f32_vec().unwrap(),
            s.train_batch_at(&ds, 5, 1).unwrap().0.to_f32_vec().unwrap(),
            "pure function of (seed, step, attempt) — a replay asks again"
        );
    }

    #[test]
    fn stream_wraps_around_pool() {
        let ds = small_dataset();
        // Train pool is 64 * 0.75 = 48 rows; batch 8 ⇒ 6 steps per lap.
        let s = BatchStream::new(99, 8, PoolSplit::of(&ds, 0.25, 99));
        let (x0, _) = s.train_batch(&ds, 0).unwrap();
        let (x6, _) = s.train_batch(&ds, 6).unwrap();
        assert_eq!(
            x0.to_f32_vec().unwrap(),
            x6.to_f32_vec().unwrap(),
            "step 6 wraps to the lap-1 start"
        );
    }

    /// `0` is a real cadence, not a clamp: the eval permutation must stay frozen
    /// era 0 forever (and `era_of` must not divide by zero).
    #[test]
    fn zero_cadence_freezes_the_eval_permutation() {
        let ds = small_dataset();
        let s = BatchStream::new(7, 8, PoolSplit::of(&ds, 0.25, 7))
            .with_eval_batch_size(4)
            .with_checkpoint_every(0);
        assert_eq!(s.checkpoint_every(), 0);
        // Every step is era 0 — including steps that would cross many boundaries
        // cadence 1 (where step 5 would be era 5).
        for step in [0u64, 1, 5, 4096] {
            let perm = s.eval_permutation(s.era_of(step));
            let frozen = s.eval_permutation(0);
            assert_eq!(perm, frozen, "step {step} must stay in the eternal era 0");
        }
        // Same batch for the same step (still pure), and the eval batch is unaffected
        // by the cadence knob beyond the permutation choice.
        assert_eq!(
            s.eval_batch(&ds, 3).unwrap().0.to_f32_vec().unwrap(),
            s.eval_batch(&ds, 3).unwrap().0.to_f32_vec().unwrap()
        );
        // A non-zero cadence still rotates (the knob kept its old meaning).
        let rotating = BatchStream::new(7, 8, PoolSplit::of(&ds, 0.25, 7))
            .with_eval_batch_size(4)
            .with_checkpoint_every(1);
        assert_ne!(
            rotating.eval_permutation(rotating.era_of(0)),
            rotating.eval_permutation(rotating.era_of(1))
        );
    }

    #[test]
    fn eval_permutation_rotates_per_era() {
        let ds = small_dataset();
        let s = BatchStream::new(99, 8, PoolSplit::of(&ds, 0.25, 99))
            .with_eval_batch_size(4)
            .with_checkpoint_every(3);
        // Same era ⇒ same eval batch for the same in-era offset.
        let (a0, _) = s.eval_batch(&ds, 0).unwrap();
        let (a1, _) = s.eval_batch(&ds, 1).unwrap();
        assert_ne!(
            a0.to_f32_vec().unwrap(),
            a1.to_f32_vec().unwrap(),
            "two steps in era 0 walk the same permutation"
        );
        // Era boundary ⇒ different permutation; the batch content changes.
        let (b0, _) = s.eval_batch(&ds, 3).unwrap();
        let (b0_again, _) = s.eval_batch(&ds, 3).unwrap();
        assert_eq!(
            b0.to_f32_vec().unwrap(),
            b0_again.to_f32_vec().unwrap(),
            "pure function of (run_seed, step)"
        );
        // Era 0's first batch must differ from era 1's: compare row *sets* across
        // several windows so the flake probability is ~0.
        let era0_rows = (0..4usize)
            .map(|i| s.eval_batch(&ds, i as u64).unwrap().0.to_f32_vec().unwrap())
            .collect::<Vec<_>>()
            .concat();
        let era1_rows = (3..7usize)
            .map(|i| s.eval_batch(&ds, i as u64).unwrap().0.to_f32_vec().unwrap())
            .collect::<Vec<_>>()
            .concat();
        assert_ne!(
            era0_rows, era1_rows,
            "eval permutation must rotate across the era boundary"
        );
    }
}