Skip to main content

rusting_brain/
shards.rs

1//! Reading the preprocessed corpus the training loop is fed from.
2//!
3//! Two binaries write the corpus once: `examples/vit_tokens.rs` produces
4//! `tokens-*.safetensors`, one tensor of frozen image tokens per render, and
5//! `examples/mesh_samples.rs` produces `samples-*.safetensors` with a surface
6//! point cloud, a set of signed-distance queries, the normalizing transform
7//! and, for a mesh that has vertex colours, one colour per surface point. Both leave an `index.json` mapping a name to the shard holding
8//! it.
9//!
10//! This module joins the two sides and hands the training loop batches.
11//! Nothing is loaded until a batch asks for it: the shard header is read once
12//! and the tensor bytes are seeked to and widened on demand, which is what
13//! keeps a corpus larger than memory usable. A worker thread stays exactly one
14//! batch ahead, so the disk and the model overlap without the queue growing.
15//!
16//! ```no_run
17//! # use rusting_brain::shards::{Corpus, BatchConfig};
18//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
19//! let corpus = Corpus::open("tokens", "samples")?;
20//! println!("{} examples", corpus.len());
21//!
22//! for batch in corpus.stream(BatchConfig { batch: 4, ..BatchConfig::default() })? {
23//!     let batch = batch?;
24//!     // batch.tokens is [batch * 197, 768], batch.queries is [batch * n, 3].
25//!     let _ = batch.tokens.rows;
26//!     break;
27//! }
28//! # Ok(())
29//! # }
30//! ```
31//!
32//! NOTE: no mmap. `SafeTensors` already reads a header once and seeks for
33//! the bytes, which is what mmap would buy here minus a dependency and minus
34//! the page-fault behaviour on a network filesystem. If a profile ever shows
35//! the seek dominating, the change is inside `safetensors.rs`, not here.
36
37use crate::matrix::Matrix;
38use crate::network::NetworkError;
39use crate::safetensors::SafeTensors;
40use rand::rngs::StdRng;
41use rand::{Rng, SeedableRng};
42use std::collections::BTreeMap;
43use std::path::{Path, PathBuf};
44
45/// One directory of shards and the index that says which holds what.
46///
47/// The open shard is cached, because an index walked in order asks for the
48/// same file many times running and reopening it would read the header again
49/// each time.
50pub struct Shards {
51    root: PathBuf,
52    index: BTreeMap<String, String>,
53    open: Option<(String, SafeTensors)>,
54}
55
56impl Shards {
57    /// Reads `index.json` from a directory written by one of the
58    /// preprocessing binaries.
59    pub fn open<P: AsRef<Path>>(root: P) -> Result<Self, NetworkError> {
60        let root = root.as_ref().to_path_buf();
61        let path = root.join("index.json");
62        let index: BTreeMap<String, String> =
63            serde_json::from_slice(&std::fs::read(&path).map_err(|error| {
64                NetworkError::InvalidDataset(format!("{}: {error}", path.display()))
65            })?)?;
66        if index.is_empty() {
67            return Err(NetworkError::InvalidDataset(format!(
68                "{} lists no shards",
69                path.display()
70            )));
71        }
72        Ok(Self {
73            root,
74            index,
75            open: None,
76        })
77    }
78
79    /// The names in the index, in order.
80    pub fn names(&self) -> impl Iterator<Item = &str> {
81        self.index.keys().map(String::as_str)
82    }
83
84    pub fn len(&self) -> usize {
85        self.index.len()
86    }
87
88    pub fn is_empty(&self) -> bool {
89        self.index.is_empty()
90    }
91
92    pub fn contains(&self, name: &str) -> bool {
93        self.index.contains_key(name)
94    }
95
96    /// One tensor, by the name the index knows and the suffix the writer used.
97    ///
98    /// The tokens shards name a tensor after the image's stem alone; the
99    /// sample shards append `.surface`, `.queries` or `.transform`, which is
100    /// what `suffix` carries.
101    pub fn tensor(
102        &mut self,
103        name: &str,
104        suffix: &str,
105    ) -> Result<(Vec<f32>, Vec<usize>), NetworkError> {
106        let file = self.shard_of(name)?;
107        file.tensor(&format!("{name}{suffix}"))
108    }
109
110    /// Whether `{name}{suffix}` is in the shard, for the tensors that are
111    /// optional — a mesh written without vertex colours carries no `.colors`.
112    pub fn has(&mut self, name: &str, suffix: &str) -> bool {
113        let full = format!("{name}{suffix}");
114        self.shard_of(name)
115            .is_ok_and(|file| file.names().any(|tensor| tensor == full))
116    }
117
118    /// The open shard holding `name`, opening it if another one is open.
119    fn shard_of(&mut self, name: &str) -> Result<&mut SafeTensors, NetworkError> {
120        let shard = self.index.get(name).cloned().ok_or_else(|| {
121            NetworkError::InvalidDataset(format!("{}: nothing named {name}", self.root.display()))
122        })?;
123        if self.open.as_ref().map(|(open, _)| open.as_str()) != Some(shard.as_str()) {
124            let file = SafeTensors::open(self.root.join(&shard))?;
125            self.open = Some((shard, file));
126        }
127        let (_, file) = self.open.as_mut().expect("just opened");
128        Ok(file)
129    }
130
131    /// The same as a [`Matrix`], which is what everything downstream wants.
132    pub fn matrix(&mut self, name: &str, suffix: &str) -> Result<Matrix, NetworkError> {
133        let (values, shape) = self.tensor(name, suffix)?;
134        match shape.as_slice() {
135            [rows, cols] => Ok(Matrix::from_vec(*rows, *cols, values)),
136            _ => Err(NetworkError::InvalidDataset(format!(
137                "{name}{suffix} has shape {shape:?}, which is not a matrix"
138            ))),
139        }
140    }
141}
142
143/// One training example: a render's tokens and the mesh they describe.
144pub struct Example {
145    /// The render's name, which is what the tokens index knows it by.
146    pub name: String,
147    /// The mesh's name, which is what the samples index knows it by.
148    pub mesh: String,
149    /// `[tokens, cond_dim]`, frozen.
150    pub tokens: Matrix,
151    /// `[points, 6]`: position then normal.
152    pub surface: Matrix,
153    /// `[points, 3]`, linear RGB, in the same order as `surface`. `None` when
154    /// the mesh was written without vertex colours.
155    pub colors: Option<Matrix>,
156    /// `[queries, 3]`.
157    pub queries: Matrix,
158    /// One signed distance per query row.
159    pub distances: Vec<f32>,
160}
161
162/// A batch of examples, stacked into the packed layout the models read.
163pub struct Batch {
164    pub names: Vec<String>,
165    /// `[batch * tokens, cond_dim]`.
166    pub tokens: Matrix,
167    /// `[batch * points, 6]`.
168    pub surface: Matrix,
169    /// `[batch * points, 3]`, present only when every example in the batch
170    /// carries colours.
171    ///
172    /// NOTE: one mixed example drops the colour targets of the whole
173    /// batch. A corpus that is mostly coloured wants a per-example mask in the
174    /// colour loss instead; a corpus that is all one way or the other does not.
175    pub colors: Option<Matrix>,
176    /// `[batch * queries, 3]`.
177    pub queries: Matrix,
178    /// One per query row, in the same order.
179    pub distances: Vec<f32>,
180}
181
182impl Batch {
183    /// How many examples were stacked.
184    pub fn len(&self) -> usize {
185        self.names.len()
186    }
187
188    pub fn is_empty(&self) -> bool {
189        self.names.is_empty()
190    }
191}
192
193/// The two halves of the corpus, paired.
194pub struct Corpus {
195    tokens: Shards,
196    samples: Shards,
197    /// Render name to mesh name, resolved once at open.
198    pairs: Vec<(String, String)>,
199}
200
201impl Corpus {
202    /// Opens a token directory and a sample directory and pairs them up.
203    ///
204    /// A render is paired with the mesh of the same name, or — since a mesh is
205    /// usually rendered from several angles and the views are named
206    /// `{mesh}_000` — with the mesh named by everything before the last
207    /// underscore. A render that matches neither is skipped, because a corpus
208    /// half-built by an interrupted preprocessing run is the normal case and
209    /// refusing the whole directory over it would be useless.
210    pub fn open<T: AsRef<Path>, S: AsRef<Path>>(
211        tokens: T,
212        samples: S,
213    ) -> Result<Self, NetworkError> {
214        let tokens = Shards::open(tokens)?;
215        let samples = Shards::open(samples)?;
216
217        let pairs: Vec<(String, String)> = tokens
218            .names()
219            .filter_map(|render| {
220                let mesh = match samples.contains(render) {
221                    true => Some(render.to_string()),
222                    false => render
223                        .rsplit_once('_')
224                        .map(|(mesh, _)| mesh.to_string())
225                        .filter(|mesh| samples.contains(mesh)),
226                }?;
227                Some((render.to_string(), mesh))
228            })
229            .collect();
230
231        if pairs.is_empty() {
232            return Err(NetworkError::InvalidDataset(
233                "no render in the token index has a mesh in the sample index".into(),
234            ));
235        }
236        Ok(Self {
237            tokens,
238            samples,
239            pairs,
240        })
241    }
242
243    /// How many paired examples there are.
244    pub fn len(&self) -> usize {
245        self.pairs.len()
246    }
247
248    pub fn is_empty(&self) -> bool {
249        self.pairs.is_empty()
250    }
251
252    /// The render and mesh names, in index order.
253    pub fn pairs(&self) -> &[(String, String)] {
254        &self.pairs
255    }
256
257    /// Loads one example whole, subsampling the surface and the queries.
258    ///
259    /// A mesh carries far more of both than a step uses — 200,000 queries is
260    /// two orders of magnitude past a batch — so the subsample is what the
261    /// record was written large for. `None` takes everything.
262    pub fn example<R: Rng>(
263        &mut self,
264        index: usize,
265        surface_points: Option<usize>,
266        queries: Option<usize>,
267        rng: &mut R,
268    ) -> Result<Example, NetworkError> {
269        let (name, mesh) = self
270            .pairs
271            .get(index)
272            .cloned()
273            .ok_or_else(|| NetworkError::InvalidDataset(format!("no example {index}")))?;
274
275        let tokens = self.tokens.matrix(&name, "")?;
276        let written = self.samples.matrix(&mesh, ".surface")?;
277        // The colours have to keep the surface's rows, so both take the same
278        // draw rather than two independent ones.
279        let chosen = choose_rows(written.rows, surface_points, rng);
280        let surface = take_rows(&written, &chosen);
281        let colors = match self.samples.has(&mesh, ".colors") {
282            true => Some(take_rows(&self.samples.matrix(&mesh, ".colors")?, &chosen)),
283            false => None,
284        };
285        let packed = subsample(&self.samples.matrix(&mesh, ".queries")?, queries, rng);
286        if packed.cols != 4 {
287            return Err(NetworkError::InvalidDataset(format!(
288                "{mesh}.queries is [{}, {}] and should be [n, 4]",
289                packed.rows, packed.cols
290            )));
291        }
292
293        // The queries are written as position and distance in one row, which
294        // is one tensor instead of two on disk and a split here.
295        let mut queries = Matrix::new(packed.rows, 3);
296        let mut distances = Vec::with_capacity(packed.rows);
297        for row in 0..packed.rows {
298            queries.row_mut(row).copy_from_slice(&packed.row(row)[..3]);
299            distances.push(packed.row(row)[3]);
300        }
301
302        Ok(Example {
303            name,
304            mesh,
305            tokens,
306            surface,
307            colors,
308            queries,
309            distances,
310        })
311    }
312
313    /// The normalizing transform the preprocessing binary applied, as
314    /// `[center_x, center_y, center_z, scale]`.
315    pub fn transform(&mut self, mesh: &str) -> Result<[f32; 4], NetworkError> {
316        let (values, _) = self.samples.tensor(mesh, ".transform")?;
317        values
318            .get(..4)
319            .map(|values| [values[0], values[1], values[2], values[3]])
320            .ok_or_else(|| {
321                NetworkError::InvalidDataset(format!(
322                    "{mesh}.transform holds {} values, not 4",
323                    values.len()
324                ))
325            })
326    }
327
328    /// Batches, on a worker thread that stays one batch ahead.
329    ///
330    /// The stream ends after `config.epochs` passes, or never if that is
331    /// `None`, which is what a training loop that counts steps rather than
332    /// epochs wants.
333    pub fn stream(self, config: BatchConfig) -> Result<BatchStream, NetworkError> {
334        BatchStream::new(self, config)
335    }
336}
337
338/// `count` row numbers out of `rows`, without replacement, or `None` for all
339/// of them.
340fn choose_rows<R: Rng>(rows: usize, count: Option<usize>, rng: &mut R) -> Option<Vec<usize>> {
341    let count = count.filter(|count| *count < rows)?;
342    // A partial Fisher-Yates over the row numbers: the first `count` entries
343    // are a uniform sample, and nothing the size of the source is allocated
344    // twice.
345    let mut order: Vec<usize> = (0..rows).collect();
346    for slot in 0..count {
347        order.swap(slot, rng.gen_range(slot..rows));
348    }
349    order.truncate(count);
350    Some(order)
351}
352
353/// The rows [`choose_rows`] picked, or the whole matrix.
354fn take_rows(source: &Matrix, chosen: &Option<Vec<usize>>) -> Matrix {
355    let Some(chosen) = chosen else {
356        return source.clone();
357    };
358    let mut out = Matrix::new(chosen.len(), source.cols);
359    for (row, taken) in chosen.iter().enumerate() {
360        out.row_mut(row).copy_from_slice(source.row(*taken));
361    }
362    out
363}
364
365/// Keeps `count` rows of a matrix, chosen at random without replacement.
366fn subsample<R: Rng>(source: &Matrix, count: Option<usize>, rng: &mut R) -> Matrix {
367    take_rows(source, &choose_rows(source.rows, count, rng))
368}
369
370/// How a [`BatchStream`] is built.
371#[derive(Clone, Copy, Debug)]
372pub struct BatchConfig {
373    pub batch: usize,
374    /// Surface points per example. `None` takes every one written.
375    pub surface_points: Option<usize>,
376    /// Distance queries per example. `None` takes every one written.
377    pub queries: Option<usize>,
378    /// Passes over the corpus. `None` never stops.
379    pub epochs: Option<usize>,
380    /// Whether each pass visits the examples in a fresh random order.
381    pub shuffle: bool,
382    pub seed: u64,
383}
384
385impl Default for BatchConfig {
386    fn default() -> Self {
387        Self {
388            batch: 8,
389            surface_points: Some(2048),
390            queries: Some(4096),
391            epochs: None,
392            shuffle: true,
393            seed: 0,
394        }
395    }
396}
397
398/// Batches arriving from a worker thread, one ahead of the one being used.
399///
400/// Dropping the stream closes the channel, which is what tells the worker to
401/// stop; it notices on its next send rather than mid-read, so a drop costs at
402/// most one wasted batch.
403pub struct BatchStream {
404    receiver: std::sync::mpsc::Receiver<Result<Batch, NetworkError>>,
405    worker: Option<std::thread::JoinHandle<()>>,
406}
407
408impl BatchStream {
409    fn new(mut corpus: Corpus, config: BatchConfig) -> Result<Self, NetworkError> {
410        if config.batch == 0 {
411            return Err(NetworkError::InvalidConfig(
412                "a batch needs at least one example".into(),
413            ));
414        }
415        // A queue of one is the whole "decode one batch ahead" requirement:
416        // the worker blocks on the second send until the first is taken.
417        let (sender, receiver) = std::sync::mpsc::sync_channel(1);
418
419        let worker = std::thread::spawn(move || {
420            let mut rng = StdRng::seed_from_u64(config.seed);
421            let mut epoch = 0;
422            while config.epochs.is_none_or(|epochs| epoch < epochs) {
423                let mut order: Vec<usize> = (0..corpus.len()).collect();
424                if config.shuffle {
425                    let examples = order.len();
426                    for slot in 0..examples {
427                        order.swap(slot, rng.gen_range(slot..examples));
428                    }
429                }
430
431                for chunk in order.chunks(config.batch) {
432                    let batch = collect(&mut corpus, chunk, &config, &mut rng);
433                    let failed = batch.is_err();
434                    // A closed channel means the reader is gone, and a failed
435                    // read means the corpus is broken; either way this is the
436                    // last thing sent.
437                    if sender.send(batch).is_err() || failed {
438                        return;
439                    }
440                }
441                epoch += 1;
442            }
443        });
444
445        Ok(Self {
446            receiver,
447            worker: Some(worker),
448        })
449    }
450}
451
452impl Iterator for BatchStream {
453    type Item = Result<Batch, NetworkError>;
454
455    fn next(&mut self) -> Option<Self::Item> {
456        self.receiver.recv().ok()
457    }
458}
459
460impl Drop for BatchStream {
461    fn drop(&mut self) {
462        // Take the channel down first, so a worker blocked on a full queue
463        // wakes up instead of being waited on forever.
464        let (_, receiver) = std::sync::mpsc::sync_channel(0);
465        drop(std::mem::replace(&mut self.receiver, receiver));
466        if let Some(worker) = self.worker.take() {
467            let _ = worker.join();
468        }
469    }
470}
471
472/// Loads the examples of one batch and stacks them.
473fn collect<R: Rng>(
474    corpus: &mut Corpus,
475    indices: &[usize],
476    config: &BatchConfig,
477    rng: &mut R,
478) -> Result<Batch, NetworkError> {
479    let mut names = Vec::with_capacity(indices.len());
480    let mut loaded = Vec::with_capacity(indices.len());
481    for index in indices {
482        let example = corpus.example(*index, config.surface_points, config.queries, rng)?;
483        names.push(example.name.clone());
484        loaded.push(example);
485    }
486
487    // Every example in a batch has to be the same shape, or the packed layout
488    // the models read cannot say where one sequence ends.
489    let shape = |matrix: &Matrix| (matrix.rows, matrix.cols);
490    for example in &loaded[1..] {
491        for (name, (left, right)) in [
492            ("tokens", (shape(&example.tokens), shape(&loaded[0].tokens))),
493            (
494                "surface",
495                (shape(&example.surface), shape(&loaded[0].surface)),
496            ),
497            (
498                "queries",
499                (shape(&example.queries), shape(&loaded[0].queries)),
500            ),
501        ] {
502            if left != right {
503                return Err(NetworkError::InvalidDataset(format!(
504                    "{}: {name} is {left:?} but {} has {right:?}",
505                    example.name, loaded[0].name
506                )));
507            }
508        }
509    }
510
511    // All or nothing: a packed batch cannot say which of its rows have a
512    // colour target and which do not.
513    let colors = loaded
514        .iter()
515        .map(|example| example.colors.as_ref())
516        .collect::<Option<Vec<_>>>()
517        .map(|parts| stack(parts.into_iter()));
518
519    Ok(Batch {
520        names,
521        tokens: stack(loaded.iter().map(|example| &example.tokens)),
522        surface: stack(loaded.iter().map(|example| &example.surface)),
523        colors,
524        queries: stack(loaded.iter().map(|example| &example.queries)),
525        distances: loaded
526            .iter()
527            .flat_map(|example| example.distances.iter().copied())
528            .collect(),
529    })
530}
531
532/// Stacks matrices of equal width into one, rows in order.
533fn stack<'a>(parts: impl Iterator<Item = &'a Matrix> + Clone) -> Matrix {
534    let cols = parts.clone().next().map_or(0, |part| part.cols);
535    let mut data = Vec::new();
536    let mut rows = 0;
537    for part in parts {
538        data.extend_from_slice(&part.data);
539        rows += part.rows;
540    }
541    Matrix::from_vec(rows, cols, data)
542}
543
544#[cfg(test)]
545mod tests {
546    use super::*;
547    use crate::safetensors::{Dtype, write_as};
548
549    /// Writes a corpus the way the two preprocessing binaries do: a token
550    /// directory of one tensor per render, a sample directory of three per
551    /// mesh, and an `index.json` on each side. Two shards on the sample side,
552    /// so the open-shard cache is exercised rather than assumed.
553    fn corpus(name: &str) -> (PathBuf, PathBuf) {
554        let root = std::env::temp_dir().join(format!(
555            "rusting_brain_shards_{name}_{}",
556            std::process::id()
557        ));
558        let tokens = root.join("tokens");
559        let samples = root.join("samples");
560        let _ = std::fs::remove_dir_all(&root);
561        std::fs::create_dir_all(&tokens).unwrap();
562        std::fs::create_dir_all(&samples).unwrap();
563
564        let meshes = ["cube", "sphere"];
565        let renders = [
566            ("cube_000", "cube"),
567            ("cube_001", "cube"),
568            ("sphere", "sphere"),
569            ("orphan_000", ""),
570        ];
571
572        let mut token_index = BTreeMap::new();
573        let held: Vec<(String, Vec<f32>)> = renders
574            .iter()
575            .enumerate()
576            .map(|(number, (render, _))| {
577                let data = (0..5 * 4)
578                    .map(|value| (value + number * 100) as f32)
579                    .collect();
580                (render.to_string(), data)
581            })
582            .collect();
583        let shard: Vec<(&str, &[usize], &[f32])> = held
584            .iter()
585            .map(|(name, data)| (name.as_str(), &[5usize, 4][..], &data[..]))
586            .collect();
587        write_as(
588            tokens.join("tokens-00000.safetensors"),
589            &shard,
590            &BTreeMap::new(),
591            Dtype::F32,
592        )
593        .unwrap();
594        for (render, _) in renders {
595            token_index.insert(render.to_string(), "tokens-00000.safetensors".to_string());
596        }
597        std::fs::write(
598            tokens.join("index.json"),
599            serde_json::to_vec(&token_index).unwrap(),
600        )
601        .unwrap();
602
603        // One mesh per shard, so a batch holding both has to reopen.
604        let mut sample_index = BTreeMap::new();
605        for (number, mesh) in meshes.iter().enumerate() {
606            let surface: Vec<f32> = (0..16 * 6).map(|value| value as f32).collect();
607            // Row `q` is `(q, q, q, -q)`: the distance is the negated first
608            // column, which makes the split checkable by eye.
609            let queries: Vec<f32> = (0..32)
610                .flat_map(|q| [q as f32, q as f32, q as f32, -(q as f32)])
611                .collect();
612            let transform = [0.5f32, -0.25, 0.75, 2.0];
613            // Row `p` is `(p, 0, 1)`, and surface row `p` starts at `6p`, so a
614            // colour still matched to its point is checkable by eye. Only the
615            // first mesh gets colours, which is the mixed corpus the loader has
616            // to cope with.
617            let colors: Vec<f32> = (0..16).flat_map(|p| [p as f32, 0.0, 1.0]).collect();
618            let file = format!("samples-{number:05}.safetensors");
619            let (points, distances, placement, colored) = (
620                format!("{mesh}.surface"),
621                format!("{mesh}.queries"),
622                format!("{mesh}.transform"),
623                format!("{mesh}.colors"),
624            );
625            let mut tensors: Vec<(&str, &[usize], &[f32])> = vec![
626                (&points, &[16, 6][..], &surface[..]),
627                (&distances, &[32, 4][..], &queries[..]),
628                (&placement, &[1, 4][..], &transform[..]),
629            ];
630            if number == 0 {
631                tensors.push((&colored, &[16, 3][..], &colors[..]));
632            }
633            write_as(samples.join(&file), &tensors, &BTreeMap::new(), Dtype::F32).unwrap();
634            sample_index.insert(mesh.to_string(), file);
635        }
636        std::fs::write(
637            samples.join("index.json"),
638            serde_json::to_vec(&sample_index).unwrap(),
639        )
640        .unwrap();
641
642        (tokens, samples)
643    }
644
645    #[test]
646    fn renders_pair_with_their_mesh_and_strays_are_skipped() {
647        let (tokens, samples) = corpus("pairing");
648        let corpus = Corpus::open(&tokens, &samples).unwrap();
649
650        // `cube_000` and `cube_001` fall back to `cube`; `sphere` matches
651        // outright; `orphan_000` has no mesh and is dropped.
652        assert_eq!(corpus.len(), 3);
653        let pairs: Vec<(&str, &str)> = corpus
654            .pairs()
655            .iter()
656            .map(|(render, mesh)| (render.as_str(), mesh.as_str()))
657            .collect();
658        assert_eq!(
659            pairs,
660            vec![
661                ("cube_000", "cube"),
662                ("cube_001", "cube"),
663                ("sphere", "sphere")
664            ]
665        );
666    }
667
668    #[test]
669    fn a_missing_index_is_refused_by_path() {
670        let error = match Shards::open(std::env::temp_dir().join("rusting_brain_no_such_corpus")) {
671            Ok(_) => panic!("a directory with no index was opened"),
672            Err(error) => error.to_string(),
673        };
674        assert!(error.contains("index.json"), "{error}");
675    }
676
677    /// A colour is only a target if it still belongs to its point, so the
678    /// colours have to survive the surface's subsample row for row.
679    #[test]
680    fn colours_follow_the_surface_points_they_belong_to() {
681        let (tokens, samples) = corpus("colors");
682        let mut corpus = Corpus::open(&tokens, &samples).unwrap();
683        let mut rng = StdRng::seed_from_u64(3);
684
685        // `cube` was written with colours; `sphere` was not.
686        let example = corpus.example(0, Some(5), Some(4), &mut rng).unwrap();
687        let colors = example.colors.expect("cube carries colours");
688        assert_eq!((colors.rows, colors.cols), (5, 3));
689        for row in 0..colors.rows {
690            // Surface row `p` starts at `6p` and colour row `p` starts at `p`.
691            assert_eq!(colors.row(row)[0] * 6.0, example.surface.row(row)[0]);
692            assert_eq!(colors.row(row)[2], 1.0);
693        }
694
695        let plain = corpus.example(2, Some(5), Some(4), &mut rng).unwrap();
696        assert_eq!(plain.mesh, "sphere");
697        assert!(plain.colors.is_none());
698
699        // A batch that mixes the two carries no colours at all.
700        let mixed = collect(
701            &mut corpus,
702            &[0, 2],
703            &BatchConfig {
704                batch: 2,
705                surface_points: Some(5),
706                queries: Some(4),
707                ..BatchConfig::default()
708            },
709            &mut rng,
710        )
711        .unwrap();
712        assert!(mixed.colors.is_none());
713
714        let colored = collect(
715            &mut corpus,
716            &[0, 1],
717            &BatchConfig {
718                batch: 2,
719                surface_points: Some(5),
720                queries: Some(4),
721                ..BatchConfig::default()
722            },
723            &mut rng,
724        )
725        .unwrap();
726        let stacked = colored.colors.expect("both examples are the same mesh");
727        assert_eq!((stacked.rows, stacked.cols), (10, 3));
728    }
729
730    #[test]
731    fn an_example_subsamples_and_splits_its_queries() {
732        let (tokens, samples) = corpus("example");
733        let mut corpus = Corpus::open(&tokens, &samples).unwrap();
734        let mut rng = StdRng::seed_from_u64(0);
735
736        let example = corpus.example(0, Some(5), Some(7), &mut rng).unwrap();
737        assert_eq!(example.name, "cube_000");
738        assert_eq!(example.mesh, "cube");
739        assert_eq!((example.tokens.rows, example.tokens.cols), (5, 4));
740        assert_eq!((example.surface.rows, example.surface.cols), (5, 6));
741        assert_eq!((example.queries.rows, example.queries.cols), (7, 3));
742        assert_eq!(example.distances.len(), 7);
743
744        // Every row kept its own distance through the subsample and the split.
745        for row in 0..example.queries.rows {
746            let point = example.queries.row(row);
747            assert_eq!(point, [point[0]; 3]);
748            assert_eq!(example.distances[row], -point[0]);
749        }
750
751        // Asking for more than there is takes everything, once.
752        let whole = corpus.example(0, None, Some(1_000), &mut rng).unwrap();
753        assert_eq!(whole.surface.rows, 16);
754        assert_eq!(whole.queries.rows, 32);
755        let mut seen: Vec<f32> = whole.distances.clone();
756        seen.sort_by(f32::total_cmp);
757        seen.dedup();
758        assert_eq!(seen.len(), 32);
759    }
760
761    #[test]
762    fn the_transform_comes_back_as_the_preprocessing_wrote_it() {
763        let (tokens, samples) = corpus("transform");
764        let mut corpus = Corpus::open(&tokens, &samples).unwrap();
765        assert_eq!(corpus.transform("sphere").unwrap(), [0.5, -0.25, 0.75, 2.0]);
766        assert!(corpus.transform("nothing").is_err());
767    }
768
769    #[test]
770    fn a_batch_stacks_its_examples_in_order() {
771        let (tokens, samples) = corpus("batch");
772        let corpus = Corpus::open(&tokens, &samples).unwrap();
773        let config = BatchConfig {
774            batch: 3,
775            surface_points: Some(4),
776            queries: Some(6),
777            epochs: Some(1),
778            shuffle: false,
779            seed: 1,
780        };
781
782        let batches: Vec<Batch> = corpus.stream(config).unwrap().map(Result::unwrap).collect();
783        assert_eq!(batches.len(), 1);
784        let batch = &batches[0];
785
786        assert_eq!(batch.len(), 3);
787        assert_eq!(batch.names, vec!["cube_000", "cube_001", "sphere"]);
788        assert_eq!((batch.tokens.rows, batch.tokens.cols), (15, 4));
789        assert_eq!((batch.surface.rows, batch.surface.cols), (12, 6));
790        assert_eq!((batch.queries.rows, batch.queries.cols), (18, 3));
791        assert_eq!(batch.distances.len(), 18);
792
793        // The second example's tokens are the second render's, unshuffled and
794        // in place: the packed layout is what the models index into.
795        assert_eq!(batch.tokens.row(5)[0], 100.0);
796        assert_eq!(batch.tokens.row(10)[0], 200.0);
797    }
798
799    #[test]
800    fn a_short_last_batch_still_arrives() {
801        let (tokens, samples) = corpus("partial");
802        let corpus = Corpus::open(&tokens, &samples).unwrap();
803        let sizes: Vec<usize> = corpus
804            .stream(BatchConfig {
805                batch: 2,
806                epochs: Some(1),
807                shuffle: false,
808                surface_points: Some(2),
809                queries: Some(2),
810                seed: 0,
811            })
812            .unwrap()
813            .map(|batch| batch.unwrap().len())
814            .collect();
815        assert_eq!(sizes, vec![2, 1]);
816    }
817
818    #[test]
819    fn a_stream_with_no_epoch_limit_keeps_going_and_shuffles() {
820        let (tokens, samples) = corpus("endless");
821        let corpus = Corpus::open(&tokens, &samples).unwrap();
822        let mut stream = corpus
823            .stream(BatchConfig {
824                batch: 1,
825                epochs: None,
826                shuffle: true,
827                surface_points: Some(2),
828                queries: Some(2),
829                seed: 7,
830            })
831            .unwrap();
832
833        // Well past one pass over three examples.
834        let names: Vec<String> = (0..12)
835            .map(|_| stream.next().unwrap().unwrap().names[0].clone())
836            .collect();
837        assert_eq!(names.len(), 12);
838        // Shuffled, so the order is not the index order repeated.
839        let ordered: Vec<String> = names[..3].to_vec();
840        assert!(
841            names.chunks(3).any(|chunk| chunk.to_vec() != ordered),
842            "every pass came out in the same order: {names:?}"
843        );
844
845        // Dropping the stream mid-flight has to stop the worker rather than
846        // hang on a full queue.
847        drop(stream);
848    }
849
850    #[test]
851    fn a_batch_of_nothing_is_refused() {
852        let (tokens, samples) = corpus("empty");
853        let corpus = Corpus::open(&tokens, &samples).unwrap();
854        assert!(
855            corpus
856                .stream(BatchConfig {
857                    batch: 0,
858                    ..BatchConfig::default()
859                })
860                .is_err()
861        );
862    }
863}