1use crate::matrix::Matrix;
38use crate::network::NetworkError;
39use crate::safetensors::SafeTensors;
40use rand::rngs::StdRng;
41use rand::{Rng, SeedableRng};
42use std::collections::BTreeMap;
43use std::path::{Path, PathBuf};
44
45pub struct Shards {
51 root: PathBuf,
52 index: BTreeMap<String, String>,
53 open: Option<(String, SafeTensors)>,
54}
55
56impl Shards {
57 pub fn open<P: AsRef<Path>>(root: P) -> Result<Self, NetworkError> {
60 let root = root.as_ref().to_path_buf();
61 let path = root.join("index.json");
62 let index: BTreeMap<String, String> =
63 serde_json::from_slice(&std::fs::read(&path).map_err(|error| {
64 NetworkError::InvalidDataset(format!("{}: {error}", path.display()))
65 })?)?;
66 if index.is_empty() {
67 return Err(NetworkError::InvalidDataset(format!(
68 "{} lists no shards",
69 path.display()
70 )));
71 }
72 Ok(Self {
73 root,
74 index,
75 open: None,
76 })
77 }
78
79 pub fn names(&self) -> impl Iterator<Item = &str> {
81 self.index.keys().map(String::as_str)
82 }
83
84 pub fn len(&self) -> usize {
85 self.index.len()
86 }
87
88 pub fn is_empty(&self) -> bool {
89 self.index.is_empty()
90 }
91
92 pub fn contains(&self, name: &str) -> bool {
93 self.index.contains_key(name)
94 }
95
96 pub fn tensor(
102 &mut self,
103 name: &str,
104 suffix: &str,
105 ) -> Result<(Vec<f32>, Vec<usize>), NetworkError> {
106 let file = self.shard_of(name)?;
107 file.tensor(&format!("{name}{suffix}"))
108 }
109
110 pub fn has(&mut self, name: &str, suffix: &str) -> bool {
113 let full = format!("{name}{suffix}");
114 self.shard_of(name)
115 .is_ok_and(|file| file.names().any(|tensor| tensor == full))
116 }
117
118 fn shard_of(&mut self, name: &str) -> Result<&mut SafeTensors, NetworkError> {
120 let shard = self.index.get(name).cloned().ok_or_else(|| {
121 NetworkError::InvalidDataset(format!("{}: nothing named {name}", self.root.display()))
122 })?;
123 if self.open.as_ref().map(|(open, _)| open.as_str()) != Some(shard.as_str()) {
124 let file = SafeTensors::open(self.root.join(&shard))?;
125 self.open = Some((shard, file));
126 }
127 let (_, file) = self.open.as_mut().expect("just opened");
128 Ok(file)
129 }
130
131 pub fn matrix(&mut self, name: &str, suffix: &str) -> Result<Matrix, NetworkError> {
133 let (values, shape) = self.tensor(name, suffix)?;
134 match shape.as_slice() {
135 [rows, cols] => Ok(Matrix::from_vec(*rows, *cols, values)),
136 _ => Err(NetworkError::InvalidDataset(format!(
137 "{name}{suffix} has shape {shape:?}, which is not a matrix"
138 ))),
139 }
140 }
141}
142
143pub struct Example {
145 pub name: String,
147 pub mesh: String,
149 pub tokens: Matrix,
151 pub surface: Matrix,
153 pub colors: Option<Matrix>,
156 pub queries: Matrix,
158 pub distances: Vec<f32>,
160}
161
162pub struct Batch {
164 pub names: Vec<String>,
165 pub tokens: Matrix,
167 pub surface: Matrix,
169 pub colors: Option<Matrix>,
176 pub queries: Matrix,
178 pub distances: Vec<f32>,
180}
181
182impl Batch {
183 pub fn len(&self) -> usize {
185 self.names.len()
186 }
187
188 pub fn is_empty(&self) -> bool {
189 self.names.is_empty()
190 }
191}
192
193pub struct Corpus {
195 tokens: Shards,
196 samples: Shards,
197 pairs: Vec<(String, String)>,
199}
200
201impl Corpus {
202 pub fn open<T: AsRef<Path>, S: AsRef<Path>>(
211 tokens: T,
212 samples: S,
213 ) -> Result<Self, NetworkError> {
214 let tokens = Shards::open(tokens)?;
215 let samples = Shards::open(samples)?;
216
217 let pairs: Vec<(String, String)> = tokens
218 .names()
219 .filter_map(|render| {
220 let mesh = match samples.contains(render) {
221 true => Some(render.to_string()),
222 false => render
223 .rsplit_once('_')
224 .map(|(mesh, _)| mesh.to_string())
225 .filter(|mesh| samples.contains(mesh)),
226 }?;
227 Some((render.to_string(), mesh))
228 })
229 .collect();
230
231 if pairs.is_empty() {
232 return Err(NetworkError::InvalidDataset(
233 "no render in the token index has a mesh in the sample index".into(),
234 ));
235 }
236 Ok(Self {
237 tokens,
238 samples,
239 pairs,
240 })
241 }
242
243 pub fn len(&self) -> usize {
245 self.pairs.len()
246 }
247
248 pub fn is_empty(&self) -> bool {
249 self.pairs.is_empty()
250 }
251
252 pub fn pairs(&self) -> &[(String, String)] {
254 &self.pairs
255 }
256
257 pub fn example<R: Rng>(
263 &mut self,
264 index: usize,
265 surface_points: Option<usize>,
266 queries: Option<usize>,
267 rng: &mut R,
268 ) -> Result<Example, NetworkError> {
269 let (name, mesh) = self
270 .pairs
271 .get(index)
272 .cloned()
273 .ok_or_else(|| NetworkError::InvalidDataset(format!("no example {index}")))?;
274
275 let tokens = self.tokens.matrix(&name, "")?;
276 let written = self.samples.matrix(&mesh, ".surface")?;
277 let chosen = choose_rows(written.rows, surface_points, rng);
280 let surface = take_rows(&written, &chosen);
281 let colors = match self.samples.has(&mesh, ".colors") {
282 true => Some(take_rows(&self.samples.matrix(&mesh, ".colors")?, &chosen)),
283 false => None,
284 };
285 let packed = subsample(&self.samples.matrix(&mesh, ".queries")?, queries, rng);
286 if packed.cols != 4 {
287 return Err(NetworkError::InvalidDataset(format!(
288 "{mesh}.queries is [{}, {}] and should be [n, 4]",
289 packed.rows, packed.cols
290 )));
291 }
292
293 let mut queries = Matrix::new(packed.rows, 3);
296 let mut distances = Vec::with_capacity(packed.rows);
297 for row in 0..packed.rows {
298 queries.row_mut(row).copy_from_slice(&packed.row(row)[..3]);
299 distances.push(packed.row(row)[3]);
300 }
301
302 Ok(Example {
303 name,
304 mesh,
305 tokens,
306 surface,
307 colors,
308 queries,
309 distances,
310 })
311 }
312
313 pub fn transform(&mut self, mesh: &str) -> Result<[f32; 4], NetworkError> {
316 let (values, _) = self.samples.tensor(mesh, ".transform")?;
317 values
318 .get(..4)
319 .map(|values| [values[0], values[1], values[2], values[3]])
320 .ok_or_else(|| {
321 NetworkError::InvalidDataset(format!(
322 "{mesh}.transform holds {} values, not 4",
323 values.len()
324 ))
325 })
326 }
327
328 pub fn stream(self, config: BatchConfig) -> Result<BatchStream, NetworkError> {
334 BatchStream::new(self, config)
335 }
336}
337
338fn choose_rows<R: Rng>(rows: usize, count: Option<usize>, rng: &mut R) -> Option<Vec<usize>> {
341 let count = count.filter(|count| *count < rows)?;
342 let mut order: Vec<usize> = (0..rows).collect();
346 for slot in 0..count {
347 order.swap(slot, rng.gen_range(slot..rows));
348 }
349 order.truncate(count);
350 Some(order)
351}
352
353fn take_rows(source: &Matrix, chosen: &Option<Vec<usize>>) -> Matrix {
355 let Some(chosen) = chosen else {
356 return source.clone();
357 };
358 let mut out = Matrix::new(chosen.len(), source.cols);
359 for (row, taken) in chosen.iter().enumerate() {
360 out.row_mut(row).copy_from_slice(source.row(*taken));
361 }
362 out
363}
364
365fn subsample<R: Rng>(source: &Matrix, count: Option<usize>, rng: &mut R) -> Matrix {
367 take_rows(source, &choose_rows(source.rows, count, rng))
368}
369
370#[derive(Clone, Copy, Debug)]
372pub struct BatchConfig {
373 pub batch: usize,
374 pub surface_points: Option<usize>,
376 pub queries: Option<usize>,
378 pub epochs: Option<usize>,
380 pub shuffle: bool,
382 pub seed: u64,
383}
384
385impl Default for BatchConfig {
386 fn default() -> Self {
387 Self {
388 batch: 8,
389 surface_points: Some(2048),
390 queries: Some(4096),
391 epochs: None,
392 shuffle: true,
393 seed: 0,
394 }
395 }
396}
397
398pub struct BatchStream {
404 receiver: std::sync::mpsc::Receiver<Result<Batch, NetworkError>>,
405 worker: Option<std::thread::JoinHandle<()>>,
406}
407
408impl BatchStream {
409 fn new(mut corpus: Corpus, config: BatchConfig) -> Result<Self, NetworkError> {
410 if config.batch == 0 {
411 return Err(NetworkError::InvalidConfig(
412 "a batch needs at least one example".into(),
413 ));
414 }
415 let (sender, receiver) = std::sync::mpsc::sync_channel(1);
418
419 let worker = std::thread::spawn(move || {
420 let mut rng = StdRng::seed_from_u64(config.seed);
421 let mut epoch = 0;
422 while config.epochs.is_none_or(|epochs| epoch < epochs) {
423 let mut order: Vec<usize> = (0..corpus.len()).collect();
424 if config.shuffle {
425 let examples = order.len();
426 for slot in 0..examples {
427 order.swap(slot, rng.gen_range(slot..examples));
428 }
429 }
430
431 for chunk in order.chunks(config.batch) {
432 let batch = collect(&mut corpus, chunk, &config, &mut rng);
433 let failed = batch.is_err();
434 if sender.send(batch).is_err() || failed {
438 return;
439 }
440 }
441 epoch += 1;
442 }
443 });
444
445 Ok(Self {
446 receiver,
447 worker: Some(worker),
448 })
449 }
450}
451
452impl Iterator for BatchStream {
453 type Item = Result<Batch, NetworkError>;
454
455 fn next(&mut self) -> Option<Self::Item> {
456 self.receiver.recv().ok()
457 }
458}
459
460impl Drop for BatchStream {
461 fn drop(&mut self) {
462 let (_, receiver) = std::sync::mpsc::sync_channel(0);
465 drop(std::mem::replace(&mut self.receiver, receiver));
466 if let Some(worker) = self.worker.take() {
467 let _ = worker.join();
468 }
469 }
470}
471
472fn collect<R: Rng>(
474 corpus: &mut Corpus,
475 indices: &[usize],
476 config: &BatchConfig,
477 rng: &mut R,
478) -> Result<Batch, NetworkError> {
479 let mut names = Vec::with_capacity(indices.len());
480 let mut loaded = Vec::with_capacity(indices.len());
481 for index in indices {
482 let example = corpus.example(*index, config.surface_points, config.queries, rng)?;
483 names.push(example.name.clone());
484 loaded.push(example);
485 }
486
487 let shape = |matrix: &Matrix| (matrix.rows, matrix.cols);
490 for example in &loaded[1..] {
491 for (name, (left, right)) in [
492 ("tokens", (shape(&example.tokens), shape(&loaded[0].tokens))),
493 (
494 "surface",
495 (shape(&example.surface), shape(&loaded[0].surface)),
496 ),
497 (
498 "queries",
499 (shape(&example.queries), shape(&loaded[0].queries)),
500 ),
501 ] {
502 if left != right {
503 return Err(NetworkError::InvalidDataset(format!(
504 "{}: {name} is {left:?} but {} has {right:?}",
505 example.name, loaded[0].name
506 )));
507 }
508 }
509 }
510
511 let colors = loaded
514 .iter()
515 .map(|example| example.colors.as_ref())
516 .collect::<Option<Vec<_>>>()
517 .map(|parts| stack(parts.into_iter()));
518
519 Ok(Batch {
520 names,
521 tokens: stack(loaded.iter().map(|example| &example.tokens)),
522 surface: stack(loaded.iter().map(|example| &example.surface)),
523 colors,
524 queries: stack(loaded.iter().map(|example| &example.queries)),
525 distances: loaded
526 .iter()
527 .flat_map(|example| example.distances.iter().copied())
528 .collect(),
529 })
530}
531
532fn stack<'a>(parts: impl Iterator<Item = &'a Matrix> + Clone) -> Matrix {
534 let cols = parts.clone().next().map_or(0, |part| part.cols);
535 let mut data = Vec::new();
536 let mut rows = 0;
537 for part in parts {
538 data.extend_from_slice(&part.data);
539 rows += part.rows;
540 }
541 Matrix::from_vec(rows, cols, data)
542}
543
544#[cfg(test)]
545mod tests {
546 use super::*;
547 use crate::safetensors::{Dtype, write_as};
548
549 fn corpus(name: &str) -> (PathBuf, PathBuf) {
554 let root = std::env::temp_dir().join(format!(
555 "rusting_brain_shards_{name}_{}",
556 std::process::id()
557 ));
558 let tokens = root.join("tokens");
559 let samples = root.join("samples");
560 let _ = std::fs::remove_dir_all(&root);
561 std::fs::create_dir_all(&tokens).unwrap();
562 std::fs::create_dir_all(&samples).unwrap();
563
564 let meshes = ["cube", "sphere"];
565 let renders = [
566 ("cube_000", "cube"),
567 ("cube_001", "cube"),
568 ("sphere", "sphere"),
569 ("orphan_000", ""),
570 ];
571
572 let mut token_index = BTreeMap::new();
573 let held: Vec<(String, Vec<f32>)> = renders
574 .iter()
575 .enumerate()
576 .map(|(number, (render, _))| {
577 let data = (0..5 * 4)
578 .map(|value| (value + number * 100) as f32)
579 .collect();
580 (render.to_string(), data)
581 })
582 .collect();
583 let shard: Vec<(&str, &[usize], &[f32])> = held
584 .iter()
585 .map(|(name, data)| (name.as_str(), &[5usize, 4][..], &data[..]))
586 .collect();
587 write_as(
588 tokens.join("tokens-00000.safetensors"),
589 &shard,
590 &BTreeMap::new(),
591 Dtype::F32,
592 )
593 .unwrap();
594 for (render, _) in renders {
595 token_index.insert(render.to_string(), "tokens-00000.safetensors".to_string());
596 }
597 std::fs::write(
598 tokens.join("index.json"),
599 serde_json::to_vec(&token_index).unwrap(),
600 )
601 .unwrap();
602
603 let mut sample_index = BTreeMap::new();
605 for (number, mesh) in meshes.iter().enumerate() {
606 let surface: Vec<f32> = (0..16 * 6).map(|value| value as f32).collect();
607 let queries: Vec<f32> = (0..32)
610 .flat_map(|q| [q as f32, q as f32, q as f32, -(q as f32)])
611 .collect();
612 let transform = [0.5f32, -0.25, 0.75, 2.0];
613 let colors: Vec<f32> = (0..16).flat_map(|p| [p as f32, 0.0, 1.0]).collect();
618 let file = format!("samples-{number:05}.safetensors");
619 let (points, distances, placement, colored) = (
620 format!("{mesh}.surface"),
621 format!("{mesh}.queries"),
622 format!("{mesh}.transform"),
623 format!("{mesh}.colors"),
624 );
625 let mut tensors: Vec<(&str, &[usize], &[f32])> = vec![
626 (&points, &[16, 6][..], &surface[..]),
627 (&distances, &[32, 4][..], &queries[..]),
628 (&placement, &[1, 4][..], &transform[..]),
629 ];
630 if number == 0 {
631 tensors.push((&colored, &[16, 3][..], &colors[..]));
632 }
633 write_as(samples.join(&file), &tensors, &BTreeMap::new(), Dtype::F32).unwrap();
634 sample_index.insert(mesh.to_string(), file);
635 }
636 std::fs::write(
637 samples.join("index.json"),
638 serde_json::to_vec(&sample_index).unwrap(),
639 )
640 .unwrap();
641
642 (tokens, samples)
643 }
644
645 #[test]
646 fn renders_pair_with_their_mesh_and_strays_are_skipped() {
647 let (tokens, samples) = corpus("pairing");
648 let corpus = Corpus::open(&tokens, &samples).unwrap();
649
650 assert_eq!(corpus.len(), 3);
653 let pairs: Vec<(&str, &str)> = corpus
654 .pairs()
655 .iter()
656 .map(|(render, mesh)| (render.as_str(), mesh.as_str()))
657 .collect();
658 assert_eq!(
659 pairs,
660 vec![
661 ("cube_000", "cube"),
662 ("cube_001", "cube"),
663 ("sphere", "sphere")
664 ]
665 );
666 }
667
668 #[test]
669 fn a_missing_index_is_refused_by_path() {
670 let error = match Shards::open(std::env::temp_dir().join("rusting_brain_no_such_corpus")) {
671 Ok(_) => panic!("a directory with no index was opened"),
672 Err(error) => error.to_string(),
673 };
674 assert!(error.contains("index.json"), "{error}");
675 }
676
677 #[test]
680 fn colours_follow_the_surface_points_they_belong_to() {
681 let (tokens, samples) = corpus("colors");
682 let mut corpus = Corpus::open(&tokens, &samples).unwrap();
683 let mut rng = StdRng::seed_from_u64(3);
684
685 let example = corpus.example(0, Some(5), Some(4), &mut rng).unwrap();
687 let colors = example.colors.expect("cube carries colours");
688 assert_eq!((colors.rows, colors.cols), (5, 3));
689 for row in 0..colors.rows {
690 assert_eq!(colors.row(row)[0] * 6.0, example.surface.row(row)[0]);
692 assert_eq!(colors.row(row)[2], 1.0);
693 }
694
695 let plain = corpus.example(2, Some(5), Some(4), &mut rng).unwrap();
696 assert_eq!(plain.mesh, "sphere");
697 assert!(plain.colors.is_none());
698
699 let mixed = collect(
701 &mut corpus,
702 &[0, 2],
703 &BatchConfig {
704 batch: 2,
705 surface_points: Some(5),
706 queries: Some(4),
707 ..BatchConfig::default()
708 },
709 &mut rng,
710 )
711 .unwrap();
712 assert!(mixed.colors.is_none());
713
714 let colored = collect(
715 &mut corpus,
716 &[0, 1],
717 &BatchConfig {
718 batch: 2,
719 surface_points: Some(5),
720 queries: Some(4),
721 ..BatchConfig::default()
722 },
723 &mut rng,
724 )
725 .unwrap();
726 let stacked = colored.colors.expect("both examples are the same mesh");
727 assert_eq!((stacked.rows, stacked.cols), (10, 3));
728 }
729
730 #[test]
731 fn an_example_subsamples_and_splits_its_queries() {
732 let (tokens, samples) = corpus("example");
733 let mut corpus = Corpus::open(&tokens, &samples).unwrap();
734 let mut rng = StdRng::seed_from_u64(0);
735
736 let example = corpus.example(0, Some(5), Some(7), &mut rng).unwrap();
737 assert_eq!(example.name, "cube_000");
738 assert_eq!(example.mesh, "cube");
739 assert_eq!((example.tokens.rows, example.tokens.cols), (5, 4));
740 assert_eq!((example.surface.rows, example.surface.cols), (5, 6));
741 assert_eq!((example.queries.rows, example.queries.cols), (7, 3));
742 assert_eq!(example.distances.len(), 7);
743
744 for row in 0..example.queries.rows {
746 let point = example.queries.row(row);
747 assert_eq!(point, [point[0]; 3]);
748 assert_eq!(example.distances[row], -point[0]);
749 }
750
751 let whole = corpus.example(0, None, Some(1_000), &mut rng).unwrap();
753 assert_eq!(whole.surface.rows, 16);
754 assert_eq!(whole.queries.rows, 32);
755 let mut seen: Vec<f32> = whole.distances.clone();
756 seen.sort_by(f32::total_cmp);
757 seen.dedup();
758 assert_eq!(seen.len(), 32);
759 }
760
761 #[test]
762 fn the_transform_comes_back_as_the_preprocessing_wrote_it() {
763 let (tokens, samples) = corpus("transform");
764 let mut corpus = Corpus::open(&tokens, &samples).unwrap();
765 assert_eq!(corpus.transform("sphere").unwrap(), [0.5, -0.25, 0.75, 2.0]);
766 assert!(corpus.transform("nothing").is_err());
767 }
768
769 #[test]
770 fn a_batch_stacks_its_examples_in_order() {
771 let (tokens, samples) = corpus("batch");
772 let corpus = Corpus::open(&tokens, &samples).unwrap();
773 let config = BatchConfig {
774 batch: 3,
775 surface_points: Some(4),
776 queries: Some(6),
777 epochs: Some(1),
778 shuffle: false,
779 seed: 1,
780 };
781
782 let batches: Vec<Batch> = corpus.stream(config).unwrap().map(Result::unwrap).collect();
783 assert_eq!(batches.len(), 1);
784 let batch = &batches[0];
785
786 assert_eq!(batch.len(), 3);
787 assert_eq!(batch.names, vec!["cube_000", "cube_001", "sphere"]);
788 assert_eq!((batch.tokens.rows, batch.tokens.cols), (15, 4));
789 assert_eq!((batch.surface.rows, batch.surface.cols), (12, 6));
790 assert_eq!((batch.queries.rows, batch.queries.cols), (18, 3));
791 assert_eq!(batch.distances.len(), 18);
792
793 assert_eq!(batch.tokens.row(5)[0], 100.0);
796 assert_eq!(batch.tokens.row(10)[0], 200.0);
797 }
798
799 #[test]
800 fn a_short_last_batch_still_arrives() {
801 let (tokens, samples) = corpus("partial");
802 let corpus = Corpus::open(&tokens, &samples).unwrap();
803 let sizes: Vec<usize> = corpus
804 .stream(BatchConfig {
805 batch: 2,
806 epochs: Some(1),
807 shuffle: false,
808 surface_points: Some(2),
809 queries: Some(2),
810 seed: 0,
811 })
812 .unwrap()
813 .map(|batch| batch.unwrap().len())
814 .collect();
815 assert_eq!(sizes, vec![2, 1]);
816 }
817
818 #[test]
819 fn a_stream_with_no_epoch_limit_keeps_going_and_shuffles() {
820 let (tokens, samples) = corpus("endless");
821 let corpus = Corpus::open(&tokens, &samples).unwrap();
822 let mut stream = corpus
823 .stream(BatchConfig {
824 batch: 1,
825 epochs: None,
826 shuffle: true,
827 surface_points: Some(2),
828 queries: Some(2),
829 seed: 7,
830 })
831 .unwrap();
832
833 let names: Vec<String> = (0..12)
835 .map(|_| stream.next().unwrap().unwrap().names[0].clone())
836 .collect();
837 assert_eq!(names.len(), 12);
838 let ordered: Vec<String> = names[..3].to_vec();
840 assert!(
841 names.chunks(3).any(|chunk| chunk.to_vec() != ordered),
842 "every pass came out in the same order: {names:?}"
843 );
844
845 drop(stream);
848 }
849
850 #[test]
851 fn a_batch_of_nothing_is_refused() {
852 let (tokens, samples) = corpus("empty");
853 let corpus = Corpus::open(&tokens, &samples).unwrap();
854 assert!(
855 corpus
856 .stream(BatchConfig {
857 batch: 0,
858 ..BatchConfig::default()
859 })
860 .is_err()
861 );
862 }
863}