use crate::batch::TokenBatch;
use crate::network::NetworkError;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use std::fs::File;
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::Path;
#[derive(Debug)]
pub struct TokenFile {
file: File,
tokens: u64,
seed: u64,
}
impl TokenFile {
pub fn write(path: impl AsRef<Path>, ids: &[u32]) -> Result<(), NetworkError> {
let mut file = std::io::BufWriter::new(File::create(path)?);
for &id in ids {
file.write_all(&id.to_le_bytes())?;
}
file.flush()?;
Ok(())
}
pub fn write_jsonl(
path: impl AsRef<Path>,
jsonl: impl AsRef<Path>,
field: &str,
separator: Option<u32>,
mut tokenize: impl FnMut(&str) -> Vec<u32>,
) -> Result<u64, NetworkError> {
let source = std::io::BufReader::new(File::open(jsonl)?);
let mut out = std::io::BufWriter::new(File::create(path)?);
let mut written = 0u64;
for (index, line) in std::io::BufRead::lines(source).enumerate() {
let line = line?;
if line.trim().is_empty() {
continue;
}
let document: serde_json::Value = serde_json::from_str(&line).map_err(|error| {
NetworkError::InvalidDataset(format!("line {}: {error}", index + 1))
})?;
let text = document
.get(field)
.and_then(serde_json::Value::as_str)
.ok_or_else(|| {
NetworkError::InvalidDataset(format!(
"line {} has no string field {field:?}",
index + 1
))
})?;
for id in tokenize(text).into_iter().chain(separator) {
out.write_all(&id.to_le_bytes())?;
written += 1;
}
}
out.flush()?;
Ok(written)
}
pub fn open(path: impl AsRef<Path>, seed: u64) -> Result<Self, NetworkError> {
let file = File::open(path)?;
let bytes = file.metadata()?.len();
if bytes == 0 || bytes % 4 != 0 {
return Err(NetworkError::InvalidDataset(format!(
"a token file holds 4-byte ids, so its length cannot be {bytes} bytes"
)));
}
Ok(Self {
file,
tokens: bytes / 4,
seed,
})
}
pub fn tokens(&self) -> u64 {
self.tokens
}
pub fn batch(
&mut self,
step: u64,
sequences: usize,
seq_len: usize,
) -> Result<TokenBatch, NetworkError> {
if seq_len < 2 || sequences == 0 {
return Err(NetworkError::InvalidConfig(format!(
"a batch of {sequences} sequences of {seq_len} tokens has nothing to predict"
)));
}
if (seq_len as u64) > self.tokens {
return Err(NetworkError::InvalidDataset(format!(
"a window of {seq_len} tokens does not fit in a corpus of {}",
self.tokens
)));
}
let mut rng = StdRng::seed_from_u64(self.seed ^ step.wrapping_mul(0x9E37_79B9_7F4A_7C15));
let last = self.tokens - seq_len as u64;
let mut window = vec![0u8; seq_len * 4];
let mut windows = Vec::with_capacity(sequences);
for _ in 0..sequences {
let offset = if last == 0 {
0
} else {
rng.gen_range(0..=last)
};
self.file.seek(SeekFrom::Start(offset * 4))?;
self.file.read_exact(&mut window)?;
windows.push(
window
.chunks_exact(4)
.map(|id| u32::from_le_bytes([id[0], id[1], id[2], id[3]]))
.collect::<Vec<u32>>(),
);
}
TokenBatch::new(&windows)
}
pub fn chunk(
&mut self,
index: u64,
sequences: usize,
seq_len: usize,
) -> Result<Option<TokenBatch>, NetworkError> {
if seq_len < 2 || sequences == 0 {
return Err(NetworkError::InvalidConfig(format!(
"a batch of {sequences} sequences of {seq_len} tokens has nothing to predict"
)));
}
let start = index * (sequences * seq_len) as u64;
let remaining = self.tokens.saturating_sub(start);
let available = (remaining / seq_len as u64).min(sequences as u64) as usize;
if available == 0 {
return Ok(None);
}
self.file.seek(SeekFrom::Start(start * 4))?;
let mut window = vec![0u8; seq_len * 4];
let mut windows = Vec::with_capacity(available);
for _ in 0..available {
self.file.read_exact(&mut window)?;
windows.push(
window
.chunks_exact(4)
.map(|id| u32::from_le_bytes([id[0], id[1], id[2], id[3]]))
.collect::<Vec<u32>>(),
);
}
Ok(Some(TokenBatch::new(&windows)?))
}
}
pub struct TokenStream<T: FnMut(&str) -> Vec<u32>> {
path: std::path::PathBuf,
lines: std::io::Lines<std::io::BufReader<File>>,
field: Option<String>,
separator: Option<u32>,
tokenize: T,
pending: Vec<u32>,
exhausted: bool,
line_number: usize,
}
impl<T: FnMut(&str) -> Vec<u32>> TokenStream<T> {
pub fn text(
path: impl AsRef<Path>,
separator: Option<u32>,
tokenize: T,
) -> Result<Self, NetworkError> {
Self::open(path.as_ref(), None, separator, tokenize)
}
pub fn jsonl(
path: impl AsRef<Path>,
field: &str,
separator: Option<u32>,
tokenize: T,
) -> Result<Self, NetworkError> {
Self::open(path.as_ref(), Some(field.to_string()), separator, tokenize)
}
fn open(
path: &Path,
field: Option<String>,
separator: Option<u32>,
tokenize: T,
) -> Result<Self, NetworkError> {
Ok(Self {
lines: read_lines(path)?,
path: path.to_path_buf(),
field,
separator,
tokenize,
pending: Vec::new(),
exhausted: false,
line_number: 0,
})
}
pub fn batch(
&mut self,
sequences: usize,
seq_len: usize,
) -> Result<Option<TokenBatch>, NetworkError> {
if seq_len < 2 || sequences == 0 {
return Err(NetworkError::InvalidConfig(format!(
"a batch of {sequences} sequences of {seq_len} tokens has nothing to predict"
)));
}
self.fill(sequences * seq_len)?;
let mut windows = Vec::with_capacity(sequences);
while windows.len() < sequences && self.pending.len() >= seq_len {
windows.push(self.pending.drain(..seq_len).collect::<Vec<u32>>());
}
if windows.len() < sequences && self.exhausted && self.pending.len() >= 2 {
windows.push(std::mem::take(&mut self.pending));
}
if windows.is_empty() {
return Ok(None);
}
Ok(Some(TokenBatch::new(&windows)?))
}
pub fn restart(&mut self) -> Result<(), NetworkError> {
self.lines = read_lines(&self.path)?;
self.pending.clear();
self.exhausted = false;
self.line_number = 0;
Ok(())
}
fn fill(&mut self, wanted: usize) -> Result<(), NetworkError> {
while self.pending.len() < wanted {
let Some(line) = self.lines.next() else {
self.exhausted = true;
break;
};
let line = line?;
self.line_number += 1;
if line.trim().is_empty() {
continue;
}
let document = match &self.field {
None => line,
Some(field) => {
let parsed: serde_json::Value =
serde_json::from_str(&line).map_err(|error| {
NetworkError::InvalidDataset(format!(
"{}:{}: {error}",
self.path.display(),
self.line_number
))
})?;
parsed
.get(field)
.and_then(serde_json::Value::as_str)
.ok_or_else(|| {
NetworkError::InvalidDataset(format!(
"{}:{}: no string field `{field}`",
self.path.display(),
self.line_number
))
})?
.to_string()
}
};
let ids = (self.tokenize)(&document);
self.pending.extend(ids);
if let Some(separator) = self.separator {
self.pending.push(separator);
}
}
Ok(())
}
}
fn read_lines(path: &Path) -> Result<std::io::Lines<std::io::BufReader<File>>, NetworkError> {
use std::io::BufRead;
Ok(std::io::BufReader::new(File::open(path)?).lines())
}
#[cfg(test)]
mod tests {
use super::*;
fn corpus(name: &str, count: u32) -> std::path::PathBuf {
let path =
std::env::temp_dir().join(format!("rusting_brain_{name}_{}.bin", std::process::id()));
TokenFile::write(&path, &(0..count).collect::<Vec<u32>>()).unwrap();
path
}
#[test]
fn a_batch_holds_contiguous_windows_of_the_corpus() {
let path = corpus("contiguous", 1_000);
let mut file = TokenFile::open(&path, 7).unwrap();
assert_eq!(file.tokens(), 1_000);
let batch = file.batch(0, 4, 16).unwrap();
assert_eq!(batch.batch(), 4);
assert_eq!(batch.seq_len(), 16);
assert!(!batch.is_padded());
for window in batch.ids().chunks_exact(16) {
assert!(window.windows(2).all(|pair| pair[1] == pair[0] + 1));
assert!(window[15] < 1_000);
}
std::fs::remove_file(path).unwrap();
}
#[test]
fn the_same_step_draws_the_same_windows_and_other_steps_do_not() {
let path = corpus("steps", 10_000);
let mut file = TokenFile::open(&path, 7).unwrap();
assert_eq!(file.batch(3, 4, 16).unwrap(), file.batch(3, 4, 16).unwrap());
assert_ne!(file.batch(3, 4, 16).unwrap(), file.batch(4, 4, 16).unwrap());
let mut other = TokenFile::open(&path, 8).unwrap();
assert_ne!(
file.batch(3, 4, 16).unwrap(),
other.batch(3, 4, 16).unwrap()
);
std::fs::remove_file(path).unwrap();
}
#[test]
fn chunks_walk_the_corpus_once_in_order_and_then_stop() {
let path = corpus("chunks", 50);
let mut file = TokenFile::open(&path, 7).unwrap();
let first = file.chunk(0, 4, 8).unwrap().unwrap();
assert_eq!(first.batch(), 4);
assert_eq!(&first.ids()[..8], &[0, 1, 2, 3, 4, 5, 6, 7]);
assert_eq!(&first.ids()[24..], &[24, 25, 26, 27, 28, 29, 30, 31]);
let second = file.chunk(1, 4, 8).unwrap().unwrap();
assert_eq!(second.batch(), 2);
assert!(!second.is_padded());
assert_eq!(&second.ids()[..8], &[32, 33, 34, 35, 36, 37, 38, 39]);
assert!(file.chunk(2, 4, 8).unwrap().is_none());
assert_eq!(file.chunk(0, 4, 8).unwrap(), Some(first));
std::fs::remove_file(path).unwrap();
}
#[test]
fn a_window_as_long_as_the_corpus_is_the_whole_corpus() {
let path = corpus("exact", 8);
let mut file = TokenFile::open(&path, 1).unwrap();
let batch = file.batch(0, 2, 8).unwrap();
assert_eq!(
batch.ids(),
&[0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7]
);
assert!(file.batch(0, 2, 9).is_err());
std::fs::remove_file(path).unwrap();
}
#[test]
fn a_streamed_corpus_holds_the_same_ids_as_the_file_it_replaces() {
let jsonl =
std::env::temp_dir().join(format!("rusting_brain_stream_{}.jsonl", std::process::id()));
std::fs::write(&jsonl, "{\"text\":\"abcd\"}\n\n{\"text\":\"efgh\"}\n").unwrap();
let bin =
std::env::temp_dir().join(format!("rusting_brain_stream_{}.bin", std::process::id()));
let tokenize = |text: &str| text.bytes().map(u32::from).collect::<Vec<u32>>();
TokenFile::write_jsonl(&bin, &jsonl, "text", Some(0), tokenize).unwrap();
let expected: Vec<u32> = std::fs::read(&bin)
.unwrap()
.chunks_exact(4)
.map(|id| u32::from_le_bytes([id[0], id[1], id[2], id[3]]))
.collect();
assert_eq!(expected.len(), 10, "four bytes and a separator, twice");
let mut corpus = TokenStream::jsonl(&jsonl, "text", Some(0), tokenize).unwrap();
let mut streamed = Vec::new();
let mut batches = 0;
while let Some(batch) = corpus.batch(2, 4).unwrap() {
for (sequence, &length) in batch.lengths().iter().enumerate() {
let start = sequence * batch.seq_len();
streamed.extend_from_slice(&batch.ids()[start..start + length]);
}
batches += 1;
}
assert_eq!(streamed, expected);
assert_eq!(batches, 2, "two full windows, then the two-token tail");
corpus.restart().unwrap();
assert!(corpus.batch(2, 4).unwrap().is_some());
let missing = TokenStream::jsonl(&jsonl, "body", None, tokenize)
.unwrap()
.batch(1, 2)
.unwrap_err()
.to_string();
assert!(
missing.contains("no string field `body`"),
"the error was {missing}"
);
std::fs::remove_file(jsonl).unwrap();
std::fs::remove_file(bin).unwrap();
}
#[test]
fn a_jsonl_corpus_is_tokenized_document_by_document() {
let jsonl =
std::env::temp_dir().join(format!("rusting_brain_{}.jsonl", std::process::id()));
std::fs::write(&jsonl, "{\"text\":\"ab\",\"id\":1}\n\n{\"text\":\"c\"}\n").unwrap();
let bin =
std::env::temp_dir().join(format!("rusting_brain_jsonl_{}.bin", std::process::id()));
let written = TokenFile::write_jsonl(&bin, &jsonl, "text", Some(255), |text| {
text.bytes().map(u32::from).collect()
})
.unwrap();
assert_eq!(written, 5);
let mut file = TokenFile::open(&bin, 0).unwrap();
assert_eq!(file.tokens(), 5);
assert_eq!(file.batch(0, 1, 5).unwrap().ids(), &[97, 98, 255, 99, 255]);
assert!(matches!(
TokenFile::write_jsonl(&bin, &jsonl, "body", None, |_| vec![1]),
Err(NetworkError::InvalidDataset(message)) if message.contains("line 1")
));
std::fs::remove_file(jsonl).unwrap();
std::fs::remove_file(bin).unwrap();
}
#[test]
fn a_file_that_is_not_whole_ids_is_rejected() {
let path =
std::env::temp_dir().join(format!("rusting_brain_ragged_{}.bin", std::process::id()));
std::fs::write(&path, [1u8, 2, 3]).unwrap();
assert!(matches!(
TokenFile::open(&path, 0),
Err(NetworkError::InvalidDataset(_))
));
std::fs::remove_file(path).unwrap();
}
}