oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
//! Training corpus discovery: single `.txt`, JSONL (`{"text":…}`), or a directory
//! of those files concatenated in deterministic sorted order.

use serde::Deserialize;
use serde::Serialize;
use std::error::Error;
use std::fmt;
use std::fs::{self, File};
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};

const MATERIALIZED_NAME: &str = "corpus.materialized";
const MANIFEST_NAME: &str = "corpus.manifest.json";
const MANIFEST_VERSION: u32 = 1;

#[derive(Debug)]
pub enum CorpusError {
    InvalidData(String),
    Io(std::io::Error),
    Json(serde_json::Error),
}

impl fmt::Display for CorpusError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::InvalidData(message) => write!(formatter, "corpus error: {message}"),
            Self::Io(error) => error.fmt(formatter),
            Self::Json(error) => write!(formatter, "corpus JSON error: {error}"),
        }
    }
}

impl Error for CorpusError {}

impl From<std::io::Error> for CorpusError {
    fn from(error: std::io::Error) -> Self {
        Self::Io(error)
    }
}

impl From<serde_json::Error> for CorpusError {
    fn from(error: serde_json::Error) -> Self {
        Self::Json(error)
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CorpusFileKind {
    Txt,
    Jsonl,
}

#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CorpusSourceEntry {
    pub path: String,
    pub kind: CorpusFileKind,
    pub bytes: u64,
}

#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CorpusManifest {
    pub version: u32,
    pub sources: Vec<CorpusSourceEntry>,
    pub total_bytes: u64,
}

/// A resolved contiguous corpus file ready for memory-mapping.
#[derive(Clone, Debug)]
pub struct ResolvedCorpus {
    /// Path to open (original `.txt` or materialized concat).
    pub path: PathBuf,
    /// Original user path (file or directory).
    pub input: PathBuf,
    pub sources: Vec<CorpusSourceEntry>,
    pub materialized: bool,
}

impl ResolvedCorpus {
    pub fn total_bytes(&self) -> u64 {
        self.sources.iter().map(|source| source.bytes).sum()
    }

    pub fn file_count(&self) -> usize {
        self.sources.len()
    }
}

#[derive(Deserialize)]
struct JsonlRecord {
    text: String,
}

/// Discover corpus inputs from `input` (file or directory) and return a single
/// contiguous text path. Multi-file and JSONL inputs are concatenated into
/// `materialize_dir` in sorted path order so token sharding is deterministic.
pub fn resolve_corpus(
    input: impl AsRef<Path>,
    materialize_dir: impl AsRef<Path>,
) -> Result<ResolvedCorpus, CorpusError> {
    let input = input.as_ref();
    let materialize_dir = materialize_dir.as_ref();
    let sources = discover_sources(input)?;
    if sources.is_empty() {
        return Err(CorpusError::InvalidData(format!(
            "no .txt or .jsonl corpus files found at {}",
            input.display()
        )));
    }

    let needs_materialize = sources.len() > 1
        || sources
            .iter()
            .any(|source| source.kind == CorpusFileKind::Jsonl);

    if !needs_materialize {
        let source = &sources[0];
        let bytes = fs::metadata(&source.path)?.len();
        return Ok(ResolvedCorpus {
            path: source.path.clone(),
            input: input.to_path_buf(),
            sources: vec![CorpusSourceEntry {
                path: source.path.display().to_string(),
                kind: source.kind,
                bytes,
            }],
            materialized: false,
        });
    }

    fs::create_dir_all(materialize_dir)?;
    let materialized_path = materialize_dir.join(MATERIALIZED_NAME);
    let manifest_path = materialize_dir.join(MANIFEST_NAME);
    let written = materialize_sources(&sources, &materialized_path)?;
    let manifest = CorpusManifest {
        version: MANIFEST_VERSION,
        sources: written,
        total_bytes: fs::metadata(&materialized_path)?.len(),
    };
    serde_json::to_writer_pretty(BufWriter::new(File::create(&manifest_path)?), &manifest)?;

    Ok(ResolvedCorpus {
        path: materialized_path,
        input: input.to_path_buf(),
        sources: manifest.sources,
        materialized: true,
    })
}

fn discover_sources(input: &Path) -> Result<Vec<CorpusSource>, CorpusError> {
    if input.is_dir() {
        let mut paths = Vec::new();
        for entry in fs::read_dir(input)? {
            let entry = entry?;
            if !entry.file_type()?.is_file() {
                continue;
            }
            let path = entry.path();
            if let Some(kind) = classify_path(&path) {
                paths.push((path, kind));
            }
        }
        paths.sort_by(|left, right| left.0.cmp(&right.0));
        if paths.is_empty() {
            return Err(CorpusError::InvalidData(format!(
                "directory {} contains no .txt or .jsonl files",
                input.display()
            )));
        }
        return Ok(paths
            .into_iter()
            .map(|(path, kind)| CorpusSource { path, kind })
            .collect());
    }

    if !input.is_file() {
        return Err(CorpusError::InvalidData(format!(
            "corpus path {} is not a file or directory",
            input.display()
        )));
    }

    let kind = classify_path(input).unwrap_or(CorpusFileKind::Txt);
    Ok(vec![CorpusSource {
        path: input.to_path_buf(),
        kind,
    }])
}

fn classify_path(path: &Path) -> Option<CorpusFileKind> {
    let ext = path.extension()?.to_string_lossy().to_ascii_lowercase();
    match ext.as_str() {
        "txt" => Some(CorpusFileKind::Txt),
        "jsonl" => Some(CorpusFileKind::Jsonl),
        _ => None,
    }
}

struct CorpusSource {
    path: PathBuf,
    kind: CorpusFileKind,
}

fn materialize_sources(
    sources: &[CorpusSource],
    destination: &Path,
) -> Result<Vec<CorpusSourceEntry>, CorpusError> {
    let temporary = destination.with_extension("tmp");
    let mut writer = BufWriter::new(File::create(&temporary)?);
    let mut entries = Vec::with_capacity(sources.len());

    for source in sources {
        let bytes = match source.kind {
            CorpusFileKind::Txt => copy_txt(&source.path, &mut writer)?,
            CorpusFileKind::Jsonl => copy_jsonl(&source.path, &mut writer)?,
        };
        entries.push(CorpusSourceEntry {
            path: source.path.display().to_string(),
            kind: source.kind,
            bytes,
        });
    }
    writer.flush()?;
    drop(writer);
    fs::rename(&temporary, destination)?;
    Ok(entries)
}

fn copy_txt(path: &Path, writer: &mut BufWriter<File>) -> Result<u64, CorpusError> {
    let mut reader = BufReader::new(File::open(path)?);
    let written = std::io::copy(&mut reader, writer)?;
    Ok(written)
}

fn copy_jsonl(path: &Path, writer: &mut BufWriter<File>) -> Result<u64, CorpusError> {
    let reader = BufReader::new(File::open(path)?);
    let mut written = 0u64;
    for (line_number, line) in reader.lines().enumerate() {
        let line = line?;
        if line.trim().is_empty() {
            continue;
        }
        let record: JsonlRecord = serde_json::from_str(&line).map_err(|error| {
            CorpusError::InvalidData(format!(
                "{}:{}: {error}",
                path.display(),
                line_number + 1
            ))
        })?;
        writer.write_all(record.text.as_bytes())?;
        writer.write_all(b"\n")?;
        written += record.text.len() as u64 + 1;
    }
    if written == 0 {
        return Err(CorpusError::InvalidData(format!(
            "JSONL file {} produced no text records",
            path.display()
        )));
    }
    Ok(written)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::time::{SystemTime, UNIX_EPOCH};

    fn temporary_directory(label: &str) -> PathBuf {
        let nanos = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap()
            .as_nanos();
        let path = std::env::temp_dir().join(format!("oxidelm-corpus-{label}-{nanos}"));
        fs::create_dir_all(&path).unwrap();
        path
    }

    #[test]
    fn single_txt_is_not_materialized() {
        let directory = temporary_directory("single");
        let input = directory.join("input.txt");
        fs::write(&input, b"abcd").unwrap();
        let cache = directory.join("cache");

        let resolved = resolve_corpus(&input, &cache).unwrap();

        assert!(!resolved.materialized);
        assert_eq!(resolved.path, input);
        assert_eq!(resolved.file_count(), 1);
        assert!(!cache.join(MATERIALIZED_NAME).exists());
    }

    #[test]
    fn directory_of_txt_concats_in_sorted_order() {
        let directory = temporary_directory("dir-txt");
        fs::write(directory.join("b.txt"), b"BBB").unwrap();
        fs::write(directory.join("a.txt"), b"AAA").unwrap();
        fs::write(directory.join("ignore.md"), b"nope").unwrap();
        let cache = directory.join("cache");

        let resolved = resolve_corpus(&directory, &cache).unwrap();
        let bytes = fs::read(&resolved.path).unwrap();

        assert!(resolved.materialized);
        assert_eq!(bytes, b"AAABBB");
        assert_eq!(resolved.file_count(), 2);
        assert_eq!(resolved.sources[0].path.contains("a.txt"), true);
        assert_eq!(resolved.sources[1].path.contains("b.txt"), true);
    }

    #[test]
    fn jsonl_extracts_text_fields() {
        let directory = temporary_directory("jsonl");
        let input = directory.join("data.jsonl");
        fs::write(
            &input,
            "{\"text\":\"hello\"}\n\n{\"text\":\"world\",\"meta\":1}\n",
        )
        .unwrap();
        let cache = directory.join("cache");

        let resolved = resolve_corpus(&input, &cache).unwrap();
        let bytes = fs::read(&resolved.path).unwrap();

        assert!(resolved.materialized);
        assert_eq!(bytes, b"hello\nworld\n");
    }

    #[test]
    fn mixed_directory_is_deterministic_across_calls() {
        let directory = temporary_directory("mixed");
        fs::write(directory.join("a.txt"), b"A").unwrap();
        fs::write(
            directory.join("b.jsonl"),
            "{\"text\":\"B1\"}\n{\"text\":\"B2\"}\n",
        )
        .unwrap();
        let cache = directory.join("cache");

        let first = resolve_corpus(&directory, &cache).unwrap();
        let first_bytes = fs::read(&first.path).unwrap();
        let second = resolve_corpus(&directory, &cache).unwrap();
        let second_bytes = fs::read(&second.path).unwrap();

        assert_eq!(first_bytes, b"AB1\nB2\n");
        assert_eq!(first_bytes, second_bytes);
        assert_eq!(first.sources, second.sources);
    }

    #[test]
    fn rejects_empty_directory() {
        let directory = temporary_directory("empty");
        let cache = directory.join("cache");
        let error = resolve_corpus(&directory, &cache).unwrap_err();
        assert!(error.to_string().contains("no .txt or .jsonl"));
    }

    #[test]
    fn rejects_jsonl_without_text() {
        let directory = temporary_directory("bad-jsonl");
        let input = directory.join("bad.jsonl");
        fs::write(&input, "{\"content\":\"x\"}\n").unwrap();
        let cache = directory.join("cache");
        let error = resolve_corpus(&input, &cache).unwrap_err();
        assert!(error.to_string().contains("bad.jsonl:1"));
    }
}