use serde::Deserialize;
use serde::Serialize;
use std::error::Error;
use std::fmt;
use std::fs::{self, File};
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};
const MATERIALIZED_NAME: &str = "corpus.materialized";
const MANIFEST_NAME: &str = "corpus.manifest.json";
const MANIFEST_VERSION: u32 = 1;
#[derive(Debug)]
pub enum CorpusError {
InvalidData(String),
Io(std::io::Error),
Json(serde_json::Error),
}
impl fmt::Display for CorpusError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidData(message) => write!(formatter, "corpus error: {message}"),
Self::Io(error) => error.fmt(formatter),
Self::Json(error) => write!(formatter, "corpus JSON error: {error}"),
}
}
}
impl Error for CorpusError {}
impl From<std::io::Error> for CorpusError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<serde_json::Error> for CorpusError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CorpusFileKind {
Txt,
Jsonl,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CorpusSourceEntry {
pub path: String,
pub kind: CorpusFileKind,
pub bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CorpusManifest {
pub version: u32,
pub sources: Vec<CorpusSourceEntry>,
pub total_bytes: u64,
}
#[derive(Clone, Debug)]
pub struct ResolvedCorpus {
pub path: PathBuf,
pub input: PathBuf,
pub sources: Vec<CorpusSourceEntry>,
pub materialized: bool,
}
impl ResolvedCorpus {
pub fn total_bytes(&self) -> u64 {
self.sources.iter().map(|source| source.bytes).sum()
}
pub fn file_count(&self) -> usize {
self.sources.len()
}
}
#[derive(Deserialize)]
struct JsonlRecord {
text: String,
}
pub fn resolve_corpus(
input: impl AsRef<Path>,
materialize_dir: impl AsRef<Path>,
) -> Result<ResolvedCorpus, CorpusError> {
let input = input.as_ref();
let materialize_dir = materialize_dir.as_ref();
let sources = discover_sources(input)?;
if sources.is_empty() {
return Err(CorpusError::InvalidData(format!(
"no .txt or .jsonl corpus files found at {}",
input.display()
)));
}
let needs_materialize = sources.len() > 1
|| sources
.iter()
.any(|source| source.kind == CorpusFileKind::Jsonl);
if !needs_materialize {
let source = &sources[0];
let bytes = fs::metadata(&source.path)?.len();
return Ok(ResolvedCorpus {
path: source.path.clone(),
input: input.to_path_buf(),
sources: vec![CorpusSourceEntry {
path: source.path.display().to_string(),
kind: source.kind,
bytes,
}],
materialized: false,
});
}
fs::create_dir_all(materialize_dir)?;
let materialized_path = materialize_dir.join(MATERIALIZED_NAME);
let manifest_path = materialize_dir.join(MANIFEST_NAME);
let written = materialize_sources(&sources, &materialized_path)?;
let manifest = CorpusManifest {
version: MANIFEST_VERSION,
sources: written,
total_bytes: fs::metadata(&materialized_path)?.len(),
};
serde_json::to_writer_pretty(BufWriter::new(File::create(&manifest_path)?), &manifest)?;
Ok(ResolvedCorpus {
path: materialized_path,
input: input.to_path_buf(),
sources: manifest.sources,
materialized: true,
})
}
fn discover_sources(input: &Path) -> Result<Vec<CorpusSource>, CorpusError> {
if input.is_dir() {
let mut paths = Vec::new();
for entry in fs::read_dir(input)? {
let entry = entry?;
if !entry.file_type()?.is_file() {
continue;
}
let path = entry.path();
if let Some(kind) = classify_path(&path) {
paths.push((path, kind));
}
}
paths.sort_by(|left, right| left.0.cmp(&right.0));
if paths.is_empty() {
return Err(CorpusError::InvalidData(format!(
"directory {} contains no .txt or .jsonl files",
input.display()
)));
}
return Ok(paths
.into_iter()
.map(|(path, kind)| CorpusSource { path, kind })
.collect());
}
if !input.is_file() {
return Err(CorpusError::InvalidData(format!(
"corpus path {} is not a file or directory",
input.display()
)));
}
let kind = classify_path(input).unwrap_or(CorpusFileKind::Txt);
Ok(vec![CorpusSource {
path: input.to_path_buf(),
kind,
}])
}
fn classify_path(path: &Path) -> Option<CorpusFileKind> {
let ext = path.extension()?.to_string_lossy().to_ascii_lowercase();
match ext.as_str() {
"txt" => Some(CorpusFileKind::Txt),
"jsonl" => Some(CorpusFileKind::Jsonl),
_ => None,
}
}
struct CorpusSource {
path: PathBuf,
kind: CorpusFileKind,
}
fn materialize_sources(
sources: &[CorpusSource],
destination: &Path,
) -> Result<Vec<CorpusSourceEntry>, CorpusError> {
let temporary = destination.with_extension("tmp");
let mut writer = BufWriter::new(File::create(&temporary)?);
let mut entries = Vec::with_capacity(sources.len());
for source in sources {
let bytes = match source.kind {
CorpusFileKind::Txt => copy_txt(&source.path, &mut writer)?,
CorpusFileKind::Jsonl => copy_jsonl(&source.path, &mut writer)?,
};
entries.push(CorpusSourceEntry {
path: source.path.display().to_string(),
kind: source.kind,
bytes,
});
}
writer.flush()?;
drop(writer);
fs::rename(&temporary, destination)?;
Ok(entries)
}
fn copy_txt(path: &Path, writer: &mut BufWriter<File>) -> Result<u64, CorpusError> {
let mut reader = BufReader::new(File::open(path)?);
let written = std::io::copy(&mut reader, writer)?;
Ok(written)
}
fn copy_jsonl(path: &Path, writer: &mut BufWriter<File>) -> Result<u64, CorpusError> {
let reader = BufReader::new(File::open(path)?);
let mut written = 0u64;
for (line_number, line) in reader.lines().enumerate() {
let line = line?;
if line.trim().is_empty() {
continue;
}
let record: JsonlRecord = serde_json::from_str(&line).map_err(|error| {
CorpusError::InvalidData(format!(
"{}:{}: {error}",
path.display(),
line_number + 1
))
})?;
writer.write_all(record.text.as_bytes())?;
writer.write_all(b"\n")?;
written += record.text.len() as u64 + 1;
}
if written == 0 {
return Err(CorpusError::InvalidData(format!(
"JSONL file {} produced no text records",
path.display()
)));
}
Ok(written)
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{SystemTime, UNIX_EPOCH};
fn temporary_directory(label: &str) -> PathBuf {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let path = std::env::temp_dir().join(format!("oxidelm-corpus-{label}-{nanos}"));
fs::create_dir_all(&path).unwrap();
path
}
#[test]
fn single_txt_is_not_materialized() {
let directory = temporary_directory("single");
let input = directory.join("input.txt");
fs::write(&input, b"abcd").unwrap();
let cache = directory.join("cache");
let resolved = resolve_corpus(&input, &cache).unwrap();
assert!(!resolved.materialized);
assert_eq!(resolved.path, input);
assert_eq!(resolved.file_count(), 1);
assert!(!cache.join(MATERIALIZED_NAME).exists());
}
#[test]
fn directory_of_txt_concats_in_sorted_order() {
let directory = temporary_directory("dir-txt");
fs::write(directory.join("b.txt"), b"BBB").unwrap();
fs::write(directory.join("a.txt"), b"AAA").unwrap();
fs::write(directory.join("ignore.md"), b"nope").unwrap();
let cache = directory.join("cache");
let resolved = resolve_corpus(&directory, &cache).unwrap();
let bytes = fs::read(&resolved.path).unwrap();
assert!(resolved.materialized);
assert_eq!(bytes, b"AAABBB");
assert_eq!(resolved.file_count(), 2);
assert_eq!(resolved.sources[0].path.contains("a.txt"), true);
assert_eq!(resolved.sources[1].path.contains("b.txt"), true);
}
#[test]
fn jsonl_extracts_text_fields() {
let directory = temporary_directory("jsonl");
let input = directory.join("data.jsonl");
fs::write(
&input,
"{\"text\":\"hello\"}\n\n{\"text\":\"world\",\"meta\":1}\n",
)
.unwrap();
let cache = directory.join("cache");
let resolved = resolve_corpus(&input, &cache).unwrap();
let bytes = fs::read(&resolved.path).unwrap();
assert!(resolved.materialized);
assert_eq!(bytes, b"hello\nworld\n");
}
#[test]
fn mixed_directory_is_deterministic_across_calls() {
let directory = temporary_directory("mixed");
fs::write(directory.join("a.txt"), b"A").unwrap();
fs::write(
directory.join("b.jsonl"),
"{\"text\":\"B1\"}\n{\"text\":\"B2\"}\n",
)
.unwrap();
let cache = directory.join("cache");
let first = resolve_corpus(&directory, &cache).unwrap();
let first_bytes = fs::read(&first.path).unwrap();
let second = resolve_corpus(&directory, &cache).unwrap();
let second_bytes = fs::read(&second.path).unwrap();
assert_eq!(first_bytes, b"AB1\nB2\n");
assert_eq!(first_bytes, second_bytes);
assert_eq!(first.sources, second.sources);
}
#[test]
fn rejects_empty_directory() {
let directory = temporary_directory("empty");
let cache = directory.join("cache");
let error = resolve_corpus(&directory, &cache).unwrap_err();
assert!(error.to_string().contains("no .txt or .jsonl"));
}
#[test]
fn rejects_jsonl_without_text() {
let directory = temporary_directory("bad-jsonl");
let input = directory.join("bad.jsonl");
fs::write(&input, "{\"content\":\"x\"}\n").unwrap();
let cache = directory.join("cache");
let error = resolve_corpus(&input, &cache).unwrap_err();
assert!(error.to_string().contains("bad.jsonl:1"));
}
}