use std::fs;
use std::fs::File;
use std::io::{BufReader, Read};
use std::path::Path;
use crate::ConversionOptions;
use crate::incremental::IncrementalCompiler;
use crate::output::{OutputFormat, render_elements_in_dir};
use crate::packages::PackageFetcher;
pub trait ProgressReporter: Send {
fn stage_started(&mut self, name: &str, pct: f64);
fn stage_finished(&mut self, name: &str);
}
pub struct NoOpReporter;
impl ProgressReporter for NoOpReporter {
fn stage_started(&mut self, _name: &str, _pct: f64) {}
fn stage_finished(&mut self, _name: &str) {}
}
pub struct ConsoleReporter;
impl ProgressReporter for ConsoleReporter {
fn stage_started(&mut self, name: &str, pct: f64) {
eprintln!("[{:>5.1}%] {}", pct, name);
}
fn stage_finished(&mut self, name: &str) {
eprintln!("[ done ] {}", name);
}
}
pub struct StreamingConverter<R: ProgressReporter> {
reporter: R,
chunk_size: usize,
incremental: IncrementalCompiler,
force_rebuild: bool,
keep_intermediate: bool,
options: ConversionOptions,
}
impl StreamingConverter<NoOpReporter> {
pub fn new() -> Self {
Self {
reporter: NoOpReporter,
chunk_size: 1024 * 1024, incremental: IncrementalCompiler::new(),
force_rebuild: false,
keep_intermediate: false,
options: ConversionOptions::default(),
}
}
}
fn package_cache_dir(options: &ConversionOptions) -> std::path::PathBuf {
options
.package_cache
.clone()
.unwrap_or_else(|| std::path::PathBuf::from(".rtex/cache"))
}
impl<R: ProgressReporter> StreamingConverter<R> {
pub fn with_reporter(reporter: R) -> Self {
Self {
reporter,
chunk_size: 1024 * 1024,
incremental: IncrementalCompiler::new(),
force_rebuild: false,
keep_intermediate: false,
options: ConversionOptions::default(),
}
}
pub fn with_options(mut self, options: ConversionOptions) -> Self {
self.options = options;
self
}
pub fn with_chunk_size(mut self, size: usize) -> Self {
self.chunk_size = size;
self
}
pub fn with_incremental(mut self, enabled: bool) -> Self {
self.incremental = if enabled {
IncrementalCompiler::new()
} else {
IncrementalCompiler::disabled()
};
self
}
pub fn with_force_rebuild(mut self, force: bool) -> Self {
self.force_rebuild = force;
self
}
pub fn with_keep_intermediate(mut self, keep: bool) -> Self {
self.keep_intermediate = keep;
self
}
pub fn incremental_stats(&self) -> (u64, u64) {
self.incremental.stats()
}
pub fn convert(&mut self, input: &Path, output: &Path) -> Result<(), crate::error::LatexError> {
if !input.exists() {
return Err(crate::error::LatexError::InvalidPath);
}
if !self.force_rebuild && output.exists() {
let deps = crate::watch::discover_inputs(input);
if !self.incremental.needs_rebuild(input)
|| IncrementalCompiler::outputs_up_to_date(input, output, &deps)
{
self.reporter.stage_started("skipped (up to date)", 100.0);
self.reporter.stage_finished("skipped (up to date)");
self.incremental
.mark_built(input, vec![output.to_path_buf()], deps);
return Ok(());
}
}
self.reporter.stage_started("reading source", 0.0);
let content = self.read_file(input)?;
self.reporter.stage_finished("reading source");
let expanded = crate::macros::expand_document(&content);
let mut search_paths = Vec::new();
if self.options.fetch_packages {
let fetcher = PackageFetcher::new(package_cache_dir(&self.options));
let _ = fetcher.prepare_source(&content);
search_paths = fetcher.search_paths();
}
self.reporter.stage_started("parsing", 30.0);
let mut parser = crate::parser::TexParser::new(content);
if let Some(parent) = input.parent() {
parser = parser.with_base_dir(parent);
}
if !search_paths.is_empty() {
parser = parser.with_search_paths(search_paths);
}
let elements = parser.parse();
self.reporter.stage_finished("parsing");
let build_stage = match self.options.format {
OutputFormat::Pdf => "building PDF",
OutputFormat::Html => "building HTML",
OutputFormat::Docx => "building DOCX",
OutputFormat::Epub => "building EPUB",
};
self.reporter.stage_started(build_stage, 60.0);
match self.options.format {
OutputFormat::Pdf | OutputFormat::Html | OutputFormat::Docx | OutputFormat::Epub => {
let bytes =
render_elements_in_dir(elements.clone(), self.options.format, input.parent())?;
fs::write(output, bytes).map_err(|source| crate::error::LatexError::IoError {
path: output.to_path_buf(),
source,
})?;
}
}
self.reporter.stage_finished(build_stage);
if self.keep_intermediate {
crate::intermediate::write_intermediates(
output,
&expanded,
&elements,
self.options.format,
)?;
}
let deps = crate::watch::discover_inputs(input);
self.incremental
.mark_built(input, vec![output.to_path_buf()], deps);
Ok(())
}
fn read_file(&mut self, path: &Path) -> Result<String, crate::error::LatexError> {
let file = File::open(path)?;
let meta = file.metadata()?;
let len = meta.len() as usize;
if len <= self.chunk_size {
let mut buf = String::with_capacity(len);
let mut reader = BufReader::new(file);
reader.read_to_string(&mut buf)?;
return Ok(buf);
}
let mut reader = BufReader::with_capacity(self.chunk_size, file);
let mut bytes = Vec::with_capacity(len);
let mut chunk = vec![0u8; self.chunk_size];
let mut total_read = 0usize;
loop {
let n = reader.read(&mut chunk)?;
if n == 0 {
break;
}
bytes.extend_from_slice(&chunk[..n]);
total_read += n;
if len > 0 {
let pct = 30.0 * (total_read as f64 / len as f64).min(1.0);
self.reporter.stage_started("reading source", pct);
}
}
String::from_utf8(bytes).map_err(|error| crate::error::LatexError::ParseError {
message: format!("source is not valid UTF-8: {error}"),
line: None,
column: None,
context: Some(path.display().to_string()),
})
}
}
impl Default for StreamingConverter<NoOpReporter> {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Arc, Mutex};
#[derive(Debug, Default)]
struct CollectingReporter {
events: Arc<Mutex<Vec<(String, f64)>>>,
}
impl ProgressReporter for CollectingReporter {
fn stage_started(&mut self, name: &str, pct: f64) {
self.events.lock().unwrap().push((name.to_string(), pct));
}
fn stage_finished(&mut self, name: &str) {
self.events
.lock()
.unwrap()
.push((format!("done:{}", name), 0.0));
}
}
#[test]
fn test_converter_reports_stages() {
let reporter = CollectingReporter::default();
let events = reporter.events.clone();
let mut converter = StreamingConverter::with_reporter(reporter);
let input = Path::new("/nonexistent/streaming_test.tex");
let output = Path::new("/nonexistent/streaming_test.pdf");
let result = converter.convert(input, output);
assert!(result.is_err());
let log = events.lock().unwrap();
assert!(log.is_empty());
}
#[test]
fn test_noop_reporter_does_not_panic() {
let mut r = NoOpReporter;
r.stage_started("x", 50.0);
r.stage_finished("x");
}
#[test]
fn test_console_reporter_does_not_panic() {
let mut r = ConsoleReporter;
r.stage_started("x", 50.0);
r.stage_finished("x");
}
#[test]
fn incremental_skips_unchanged_conversion() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("doc.tex");
let dst = tmp.path().join("doc.pdf");
std::fs::write(
&src,
"\\documentclass{article}\\begin{document}Hi\\end{document}",
)
.unwrap();
let mut first = StreamingConverter::new();
first.convert(&src, &dst).expect("first conversion");
assert!(dst.exists());
let reporter = CollectingReporter::default();
let events = reporter.events.clone();
let mut second = StreamingConverter::with_reporter(reporter);
second
.convert(&src, &dst)
.expect("second conversion should skip");
let log = events.lock().unwrap();
assert!(log.iter().any(|(name, _)| name == "skipped (up to date)"));
}
#[test]
fn force_rebuild_runs_even_when_unchanged() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("doc.tex");
let dst = tmp.path().join("doc.pdf");
std::fs::write(
&src,
"\\documentclass{article}\\begin{document}Hi\\end{document}",
)
.unwrap();
let mut converter = StreamingConverter::new();
converter.convert(&src, &dst).expect("first conversion");
let reporter = CollectingReporter::default();
let events = reporter.events.clone();
let mut converter = StreamingConverter::with_reporter(reporter).with_force_rebuild(true);
converter.convert(&src, &dst).expect("forced conversion");
let log = events.lock().unwrap();
assert!(!log.iter().any(|(name, _)| name == "skipped (up to date)"));
assert!(log.iter().any(|(name, _)| name == "reading source"));
}
#[test]
fn streaming_converts_html() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("doc.tex");
let dst = tmp.path().join("doc.html");
std::fs::write(
&src,
"\\documentclass{article}\\begin{document}Hello\\end{document}",
)
.unwrap();
let mut converter = StreamingConverter::new()
.with_options(ConversionOptions::default().with_format(OutputFormat::Html));
converter.convert(&src, &dst).expect("html conversion");
let html = std::fs::read_to_string(&dst).unwrap();
assert!(html.contains("Hello"));
assert!(html.contains("<html"));
}
#[test]
fn streaming_writes_intermediate_artifacts() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("doc.tex");
let dst = tmp.path().join("doc.pdf");
std::fs::write(
&src,
"\\documentclass{article}\\title{T}\\begin{document}Hi\\end{document}",
)
.unwrap();
let mut converter = StreamingConverter::new().with_keep_intermediate(true);
converter.convert(&src, &dst).expect("conversion");
assert!(dst.exists());
assert!(tmp.path().join("doc.expanded.tex").exists());
assert!(tmp.path().join("doc.ast.json").exists());
assert!(tmp.path().join("doc.meta.json").exists());
}
#[test]
fn streaming_converts_epub() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("doc.tex");
let dst = tmp.path().join("doc.epub");
std::fs::write(
&src,
"\\documentclass{article}\\begin{document}EPUB content\\end{document}",
)
.unwrap();
let mut converter = StreamingConverter::new()
.with_options(ConversionOptions::default().with_format(OutputFormat::Epub));
converter.convert(&src, &dst).expect("epub conversion");
assert!(dst.exists());
let data = std::fs::read(&dst).unwrap();
assert!(data.starts_with(b"PK"), "EPUB should be a ZIP");
}
#[test]
fn streaming_converts_docx() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("doc.tex");
let dst = tmp.path().join("doc.docx");
std::fs::write(
&src,
"\\documentclass{article}\\begin{document}DOCX content\\end{document}",
)
.unwrap();
let mut converter = StreamingConverter::new()
.with_options(ConversionOptions::default().with_format(OutputFormat::Docx));
converter.convert(&src, &dst).expect("docx conversion");
assert!(dst.exists());
let data = std::fs::read(&dst).unwrap();
assert!(data.starts_with(b"PK"), "DOCX should be a ZIP");
}
#[test]
fn streaming_custom_chunk_size() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("doc.tex");
let dst = tmp.path().join("doc.pdf");
std::fs::write(
&src,
"\\documentclass{article}\\begin{document}Chunked\\end{document}",
)
.unwrap();
let mut converter = StreamingConverter::new().with_chunk_size(256);
converter.convert(&src, &dst).expect("conversion with small chunk size");
assert!(dst.exists());
assert!(std::fs::read(&dst).unwrap().starts_with(b"%PDF"));
}
#[test]
fn incremental_rebuilds_on_source_change() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("doc.tex");
let dst = tmp.path().join("doc.pdf");
std::fs::write(
&src,
"\\documentclass{article}\\begin{document}Version 1\\end{document}",
)
.unwrap();
let mut first = StreamingConverter::new();
first.convert(&src, &dst).expect("first conversion");
assert!(dst.exists());
std::fs::write(
&src,
"\\documentclass{article}\\begin{document}Version 2\\end{document}",
)
.unwrap();
let reporter = CollectingReporter::default();
let events = reporter.events.clone();
let mut second = StreamingConverter::with_reporter(reporter);
second.convert(&src, &dst).expect("second conversion after change");
let log = events.lock().unwrap();
assert!(log.iter().any(|(name, _)| name == "reading source"),
"Should rebuild when source changes");
}
#[test]
fn streaming_nonexistent_file_returns_error() {
let tmp = tempfile::tempdir().unwrap();
let src = tmp.path().join("nonexistent.tex");
let dst = tmp.path().join("out.pdf");
let mut converter = StreamingConverter::new();
assert!(converter.convert(&src, &dst).is_err());
}
}