retex_engine/

files.rs

//! Files: `\input`, the log file, `\openin`/`\read`, `\scantokens`
//! pseudo-files and the record of files read and written that the pass
//! driver uses (*TeX: The Program* §511–§538, §480–§486; e-TeX).

use std::collections::{HashMap, VecDeque};
use std::path::{Path, PathBuf};
use std::rc::Rc;

use crate::Engine;
use crate::cmd::Cmd;
use crate::input::{Level, LineSource, ScannerStatus};
use crate::primitives_gen as pg;
use crate::print::Selector;
use crate::token::Token;

/// The state of a `\read` stream.
#[derive(Clone, Debug)]
pub enum ReadFile {
    /// Closed.
    Closed,
    /// Open; no line read yet.
    JustOpen(LineReader),
    /// Open, at least one line read.
    Normal(LineReader),
}

/// The lines of a file opened by `\openin`, split when they are read:
/// files are often opened only to test that they exist.
#[derive(Clone, Debug, Default)]
pub struct LineReader {
    data: Vec<u8>,
    pos: usize,
}

impl LineReader {
    /// Reads the lines of `data`.
    pub fn new(data: Vec<u8>) -> Self {
        LineReader { data, pos: 0 }
    }

    /// The next line (without its terminator), if any.
    pub fn pop_front(&mut self) -> Option<Vec<u8>> {
        let (line, next) = crate::input::next_line(&self.data, self.pos)?;
        let line = line.to_vec();
        self.pos = next;
        Some(line)
    }
}

/// A file that was read or written during a pass, with a content hash.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FileRecord {
    /// The path as opened.
    pub path: PathBuf,
    /// FNV-1a hash of the content, or `None` if the file did not exist.
    pub hash: Option<u64>,
}

/// File-related state.
#[derive(Debug)]
pub struct FileState {
    /// `\read` streams 0–15.
    pub read_files: Vec<ReadFile>,
    /// Whether a file name is being scanned (`name_in_progress`).
    pub name_in_progress: bool,
    /// The directory for output files (`-output-directory`).
    pub output_dir: Option<PathBuf>,
    /// Files read during this pass, in order of first access.
    pub read_log: Vec<FileRecord>,
    /// Content of files written during this pass, by path.
    pub written: HashMap<PathBuf, Vec<u8>>,
    /// The first line given on the command line.
    pub command_line: Vec<u8>,
    /// The log file name, once opened.
    pub log_name: Option<PathBuf>,
    /// Further files the job depends on that the pass driver does not
    /// compare between passes (images), for `--watch`.
    pub watched: Vec<PathBuf>,
}

impl Default for FileState {
    fn default() -> Self {
        FileState {
            read_files: vec![ReadFile::Closed; 16],
            name_in_progress: false,
            output_dir: None,
            read_log: Vec::new(),
            written: HashMap::new(),
            command_line: Vec::new(),
            log_name: None,
            watched: Vec::new(),
        }
    }
}

impl FileState {
    /// Whether `\read` stream `n` is open.
    pub fn read_open(&self, n: u32) -> bool {
        !matches!(
            self.read_files.get(n as usize),
            None | Some(ReadFile::Closed)
        )
    }
}

/// FNV-1a hash used to detect content changes between passes.
pub fn content_hash(data: &[u8]) -> u64 {
    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
    for &b in data {
        h ^= u64::from(b);
        h = h.wrapping_mul(0x0100_0000_01b3);
    }
    h
}

impl Engine {
    /// Reads a whole file, recording it for the pass driver.
    pub fn read_file_recorded(&mut self, path: &Path) -> Option<Vec<u8>> {
        let data = std::fs::read(path).ok();
        if !self.files.read_log.iter().any(|r| r.path == path) {
            self.files.read_log.push(FileRecord {
                path: path.to_path_buf(),
                hash: data.as_deref().map(content_hash),
            });
        }
        data
    }

    /// Records a file lookup from Lua (`kpse.find_file`, which expl3 uses to
    /// test whether a file exists) for the pass driver: files of the job
    /// (relative names) are recorded with their content hash, or as absent.
    pub fn record_probe(&mut self, name: &str, found: Option<&Path>) {
        let path = match found {
            Some(p) if p.is_relative() => p.to_path_buf(),
            Some(_) => return,
            None if !Path::new(name).is_absolute() && !name.contains("//") => {
                self.absent_path(name)
            }
            None => return,
        };
        if self.files.read_log.iter().any(|r| r.path == path) {
            return;
        }
        let hash = found
            .and_then(|p| std::fs::read(p).ok())
            .map(|d| content_hash(&d));
        self.files.read_log.push(FileRecord { path, hash });
    }

    /// Where a job file `name` that was not found would appear: in the
    /// output directory, where the job writes its files, if there is one.
    fn absent_path(&self, name: &str) -> PathBuf {
        match &self.files.output_dir {
            Some(d) if Path::new(name).is_relative() => d.join(name),
            _ => PathBuf::from(name),
        }
    }

    /// Records that file `name` was looked for and not found, so that the
    /// pass driver reruns when this pass writes it (e.g. the `.aux` file
    /// LaTeX tests for with `\filesize`).
    pub fn record_absent_file(&mut self, name: &str) {
        let p = self.absent_path(name);
        if !self.files.read_log.iter().any(|r| r.path == p) {
            self.files.read_log.push(FileRecord {
                path: p,
                hash: None,
            });
        }
    }

    /// Looks a TeX input file up: in the output directory first, then via
    /// kpathsea.
    pub fn find_input_file(&mut self, name: &str, fmt: retex_kpath::FileFormat) -> Option<PathBuf> {
        if let Some(dir) = &self.files.output_dir {
            for cand in [format!("{name}.tex"), name.to_owned()] {
                let p = dir.join(&cand);
                if p.is_file() {
                    return Some(p);
                }
            }
        }
        if let Some(k) = &self.kpse {
            return k.borrow_mut().find_file(name, fmt);
        }
        let p = PathBuf::from(name);
        if p.is_file() {
            return Some(p);
        }
        let p = PathBuf::from(format!("{name}.tex"));
        p.is_file().then_some(p)
    }

    /// Scans a file name (`scan_file_name`). LuaTeX also accepts a name in
    /// braces and names in double quotes.
    pub fn scan_file_name(&mut self) -> Vec<u8> {
        self.files.name_in_progress = true;
        let mut name: Vec<u8> = Vec::new();
        self.get_nonblank_noncall();
        if self.cur_cmd == Cmd::LeftBrace {
            self.back_input();
            let toks = self.scan_toks_expanded();
            let s = self.print_to_string(|e| e.show_token_list(&toks, 0, toks.len(), i64::MAX));
            name = s.into_iter().filter(|&b| b != b'"').collect();
            self.files.name_in_progress = false;
            return name;
        }
        let mut quoted = false;
        loop {
            if self.cur_cmd > Cmd::OtherChar || self.cur_cs != crate::hash::NO_CS {
                self.back_input();
                break;
            }
            let c = self.cur_chr;
            if c == ' ' as u32 && !quoted {
                break;
            }
            if c == '"' as u32 {
                quoted = !quoted;
            } else {
                crate::utf8x::push(&mut name, c);
            }
            self.get_x_token();
        }
        self.files.name_in_progress = false;
        name
    }

    /// `\input` (`start_input`).
    pub fn start_input(&mut self) {
        let name = self.scan_file_name();
        let name_str = String::from_utf8_lossy(&name).into_owned();
        let found = match self.find_input_file(&name_str, retex_kpath::FileFormat::Tex) {
            Some(p) => p,
            None => self.prompt_file_name(&name, "input file name", ".tex"),
        };
        let data = self.read_file_recorded(&found).unwrap_or_default();
        if !self.printer.log_opened() && self.job_name.is_some() {
            self.open_log_file();
        }
        if self.job_name.is_none() {
            let base = Path::new(&name_str)
                .file_stem()
                .map(|s| s.to_string_lossy().into_owned())
                .unwrap_or_else(|| name_str.clone());
            self.job_name = Some(base.into_bytes());
            self.open_log_file();
        }
        let display = found.to_string_lossy().into_owned().into_bytes();
        if self.printer.term_offset + display.len() > self.printer.max_print_line.saturating_sub(2)
        {
            self.printer.print_ln();
        } else if self.printer.term_offset > 0 || self.printer.file_offset > 0 {
            self.printer.print_char(' ' as u32);
        }
        self.printer.print_char('(' as u32);
        self.input.open_parens += 1;
        self.printer.print_bytes(&display);
        self.printer.update_terminal();
        let source = LineSource::File {
            data: Rc::from(data),
            pos: 0,
        };
        self.begin_file_reading(source, display, true);
        // Read the first line of the new file.
        self.read_first_line();
    }

    fn read_first_line(&mut self) {
        let end_line_char = self.eqtb.int(pg::assign_int::ENDLINECHAR);
        if let Some(Level::Text(t)) = self.input.stack.last_mut() {
            t.line = 0;
            let line = match &mut t.source {
                LineSource::File { data, pos } => match crate::input::next_line(data, *pos) {
                    Some((bytes, next)) => {
                        let l = crate::input::decode_line(bytes);
                        *pos = next;
                        Some(l)
                    }
                    None => None,
                },
                _ => None,
            };
            t.line = 1;
            t.buf = line.unwrap_or_default();
            if (0..=0x10_FFFF).contains(&end_line_char) {
                t.buf.push(end_line_char as u32);
            }
            t.loc = 0;
            t.state = crate::input::LexState::NewLine;
        }
    }

    /// Asks for a file name after a lookup failure (`prompt_file_name`). In
    /// non-interactive modes this is fatal.
    pub fn prompt_file_name(&mut self, name: &[u8], s: &str, e: &str) -> ! {
        if s == "input file name" {
            self.print_err("I can't find file `");
        } else {
            self.print_err("I can't write on file `");
        }
        self.printer.print_bytes(name);
        self.printer.print_str("'.");
        if e == ".tex" {
            self.show_context();
        }
        self.printer.print_ln();
        self.printer
            .print_str("(Press Enter to retry, or Control-D to exit");
        if !e.is_empty() {
            self.printer.print_str("; default file extension is `");
            self.printer.print_str(e);
            self.printer.print_str("'");
        }
        self.printer.print_str(")");
        self.printer.print_nl("Please type another ");
        self.printer.print_str(s);
        self.fatal_error("*** (job aborted, file error in nonstop mode)");
    }

    /// Opens the log file and prints the banner (`open_log_file`).
    pub fn open_log_file(&mut self) {
        let old = self.printer.selector;
        if self.job_name.is_none() {
            self.job_name = Some(b"texput".to_vec());
        }
        let job = String::from_utf8_lossy(self.job_name.as_ref().unwrap()).into_owned();
        let mut path = PathBuf::from(format!("{job}.log"));
        if let Some(dir) = &self.files.output_dir {
            path = dir.join(path);
        }
        match std::fs::File::create(&path) {
            Ok(f) => self
                .printer
                .set_log(Box::new(std::io::BufWriter::with_capacity(1 << 16, f))),
            Err(_) => {
                // TeX would prompt; without a log we continue silently.
                self.printer.set_log(Box::new(std::io::sink()));
            }
        }
        self.files.log_name = Some(path);
        // Print the banner line, including the date and time.
        let banner = self.log_banner();
        self.printer.log_raw(banner.as_bytes());
        self.printer
            .log_raw(b"\n restricted system commands enabled.");
        self.printer.file_offset = " restricted system commands enabled.".len();
        self.printer.selector = Selector::LogOnly;
        self.printer.print_nl("**");
        let line = self.files.command_line.clone();
        self.printer.print_bytes(&line);
        self.printer.print_ln();
        self.printer.selector = match old {
            Selector::NoPrint => Selector::LogOnly,
            Selector::TermOnly => Selector::TermAndLog,
            s => s,
        };
    }

    /// The first line of the log file.
    fn log_banner(&self) -> String {
        let c = self.start_time.civil();
        const MONTHS: [&str; 12] = [
            "JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC",
        ];
        format!(
            "This is retex, Version {} (LuaTeX 1.22.0 compatible)  ({})  {} {} {} {:02}:{:02}",
            env!("CARGO_PKG_VERSION"),
            if self.ini_version {
                "INITEX".to_owned()
            } else {
                self.format_ident
                    .trim()
                    .trim_start_matches('(')
                    .trim_end_matches(')')
                    .to_owned()
            },
            c.day,
            MONTHS[(c.month as usize).saturating_sub(1).min(11)],
            c.year,
            c.hour,
            c.minute
        )
    }

    /// The terminal line is exhausted: in `\errorstopmode`/`\scrollmode`
    /// ask for more input (§360), otherwise stop.
    pub fn terminal_exhausted(&mut self) {
        if self.printer.selector == Selector::NoPrint || self.printer.selector == Selector::TermOnly
        {
            self.open_log_file();
        }
        if self.err.interaction > crate::error::Interaction::NonStop {
            let empty = matches!(self.input.stack.last(), Some(Level::Text(t)) if t.buf.is_empty());
            if empty {
                self.printer
                    .print_nl("(Please type a command or say `\\end')");
            }
            self.printer.print_ln();
            let mut line = self.prompt_input("*");
            let elc = self.eqtb.int(pg::assign_int::ENDLINECHAR);
            if (0..=0x10_FFFF).contains(&elc) {
                line.push(elc as u32);
            }
            if let Some(Level::Text(t)) = self.input.stack.last_mut() {
                t.buf = line;
                t.loc = 0;
                t.state = crate::input::LexState::NewLine;
            }
            return;
        }
        self.fatal_error("*** (job aborted, no legal \\end found)");
    }

    /// `\openin` and `\closein` (`open_or_close_in`).
    pub fn open_or_close_in(&mut self) {
        let c = self.cur_chr;
        self.scan_four_bit_int();
        let n = self.cur_val as usize;
        self.files.read_files[n] = ReadFile::Closed;
        if c != 0 {
            self.scan_optional_equals();
            let name = self.scan_file_name();
            let name = String::from_utf8_lossy(&name).into_owned();
            if let Some(p) = self.find_input_file(&name, retex_kpath::FileFormat::Tex)
                && let Some(data) = self.read_file_recorded(&p)
            {
                self.files.read_files[n] = ReadFile::JustOpen(LineReader::new(data));
            } else {
                self.record_absent_file(&name);
            }
        }
    }

    /// Reads a line from a `\read` stream into a token list (`read_toks`,
    /// §482). `j` is 1 for `\readline` (e-TeX). The result starts with the
    /// `END_MATCH` token, as a macro body without parameters.
    pub fn read_toks(&mut self, n: i32, r: crate::hash::CsId, j: u32) -> Vec<Token> {
        self.input.scanner_status = ScannerStatus::Defining;
        self.input.warning_index = r;
        let mut out: Vec<Token> = vec![Token::END_MATCH];
        let m: usize = if (0..=15).contains(&n) {
            n as usize
        } else {
            16
        };
        let s = self.input.align_state;
        self.input.align_state = 1_000_000;
        loop {
            // Input and store tokens from the next line of the file.
            let line: Vec<u8> = match self.files.read_files.get_mut(m) {
                None | Some(ReadFile::Closed) => {
                    if self.err.interaction > crate::error::Interaction::NonStop {
                        // Interactive terminal reading is not supported.
                        self.fatal_error("*** (cannot \\read from terminal in nonstop modes)");
                    }
                    self.fatal_error("*** (cannot \\read from terminal in nonstop modes)");
                }
                Some(ReadFile::JustOpen(lines)) => {
                    let first = lines.pop_front();
                    match first {
                        Some(l) => {
                            let rest = std::mem::take(lines);
                            self.files.read_files[m] = ReadFile::Normal(rest);
                            l
                        }
                        None => {
                            self.files.read_files[m] = ReadFile::Closed;
                            Vec::new()
                        }
                    }
                }
                Some(ReadFile::Normal(lines)) => match lines.pop_front() {
                    Some(l) => l,
                    None => {
                        self.files.read_files[m] = ReadFile::Closed;
                        if self.input.align_state != 1_000_000 {
                            self.runaway();
                            self.print_err("File ended within ");
                            self.print_esc_str("read");
                            self.help(&["This \\read has unbalanced braces."]);
                            self.input.align_state = 1_000_000;
                            self.error();
                        }
                        Vec::new()
                    }
                },
            };
            let end_line_char = self.eqtb.int(pg::assign_int::ENDLINECHAR);
            let mut buf = crate::input::decode_line(&line);
            if (0..=0x10_FFFF).contains(&end_line_char) {
                buf.push(end_line_char as u32);
            }
            if j == 1 {
                // \readline: spaces become space tokens, the rest other.
                for &c in &buf {
                    out.push(if c == ' ' as u32 {
                        Token::SPACE
                    } else {
                        Token::other(c)
                    });
                }
                break;
            }
            self.begin_file_reading(LineSource::Read(m as u32), Vec::new(), false);
            if let Some(Level::Text(t)) = self.input.stack.last_mut() {
                t.buf = buf;
                t.loc = 0;
                t.state = crate::input::LexState::NewLine;
            }
            loop {
                self.get_token();
                if self.cur_tok.0 == 0 {
                    break;
                }
                if self.input.align_state < 1_000_000 {
                    // An unmatched `}` aborts the line.
                    loop {
                        self.get_token();
                        if self.cur_tok.0 == 0 {
                            break;
                        }
                    }
                    self.input.align_state = 1_000_000;
                    break;
                }
                out.push(self.cur_tok);
            }
            self.end_file_reading();
            if self.input.align_state == 1_000_000 {
                break;
            }
        }
        self.input.scanner_status = ScannerStatus::Normal;
        self.input.align_state = s;
        out
    }

    /// `\scantokens` and `\scantextokens` (e-TeX `pseudo_start`).
    pub fn pseudo_start(&mut self, text_tokens: bool) {
        let list = self.scan_general_text();
        let s = self.print_to_string(|e| e.show_token_list(&list, 0, list.len(), i64::MAX));
        // Split at the new-line character.
        let nl = self.eqtb.int(pg::assign_int::NEWLINECHAR);
        let mut lines: VecDeque<Vec<u8>> = VecDeque::new();
        if (0..128).contains(&nl) {
            for part in s.split(|&b| i32::from(b) == nl) {
                lines.push_back(part.to_vec());
            }
        } else {
            lines.push_back(s);
        }
        let tracing = self.eqtb.int(pg::assign_int::TRACINGSCANTOKENS) > 0 && !text_tokens;
        self.begin_file_reading(
            LineSource::Lines {
                lines,
                catcodes: None,
                pseudo: !text_tokens,
            },
            Vec::new(),
            tracing,
        );
        if let Some(Level::Text(t)) = self.input.stack.last_mut() {
            t.line = 0;
            t.buf.clear();
            t.loc = 0;
            t.eof_seen = text_tokens;
        }
        if tracing {
            if self.printer.term_offset > self.printer.max_print_line.saturating_sub(3) {
                self.printer.print_ln();
            } else if self.printer.term_offset > 0 || self.printer.file_offset > 0 {
                self.printer.print_char(' ' as u32);
            }
            self.printer.print_str("( ");
            self.input.open_parens += 1;
            self.printer.update_terminal();
        }
    }

    /// e-TeX's check when a file ends inside groups or conditionals.
    pub fn file_closed_checks(&mut self) {}

    /// Closes everything at the end of the run
    /// (`close_files_and_terminate`).
    pub fn close_files_and_terminate(&mut self) {
        for n in 0..128u32 {
            if self.printer.write_open(n) {
                self.printer.close_write(n);
            }
        }
        self.finish_output();
        if self.total_pages == 0 {
            self.printer.print_nl(
                if self.eqtb.int(crate::primitives_gen::assign_int::OUTPUTMODE) > 0 {
                    "warning  (pdf backend): no pages of output."
                } else {
                    "No pages of output."
                },
            );
            self.printer.print_ln();
            if self.pdf.stream_written
                && self.eqtb.int(crate::primitives_gen::assign_int::OUTPUTMODE) > 0
            {
                self.pdf_fatal("already written content discarded, no output file produced.");
            }
        }
        // LuaTeX reports the objects it allocated (log only).
        if self.eqtb.int(crate::primitives_gen::assign_int::OUTPUTMODE) > 0
            && self.pdf.max_obj() > 0
            && !self.pdf.fatal
            && self.printer.log_opened()
        {
            if self.printer.file_offset > 0 {
                self.printer.print_ln();
            }
            let n = self.pdf.max_obj();
            let names = self.pdf.named_dests();
            // The extra memory is not modelled exactly: LuaTeX uses 8
            // words per outline entry, ten per box resource and five per
            // object (observed), and more for threads.
            let extra = 1
                + 8 * self.pdf.outlines.len()
                + 10 * self.box_resources.len()
                + self.pdf.obj_words;
            let (objs, streams) = self.pdf.objstm_stats;
            let compressed = if streams > 0 {
                format!(
                    " {objs} compressed objects within {streams} object stream{}\n",
                    if streams == 1 { "" } else { "s" }
                )
            } else {
                String::new()
            };
            self.printer.log_raw(
                format!(
                    "\nPDF statistics: {n} PDF objects out of {} (max. 8388607)\n{compressed} \
                     {names} named destinations out of {} (max. 131072)\n \
                     {extra} words of extra memory for PDF output out of {} (max. 100000000)\n",
                    n.max(1000),
                    names.max(1000),
                    extra.max(10000),
                )
                .as_bytes(),
            );
        }
        if self.printer.log_opened() {
            self.printer.log_raw(b"\n");
            self.printer.close_log();
            self.printer.selector = Selector::TermOnly;
            if let Some(name) = self.files.log_name.clone() {
                self.printer.print_nl("Transcript written on ");
                self.printer.print_bytes(name.to_string_lossy().as_bytes());
                self.printer.print_char('.' as u32);
            }
        }
        self.printer.print_ln();
        self.printer.flush();
    }
}

/// Shared record of the contents written to files during a pass.
pub type WrittenSink = std::rc::Rc<std::cell::RefCell<HashMap<PathBuf, Vec<u8>>>>;

/// A file writer that also records everything written, so that the pass
/// driver can compare it with what the next pass reads.
pub struct RecordingWriter {
    inner: std::io::BufWriter<std::fs::File>,
    path: PathBuf,
    sink: WrittenSink,
}

impl RecordingWriter {
    /// Wraps `f`, recording its content under `path`.
    pub fn new(f: std::fs::File, path: PathBuf, sink: WrittenSink) -> Self {
        sink.borrow_mut().insert(path.clone(), Vec::new());
        RecordingWriter {
            inner: std::io::BufWriter::new(f),
            path,
            sink,
        }
    }
}

impl RecordingWriter {
    /// Wraps `f`, appending to what is already recorded under `path` (a
    /// file reopened after resuming from a snapshot).
    pub fn resume(f: std::fs::File, path: PathBuf, sink: WrittenSink) -> Self {
        sink.borrow_mut().entry(path.clone()).or_default();
        RecordingWriter {
            inner: std::io::BufWriter::new(f),
            path,
            sink,
        }
    }
}

impl std::io::Write for RecordingWriter {
    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
        if let Some(v) = self.sink.borrow_mut().get_mut(&self.path) {
            v.extend_from_slice(buf);
        }
        self.inner.write(buf)
    }

    fn flush(&mut self) -> std::io::Result<()> {
        self.inner.flush()
    }
}