text_mirror/convert/
mod.rs

1//! The converter boundary and the plain text passthrough.
2//!
3//! Every converter returns the same outcome shape. The registry in
4//! `rules/converters.toml` maps format ids to converters. A format no
5//! converter claims is unsupported. A converter error or invalid
6//! output is a failure with a machine-readable reason and no text
7//! artifact. There is no silent skip and no silently empty file. A
8//! non-empty source that converts to empty text fails with reason
9//! `empty_output` in every converter.
10//!
11//! In-process conversion runs under a panic boundary and explicit
12//! ceilings that apply before allocation in this crate's own code:
13//! the source size before read, the decompressed size of every
14//! archive part and compound stream, the cell extent of every sheet
15//! before the workbook parser runs, and the rendered output size. A
16//! caught panic or an exceeded ceiling becomes one failed record and
17//! the run continues. The panic boundary requires the default
18//! `panic = "unwind"` profile. A downstream `panic = "abort"` build
19//! turns any converter panic into process death, so keep unwinding
20//! enabled wherever this crate converts untrusted bytes.
21
22/// Ceiling on source bytes read for conversion, checked before read.
23pub const MAX_SOURCE_BYTES: u64 = 128 * 1024 * 1024;
24
25/// Ceiling on the decompressed bytes of one archive part or one
26/// compound file stream.
27pub const MAX_PART_BYTES: u64 = 64 * 1024 * 1024;
28
29/// Ceiling on rendered artifact text bytes.
30pub const MAX_OUTPUT_BYTES: usize = 64 * 1024 * 1024;
31
32/// Ceiling on 0-based row indexes in one sheet.
33pub const MAX_SHEET_ROWS: u32 = 1_048_576;
34
35/// Ceiling on 0-based column indexes in one sheet.
36pub const MAX_SHEET_COLUMNS: u32 = 16_384;
37
38/// Ceiling on the row-column product of one sheet, checked from the
39/// source's own references before the workbook parser allocates.
40pub const MAX_SHEET_CELLS: u64 = 10_000_000;
41
42use std::collections::HashMap;
43use std::fmt;
44
45use serde::Deserialize;
46use unicode_normalization::UnicodeNormalization;
47
48use crate::segments::Segment;
49use crate::{Error, Result};
50
51mod anydoc_log;
52mod differential;
53mod document;
54pub mod pdf;
55pub mod subprocess;
56mod visibility;
57mod workbook;
58
59pub use differential::{
60    DiffMetrics, DiffVerdict, DifferentialOutcome, compare_texts, normalize_for_diff,
61};
62pub use document::{AnydocDocument, markdown_to_plain};
63pub use subprocess::PdfSubprocess;
64pub use visibility::{IndexRange, SheetVisibility, WorkbookVisibility, read_visibility};
65pub use workbook::WorkbookIr;
66
67/// What every converter returns on success.
68#[derive(Debug, Clone)]
69pub struct Outcome {
70    /// Stable converter id from the registry.
71    pub converter_id: String,
72    /// Converter version from the registry.
73    pub converter_version: String,
74    /// Format id the converter was asked to handle.
75    pub detected_format: String,
76    /// The converted text, UTF-8, NFC, LF line endings.
77    pub text: String,
78    /// Non-fatal notes about the conversion.
79    pub warnings: Vec<String>,
80    /// Structure spans over `text` for the segments file.
81    pub segments: Vec<Segment>,
82}
83
84/// A conversion failure with a machine-readable reason.
85#[derive(Debug, Clone)]
86pub struct ConvertError {
87    /// Stable reason code, such as `invalid_utf8` or `empty_output`.
88    pub code: &'static str,
89    /// Detail for a human reading the manifest.
90    pub message: String,
91}
92
93impl fmt::Display for ConvertError {
94    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
95        write!(f, "{}: {}", self.code, self.message)
96    }
97}
98
99impl std::error::Error for ConvertError {}
100
101/// A converter takes one source's bytes and returns text or a reason.
102///
103/// The caller supplies the bytes, so the hash it records and the bytes
104/// the converter consumed are one snapshot. Implementations must be
105/// deterministic. The same input bytes under the same rules produce
106/// byte-identical output.
107pub trait Converter {
108    /// Stable converter id.
109    fn id(&self) -> &'static str;
110    /// Converter version, part of the incremental checkpoint key.
111    fn version(&self) -> &'static str;
112    /// Converts one source from its bytes.
113    fn convert(
114        &self,
115        source: &[u8],
116        detected_format: &str,
117    ) -> std::result::Result<Outcome, ConvertError>;
118}
119
120/// Normalizes text to the output contract.
121///
122/// CRLF and lone CR become LF, then the text is normalized to NFC.
123/// NFC only. NFKC rewrites identifiers, so it is prohibited.
124pub fn normalize_text(input: &str) -> String {
125    let mut unified = String::with_capacity(input.len());
126    let mut chars = input.chars().peekable();
127    while let Some(c) = chars.next() {
128        if c == '\r' {
129            if chars.peek() == Some(&'\n') {
130                chars.next();
131            }
132            unified.push('\n');
133        } else {
134            unified.push(c);
135        }
136    }
137    unified.nfc().collect()
138}
139
140/// Passthrough for text-native formats.
141///
142/// Reads the source as plain text, validates UTF-8, strips a leading
143/// byte order mark, and applies [`normalize_text`]. It parses nothing
144/// and strips no markup.
145pub struct PlainTextPassthrough;
146
147/// Registry id of the passthrough converter.
148pub const PASSTHROUGH_ID: &str = "text-passthrough";
149/// Version of the passthrough converter.
150pub const PASSTHROUGH_VERSION: &str = "1.0.0";
151
152impl Converter for PlainTextPassthrough {
153    fn id(&self) -> &'static str {
154        PASSTHROUGH_ID
155    }
156
157    fn version(&self) -> &'static str {
158        PASSTHROUGH_VERSION
159    }
160
161    fn convert(
162        &self,
163        source: &[u8],
164        detected_format: &str,
165    ) -> std::result::Result<Outcome, ConvertError> {
166        let source_len = source.len();
167        let raw = String::from_utf8(source.to_vec()).map_err(|e| ConvertError {
168            code: "invalid_utf8",
169            message: format!("invalid UTF-8 at byte {}", e.utf8_error().valid_up_to()),
170        })?;
171        let mut warnings = Vec::new();
172        let raw = match raw.strip_prefix('\u{feff}') {
173            Some(stripped) => {
174                warnings.push("stripped leading byte order mark".to_string());
175                stripped.to_string()
176            }
177            None => raw,
178        };
179        let text = normalize_text(&raw);
180        if source_len > 0 && text.is_empty() {
181            return Err(ConvertError {
182                code: "empty_output",
183                message: format!("source is {source_len} bytes but conversion produced no text"),
184            });
185        }
186        let segments = vec![Segment::span(0, text.len(), "document")];
187        Ok(Outcome {
188            converter_id: PASSTHROUGH_ID.to_string(),
189            converter_version: PASSTHROUGH_VERSION.to_string(),
190            detected_format: detected_format.to_string(),
191            text,
192            warnings,
193            segments,
194        })
195    }
196}
197
198#[derive(Debug, Deserialize)]
199#[serde(deny_unknown_fields)]
200struct RawRegistry {
201    version: String,
202    converters: Vec<RawEntry>,
203    #[serde(default)]
204    unsupported: Vec<RawUnsupported>,
205}
206
207#[derive(Debug, Deserialize)]
208#[serde(deny_unknown_fields)]
209struct RawEntry {
210    id: String,
211    version: String,
212    formats: Vec<String>,
213}
214
215#[derive(Debug, Deserialize)]
216#[serde(deny_unknown_fields)]
217struct RawUnsupported {
218    reason: String,
219    formats: Vec<String>,
220}
221
222/// The converter registry from `rules/converters.toml`.
223///
224/// The registry is data. A rules bump is a visible versioned event,
225/// and the pinned entry version must match the implementation, so a
226/// code change cannot ship silently under an old version.
227pub struct Registry {
228    version: String,
229    converters: Vec<Box<dyn Converter>>,
230    by_format: HashMap<String, usize>,
231    unsupported_reasons: HashMap<String, String>,
232}
233
234impl Registry {
235    /// The registry compiled into the binary from `rules/converters.toml`.
236    pub fn builtin() -> Result<Self> {
237        Self::parse(
238            include_str!("../../rules/converters.toml"),
239            "converters.toml",
240        )
241    }
242
243    /// Parses a registry and binds entries to implementations.
244    pub fn parse(text: &str, name: &str) -> Result<Self> {
245        let raw: RawRegistry = toml::from_str(text).map_err(|e| Error::Rules {
246            name: name.to_string(),
247            message: e.to_string(),
248        })?;
249        let mut converters: Vec<Box<dyn Converter>> = Vec::new();
250        let mut by_format = HashMap::new();
251        for entry in &raw.converters {
252            let converter: Box<dyn Converter> = match entry.id.as_str() {
253                PASSTHROUGH_ID => Box::new(PlainTextPassthrough),
254                document::ANYDOC_ID => Box::new(AnydocDocument),
255                workbook::WORKBOOK_ID => Box::new(WorkbookIr),
256                subprocess::PDF_SUBPROCESS_ID => Box::new(PdfSubprocess),
257                other => {
258                    return Err(Error::Rules {
259                        name: name.to_string(),
260                        message: format!("unknown converter id {other:?}"),
261                    });
262                }
263            };
264            if entry.version != converter.version() {
265                return Err(Error::Rules {
266                    name: name.to_string(),
267                    message: format!(
268                        "converter {:?} pins version {:?} but the implementation is {:?}",
269                        entry.id,
270                        entry.version,
271                        converter.version()
272                    ),
273                });
274            }
275            let index = converters.len();
276            converters.push(converter);
277            for format in &entry.formats {
278                if by_format.insert(format.clone(), index).is_some() {
279                    return Err(Error::Rules {
280                        name: name.to_string(),
281                        message: format!("format {format:?} claimed twice"),
282                    });
283                }
284            }
285        }
286        let mut unsupported_reasons = HashMap::new();
287        for entry in &raw.unsupported {
288            for format in &entry.formats {
289                if by_format.contains_key(format)
290                    || unsupported_reasons
291                        .insert(format.clone(), entry.reason.clone())
292                        .is_some()
293                {
294                    return Err(Error::Rules {
295                        name: name.to_string(),
296                        message: format!("format {format:?} claimed twice"),
297                    });
298                }
299            }
300        }
301        Ok(Registry {
302            version: raw.version,
303            converters,
304            by_format,
305            unsupported_reasons,
306        })
307    }
308
309    /// The registry version, part of the run's rules version.
310    pub fn version(&self) -> &str {
311        &self.version
312    }
313
314    /// The converter that claims a format id, if any.
315    pub fn converter_for(&self, format_id: &str) -> Option<&dyn Converter> {
316        self.by_format
317            .get(format_id)
318            .map(|index| self.converters[*index].as_ref())
319    }
320
321    /// The declared reason a format is unsupported, when the registry
322    /// names one, such as a spreadsheet format with no visibility
323    /// reader yet.
324    pub fn unsupported_reason(&self, format_id: &str) -> Option<&str> {
325        self.unsupported_reasons.get(format_id).map(String::as_str)
326    }
327
328    /// A registry built directly from converter instances, for tests
329    /// that need behavior no shipped converter exhibits, such as a
330    /// panicking converter.
331    #[cfg(test)]
332    pub(crate) fn for_tests(
333        version: &str,
334        entries: Vec<(Box<dyn Converter>, Vec<&str>)>,
335    ) -> Registry {
336        let mut converters = Vec::new();
337        let mut by_format = HashMap::new();
338        for (converter, formats) in entries {
339            let index = converters.len();
340            converters.push(converter);
341            for format in formats {
342                by_format.insert(format.to_string(), index);
343            }
344        }
345        Registry {
346            version: version.to_string(),
347            converters,
348            by_format,
349            unsupported_reasons: HashMap::new(),
350        }
351    }
352}
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357
358    #[test]
359    fn normalize_converts_crlf_to_lf() {
360        assert_eq!(normalize_text("a\r\nb\r\n"), "a\nb\n");
361    }
362
363    #[test]
364    fn normalize_converts_lone_cr_to_lf() {
365        assert_eq!(normalize_text("a\rb\r"), "a\nb\n");
366        assert_eq!(normalize_text("a\r\rb"), "a\n\nb");
367    }
368
369    #[test]
370    fn normalize_composes_to_nfc() {
371        assert_eq!(normalize_text("cafe\u{301}"), "caf\u{e9}");
372    }
373
374    #[test]
375    fn normalize_never_applies_nfkc() {
376        assert_eq!(normalize_text("\u{fb01}le \u{2460}"), "\u{fb01}le \u{2460}");
377    }
378
379    #[test]
380    fn passthrough_rejects_invalid_utf8() {
381        let err = PlainTextPassthrough
382            .convert(b"hello \xff world", "text")
383            .unwrap_err();
384        assert_eq!(err.code, "invalid_utf8");
385    }
386
387    #[test]
388    fn passthrough_fails_on_suspiciously_empty_output() {
389        let err = PlainTextPassthrough
390            .convert("\u{feff}".as_bytes(), "text")
391            .unwrap_err();
392        assert_eq!(err.code, "empty_output");
393    }
394
395    #[test]
396    fn passthrough_accepts_an_empty_source() {
397        let outcome = PlainTextPassthrough.convert(b"", "text").unwrap();
398        assert_eq!(outcome.text, "");
399    }
400
401    #[test]
402    fn builtin_registry_claims_the_text_family() {
403        let registry = Registry::builtin().unwrap();
404        assert_eq!(registry.version(), "3");
405        assert!(registry.converter_for("text").is_some());
406        assert!(registry.converter_for("markdown").is_some());
407        assert!(registry.converter_for("csv").is_some());
408        assert!(registry.converter_for("pdf").is_some());
409        assert_eq!(
410            registry.converter_for("pdf").map(|c| c.id()),
411            Some("pdf-subprocess")
412        );
413        assert!(registry.converter_for("docx").is_some());
414        assert!(registry.converter_for("xlsx").is_some());
415        assert!(registry.converter_for("html").is_none());
416        assert!(registry.converter_for("xlsb").is_none());
417        assert_eq!(
418            registry.unsupported_reason("xlsb"),
419            Some("hidden-visibility-unresolved")
420        );
421        assert_eq!(registry.unsupported_reason("png"), Some("engine-unpinned"));
422        assert_eq!(registry.unsupported_reason("mp4"), Some("engine-unpinned"));
423        assert_eq!(registry.unsupported_reason("html"), None);
424    }
425
426    #[test]
427    fn registry_rejects_a_version_that_contradicts_the_implementation() {
428        let toml = r#"
429version = "1"
430
431[[converters]]
432id = "text-passthrough"
433version = "9.9.9"
434formats = ["text"]
435"#;
436        assert!(Registry::parse(toml, "converters.toml").is_err());
437    }
438}
text_mirror/convert/mod.rs

text_mirror/convert/
mod.rs
