text_mirror/convert/
mod.rs

1//! The converter boundary and the plain text passthrough.
2//!
3//! Every converter returns the same outcome shape. The registry in
4//! `rules/converters.toml` maps format ids to converters. A format no
5//! converter claims is unsupported, with a declared reason or the
6//! `no-converter` floor. A converter error or invalid
7//! output is a failure with a machine-readable reason and no text
8//! artifact. There is no silent skip and no silently empty file. A
9//! non-empty source that converts to empty text fails with reason
10//! `empty_output` in every converter.
11//!
12//! In-process conversion runs under a panic boundary and explicit
13//! ceilings that apply before allocation in this crate's own code:
14//! the source size before read, the decompressed size of every
15//! archive part and compound stream, the cell extent of every sheet
16//! before the workbook parser runs, and the rendered output size. A
17//! caught panic or an exceeded ceiling becomes one failed record and
18//! the run continues. The panic boundary requires the default
19//! `panic = "unwind"` profile. A downstream `panic = "abort"` build
20//! turns any converter panic into process death, so keep unwinding
21//! enabled wherever this crate converts untrusted bytes.
22
23/// Ceiling on source bytes read for conversion, checked before read.
24pub const MAX_SOURCE_BYTES: u64 = 128 * 1024 * 1024;
25
26/// Ceiling on the decompressed bytes of one archive part or one
27/// compound file stream.
28pub const MAX_PART_BYTES: u64 = 64 * 1024 * 1024;
29
30/// Ceiling on rendered artifact text bytes.
31pub const MAX_OUTPUT_BYTES: usize = 64 * 1024 * 1024;
32
33/// Ceiling on 0-based row indexes in one sheet.
34pub const MAX_SHEET_ROWS: u32 = 1_048_576;
35
36/// Ceiling on 0-based column indexes in one sheet.
37pub const MAX_SHEET_COLUMNS: u32 = 16_384;
38
39/// Ceiling on the row-column product of one sheet, checked from the
40/// source's own references before the workbook parser allocates.
41pub const MAX_SHEET_CELLS: u64 = 10_000_000;
42
43use std::collections::HashMap;
44use std::fmt;
45
46use serde::Deserialize;
47use unicode_normalization::UnicodeNormalization;
48
49use crate::detect::RESERVED_FORMAT_IDS;
50use crate::segments::Segment;
51use crate::{Error, Result};
52
53mod anydoc_log;
54mod differential;
55mod document;
56pub mod pdf;
57pub mod subprocess;
58mod visibility;
59mod workbook;
60
61pub use differential::{
62    DiffMetrics, DiffVerdict, DifferentialOutcome, compare_texts, normalize_for_diff,
63};
64pub use document::{AnydocDocument, markdown_to_plain};
65pub use subprocess::PdfSubprocess;
66pub use visibility::{IndexRange, SheetVisibility, WorkbookVisibility, read_visibility};
67pub use workbook::WorkbookIr;
68
69/// What every converter returns on success.
70#[derive(Debug, Clone)]
71pub struct Outcome {
72    /// Stable converter id from the registry.
73    pub converter_id: String,
74    /// Converter version from the registry.
75    pub converter_version: String,
76    /// Format id the converter was asked to handle.
77    pub detected_format: String,
78    /// The converted text, UTF-8, NFC, LF line endings.
79    pub text: String,
80    /// Non-fatal notes about the conversion.
81    pub warnings: Vec<String>,
82    /// Structure spans over `text` for the segments file.
83    pub segments: Vec<Segment>,
84}
85
86/// A conversion failure with a machine-readable reason.
87#[derive(Debug, Clone)]
88pub struct ConvertError {
89    /// Stable reason code, such as `invalid_utf8` or `empty_output`.
90    pub code: &'static str,
91    /// Detail for a human reading the manifest.
92    pub message: String,
93}
94
95impl fmt::Display for ConvertError {
96    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
97        write!(f, "{}: {}", self.code, self.message)
98    }
99}
100
101impl std::error::Error for ConvertError {}
102
103/// A converter takes one source's bytes and returns text or a reason.
104///
105/// The caller supplies the bytes, so the hash it records and the bytes
106/// the converter consumed are one snapshot. Implementations must be
107/// deterministic. The same input bytes under the same rules produce
108/// byte-identical output.
109pub trait Converter {
110    /// Stable converter id.
111    fn id(&self) -> &'static str;
112    /// Converter version, part of the incremental checkpoint key.
113    fn version(&self) -> &'static str;
114    /// Converts one source from its bytes.
115    fn convert(
116        &self,
117        source: &[u8],
118        detected_format: &str,
119    ) -> std::result::Result<Outcome, ConvertError>;
120}
121
122/// Normalizes text to the output contract.
123///
124/// CRLF and lone CR become LF, then the text is normalized to NFC.
125/// NFC only. NFKC rewrites identifiers, so it is prohibited.
126pub fn normalize_text(input: &str) -> String {
127    let mut unified = String::with_capacity(input.len());
128    let mut chars = input.chars().peekable();
129    while let Some(c) = chars.next() {
130        if c == '\r' {
131            if chars.peek() == Some(&'\n') {
132                chars.next();
133            }
134            unified.push('\n');
135        } else {
136            unified.push(c);
137        }
138    }
139    unified.nfc().collect()
140}
141
142/// Passthrough for text-native formats.
143///
144/// Reads the source as plain text and applies [`normalize_text`]. It
145/// parses nothing and strips no markup. Input encoding is UTF-8, or
146/// UTF-16 behind a required byte order mark: a leading `FF FE`
147/// decodes as UTF-16LE and `FE FF` as UTF-16BE, the mark is consumed
148/// before decoding, and the outcome carries a transcoding warning. A
149/// UTF-32 mark fails closed with reason `unsupported-encoding`, an
150/// odd trailing byte or unpaired surrogate fails with `invalid_utf16`
151/// naming the byte offset, and a file with no mark must be valid
152/// UTF-8. A leading UTF-8 byte order mark is stripped with a warning.
153pub struct PlainTextPassthrough;
154
155/// The floor reason for an unsupported record whose detected format
156/// has no converter entry and no declared reason, including true
157/// unknowns.
158pub const NO_CONVERTER_REASON: &str = "no-converter";
159
160/// Registry id of the passthrough converter.
161pub const PASSTHROUGH_ID: &str = "text-passthrough";
162/// Version of the passthrough converter.
163pub const PASSTHROUGH_VERSION: &str = "1.1.0";
164
165/// Decodes UTF-16 byte pairs after a consumed byte order mark.
166///
167/// `mark_len` is the consumed mark's length, so failure offsets name
168/// positions in the original source bytes.
169fn decode_utf16_pairs(
170    bytes: &[u8],
171    big_endian: bool,
172    mark_len: usize,
173) -> std::result::Result<String, ConvertError> {
174    if bytes.len() % 2 != 0 {
175        return Err(ConvertError {
176            code: "invalid_utf16",
177            message: format!("odd trailing byte at byte {}", mark_len + bytes.len() - 1),
178        });
179    }
180    let units = bytes.chunks_exact(2).map(|pair| {
181        if big_endian {
182            u16::from_be_bytes([pair[0], pair[1]])
183        } else {
184            u16::from_le_bytes([pair[0], pair[1]])
185        }
186    });
187    let mut decoded = String::with_capacity(bytes.len() / 2);
188    let mut consumed_units = 0usize;
189    for result in char::decode_utf16(units) {
190        match result {
191            Ok(c) => {
192                decoded.push(c);
193                consumed_units += c.len_utf16();
194            }
195            Err(_) => {
196                return Err(ConvertError {
197                    code: "invalid_utf16",
198                    message: format!(
199                        "unpaired surrogate at byte {}",
200                        mark_len + consumed_units * 2
201                    ),
202                });
203            }
204        }
205    }
206    Ok(decoded)
207}
208
209impl Converter for PlainTextPassthrough {
210    fn id(&self) -> &'static str {
211        PASSTHROUGH_ID
212    }
213
214    fn version(&self) -> &'static str {
215        PASSTHROUGH_VERSION
216    }
217
218    fn convert(
219        &self,
220        source: &[u8],
221        detected_format: &str,
222    ) -> std::result::Result<Outcome, ConvertError> {
223        let source_len = source.len();
224        let mut warnings = Vec::new();
225        // The four-byte UTF-32 marks come first, because the UTF-32LE
226        // mark begins with the UTF-16LE one and would otherwise decode
227        // as UTF-16 with interleaved NULs.
228        let raw = if source.starts_with(&[0xFF, 0xFE, 0x00, 0x00])
229            || source.starts_with(&[0x00, 0x00, 0xFE, 0xFF])
230        {
231            return Err(ConvertError {
232                code: "unsupported-encoding",
233                message: "utf-32 byte order mark".to_string(),
234            });
235        } else if let Some(rest) = source.strip_prefix(&[0xFF, 0xFE][..]) {
236            warnings.push("transcoded from utf-16le".to_string());
237            decode_utf16_pairs(rest, false, 2)?
238        } else if let Some(rest) = source.strip_prefix(&[0xFE, 0xFF][..]) {
239            warnings.push("transcoded from utf-16be".to_string());
240            decode_utf16_pairs(rest, true, 2)?
241        } else {
242            let raw = String::from_utf8(source.to_vec()).map_err(|e| ConvertError {
243                code: "invalid_utf8",
244                message: format!("invalid UTF-8 at byte {}", e.utf8_error().valid_up_to()),
245            })?;
246            match raw.strip_prefix('\u{feff}') {
247                Some(stripped) => {
248                    warnings.push("stripped leading byte order mark".to_string());
249                    stripped.to_string()
250                }
251                None => raw,
252            }
253        };
254        let text = normalize_text(&raw);
255        if source_len > 0 && text.is_empty() {
256            return Err(ConvertError {
257                code: "empty_output",
258                message: format!("source is {source_len} bytes but conversion produced no text"),
259            });
260        }
261        let segments = vec![Segment::span(0, text.len(), "document")];
262        Ok(Outcome {
263            converter_id: PASSTHROUGH_ID.to_string(),
264            converter_version: PASSTHROUGH_VERSION.to_string(),
265            detected_format: detected_format.to_string(),
266            text,
267            warnings,
268            segments,
269        })
270    }
271}
272
273#[derive(Debug, Deserialize)]
274#[serde(deny_unknown_fields)]
275struct RawRegistry {
276    version: String,
277    converters: Vec<RawEntry>,
278    #[serde(default)]
279    unsupported: Vec<RawUnsupported>,
280}
281
282#[derive(Debug, Deserialize)]
283#[serde(deny_unknown_fields)]
284struct RawEntry {
285    id: String,
286    version: String,
287    formats: Vec<String>,
288}
289
290#[derive(Debug, Deserialize)]
291#[serde(deny_unknown_fields)]
292struct RawUnsupported {
293    reason: String,
294    formats: Vec<String>,
295}
296
297/// The converter registry from `rules/converters.toml`.
298///
299/// The registry is data. A rules bump is a visible versioned event,
300/// and the pinned entry version must match the implementation, so a
301/// code change cannot ship silently under an old version.
302pub struct Registry {
303    version: String,
304    converters: Vec<Box<dyn Converter>>,
305    by_format: HashMap<String, usize>,
306    unsupported_reasons: HashMap<String, String>,
307}
308
309impl Registry {
310    /// The registry compiled into the binary from `rules/converters.toml`.
311    pub fn builtin() -> Result<Self> {
312        Self::parse(
313            include_str!("../../rules/converters.toml"),
314            "converters.toml",
315        )
316    }
317
318    /// Parses a registry and binds entries to implementations.
319    pub fn parse(text: &str, name: &str) -> Result<Self> {
320        let raw: RawRegistry = toml::from_str(text).map_err(|e| Error::Rules {
321            name: name.to_string(),
322            message: e.to_string(),
323        })?;
324        let mut converters: Vec<Box<dyn Converter>> = Vec::new();
325        let mut by_format = HashMap::new();
326        for entry in &raw.converters {
327            let converter: Box<dyn Converter> = match entry.id.as_str() {
328                PASSTHROUGH_ID => Box::new(PlainTextPassthrough),
329                document::ANYDOC_ID => Box::new(AnydocDocument),
330                workbook::WORKBOOK_ID => Box::new(WorkbookIr),
331                subprocess::PDF_SUBPROCESS_ID => Box::new(PdfSubprocess),
332                other => {
333                    return Err(Error::Rules {
334                        name: name.to_string(),
335                        message: format!("unknown converter id {other:?}"),
336                    });
337                }
338            };
339            if entry.version != converter.version() {
340                return Err(Error::Rules {
341                    name: name.to_string(),
342                    message: format!(
343                        "converter {:?} pins version {:?} but the implementation is {:?}",
344                        entry.id,
345                        entry.version,
346                        converter.version()
347                    ),
348                });
349            }
350            let index = converters.len();
351            converters.push(converter);
352            for format in &entry.formats {
353                if RESERVED_FORMAT_IDS.contains(&format.as_str()) {
354                    return Err(Error::Rules {
355                        name: name.to_string(),
356                        message: format!("format id {format:?} is reserved"),
357                    });
358                }
359                if by_format.insert(format.clone(), index).is_some() {
360                    return Err(Error::Rules {
361                        name: name.to_string(),
362                        message: format!("format {format:?} claimed twice"),
363                    });
364                }
365            }
366        }
367        let mut unsupported_reasons = HashMap::new();
368        for entry in &raw.unsupported {
369            for format in &entry.formats {
370                if RESERVED_FORMAT_IDS.contains(&format.as_str()) {
371                    return Err(Error::Rules {
372                        name: name.to_string(),
373                        message: format!("format id {format:?} is reserved"),
374                    });
375                }
376                if by_format.contains_key(format)
377                    || unsupported_reasons
378                        .insert(format.clone(), entry.reason.clone())
379                        .is_some()
380                {
381                    return Err(Error::Rules {
382                        name: name.to_string(),
383                        message: format!("format {format:?} claimed twice"),
384                    });
385                }
386            }
387        }
388        Ok(Registry {
389            version: raw.version,
390            converters,
391            by_format,
392            unsupported_reasons,
393        })
394    }
395
396    /// The registry version, part of the run's rules version.
397    pub fn version(&self) -> &str {
398        &self.version
399    }
400
401    /// The converter that claims a format id, if any.
402    pub fn converter_for(&self, format_id: &str) -> Option<&dyn Converter> {
403        self.by_format
404            .get(format_id)
405            .map(|index| self.converters[*index].as_ref())
406    }
407
408    /// The reason a format is unsupported.
409    ///
410    /// A declared entry from the registry wins, such as a spreadsheet
411    /// format with no visibility reader yet. Every other format no
412    /// converter claims gets the generic floor
413    /// [`NO_CONVERTER_REASON`], so an unsupported record is never
414    /// reason-less. A format a converter claims returns `None`.
415    pub fn unsupported_reason(&self, format_id: &str) -> Option<&str> {
416        if self.by_format.contains_key(format_id) {
417            return None;
418        }
419        Some(
420            self.unsupported_reasons
421                .get(format_id)
422                .map(String::as_str)
423                .unwrap_or(NO_CONVERTER_REASON),
424        )
425    }
426
427    /// A registry built directly from converter instances, for tests
428    /// that need behavior no shipped converter exhibits, such as a
429    /// panicking converter.
430    #[cfg(test)]
431    pub(crate) fn for_tests(
432        version: &str,
433        entries: Vec<(Box<dyn Converter>, Vec<&str>)>,
434    ) -> Registry {
435        let mut converters = Vec::new();
436        let mut by_format = HashMap::new();
437        for (converter, formats) in entries {
438            let index = converters.len();
439            converters.push(converter);
440            for format in formats {
441                by_format.insert(format.to_string(), index);
442            }
443        }
444        Registry {
445            version: version.to_string(),
446            converters,
447            by_format,
448            unsupported_reasons: HashMap::new(),
449        }
450    }
451}
452
453#[cfg(test)]
454mod tests {
455    use super::*;
456
457    #[test]
458    fn normalize_converts_crlf_to_lf() {
459        assert_eq!(normalize_text("a\r\nb\r\n"), "a\nb\n");
460    }
461
462    #[test]
463    fn normalize_converts_lone_cr_to_lf() {
464        assert_eq!(normalize_text("a\rb\r"), "a\nb\n");
465        assert_eq!(normalize_text("a\r\rb"), "a\n\nb");
466    }
467
468    #[test]
469    fn normalize_composes_to_nfc() {
470        assert_eq!(normalize_text("cafe\u{301}"), "caf\u{e9}");
471    }
472
473    #[test]
474    fn normalize_never_applies_nfkc() {
475        assert_eq!(normalize_text("\u{fb01}le \u{2460}"), "\u{fb01}le \u{2460}");
476    }
477
478    #[test]
479    fn passthrough_rejects_invalid_utf8() {
480        let err = PlainTextPassthrough
481            .convert(b"hello \xff world", "text")
482            .unwrap_err();
483        assert_eq!(err.code, "invalid_utf8");
484    }
485
486    #[test]
487    fn passthrough_fails_on_suspiciously_empty_output() {
488        let err = PlainTextPassthrough
489            .convert("\u{feff}".as_bytes(), "text")
490            .unwrap_err();
491        assert_eq!(err.code, "empty_output");
492    }
493
494    #[test]
495    fn passthrough_accepts_an_empty_source() {
496        let outcome = PlainTextPassthrough.convert(b"", "text").unwrap();
497        assert_eq!(outcome.text, "");
498    }
499
500    fn utf16_bytes(text: &str, big_endian: bool) -> Vec<u8> {
501        let mut bytes = if big_endian {
502            vec![0xFE, 0xFF]
503        } else {
504            vec![0xFF, 0xFE]
505        };
506        for unit in text.encode_utf16() {
507            bytes.extend_from_slice(&if big_endian {
508                unit.to_be_bytes()
509            } else {
510                unit.to_le_bytes()
511            });
512        }
513        bytes
514    }
515
516    #[test]
517    fn passthrough_transcodes_utf16le_behind_a_byte_order_mark() {
518        let source = utf16_bytes("a,b\r\n1,caf\u{e9}\r\n", false);
519        let outcome = PlainTextPassthrough.convert(&source, "csv").unwrap();
520        assert_eq!(outcome.text, "a,b\n1,caf\u{e9}\n");
521        assert_eq!(
522            outcome.warnings,
523            vec!["transcoded from utf-16le".to_string()]
524        );
525    }
526
527    #[test]
528    fn passthrough_transcodes_utf16be_behind_a_byte_order_mark() {
529        let source = utf16_bytes("total \u{5317}\u{4eac}\n", true);
530        let outcome = PlainTextPassthrough.convert(&source, "text").unwrap();
531        assert_eq!(outcome.text, "total \u{5317}\u{4eac}\n");
532        assert_eq!(
533            outcome.warnings,
534            vec!["transcoded from utf-16be".to_string()]
535        );
536    }
537
538    #[test]
539    fn passthrough_refuses_utf32_byte_order_marks() {
540        for source in [
541            b"\xFF\xFE\x00\x00A\x00\x00\x00".as_slice(),
542            b"\x00\x00\xFE\xFF\x00\x00\x00A".as_slice(),
543        ] {
544            let err = PlainTextPassthrough.convert(source, "text").unwrap_err();
545            assert_eq!(err.code, "unsupported-encoding");
546            assert!(err.to_string().starts_with("unsupported-encoding: utf-32"));
547        }
548    }
549
550    #[test]
551    fn passthrough_fails_an_odd_utf16_tail_at_its_offset() {
552        let err = PlainTextPassthrough
553            .convert(b"\xFF\xFEA\x00B", "text")
554            .unwrap_err();
555        assert_eq!(err.code, "invalid_utf16");
556        assert_eq!(err.message, "odd trailing byte at byte 4");
557    }
558
559    #[test]
560    fn passthrough_fails_an_unpaired_surrogate_at_its_offset() {
561        let err = PlainTextPassthrough
562            .convert(b"\xFF\xFEA\x00\x00\xD8", "text")
563            .unwrap_err();
564        assert_eq!(err.code, "invalid_utf16");
565        assert_eq!(err.message, "unpaired surrogate at byte 4");
566    }
567
568    #[test]
569    fn passthrough_still_fails_bomless_utf16_as_invalid_utf8() {
570        let mut source = utf16_bytes("R\u{e9}sum\u{e9}\n", false);
571        source.drain(..2);
572        let err = PlainTextPassthrough.convert(&source, "text").unwrap_err();
573        assert_eq!(err.code, "invalid_utf8");
574    }
575
576    #[test]
577    fn builtin_registry_claims_the_text_family() {
578        let registry = Registry::builtin().unwrap();
579        assert_eq!(registry.version(), "4");
580        assert!(registry.converter_for("json").is_some());
581        assert!(registry.converter_for("yaml").is_some());
582        assert!(registry.converter_for("svg").is_some());
583        assert!(registry.converter_for("ipynb").is_some());
584        assert!(registry.converter_for("python").is_some());
585        assert!(registry.converter_for("pickle").is_none());
586        assert!(registry.converter_for("text").is_some());
587        assert!(registry.converter_for("markdown").is_some());
588        assert!(registry.converter_for("csv").is_some());
589        assert!(registry.converter_for("pdf").is_some());
590        assert_eq!(
591            registry.converter_for("pdf").map(|c| c.id()),
592            Some("pdf-subprocess")
593        );
594        assert!(registry.converter_for("docx").is_some());
595        assert!(registry.converter_for("xlsx").is_some());
596        assert!(registry.converter_for("html").is_none());
597        assert!(registry.converter_for("xlsb").is_none());
598        assert_eq!(
599            registry.unsupported_reason("xlsb"),
600            Some("hidden-visibility-unresolved")
601        );
602        assert_eq!(
603            registry.unsupported_reason("pickle"),
604            Some("pickle-deserialization-unsafe")
605        );
606        assert_eq!(registry.unsupported_reason("webp"), Some("engine-unpinned"));
607        assert_eq!(
608            registry.unsupported_reason("parquet"),
609            Some("converter-deferred")
610        );
611        assert_eq!(
612            registry.unsupported_reason("tar"),
613            Some("container-deferred")
614        );
615        assert_eq!(
616            registry.unsupported_reason("psd"),
617            Some("proprietary-binary")
618        );
619        // The floor: a format nothing claims and nothing declares.
620        assert_eq!(
621            registry.unsupported_reason("unknown"),
622            Some(NO_CONVERTER_REASON)
623        );
624        // A claimed format has no unsupported reason.
625        assert_eq!(registry.unsupported_reason("text"), None);
626        assert_eq!(registry.unsupported_reason("png"), Some("engine-unpinned"));
627        assert_eq!(registry.unsupported_reason("mp4"), Some("engine-unpinned"));
628        // html is detected but unclaimed until a strip converter
629        // lands, so it sits on the floor, not outside the vocabulary.
630        assert_eq!(
631            registry.unsupported_reason("html"),
632            Some(NO_CONVERTER_REASON)
633        );
634    }
635
636    #[test]
637    fn registry_rejects_reserved_format_ids() {
638        for (section, id) in [
639            ("converters", "symlink"),
640            ("converters", "unknown"),
641            ("unsupported", "symlink"),
642            ("unsupported", "unknown"),
643        ] {
644            let toml = if section == "converters" {
645                format!(
646                    "version = \"1\"\n[[converters]]\nid = \"text-passthrough\"\nversion = \"1.1.0\"\nformats = [\"{id}\"]\n"
647                )
648            } else {
649                format!(
650                    "version = \"1\"\nconverters = []\n[[unsupported]]\nreason = \"r\"\nformats = [\"{id}\"]\n"
651                )
652            };
653            let err = match Registry::parse(&toml, "converters.toml") {
654                Ok(_) => panic!("{section} {id}: reserved id was accepted"),
655                Err(e) => e,
656            };
657            assert!(
658                err.to_string().contains("reserved"),
659                "{section} {id}: {err}"
660            );
661        }
662    }
663
664    #[test]
665    fn registry_rejects_a_version_that_contradicts_the_implementation() {
666        let toml = r#"
667version = "1"
668
669[[converters]]
670id = "text-passthrough"
671version = "9.9.9"
672formats = ["text"]
673"#;
674        assert!(Registry::parse(toml, "converters.toml").is_err());
675    }
676}
text_mirror/convert/mod.rs

text_mirror/convert/
mod.rs
