text_mirror/convert/
document.rs

1//! The document adapter over anydoc.
2//!
3//! One converter covers documents, presentations, rich text, ebooks,
4//! and OpenDocument files. anydoc emits Markdown, and one
5//! deterministic normalizer flattens that Markdown to plain text
6//! before the shared NFC and LF pass. anydoc's stable error class is
7//! preserved as the machine-readable reason of a failed record.
8//!
9//! Text-layer PDF is no longer an in-process format here. It runs
10//! behind the sandbox runner in the worker binary, because hostile
11//! PDF bytes must crash a jailed child and never the pipeline. See
12//! the sibling `pdf` and `subprocess` modules.
13//!
14//! Legacy `doc` and `ppt` conversions run the differential check
15//! against a second extraction. See the sibling differential module
16//! for the policy.
17//!
18//! anydoc reports recovered and skipped content through the log
19//! facade. The shared bridge in `anydoc_log` captures those records
20//! per conversion and promotes them to manifest warnings, so a
21//! partially extracted document never reads as silently complete.
22
23use crate::segments::Segment;
24
25use super::anydoc_log::capture_anydoc;
26use super::differential::{self, DifferentialOutcome};
27use super::{ConvertError, Converter, Outcome, normalize_text};
28
29/// Turns a captured anydoc message into a manifest warning.
30fn promote_warning(message: &str) -> String {
31    format!("anydoc_recovery: {message}")
32}
33
34/// Registry id of the document adapter.
35pub const ANYDOC_ID: &str = "anydoc-document";
36
37/// Version of the document adapter.
38pub const ANYDOC_VERSION: &str = "1.0.0";
39
40fn anydoc_format(format_id: &str) -> Option<anydoc::Format> {
41    Some(match format_id {
42        "doc" => anydoc::Format::Doc,
43        "docx" => anydoc::Format::Docx,
44        "odt" => anydoc::Format::Odt,
45        "ppt" => anydoc::Format::Ppt,
46        "pptx" => anydoc::Format::Pptx,
47        "rtf" => anydoc::Format::Rtf,
48        "epub" => anydoc::Format::Epub,
49        "odp" => anydoc::Format::Odp,
50        _ => return None,
51    })
52}
53
54fn error_code(error: &anydoc::ConvertError) -> &'static str {
55    match error.code() {
56        "unsupported" => "unsupported",
57        "malformed" => "malformed",
58        "encrypted" => "encrypted",
59        "resourceLimit" => "resourceLimit",
60        "missingPart" => "missingPart",
61        _ => "io",
62    }
63}
64
65/// Renders Markdown to plain text through a real parser.
66///
67/// The GFM event stream renders explicitly: text nodes, code span and
68/// code block content, and link and image labels come through
69/// verbatim, table cells join with tabs and rows with newlines, block
70/// ends separate with one blank line, and destinations, heading
71/// markers, emphasis delimiters, and rules leave nothing behind.
72/// Escapes resolve in the parser, so literal content such as an
73/// intraword underscore or an escaped pipe survives byte for byte.
74pub fn markdown_to_plain(markdown: &str) -> String {
75    use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
76
77    let mut options = Options::empty();
78    options.insert(Options::ENABLE_TABLES);
79    options.insert(Options::ENABLE_STRIKETHROUGH);
80    options.insert(Options::ENABLE_TASKLISTS);
81
82    fn close_block(out: &mut String) {
83        while out.ends_with('\n') {
84            out.pop();
85        }
86        if !out.is_empty() {
87            out.push_str("\n\n");
88        }
89    }
90
91    fn close_line(out: &mut String) {
92        if !out.is_empty() && !out.ends_with('\n') {
93            out.push('\n');
94        }
95    }
96
97    let mut out = String::with_capacity(markdown.len());
98    let mut first_cell = true;
99    for event in Parser::new_ext(markdown, options) {
100        match event {
101            Event::Start(tag) => match tag {
102                Tag::TableHead | Tag::TableRow => first_cell = true,
103                Tag::TableCell => {
104                    if !first_cell {
105                        out.push('\t');
106                    }
107                    first_cell = false;
108                }
109                _ => {}
110            },
111            Event::End(tag) => match tag {
112                TagEnd::Paragraph
113                | TagEnd::Heading(_)
114                | TagEnd::List(_)
115                | TagEnd::Table
116                | TagEnd::CodeBlock => close_block(&mut out),
117                TagEnd::Item | TagEnd::TableHead | TagEnd::TableRow => close_line(&mut out),
118                _ => {}
119            },
120            Event::Text(text) => out.push_str(&text),
121            Event::Code(code) => out.push_str(&code),
122            Event::Html(html) | Event::InlineHtml(html) => out.push_str(&html),
123            Event::SoftBreak | Event::HardBreak => out.push('\n'),
124            Event::Rule => {}
125            Event::TaskListMarker(checked) => {
126                out.push_str(if checked { "[x] " } else { "[ ] " });
127            }
128            Event::FootnoteReference(label) => {
129                out.push('[');
130                out.push_str(&label);
131                out.push(']');
132            }
133            _ => {}
134        }
135    }
136    while out.ends_with('\n') {
137        out.pop();
138    }
139    if !out.is_empty() {
140        out.push('\n');
141    }
142    out
143}
144
145/// The document adapter. See the module documentation.
146pub struct AnydocDocument;
147
148impl Converter for AnydocDocument {
149    fn id(&self) -> &'static str {
150        ANYDOC_ID
151    }
152
153    fn version(&self) -> &'static str {
154        ANYDOC_VERSION
155    }
156
157    fn convert(
158        &self,
159        source: &[u8],
160        detected_format: &str,
161    ) -> std::result::Result<Outcome, ConvertError> {
162        let Some(format) = anydoc_format(detected_format) else {
163            return Err(ConvertError {
164                code: "unclaimed_format",
165                message: format!("the document adapter does not handle {detected_format}"),
166            });
167        };
168        let (converted, captured) = capture_anydoc(|| anydoc::to_markdown_bytes(source, format));
169        let markdown = converted.map_err(|error| ConvertError {
170            code: error_code(&error),
171            message: error.to_string(),
172        })?;
173        let text = normalize_text(&markdown_to_plain(&markdown));
174        if !source.is_empty() && text.is_empty() {
175            return Err(ConvertError {
176                code: "empty_output",
177                message: format!(
178                    "source is {} bytes but conversion produced no text",
179                    source.len()
180                ),
181            });
182        }
183        let mut warnings: Vec<String> = captured
184            .iter()
185            .map(|message| promote_warning(message))
186            .collect();
187
188        // Legacy Office formats get a second, independent extraction.
189        if matches!(detected_format, "doc" | "ppt") {
190            match differential::check(&text, source, detected_format) {
191                DifferentialOutcome::Agreement => {}
192                DifferentialOutcome::Warning(warning) => warnings.push(warning),
193                DifferentialOutcome::Failure(detail) => {
194                    return Err(ConvertError {
195                        code: "differential-divergence",
196                        message: detail,
197                    });
198                }
199            }
200        }
201
202        let segments = vec![Segment::span(0, text.len(), "document")];
203        Ok(Outcome {
204            converter_id: ANYDOC_ID.to_string(),
205            converter_version: ANYDOC_VERSION.to_string(),
206            detected_format: detected_format.to_string(),
207            text,
208            warnings,
209            segments,
210        })
211    }
212}
213
214#[cfg(test)]
215mod tests {
216    use super::*;
217
218    #[test]
219    fn markdown_structure_flattens_to_plain_text() {
220        let markdown = "\
221# Title
222
223Some *emphasis* and **bold** and `code`.
224
225> quoted line
226
227- first item
228- second item
2291. numbered
230
231| a | b |
232| --- | --- |
233| 1 | 2 |
234
235[label](https://example.invalid/path) and \\*literal\\*
236
237```
238let x = 1;
239```
240";
241        let plain = markdown_to_plain(markdown);
242        assert_eq!(
243            plain,
244            "\
245Title
246
247Some emphasis and bold and code.
248
249quoted line
250
251first item
252second item
253
254numbered
255
256a\tb
2571\t2
258
259label and *literal*
260
261let x = 1;
262"
263        );
264    }
265
266    #[test]
267    fn literal_identifiers_and_operators_survive_verbatim() {
268        assert_eq!(
269            markdown_to_plain("employee_id and 2 * 3 and snake_case_name"),
270            "employee_id and 2 * 3 and snake_case_name\n"
271        );
272        assert_eq!(markdown_to_plain("`2 * 3 _x_ [y]`"), "2 * 3 _x_ [y]\n");
273        assert_eq!(
274            markdown_to_plain("ends with a backslash \\"),
275            "ends with a backslash \\\n"
276        );
277    }
278
279    #[test]
280    fn escaped_pipes_stay_inside_their_table_cell() {
281        let table = "| a \\| b | c |\n| --- | --- |\n| 1 | 2 |";
282        assert_eq!(markdown_to_plain(table), "a | b\tc\n1\t2\n");
283    }
284
285    #[test]
286    fn link_labels_survive_and_destinations_never_leak() {
287        assert_eq!(
288            markdown_to_plain("[label](https://example.invalid/a(b)) tail"),
289            "label tail\n"
290        );
291        assert_eq!(
292            markdown_to_plain("[ref label][r] tail\n\n[r]: https://example.invalid/x"),
293            "ref label tail\n"
294        );
295        assert_eq!(
296            markdown_to_plain("![alt text](https://example.invalid/i.png) tail"),
297            "alt text tail\n"
298        );
299    }
300
301    #[test]
302    fn gfm_extensions_render_their_content() {
303        assert_eq!(markdown_to_plain("~~struck~~ kept"), "struck kept\n");
304        assert_eq!(
305            markdown_to_plain("- [x] done\n- [ ] open"),
306            "[x] done\n[ ] open\n"
307        );
308    }
309
310    #[test]
311    fn the_log_bridge_captures_and_promotes_anydoc_warnings() {
312        let ((), captured) = capture_anydoc(|| {
313            log::warn!(target: "anydoc::formats::pptx", "skipped slide 3: corrupt part");
314            log::warn!(target: "unrelated::crate", "never captured");
315        });
316        assert_eq!(captured.len(), 1);
317        assert_eq!(
318            promote_warning(&captured[0]),
319            "anydoc_recovery: skipped slide 3: corrupt part"
320        );
321    }
322
323    #[test]
324    fn adapter_rejects_formats_it_does_not_map() {
325        let err = AnydocDocument.convert(b"anything", "png").unwrap_err();
326        assert_eq!(err.code, "unclaimed_format");
327    }
328
329    #[test]
330    fn malformed_bytes_preserve_the_anydoc_error_class() {
331        let err = AnydocDocument
332            .convert(b"PK\x03\x04 not a real docx", "docx")
333            .unwrap_err();
334        assert!(
335            ["malformed", "unsupported", "missingPart", "io"].contains(&err.code),
336            "unexpected class {}",
337            err.code
338        );
339    }
340}
text_mirror/convert/document.rs

text_mirror/convert/
document.rs
