Skip to main content

graphrt_doc/
lib.rs

1//! # graphrt-doc
2//!
3//! Text extraction from documents, as a library ([`extract`]) and as the
4//! `document-extractor` workflow node ([`register`]).
5//!
6//! Everything is pure Rust. Formats and the features that enable them:
7//!
8//! | Format | Feature |
9//! | --- | --- |
10//! | txt, md, json, yaml, xml, csv, tsv, html, vtt, srt | always |
11//! | pdf | `pdf` |
12//! | docx, pptx | `office` |
13//! | xlsx, xls, ods | `spreadsheet` |
14//! | epub | `epub` |
15//! | eml | `email` |
16//!
17//! The format is chosen from the MIME type, then the file extension, then the
18//! content itself. Archives are read with entry-count and decompressed-size
19//! limits (zip-bomb protection), and parsers that are known to panic on
20//! malformed input are isolated with `catch_unwind` (effective only in builds
21//! that unwind on panic).
22
23use serde::{Deserialize, Serialize};
24
25mod binary;
26pub mod detect;
27mod node;
28pub mod text;
29#[cfg(feature = "zip-formats")]
30mod zipfmt;
31
32pub use detect::Format;
33pub use node::{DocLimits, DocumentExtractorNode, KIND, descriptor, register};
34
35/// Shape of the extracted text.
36#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
37#[serde(rename_all = "lowercase")]
38pub enum OutputFormat {
39    /// Plain text; tables become tab-separated lines.
40    #[default]
41    Text,
42    /// Markdown; tables become Markdown tables, headings keep their level.
43    Markdown,
44}
45
46/// Extraction options and limits.
47#[derive(Clone, Debug, PartialEq, Eq)]
48pub struct Options {
49    pub format: OutputFormat,
50    /// Output longer than this many characters is truncated.
51    pub max_output_chars: usize,
52    /// Maximum PDF pages, presentation slides or EPUB chapters.
53    pub max_pages: usize,
54    /// Maximum spreadsheet sheets.
55    pub max_sheets: usize,
56    /// Maximum rows per spreadsheet sheet (and per CSV/TSV file).
57    pub max_rows: usize,
58    /// Maximum columns of any table (spreadsheet sheet, CSV/TSV, DOCX table).
59    pub max_cols: usize,
60    /// Maximum non-empty cells across a spreadsheet sheet. Spreadsheets whose
61    /// used range would exceed this are rejected before they are loaded.
62    pub max_cells: usize,
63    /// Maximum entries in a zip container.
64    pub max_zip_entries: usize,
65    /// Maximum total decompressed bytes read from a zip container.
66    pub max_decompressed_bytes: u64,
67}
68
69impl Default for Options {
70    fn default() -> Self {
71        Self {
72            format: OutputFormat::Text,
73            max_output_chars: 5_000_000,
74            max_pages: 2_000,
75            max_sheets: 100,
76            max_rows: 200_000,
77            max_cols: 2_048,
78            max_cells: 2_000_000,
79            max_zip_entries: 10_000,
80            max_decompressed_bytes: 256 * 1024 * 1024,
81        }
82    }
83}
84
85impl Options {
86    pub fn with_format(mut self, format: OutputFormat) -> Self {
87        self.format = format;
88        self
89    }
90
91    pub fn with_max_output_chars(mut self, max: usize) -> Self {
92        self.max_output_chars = max;
93        self
94    }
95}
96
97/// Extraction failure.
98#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
99#[non_exhaustive]
100pub enum DocError {
101    /// The content type could not be identified or is not a document format.
102    #[error("unsupported document format: {detected}")]
103    Unsupported { detected: String },
104    /// The format is known but its cargo feature is not enabled in this build.
105    #[error("{format} documents are not supported by this build (enable the `{feature}` feature)")]
106    FeatureDisabled { format: Format, feature: &'static str },
107    /// The document could not be parsed.
108    #[error("malformed {format} document: {message}")]
109    Malformed { format: String, message: String },
110    /// A size limit was exceeded.
111    #[error("document exceeds limits: {0}")]
112    LimitExceeded(String),
113}
114
115impl DocError {
116    #[cfg_attr(
117        not(any(
118            feature = "pdf",
119            feature = "spreadsheet",
120            feature = "zip-formats",
121            feature = "email",
122            feature = "tokio"
123        )),
124        allow(dead_code)
125    )]
126    pub(crate) fn malformed(format: &str, err: impl std::fmt::Display) -> Self {
127        DocError::Malformed { format: format.to_owned(), message: err.to_string() }
128    }
129
130    /// Stable machine-readable category.
131    pub fn kind(&self) -> &'static str {
132        match self {
133            DocError::Unsupported { .. } | DocError::FeatureDisabled { .. } => "unsupported_format",
134            DocError::Malformed { .. } => "malformed_document",
135            DocError::LimitExceeded(_) => "limit_exceeded",
136        }
137    }
138}
139
140/// Result of [`extract_detailed`].
141#[derive(Clone, Debug, PartialEq, Eq)]
142pub struct Extraction {
143    pub text: String,
144    pub format: Format,
145    /// Whether the text was cut at [`Options::max_output_chars`].
146    pub truncated: bool,
147}
148
149/// Extract the text of a document.
150///
151/// `mime` and `filename` are hints; either may be `None`. Text longer than
152/// [`Options::max_output_chars`] is cut; use [`extract_detailed`] to learn
153/// whether that happened.
154pub fn extract(bytes: &[u8], mime: Option<&str>, filename: Option<&str>, opts: &Options) -> Result<String, DocError> {
155    extract_detailed(bytes, mime, filename, opts).map(|e| e.text)
156}
157
158/// Like [`extract`], also reporting the detected format and truncation.
159pub fn extract_detailed(
160    bytes: &[u8],
161    mime: Option<&str>,
162    filename: Option<&str>,
163    opts: &Options,
164) -> Result<Extraction, DocError> {
165    let format = resolve_format(bytes, mime, filename, opts)?;
166    let text = extract_as(format, bytes, opts)?;
167    let (text, truncated) = truncate_chars(text, opts.max_output_chars);
168    Ok(Extraction { text, format, truncated })
169}
170
171/// Determine the format: MIME type, then extension, then content sniffing.
172pub fn resolve_format(
173    bytes: &[u8],
174    mime: Option<&str>,
175    filename: Option<&str>,
176    opts: &Options,
177) -> Result<Format, DocError> {
178    let claimed = mime
179        .and_then(detect::from_mime)
180        .or_else(|| filename.and_then(detect::extension).and_then(detect::from_extension));
181    if let Some(f) = claimed {
182        if !f.is_textual() {
183            return Ok(f);
184        }
185        // A text label on binary content (a PDF uploaded as text/plain, a zip named
186        // .txt) would otherwise "succeed" with garbage text.
187        if detect::sniff(bytes).is_some_and(|s| !s.is_textual()) || detect::is_zip(bytes) {
188            return sniffed_binary(bytes, opts);
189        }
190        if !bytes.is_empty() && !detect::looks_textual(bytes) {
191            return Err(DocError::Unsupported { detected: format!("binary data labelled as {f}") });
192        }
193        return Ok(f);
194    }
195    if let Some(f) = detect::sniff(bytes) {
196        if f == Format::Xls && !detect::ole_is_workbook(bytes) {
197            return Err(DocError::Unsupported {
198                detected: "legacy OLE document (.doc / .ppt / .msg) — only spreadsheets are supported".into(),
199            });
200        }
201        return Ok(f);
202    }
203    if detect::is_zip(bytes) {
204        #[cfg(feature = "zip-formats")]
205        if let Some(f) = zipfmt::identify(bytes, opts)? {
206            return Ok(f);
207        }
208        return Err(DocError::Unsupported { detected: "zip archive".into() });
209    }
210    let _ = opts;
211    if !bytes.is_empty() && detect::looks_textual(bytes) {
212        return Ok(Format::PlainText);
213    }
214    let detected = match (mime, filename.and_then(detect::extension)) {
215        (Some(m), _) => m.to_owned(),
216        (None, Some(ext)) => format!(".{ext}"),
217        (None, None) if bytes.is_empty() => "empty file".to_owned(),
218        (None, None) => "binary data".to_owned(),
219    };
220    Err(DocError::Unsupported { detected })
221}
222
223/// Resolve content that sniffing identified as binary, ignoring a text label.
224fn sniffed_binary(bytes: &[u8], opts: &Options) -> Result<Format, DocError> {
225    resolve_format(bytes, None, None, opts)
226}
227
228/// Extract text from bytes known to be in `format`.
229pub fn extract_as(format: Format, bytes: &[u8], opts: &Options) -> Result<String, DocError> {
230    #[allow(unused_variables)]
231    let disabled = |feature: &'static str| DocError::FeatureDisabled { format, feature };
232    match format {
233        Format::PlainText | Format::Markdown | Format::Json | Format::Yaml | Format::Xml => {
234            Ok(text::decode(bytes).trim_end().to_owned())
235        }
236        Format::Csv | Format::Tsv => {
237            let delimiter = if format == Format::Csv { ',' } else { '\t' };
238            let rows = text::parse_delimited(&text::decode(bytes), delimiter);
239            if rows.len() > opts.max_rows {
240                return Err(DocError::LimitExceeded(format!("{} rows (limit {})", rows.len(), opts.max_rows)));
241            }
242            text::render_table(&rows, opts)
243        }
244        Format::Html => Ok(text::html_to_text(&text::decode(bytes), opts.format)),
245        Format::Vtt | Format::Srt => Ok(text::subtitles(&text::decode(bytes))),
246        Format::Pdf => {
247            #[cfg(feature = "pdf")]
248            return binary::pdf(bytes, opts);
249            #[cfg(not(feature = "pdf"))]
250            return Err(disabled("pdf"));
251        }
252        Format::Docx => {
253            #[cfg(feature = "office")]
254            return zipfmt::docx(bytes, opts);
255            #[cfg(not(feature = "office"))]
256            return Err(disabled("office"));
257        }
258        Format::Pptx => {
259            #[cfg(feature = "office")]
260            return zipfmt::pptx(bytes, opts);
261            #[cfg(not(feature = "office"))]
262            return Err(disabled("office"));
263        }
264        Format::Epub => {
265            #[cfg(feature = "epub")]
266            return zipfmt::epub(bytes, opts);
267            #[cfg(not(feature = "epub"))]
268            return Err(disabled("epub"));
269        }
270        Format::Xlsx | Format::Xls | Format::Ods => {
271            #[cfg(feature = "spreadsheet")]
272            return binary::spreadsheet(bytes, format, opts);
273            #[cfg(not(feature = "spreadsheet"))]
274            return Err(disabled("spreadsheet"));
275        }
276        Format::Eml => {
277            #[cfg(feature = "email")]
278            return binary::email(bytes, opts);
279            #[cfg(not(feature = "email"))]
280            return Err(disabled("email"));
281        }
282    }
283}
284
285/// Cut `text` to at most `max` characters (on a char boundary).
286fn truncate_chars(mut text: String, max: usize) -> (String, bool) {
287    match text.char_indices().nth(max) {
288        Some((byte_index, _)) => {
289            text.truncate(byte_index);
290            (text, true)
291        }
292        None => (text, false),
293    }
294}
295
296#[cfg(test)]
297mod tests {
298    use super::*;
299
300    #[test]
301    fn truncation_respects_char_boundaries() {
302        let (t, cut) = truncate_chars("héllo".to_owned(), 2);
303        assert_eq!((t.as_str(), cut), ("hé", true));
304        let (t, cut) = truncate_chars("hi".to_owned(), 2);
305        assert_eq!((t.as_str(), cut), ("hi", false));
306    }
307
308    #[test]
309    fn format_resolution_order() {
310        let opts = Options::default();
311        // MIME wins over extension.
312        assert_eq!(resolve_format(b"x", Some("text/csv"), Some("a.json"), &opts), Ok(Format::Csv));
313        // Generic MIME falls through to the extension.
314        assert_eq!(resolve_format(b"x", Some("application/octet-stream"), Some("a.md"), &opts), Ok(Format::Markdown));
315        // Sniffing, then plain-text fallback.
316        assert_eq!(resolve_format(b"%PDF-1.7", None, None, &opts), Ok(Format::Pdf));
317        assert_eq!(resolve_format(b"just words", None, None, &opts), Ok(Format::PlainText));
318        assert!(matches!(resolve_format(&[0, 1, 2, 3], None, Some("x.bin"), &opts), Err(DocError::Unsupported { .. })));
319    }
320
321    #[test]
322    fn csv_markdown_and_text() {
323        let csv = b"name,qty\napple,3\n";
324        let md = extract(csv, Some("text/csv"), None, &Options::default().with_format(OutputFormat::Markdown));
325        assert_eq!(md.as_deref(), Ok("| name | qty |\n| --- | --- |\n| apple | 3 |"));
326        let txt = extract(csv, None, Some("f.csv"), &Options::default());
327        assert_eq!(txt.as_deref(), Ok("name\tqty\napple\t3"));
328    }
329}