Skip to main content

paper/core/extract/
mod.rs

1//! Document extraction with media-native citation anchors.
2//!
3//! # Design
4//!
5//! Extraction produces **facts**, never interpretations. A block carries the
6//! text exactly as the file states it plus the locator the file itself exposes:
7//! a PDF page number, a spreadsheet cell range, a slide index, a paragraph
8//! range, an EPUB spine position, an email message id. Nothing is classified,
9//! dated, summarised or normalised semantically. If the library cannot read a
10//! format or a page, it says so; it never guesses.
11//!
12//! Formats are handled by an ordered registry of [`Extractor`] implementations.
13//! Selection is by declared extension first and by unambiguous magic bytes
14//! second; an unrecognised file is *unsupported*, not "probably text". New
15//! formats are additive: implement `Extractor` and [`register`] it.
16//!
17//! # Anchors
18//!
19//! A chunk inherits the anchor of the block it was cut from, so a citation can
20//! name `report.pdf [page 7]`, `budget.xlsx [Q3!B4:H4]` or `deck.pptx [slide 2]`
21//! and a reader can verify it in one step. Line ranges are added where the
22//! format has real lines.
23
24#[cfg(feature = "shell-accel")]
25pub mod accel;
26mod archive;
27mod email;
28#[cfg(feature = "ocr")]
29pub mod ocr;
30mod ooxml;
31mod pdf;
32mod sheet;
33mod textfmt;
34
35use anyhow::Result;
36use std::path::Path;
37use std::sync::{Arc, Mutex, OnceLock};
38
39/// Hard cap on a single extracted text, to bound memory on hostile input.
40pub const MAX_EXTRACTED_BYTES: usize = 32 * 1024 * 1024;
41/// Hard cap on archive members and PDF pages.
42pub const MAX_ARCHIVE_ENTRIES: usize = 2_000;
43pub const MAX_PDF_PAGES: u32 = 5_000;
44
45/// A parser that blows its five-second deadline is not killed (Rust has no safe
46/// cancellation), so its thread and input copy survive. This keeps the handles
47/// of stalled parsers so finished ones can be reaped and the set can be capped.
48/// Normal concurrency is not limited: only threads that outlived the deadline
49/// occupy a slot. Worst-case retention is `MAX_STUCK_PARSERS` input copies.
50const MAX_STUCK_PARSERS: usize = 4;
51static STUCK_PARSERS: Mutex<Vec<std::thread::JoinHandle<()>>> = Mutex::new(Vec::new());
52
53/// Capabilities declared by an extractor.
54///
55/// These bounds are enforced by the extraction pipeline before and during
56/// parsing. An extractor should choose conservative values that reflect the
57/// worst-case resource consumption of its parser on malicious input.
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub struct Capabilities {
60    /// Maximum file size this extractor will accept, in bytes.
61    /// Files larger than this are rejected before parsing begins.
62    pub max_file_bytes: u64,
63    /// Maximum number of pages (for paged formats) or entries (for archives).
64    pub max_pages: u32,
65    /// Maximum number of archive members or spreadsheet rows to process.
66    pub max_entries: usize,
67    /// Whether the extractor can process input incrementally without loading
68    /// the entire file into memory.
69    pub streaming: bool,
70    /// Whether the extractor is safe to call concurrently from multiple tasks.
71    pub thread_safe: bool,
72    /// Whether the extractor requires an external binary or system dependency.
73    pub requires_external: bool,
74}
75
76impl Default for Capabilities {
77    fn default() -> Self {
78        Self {
79            max_file_bytes: 64 * 1024 * 1024, // 64 MB
80            max_pages: 5_000,
81            max_entries: 2_000,
82            streaming: false,
83            thread_safe: true,
84            requires_external: false,
85        }
86    }
87}
88
89/// One extracted document. Archives yield several, addressed as
90/// `outer.zip!/inner/path`.
91#[derive(Debug, Clone)]
92pub struct ExtractedDocument {
93    pub path: String,
94    pub format: &'static str,
95    pub blocks: Vec<Block>,
96    /// Explicit failures such as `page 3: no text layer`. Reported to the user
97    /// instead of being silently swallowed.
98    pub warnings: Vec<String>,
99}
100
101#[derive(Debug, Clone, PartialEq, Eq)]
102pub struct Block {
103    pub text: String,
104    /// 1-based page for paged formats, 0 when the format has no pages.
105    pub page: i64,
106    /// Format-specific anchor: `page 3`, `Sheet1!A5:H5`, `slide 2`,
107    /// `para 4-9`, `spine 1: chapter.xhtml`, `message <id>`.
108    pub anchor: Option<String>,
109    /// Declared section path (`Warranty / Coverage`), when the format states
110    /// one. Never inferred from layout.
111    pub section: Option<String>,
112    /// 1-based line range *within the block's text source*, 0 when the format
113    /// has no meaningful lines (DOCX, spreadsheets, slides, email).
114    pub line_start: i64,
115    pub line_end: i64,
116}
117
118impl Block {
119    pub fn text(text: impl Into<String>) -> Self {
120        let text = text.into();
121        let lines = text.lines().count().max(1) as i64;
122        Self {
123            text,
124            page: 0,
125            anchor: None,
126            section: None,
127            line_start: 1,
128            line_end: lines,
129        }
130    }
131
132    pub fn anchored(text: impl Into<String>, anchor: impl Into<String>) -> Self {
133        let mut block = Self::text(text);
134        block.anchor = Some(anchor.into());
135        block
136    }
137}
138
139/// A format handler.
140///
141/// `extract` must either return text it actually decoded or `Ok(None)`. It must
142/// not fall back to treating arbitrary bytes as text.
143pub trait Extractor: Send + Sync {
144    /// Stable format name, used for reporting and for `Hit::anchor_kind`.
145    fn name(&self) -> &'static str;
146
147    /// Extensions handled, lower-case, without the dot.
148    fn extensions(&self) -> &'static [&'static str];
149
150    /// Unambiguous content check, for files with a missing or wrong extension.
151    fn sniff(&self, _bytes: &[u8]) -> bool {
152        false
153    }
154
155    /// Resource and safety capabilities of this extractor.
156    ///
157    /// The extraction pipeline enforces `max_file_bytes` before calling
158    /// `extract`, wraps the call in a 5-second timeout, and respects
159    /// `thread_safe` when deciding on parallelism.
160    fn capabilities(&self) -> Capabilities {
161        Capabilities::default()
162    }
163
164    fn can_handle(&self, name: &str, bytes: &[u8]) -> bool {
165        self.extensions().contains(&extension_of(name).as_str()) || self.sniff(bytes)
166    }
167
168    fn extract(&self, name: &str, bytes: &[u8]) -> Result<Option<Vec<ExtractedDocument>>>;
169}
170
171fn builtin() -> Vec<Arc<dyn Extractor>> {
172    vec![
173        Arc::new(pdf::Pdf),
174        Arc::new(ooxml::Docx),
175        Arc::new(ooxml::Pptx),
176        Arc::new(ooxml::Epub),
177        Arc::new(sheet::Sheet),
178        Arc::new(archive::Archive),
179        Arc::new(email::Email),
180        Arc::new(textfmt::Html),
181        // Last: only claims extensions it declares or unambiguous text markers.
182        Arc::new(textfmt::Plain),
183    ]
184}
185
186fn custom() -> &'static Mutex<Vec<Arc<dyn Extractor>>> {
187    static CUSTOM: OnceLock<Mutex<Vec<Arc<dyn Extractor>>>> = OnceLock::new();
188    CUSTOM.get_or_init(|| Mutex::new(Vec::new()))
189}
190
191/// Add an extractor. Custom extractors are tried after the built-ins, so a
192/// custom handler can widen support without changing built-in behaviour.
193pub fn register(extractor: Arc<dyn Extractor>) {
194    if let Ok(mut list) = custom().lock() {
195        list.push(extractor);
196    }
197}
198
199/// Every registered extractor, built-ins first.
200pub fn extractors() -> Vec<Arc<dyn Extractor>> {
201    let mut all = builtin();
202    if let Ok(list) = custom().lock() {
203        all.extend(list.iter().cloned());
204    }
205    all
206}
207
208/// Every extension any extractor declares.
209pub fn supported_extensions() -> Vec<&'static str> {
210    let mut out: Vec<&'static str> = Vec::new();
211    for extractor in extractors() {
212        out.extend_from_slice(extractor.extensions());
213    }
214    out.sort_unstable();
215    out.dedup();
216    out
217}
218
219pub fn extension_of(path: &str) -> String {
220    Path::new(path)
221        .extension()
222        .and_then(|e| e.to_str())
223        .unwrap_or("")
224        .to_ascii_lowercase()
225}
226
227/// Whether any extractor claims this path by extension.
228pub fn is_supported(path: &str) -> bool {
229    let ext = extension_of(path);
230    extractors()
231        .iter()
232        .any(|e| e.extensions().contains(&ext.as_str()))
233}
234
235/// Extract a file from disk. Returns zero or more documents (archives yield
236/// many). An unsupported file is `Ok(None)`.
237pub fn extract_path(path: &Path) -> Result<Option<Vec<ExtractedDocument>>> {
238    let name = path.to_string_lossy().to_string();
239    if !is_supported(&name) {
240        return Ok(None);
241    }
242    // Pre-read size cap: refuse to load a file larger than every applicable
243    // extractor will accept, preventing memory DoS on hostile input (I7).
244    let metadata = std::fs::symlink_metadata(path)?;
245    let mut max_allowed: u64 = 0;
246    for extractor in extractors() {
247        if extractor
248            .extensions()
249            .contains(&extension_of(&name).as_str())
250        {
251            max_allowed = max_allowed.max(extractor.capabilities().max_file_bytes);
252        }
253    }
254    if max_allowed == 0 || metadata.len() > max_allowed {
255        return Ok(None);
256    }
257    let bytes = std::fs::read(path)?;
258    extract_named(&name, &bytes, false)
259}
260
261/// Extract bytes that are already in memory (archive members, tests).
262/// `ocr_enabled` controls whether OCR is attempted for scanned PDF pages.
263pub fn extract_named(
264    name: &str,
265    bytes: &[u8],
266    _ocr_enabled: bool,
267) -> Result<Option<Vec<ExtractedDocument>>> {
268    // Try shell-out accelerators first (feature-gated)
269    #[cfg(feature = "shell-accel")]
270    {
271        if let Some(path) = std::path::Path::new(name).extension() {
272            if let Some(ext) = path.to_str() {
273                if let Ok(Some(docs)) =
274                    crate::core::extract::accel::try_accelerators(std::path::Path::new(name), ext)
275                {
276                    if !docs.is_empty() {
277                        return Ok(Some(docs));
278                    }
279                }
280            }
281        }
282    }
283
284    for extractor in extractors() {
285        if !extractor.can_handle(name, bytes) {
286            continue;
287        }
288        let caps = extractor.capabilities();
289
290        // Enforce memory cap before parsing.
291        if bytes.len() as u64 > caps.max_file_bytes {
292            return Ok(Some(single(
293                name,
294                extractor.name(),
295                Vec::new(),
296                vec![format!(
297                    "file size {} exceeds maximum {} bytes",
298                    bytes.len(),
299                    caps.max_file_bytes
300                )],
301            )));
302        }
303
304        // Per-parser timeout: run extract in a separate thread with a 5-second
305        // deadline. `std::thread` + mpsc avoids nesting a tokio runtime.
306        //
307        // A parser that ignores its deadline must not be able to spawn an
308        // unbounded number of threads. Reap any that have since finished, and
309        // refuse to start more while the stalled set is at its cap. Normal
310        // concurrent parsing is never limited.
311        {
312            let mut stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
313            stuck.retain(|h| !h.is_finished());
314            if stuck.len() >= MAX_STUCK_PARSERS {
315                return Ok(Some(single(
316                    name,
317                    extractor.name(),
318                    Vec::new(),
319                    vec![format!(
320                        "skipped: {MAX_STUCK_PARSERS} parsers are stalled; restart the indexer"
321                    )],
322                )));
323            }
324        }
325
326        let extractor_clone = extractor.clone();
327        let name_owned = name.to_string();
328        let bytes_owned = bytes.to_vec();
329        let (tx, rx) = std::sync::mpsc::channel();
330        let handle = std::thread::spawn(move || {
331            let _ = tx.send(extractor_clone.extract(&name_owned, &bytes_owned));
332        });
333        let result = rx.recv_timeout(std::time::Duration::from_secs(5));
334
335        let mut docs = match result {
336            Ok(Ok(Some(docs))) => {
337                let _ = handle.join();
338                docs
339            }
340            Ok(Ok(None)) => {
341                let _ = handle.join();
342                continue;
343            }
344            Ok(Err(err)) => {
345                let _ = handle.join();
346                return Ok(Some(single(
347                    name,
348                    extractor.name(),
349                    Vec::new(),
350                    vec![format!("extraction error: {err}")],
351                )));
352            }
353            Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
354                // Deliberately do NOT join: the parser may never return. Park the
355                // handle so it occupies a stalled slot until it finishes.
356                tracing::warn!("{}: extraction timed out after 5 seconds", name);
357                STUCK_PARSERS
358                    .lock()
359                    .unwrap_or_else(|e| e.into_inner())
360                    .push(handle);
361                return Ok(Some(single(
362                    name,
363                    extractor.name(),
364                    Vec::new(),
365                    vec!["extraction timed out after 5 seconds".to_string()],
366                )));
367            }
368            Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
369                let _ = handle.join();
370                return Ok(Some(single(
371                    name,
372                    extractor.name(),
373                    Vec::new(),
374                    vec!["extraction task panicked".to_string()],
375                )));
376            }
377        };
378
379        for doc in &mut docs {
380            doc.format = extractor.name();
381            if doc.path.is_empty() {
382                doc.path = name.to_string();
383            }
384            doc.blocks = finish_blocks(std::mem::take(&mut doc.blocks));
385        }
386        docs.retain(|doc| !doc.blocks.is_empty());
387        if docs.is_empty() {
388            // Matched the format but had nothing citable (empty or unreadable).
389            return Ok(None);
390        }
391        return Ok(Some(docs));
392    }
393    Ok(None)
394}
395
396/// Drop empty blocks and cap their size. Anchors are never modified.
397pub(crate) fn finish_blocks(mut blocks: Vec<Block>) -> Vec<Block> {
398    let mut total = 0usize;
399    blocks.retain(|b| !b.text.trim().is_empty());
400    for block in &mut blocks {
401        if block.text.len() > MAX_EXTRACTED_BYTES {
402            let mut cut = MAX_EXTRACTED_BYTES;
403            while cut > 0 && !block.text.is_char_boundary(cut) {
404                cut -= 1;
405            }
406            block.text.truncate(cut);
407        }
408        total += block.text.len();
409        if total > MAX_EXTRACTED_BYTES {
410            break;
411        }
412    }
413    blocks
414}
415
416/// Helper for extractors: build a single-document result.
417pub(crate) fn single(
418    name: &str,
419    format: &'static str,
420    blocks: Vec<Block>,
421    warnings: Vec<String>,
422) -> Vec<ExtractedDocument> {
423    vec![ExtractedDocument {
424        path: name.to_string(),
425        format,
426        blocks,
427        warnings,
428    }]
429}