Skip to main content

paper/core/extract/
mod.rs

1//! Document extraction with media-native citation anchors.
2//!
3//! # Design
4//!
5//! Extraction produces **facts**, never interpretations. A block carries the
6//! text exactly as the file states it plus the locator the file itself exposes:
7//! a PDF page number, a spreadsheet cell range, a slide index, a paragraph
8//! range, an EPUB spine position, an email message id. Nothing is classified,
9//! dated, summarised or normalised semantically. If the library cannot read a
10//! format or a page, it says so; it never guesses.
11//!
12//! Formats are handled by an ordered registry of [`Extractor`] implementations.
13//! Selection is by declared extension first and by unambiguous magic bytes
14//! second; an unrecognised file is *unsupported*, not "probably text". New
15//! formats are additive: implement `Extractor` and [`register`] it.
16//!
17//! # Anchors
18//!
19//! A chunk inherits the anchor of the block it was cut from, so a citation can
20//! name `report.pdf [page 7]`, `budget.xlsx [Q3!B4:H4]` or `deck.pptx [slide 2]`
21//! and a reader can verify it in one step. Line ranges are added where the
22//! format has real lines.
23
24#[cfg(feature = "shell-accel")]
25pub mod accel;
26mod archive;
27mod email;
28#[cfg(feature = "ocr")]
29pub mod ocr;
30mod ooxml;
31mod pdf;
32mod sheet;
33mod textfmt;
34
35use anyhow::Result;
36use std::path::Path;
37use std::sync::{Arc, Mutex, OnceLock};
38
39/// Hard cap on a single extracted text, to bound memory on hostile input.
40pub const MAX_EXTRACTED_BYTES: usize = 32 * 1024 * 1024;
41/// Hard cap on archive members and PDF pages.
42pub const MAX_ARCHIVE_ENTRIES: usize = 2_000;
43pub const MAX_PDF_PAGES: u32 = 5_000;
44
45/// A parser that blows its five-second deadline is not killed (Rust has no safe
46/// cancellation), so its thread and input copy survive. This keeps the handles
47/// of stalled parsers so finished ones can be reaped and the set can be capped.
48/// Normal concurrency is not limited: only threads that outlived the deadline
49/// occupy a slot. Worst-case retention is `MAX_STUCK_PARSERS` input copies.
50const MAX_STUCK_PARSERS: usize = 4;
51static STUCK_PARSERS: Mutex<Vec<std::thread::JoinHandle<()>>> = Mutex::new(Vec::new());
52
53/// Background reaper: joins finished stuck parser threads so their resources
54/// are reclaimed instead of leaking until the next extraction attempt.
55fn reap_stuck_parsers() {
56    let mut stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
57    let mut i = 0;
58    while i < stuck.len() {
59        if stuck[i].is_finished() {
60            let handle = stuck.remove(i);
61            let _ = handle.join();
62        } else {
63            i += 1;
64        }
65    }
66    drop(stuck);
67}
68
69/// Start a lazy background reaper that drains finished stuck parsers every few
70/// seconds. Spawned on first use.
71fn ensure_reaper() {
72    static REAPER: std::sync::OnceLock<std::thread::JoinHandle<()>> = std::sync::OnceLock::new();
73    let _ = REAPER.get_or_init(|| {
74        std::thread::spawn(move || loop {
75            std::thread::sleep(std::time::Duration::from_secs(5));
76            reap_stuck_parsers();
77        })
78    });
79}
80
81/// Capabilities declared by an extractor.
82///
83/// These bounds are enforced by the extraction pipeline before and during
84/// parsing. An extractor should choose conservative values that reflect the
85/// worst-case resource consumption of its parser on malicious input.
86#[derive(Debug, Clone, Copy, PartialEq, Eq)]
87pub struct Capabilities {
88    /// Maximum file size this extractor will accept, in bytes.
89    /// Files larger than this are rejected before parsing begins.
90    pub max_file_bytes: u64,
91    /// Maximum number of pages (for paged formats) or entries (for archives).
92    pub max_pages: u32,
93    /// Maximum number of archive members or spreadsheet rows to process.
94    pub max_entries: usize,
95    /// Whether the extractor can process input incrementally without loading
96    /// the entire file into memory.
97    pub streaming: bool,
98    /// Whether the extractor is safe to call concurrently from multiple tasks.
99    pub thread_safe: bool,
100    /// Whether the extractor requires an external binary or system dependency.
101    pub requires_external: bool,
102}
103
104impl Default for Capabilities {
105    fn default() -> Self {
106        Self {
107            max_file_bytes: 64 * 1024 * 1024, // 64 MB
108            max_pages: 5_000,
109            max_entries: 2_000,
110            streaming: false,
111            thread_safe: true,
112            requires_external: false,
113        }
114    }
115}
116
117/// One extracted document. Archives yield several, addressed as
118/// `outer.zip!/inner/path`.
119#[derive(Debug, Clone)]
120pub struct ExtractedDocument {
121    pub path: String,
122    pub format: &'static str,
123    pub blocks: Vec<Block>,
124    /// Explicit failures such as `page 3: no text layer`. Reported to the user
125    /// instead of being silently swallowed.
126    pub warnings: Vec<String>,
127}
128
129#[derive(Debug, Clone, PartialEq, Eq)]
130pub struct Block {
131    pub text: String,
132    /// 1-based page for paged formats, 0 when the format has no pages.
133    pub page: i64,
134    /// Format-specific anchor: `page 3`, `Sheet1!A5:H5`, `slide 2`,
135    /// `para 4-9`, `spine 1: chapter.xhtml`, `message <id>`.
136    pub anchor: Option<String>,
137    /// Declared section path (`Warranty / Coverage`), when the format states
138    /// one. Never inferred from layout.
139    pub section: Option<String>,
140    /// 1-based line range *within the block's text source*, 0 when the format
141    /// has no meaningful lines (DOCX, spreadsheets, slides, email).
142    pub line_start: i64,
143    pub line_end: i64,
144}
145
146impl Block {
147    pub fn text(text: impl Into<String>) -> Self {
148        let text = text.into();
149        let lines = text.lines().count().max(1) as i64;
150        Self {
151            text,
152            page: 0,
153            anchor: None,
154            section: None,
155            line_start: 1,
156            line_end: lines,
157        }
158    }
159
160    pub fn anchored(text: impl Into<String>, anchor: impl Into<String>) -> Self {
161        let mut block = Self::text(text);
162        block.anchor = Some(anchor.into());
163        block
164    }
165}
166
167/// A format handler.
168///
169/// `extract` must either return text it actually decoded or `Ok(None)`. It must
170/// not fall back to treating arbitrary bytes as text.
171pub trait Extractor: Send + Sync {
172    /// Stable format name, used for reporting and for `Hit::anchor_kind`.
173    fn name(&self) -> &'static str;
174
175    /// Extensions handled, lower-case, without the dot.
176    fn extensions(&self) -> &'static [&'static str];
177
178    /// Unambiguous content check, for files with a missing or wrong extension.
179    fn sniff(&self, _bytes: &[u8]) -> bool {
180        false
181    }
182
183    /// Resource and safety capabilities of this extractor.
184    ///
185    /// The extraction pipeline enforces `max_file_bytes` before calling
186    /// `extract`, wraps the call in a 5-second timeout, and respects
187    /// `thread_safe` when deciding on parallelism.
188    fn capabilities(&self) -> Capabilities {
189        Capabilities::default()
190    }
191
192    fn can_handle(&self, name: &str, bytes: &[u8]) -> bool {
193        self.extensions().contains(&extension_of(name).as_str()) || self.sniff(bytes)
194    }
195
196    fn extract(&self, name: &str, bytes: &[u8]) -> Result<Option<Vec<ExtractedDocument>>>;
197}
198
199fn builtin() -> Vec<Arc<dyn Extractor>> {
200    vec![
201        Arc::new(pdf::Pdf),
202        Arc::new(ooxml::Docx),
203        Arc::new(ooxml::Pptx),
204        Arc::new(ooxml::Epub),
205        Arc::new(sheet::Sheet),
206        Arc::new(archive::Archive),
207        Arc::new(email::Email),
208        Arc::new(textfmt::Html),
209        // Last: only claims extensions it declares or unambiguous text markers.
210        Arc::new(textfmt::Plain),
211    ]
212}
213
214fn custom() -> &'static Mutex<Vec<Arc<dyn Extractor>>> {
215    static CUSTOM: OnceLock<Mutex<Vec<Arc<dyn Extractor>>>> = OnceLock::new();
216    CUSTOM.get_or_init(|| Mutex::new(Vec::new()))
217}
218
219/// Add an extractor. Custom extractors are tried after the built-ins, so a
220/// custom handler can widen support without changing built-in behaviour.
221pub fn register(extractor: Arc<dyn Extractor>) {
222    if let Ok(mut list) = custom().lock() {
223        list.push(extractor);
224    }
225}
226
227/// Every registered extractor, built-ins first.
228pub fn extractors() -> Vec<Arc<dyn Extractor>> {
229    let mut all = builtin();
230    if let Ok(list) = custom().lock() {
231        all.extend(list.iter().cloned());
232    }
233    all
234}
235
236/// Every extension any extractor declares.
237pub fn supported_extensions() -> Vec<&'static str> {
238    let mut out: Vec<&'static str> = Vec::new();
239    for extractor in extractors() {
240        out.extend_from_slice(extractor.extensions());
241    }
242    out.sort_unstable();
243    out.dedup();
244    out
245}
246
247pub fn extension_of(path: &str) -> String {
248    Path::new(path)
249        .extension()
250        .and_then(|e| e.to_str())
251        .unwrap_or("")
252        .to_ascii_lowercase()
253}
254
255/// Whether any extractor claims this path by extension.
256pub fn is_supported(path: &str) -> bool {
257    let ext = extension_of(path);
258    extractors()
259        .iter()
260        .any(|e| e.extensions().contains(&ext.as_str()))
261}
262
263/// Extract a file from disk. Returns zero or more documents (archives yield
264/// many). An unsupported file is `Ok(None)`.
265pub fn extract_path(path: &Path) -> Result<Option<Vec<ExtractedDocument>>> {
266    let name = path.to_string_lossy().to_string();
267    if !is_supported(&name) {
268        return Ok(None);
269    }
270    // Pre-read size cap: refuse to load a file larger than every applicable
271    // extractor will accept, preventing memory DoS on hostile input (I7).
272    let metadata = std::fs::symlink_metadata(path)?;
273    let mut max_allowed: u64 = 0;
274    for extractor in extractors() {
275        if extractor
276            .extensions()
277            .contains(&extension_of(&name).as_str())
278        {
279            max_allowed = max_allowed.max(extractor.capabilities().max_file_bytes);
280        }
281    }
282    if max_allowed == 0 || metadata.len() > max_allowed {
283        return Ok(None);
284    }
285    let bytes = std::fs::read(path)?;
286    extract_named(&name, &bytes, false)
287}
288
289/// Extract bytes that are already in memory (archive members, tests).
290/// `ocr_enabled` controls whether OCR is attempted for scanned PDF pages.
291pub fn extract_named(
292    name: &str,
293    bytes: &[u8],
294    _ocr_enabled: bool,
295) -> Result<Option<Vec<ExtractedDocument>>> {
296    // Try shell-out accelerators first (feature-gated)
297    #[cfg(feature = "shell-accel")]
298    {
299        if let Some(path) = std::path::Path::new(name).extension() {
300            if let Some(ext) = path.to_str() {
301                if let Ok(Some(docs)) =
302                    crate::core::extract::accel::try_accelerators(std::path::Path::new(name), ext)
303                {
304                    if !docs.is_empty() {
305                        return Ok(Some(docs));
306                    }
307                }
308            }
309        }
310    }
311
312    for extractor in extractors() {
313        if !extractor.can_handle(name, bytes) {
314            continue;
315        }
316        let caps = extractor.capabilities();
317
318        // Enforce memory cap before parsing.
319        if bytes.len() as u64 > caps.max_file_bytes {
320            return Ok(Some(single(
321                name,
322                extractor.name(),
323                Vec::new(),
324                vec![format!(
325                    "file size {} exceeds maximum {} bytes",
326                    bytes.len(),
327                    caps.max_file_bytes
328                )],
329            )));
330        }
331
332        // Per-parser timeout: run extract in a separate thread with a 5-second
333        // deadline. `std::thread` + mpsc avoids nesting a tokio runtime.
334        //
335        // A parser that ignores its deadline must not be able to spawn an
336        // unbounded number of threads. Reap any that have since finished, and
337        // refuse to start more while the stalled set is at its cap. Normal
338        // concurrent parsing is never limited.
339        {
340            reap_stuck_parsers();
341            ensure_reaper();
342            let stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
343            if stuck.len() >= MAX_STUCK_PARSERS {
344                return Ok(Some(single(
345                    name,
346                    extractor.name(),
347                    Vec::new(),
348                    vec![format!(
349                        "skipped: {MAX_STUCK_PARSERS} parsers are stalled; restart the indexer"
350                    )],
351                )));
352            }
353            drop(stuck);
354        }
355
356        let extractor_clone = extractor.clone();
357        let name_owned = name.to_string();
358        let bytes_owned = bytes.to_vec();
359        let (tx, rx) = std::sync::mpsc::channel();
360        let handle = std::thread::spawn(move || {
361            let _ = tx.send(extractor_clone.extract(&name_owned, &bytes_owned));
362        });
363        let result = rx.recv_timeout(std::time::Duration::from_secs(5));
364
365        let mut docs = match result {
366            Ok(Ok(Some(docs))) => {
367                let _ = handle.join();
368                docs
369            }
370            Ok(Ok(None)) => {
371                let _ = handle.join();
372                continue;
373            }
374            Ok(Err(err)) => {
375                let _ = handle.join();
376                return Ok(Some(single(
377                    name,
378                    extractor.name(),
379                    Vec::new(),
380                    vec![format!("extraction error: {err}")],
381                )));
382            }
383            Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
384                // Deliberately do NOT join: the parser may never return. Park the
385                // handle so it occupies a stalled slot until it finishes. A
386                // background reaper joins finished handles so threads don't leak.
387                tracing::warn!("{}: extraction timed out after 5 seconds", name);
388                ensure_reaper();
389                STUCK_PARSERS
390                    .lock()
391                    .unwrap_or_else(|e| e.into_inner())
392                    .push(handle);
393                return Ok(Some(single(
394                    name,
395                    extractor.name(),
396                    Vec::new(),
397                    vec!["extraction timed out after 5 seconds".to_string()],
398                )));
399            }
400            Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
401                let _ = handle.join();
402                return Ok(Some(single(
403                    name,
404                    extractor.name(),
405                    Vec::new(),
406                    vec!["extraction task panicked".to_string()],
407                )));
408            }
409        };
410
411        for doc in &mut docs {
412            doc.format = extractor.name();
413            if doc.path.is_empty() {
414                doc.path = name.to_string();
415            }
416            doc.blocks = finish_blocks(std::mem::take(&mut doc.blocks));
417        }
418        docs.retain(|doc| !doc.blocks.is_empty());
419        if docs.is_empty() {
420            // Matched the format but had nothing citable (empty or unreadable).
421            return Ok(None);
422        }
423        return Ok(Some(docs));
424    }
425    Ok(None)
426}
427
428/// Drop empty blocks and cap their size. Anchors are never modified.
429pub(crate) fn finish_blocks(mut blocks: Vec<Block>) -> Vec<Block> {
430    let mut total = 0usize;
431    blocks.retain(|b| !b.text.trim().is_empty());
432    for block in &mut blocks {
433        if block.text.len() > MAX_EXTRACTED_BYTES {
434            let mut cut = MAX_EXTRACTED_BYTES;
435            while cut > 0 && !block.text.is_char_boundary(cut) {
436                cut -= 1;
437            }
438            block.text.truncate(cut);
439        }
440        total += block.text.len();
441        if total > MAX_EXTRACTED_BYTES {
442            break;
443        }
444    }
445    blocks
446}
447
448/// Helper for extractors: build a single-document result.
449pub(crate) fn single(
450    name: &str,
451    format: &'static str,
452    blocks: Vec<Block>,
453    warnings: Vec<String>,
454) -> Vec<ExtractedDocument> {
455    vec![ExtractedDocument {
456        path: name.to_string(),
457        format,
458        blocks,
459        warnings,
460    }]
461}