paperscope 0.2.2

Paper: local-first document search with exact page and line citations. CLI + MCP server in one binary.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
//! Document extraction with media-native citation anchors.
//!
//! # Design
//!
//! Extraction produces **facts**, never interpretations. A block carries the
//! text exactly as the file states it plus the locator the file itself exposes:
//! a PDF page number, a spreadsheet cell range, a slide index, a paragraph
//! range, an EPUB spine position, an email message id. Nothing is classified,
//! dated, summarised or normalised semantically. If the library cannot read a
//! format or a page, it says so; it never guesses.
//!
//! Formats are handled by an ordered registry of [`Extractor`] implementations.
//! Selection is by declared extension first and by unambiguous magic bytes
//! second; an unrecognised file is *unsupported*, not "probably text". New
//! formats are additive: implement `Extractor` and [`register`] it.
//!
//! # Anchors
//!
//! A chunk inherits the anchor of the block it was cut from, so a citation can
//! name `report.pdf [page 7]`, `budget.xlsx [Q3!B4:H4]` or `deck.pptx [slide 2]`
//! and a reader can verify it in one step. Line ranges are added where the
//! format has real lines.

#[cfg(feature = "shell-accel")]
pub mod accel;
mod archive;
mod email;
#[cfg(feature = "ocr")]
pub mod ocr;
mod ooxml;
mod pdf;
mod sheet;
mod textfmt;

use anyhow::Result;
use std::path::Path;
use std::sync::{Arc, Mutex, OnceLock};

/// Hard cap on a single extracted text, to bound memory on hostile input.
pub const MAX_EXTRACTED_BYTES: usize = 32 * 1024 * 1024;
/// Hard cap on archive members and PDF pages.
pub const MAX_ARCHIVE_ENTRIES: usize = 2_000;
pub const MAX_PDF_PAGES: u32 = 5_000;

/// A parser that blows its five-second deadline is not killed (Rust has no safe
/// cancellation), so its thread and input copy survive. This keeps the handles
/// of stalled parsers so finished ones can be reaped and the set can be capped.
/// Normal concurrency is not limited: only threads that outlived the deadline
/// occupy a slot. Worst-case retention is `MAX_STUCK_PARSERS` input copies.
const MAX_STUCK_PARSERS: usize = 4;
static STUCK_PARSERS: Mutex<Vec<std::thread::JoinHandle<()>>> = Mutex::new(Vec::new());

/// Capabilities declared by an extractor.
///
/// These bounds are enforced by the extraction pipeline before and during
/// parsing. An extractor should choose conservative values that reflect the
/// worst-case resource consumption of its parser on malicious input.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Capabilities {
    /// Maximum file size this extractor will accept, in bytes.
    /// Files larger than this are rejected before parsing begins.
    pub max_file_bytes: u64,
    /// Maximum number of pages (for paged formats) or entries (for archives).
    pub max_pages: u32,
    /// Maximum number of archive members or spreadsheet rows to process.
    pub max_entries: usize,
    /// Whether the extractor can process input incrementally without loading
    /// the entire file into memory.
    pub streaming: bool,
    /// Whether the extractor is safe to call concurrently from multiple tasks.
    pub thread_safe: bool,
    /// Whether the extractor requires an external binary or system dependency.
    pub requires_external: bool,
}

impl Default for Capabilities {
    fn default() -> Self {
        Self {
            max_file_bytes: 64 * 1024 * 1024, // 64 MB
            max_pages: 5_000,
            max_entries: 2_000,
            streaming: false,
            thread_safe: true,
            requires_external: false,
        }
    }
}

/// One extracted document. Archives yield several, addressed as
/// `outer.zip!/inner/path`.
#[derive(Debug, Clone)]
pub struct ExtractedDocument {
    pub path: String,
    pub format: &'static str,
    pub blocks: Vec<Block>,
    /// Explicit failures such as `page 3: no text layer`. Reported to the user
    /// instead of being silently swallowed.
    pub warnings: Vec<String>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Block {
    pub text: String,
    /// 1-based page for paged formats, 0 when the format has no pages.
    pub page: i64,
    /// Format-specific anchor: `page 3`, `Sheet1!A5:H5`, `slide 2`,
    /// `para 4-9`, `spine 1: chapter.xhtml`, `message <id>`.
    pub anchor: Option<String>,
    /// Declared section path (`Warranty / Coverage`), when the format states
    /// one. Never inferred from layout.
    pub section: Option<String>,
    /// 1-based line range *within the block's text source*, 0 when the format
    /// has no meaningful lines (DOCX, spreadsheets, slides, email).
    pub line_start: i64,
    pub line_end: i64,
}

impl Block {
    pub fn text(text: impl Into<String>) -> Self {
        let text = text.into();
        let lines = text.lines().count().max(1) as i64;
        Self {
            text,
            page: 0,
            anchor: None,
            section: None,
            line_start: 1,
            line_end: lines,
        }
    }

    pub fn anchored(text: impl Into<String>, anchor: impl Into<String>) -> Self {
        let mut block = Self::text(text);
        block.anchor = Some(anchor.into());
        block
    }
}

/// A format handler.
///
/// `extract` must either return text it actually decoded or `Ok(None)`. It must
/// not fall back to treating arbitrary bytes as text.
pub trait Extractor: Send + Sync {
    /// Stable format name, used for reporting and for `Hit::anchor_kind`.
    fn name(&self) -> &'static str;

    /// Extensions handled, lower-case, without the dot.
    fn extensions(&self) -> &'static [&'static str];

    /// Unambiguous content check, for files with a missing or wrong extension.
    fn sniff(&self, _bytes: &[u8]) -> bool {
        false
    }

    /// Resource and safety capabilities of this extractor.
    ///
    /// The extraction pipeline enforces `max_file_bytes` before calling
    /// `extract`, wraps the call in a 5-second timeout, and respects
    /// `thread_safe` when deciding on parallelism.
    fn capabilities(&self) -> Capabilities {
        Capabilities::default()
    }

    fn can_handle(&self, name: &str, bytes: &[u8]) -> bool {
        self.extensions().contains(&extension_of(name).as_str()) || self.sniff(bytes)
    }

    fn extract(&self, name: &str, bytes: &[u8]) -> Result<Option<Vec<ExtractedDocument>>>;
}

fn builtin() -> Vec<Arc<dyn Extractor>> {
    vec![
        Arc::new(pdf::Pdf),
        Arc::new(ooxml::Docx),
        Arc::new(ooxml::Pptx),
        Arc::new(ooxml::Epub),
        Arc::new(sheet::Sheet),
        Arc::new(archive::Archive),
        Arc::new(email::Email),
        Arc::new(textfmt::Html),
        // Last: only claims extensions it declares or unambiguous text markers.
        Arc::new(textfmt::Plain),
    ]
}

fn custom() -> &'static Mutex<Vec<Arc<dyn Extractor>>> {
    static CUSTOM: OnceLock<Mutex<Vec<Arc<dyn Extractor>>>> = OnceLock::new();
    CUSTOM.get_or_init(|| Mutex::new(Vec::new()))
}

/// Add an extractor. Custom extractors are tried after the built-ins, so a
/// custom handler can widen support without changing built-in behaviour.
pub fn register(extractor: Arc<dyn Extractor>) {
    if let Ok(mut list) = custom().lock() {
        list.push(extractor);
    }
}

/// Every registered extractor, built-ins first.
pub fn extractors() -> Vec<Arc<dyn Extractor>> {
    let mut all = builtin();
    if let Ok(list) = custom().lock() {
        all.extend(list.iter().cloned());
    }
    all
}

/// Every extension any extractor declares.
pub fn supported_extensions() -> Vec<&'static str> {
    let mut out: Vec<&'static str> = Vec::new();
    for extractor in extractors() {
        out.extend_from_slice(extractor.extensions());
    }
    out.sort_unstable();
    out.dedup();
    out
}

pub fn extension_of(path: &str) -> String {
    Path::new(path)
        .extension()
        .and_then(|e| e.to_str())
        .unwrap_or("")
        .to_ascii_lowercase()
}

/// Whether any extractor claims this path by extension.
pub fn is_supported(path: &str) -> bool {
    let ext = extension_of(path);
    extractors()
        .iter()
        .any(|e| e.extensions().contains(&ext.as_str()))
}

/// Extract a file from disk. Returns zero or more documents (archives yield
/// many). An unsupported file is `Ok(None)`.
pub fn extract_path(path: &Path) -> Result<Option<Vec<ExtractedDocument>>> {
    let name = path.to_string_lossy().to_string();
    if !is_supported(&name) {
        return Ok(None);
    }
    // Pre-read size cap: refuse to load a file larger than every applicable
    // extractor will accept, preventing memory DoS on hostile input (I7).
    let metadata = std::fs::symlink_metadata(path)?;
    let mut max_allowed: u64 = 0;
    for extractor in extractors() {
        if extractor
            .extensions()
            .contains(&extension_of(&name).as_str())
        {
            max_allowed = max_allowed.max(extractor.capabilities().max_file_bytes);
        }
    }
    if max_allowed == 0 || metadata.len() > max_allowed {
        return Ok(None);
    }
    let bytes = std::fs::read(path)?;
    extract_named(&name, &bytes, false)
}

/// Extract bytes that are already in memory (archive members, tests).
/// `ocr_enabled` controls whether OCR is attempted for scanned PDF pages.
pub fn extract_named(
    name: &str,
    bytes: &[u8],
    _ocr_enabled: bool,
) -> Result<Option<Vec<ExtractedDocument>>> {
    // Try shell-out accelerators first (feature-gated)
    #[cfg(feature = "shell-accel")]
    {
        if let Some(path) = std::path::Path::new(name).extension() {
            if let Some(ext) = path.to_str() {
                if let Ok(Some(docs)) =
                    crate::core::extract::accel::try_accelerators(std::path::Path::new(name), ext)
                {
                    if !docs.is_empty() {
                        return Ok(Some(docs));
                    }
                }
            }
        }
    }

    for extractor in extractors() {
        if !extractor.can_handle(name, bytes) {
            continue;
        }
        let caps = extractor.capabilities();

        // Enforce memory cap before parsing.
        if bytes.len() as u64 > caps.max_file_bytes {
            return Ok(Some(single(
                name,
                extractor.name(),
                Vec::new(),
                vec![format!(
                    "file size {} exceeds maximum {} bytes",
                    bytes.len(),
                    caps.max_file_bytes
                )],
            )));
        }

        // Per-parser timeout: run extract in a separate thread with a 5-second
        // deadline. `std::thread` + mpsc avoids nesting a tokio runtime.
        //
        // A parser that ignores its deadline must not be able to spawn an
        // unbounded number of threads. Reap any that have since finished, and
        // refuse to start more while the stalled set is at its cap. Normal
        // concurrent parsing is never limited.
        {
            let mut stuck = STUCK_PARSERS.lock().unwrap_or_else(|e| e.into_inner());
            stuck.retain(|h| !h.is_finished());
            if stuck.len() >= MAX_STUCK_PARSERS {
                return Ok(Some(single(
                    name,
                    extractor.name(),
                    Vec::new(),
                    vec![format!(
                        "skipped: {MAX_STUCK_PARSERS} parsers are stalled; restart the indexer"
                    )],
                )));
            }
        }

        let extractor_clone = extractor.clone();
        let name_owned = name.to_string();
        let bytes_owned = bytes.to_vec();
        let (tx, rx) = std::sync::mpsc::channel();
        let handle = std::thread::spawn(move || {
            let _ = tx.send(extractor_clone.extract(&name_owned, &bytes_owned));
        });
        let result = rx.recv_timeout(std::time::Duration::from_secs(5));

        let mut docs = match result {
            Ok(Ok(Some(docs))) => {
                let _ = handle.join();
                docs
            }
            Ok(Ok(None)) => {
                let _ = handle.join();
                continue;
            }
            Ok(Err(err)) => {
                let _ = handle.join();
                return Ok(Some(single(
                    name,
                    extractor.name(),
                    Vec::new(),
                    vec![format!("extraction error: {err}")],
                )));
            }
            Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
                // Deliberately do NOT join: the parser may never return. Park the
                // handle so it occupies a stalled slot until it finishes.
                tracing::warn!("{}: extraction timed out after 5 seconds", name);
                STUCK_PARSERS
                    .lock()
                    .unwrap_or_else(|e| e.into_inner())
                    .push(handle);
                return Ok(Some(single(
                    name,
                    extractor.name(),
                    Vec::new(),
                    vec!["extraction timed out after 5 seconds".to_string()],
                )));
            }
            Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
                let _ = handle.join();
                return Ok(Some(single(
                    name,
                    extractor.name(),
                    Vec::new(),
                    vec!["extraction task panicked".to_string()],
                )));
            }
        };

        for doc in &mut docs {
            doc.format = extractor.name();
            if doc.path.is_empty() {
                doc.path = name.to_string();
            }
            doc.blocks = finish_blocks(std::mem::take(&mut doc.blocks));
        }
        docs.retain(|doc| !doc.blocks.is_empty());
        if docs.is_empty() {
            // Matched the format but had nothing citable (empty or unreadable).
            return Ok(None);
        }
        return Ok(Some(docs));
    }
    Ok(None)
}

/// Drop empty blocks and cap their size. Anchors are never modified.
pub(crate) fn finish_blocks(mut blocks: Vec<Block>) -> Vec<Block> {
    let mut total = 0usize;
    blocks.retain(|b| !b.text.trim().is_empty());
    for block in &mut blocks {
        if block.text.len() > MAX_EXTRACTED_BYTES {
            let mut cut = MAX_EXTRACTED_BYTES;
            while cut > 0 && !block.text.is_char_boundary(cut) {
                cut -= 1;
            }
            block.text.truncate(cut);
        }
        total += block.text.len();
        if total > MAX_EXTRACTED_BYTES {
            break;
        }
    }
    blocks
}

/// Helper for extractors: build a single-document result.
pub(crate) fn single(
    name: &str,
    format: &'static str,
    blocks: Vec<Block>,
    warnings: Vec<String>,
) -> Vec<ExtractedDocument> {
    vec![ExtractedDocument {
        path: name.to_string(),
        format,
        blocks,
        warnings,
    }]
}