spectre_pdf 1.0.0

//! `spectre_pdf` — Native Rust PDF extraction engine.
//!
//! Read-only PDF parsing + extraction. Every surface — the persistent
//! [`Document`] handle and the free functions — runs on the in-repo
//! `spectre_parse` lazy parser. No C dependencies. Full PDF Standard
//! Security Handler decryption (RC4 40/128-bit + AES-128/256 R=5/R=6).
//! Works as a pure Rust library or as a Python extension when built
//! with the `python` feature (the Python package is published as
//! `spectre-rs`).
//!
//! # Quickstart (Rust)
//!
//! ```no_run
//! use std::fs;
//! let bytes = fs::read("document.pdf").unwrap();
//! let text = spectre_pdf::extract_text(&bytes).unwrap();
//! let pages = spectre_pdf::extract_pages(&bytes).unwrap();
//! let meta = spectre_pdf::extract_metadata(&bytes).unwrap();
//! let tables = spectre_pdf::extract_tables(&bytes, None).unwrap();
//! let score = spectre_pdf::score_text(&text); // 0.0 garbage .. 1.0 clean
//! # let _ = (pages, meta, tables, score);
//! ```
//!
//! # Quickstart (Python, requires `--features python`)
//!
//! ```text
//! maturin develop --release --features python
//! ```
//!
//! ```python
//! import spectre_rs
//! text = spectre_rs.extract_text(open("document.pdf", "rb").read())
//! pages = spectre_rs.extract_pages(open("document.pdf", "rb").read())
//! ```

use rayon::prelude::*;
use regex::Regex;
use std::collections::HashMap;
use std::fmt;
use std::sync::OnceLock;

mod document;
pub mod geom;
mod markdown;
mod meta;
mod positioned;
mod scorer;
pub mod search;
mod structure;
mod tables;

#[doc(hidden)]
pub mod test_only_markdown_helpers {
    pub use crate::markdown::{
        build_heading_size_map, detect_numbered_prefix_level, detect_structural_heading,
        merge_adjacent_heading_blocks, normalize_heading_lookup, render_styled_line,
    };
}

pub(crate) use scorer::score_text_impl;

pub use document::{
    DecodedImage, Document, FormField, FormFieldType, ImageContainer, Widget,
};
pub use positioned::{PositionedPage, TextBlock, TextLine, TextSpan, Word};
pub use search::{SearchHit, SearchOptions};
pub use structure::{Annotation, DocumentInfo, ImageInfo, Link, PageInfo, TocEntry};

#[cfg(feature = "python")]
mod python;

// ── Resource bounds ─────────────────────────────────────────────────────────
//
// Hard ceilings on per-call work. Hostile PDFs can claim arbitrary page
// counts or produce arbitrarily large extraction output; these caps let
// the crate run in long-lived services without per-caller input vetting.

/// Maximum number of pages a single extraction call will process.
/// Crafted PDFs can claim millions of pages; we refuse before doing the work.
pub const MAX_PAGES: usize = 10_000;

/// Maximum byte length of the concatenated output produced by
/// [`extract_text`]. Refuses pathological documents whose extracted text
/// would exhaust available memory.
pub const MAX_OUTPUT_BYTES: usize = 256 * 1024 * 1024;

/// Maximum number of tables [`extract_tables`] will return per call. The
/// detector occasionally over-segments on noisy input; this cap keeps the
/// returned `Vec<Vec<Vec<String>>>` bounded.
pub const MAX_TABLES: usize = 10_000;

// ── Error type ───────────────────────────────────────────────────────────────

/// Errors emitted by `spectre_pdf` extraction functions.
///
/// `#[non_exhaustive]`: downstream `match` expressions must include a
/// wildcard arm so future minor releases can add variants.
#[derive(Debug)]
#[non_exhaustive]
pub enum ExtractError {
    /// Malformed bytes, unsupported version, etc.
    LoadFailed(String),
    /// Table or metadata extraction returned an internal error.
    ParseFailed(String),
    /// A specific page failed to extract. Surfaced as a typed error
    /// rather than silently dropped, so callers can route the page to
    /// OCR or human review.
    PageExtractFailed {
        /// 1-indexed page number.
        page: u32,
        /// Per-page parser error message.
        source: String,
    },
    /// A configured resource limit was hit. See [`MAX_PAGES`],
    /// [`MAX_OUTPUT_BYTES`], [`MAX_TABLES`].
    LimitExceeded {
        limit: &'static str,
        actual: usize,
        max: usize,
    },
    /// The PDF is encrypted and the supplied password did not
    /// authenticate. Use [`Document::open_with_password`] to retry.
    Encrypted,
}

impl fmt::Display for ExtractError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::LoadFailed(msg) => write!(f, "Failed to load PDF: {msg}"),
            Self::ParseFailed(msg) => write!(f, "Extraction failed: {msg}"),
            Self::PageExtractFailed { page, source } => {
                write!(f, "Failed to extract page {page}: {source}")
            }
            Self::LimitExceeded { limit, actual, max } => {
                write!(f, "{limit} limit exceeded: {actual} > {max}")
            }
            Self::Encrypted => write!(
                f,
                "PDF is encrypted and the supplied password did not authenticate"
            ),
        }
    }
}

impl std::error::Error for ExtractError {}

// ── Public API (pure Rust) ───────────────────────────────────────────────────

/// Extract all text from PDF bytes as a single string.
///
/// Page boundaries are normalized to double newlines and runs of three or
/// more blank lines are collapsed to two. Returns
/// [`ExtractError::LimitExceeded`] if the concatenated output would exceed
/// [`MAX_OUTPUT_BYTES`], or [`ExtractError::PageExtractFailed`] if any
/// individual page fails to extract.
pub fn extract_text(pdf_bytes: &[u8]) -> Result<String, ExtractError> {
    let pages = extract_pages(pdf_bytes)?;
    // Pre-check the joined output size so a hostile PDF can't OOM us
    // via thousands of small pages joined with double-newlines.
    let join_overhead = pages.len().saturating_mul(2);
    let projected = pages
        .iter()
        .map(|p| p.len())
        .sum::<usize>()
        .saturating_add(join_overhead);
    if projected > MAX_OUTPUT_BYTES {
        return Err(ExtractError::LimitExceeded {
            limit: "output bytes",
            actual: projected,
            max: MAX_OUTPUT_BYTES,
        });
    }
    Ok(normalize_text(&pages.join("\n\n")))
}

/// Extract text per page, in document order. Image-only pages return
/// empty strings so indices line up. Per-page extraction failures
/// surface as [`ExtractError::PageExtractFailed`]. Refuses documents
/// claiming more than [`MAX_PAGES`].
pub fn extract_pages(pdf_bytes: &[u8]) -> Result<Vec<String>, ExtractError> {
    let (doc, pages) = open_strict(pdf_bytes)?;
    pages
        .iter()
        .map(|n| {
            doc.extract_text(&[*n])
                .map_err(|e| ExtractError::PageExtractFailed {
                    page: *n,
                    source: e.to_string(),
                })
        })
        .collect()
}

/// Best-effort variant: pages that fail to extract (commonly CID fonts
/// missing `/ToUnicode`) return empty strings rather than erroring the
/// whole document. Document-level errors still surface.
pub fn extract_pages_lenient(pdf_bytes: &[u8]) -> Result<Vec<String>, ExtractError> {
    let (doc, pages) = open_strict(pdf_bytes)?;
    Ok(pages
        .iter()
        .map(|n| doc.extract_text(&[*n]).unwrap_or_default())
        .collect())
}

fn open_strict(pdf_bytes: &[u8]) -> Result<(spectre_parse::Document, Vec<u32>), ExtractError> {
    let doc = crate::document::open_sp_with_password(pdf_bytes, b"")?;
    let mut pages: Vec<u32> = doc.get_pages().keys().copied().collect();
    if pages.len() > MAX_PAGES {
        return Err(ExtractError::LimitExceeded {
            limit: "pages",
            actual: pages.len(),
            max: MAX_PAGES,
        });
    }
    pages.sort();
    Ok((doc, pages))
}

/// Best-effort variant of [`extract_text`]: failing pages are silently
/// dropped (no error, no placeholder). Output size limit still enforced.
pub fn extract_text_lenient(pdf_bytes: &[u8]) -> Result<String, ExtractError> {
    let pages = extract_pages_lenient(pdf_bytes)?;
    let join_overhead = pages.len().saturating_mul(2);
    let projected = pages
        .iter()
        .map(|p| p.len())
        .sum::<usize>()
        .saturating_add(join_overhead);
    if projected > MAX_OUTPUT_BYTES {
        return Err(ExtractError::LimitExceeded {
            limit: "output bytes",
            actual: projected,
            max: MAX_OUTPUT_BYTES,
        });
    }
    Ok(normalize_text(&pages.join("\n\n")))
}

/// Whitespace-column table extraction tuned for borderless tables.
/// `page` is 1-indexed matching the other `extract_*` functions;
/// `None` scans every page.
pub fn extract_tables(
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> Result<Vec<Vec<Vec<String>>>, ExtractError> {
    let page_num = page.map(|p| p.saturating_sub(1) as usize);
    let tables = tables::extract_tables_impl(pdf_bytes, page_num)?;
    if tables.len() > MAX_TABLES {
        return Err(ExtractError::LimitExceeded {
            limit: "tables",
            actual: tables.len(),
            max: MAX_TABLES,
        });
    }
    Ok(tables)
}

/// PDF `/Info` dictionary plus page count and PDF version. Keys:
/// `pages`, `pdf_version`, optionally `title`, `author`, `subject`,
/// `keywords`, `creator`, `producer`, `creation_date`, `mod_date`.
pub fn extract_metadata(pdf_bytes: &[u8]) -> Result<HashMap<String, String>, ExtractError> {
    meta::extract_metadata_impl(pdf_bytes).map_err(ExtractError::ParseFailed)
}

/// Text-quality score: 0.0 (binary noise) to 1.0 (clean prose). Useful
/// for filtering pages where extraction returned a CID dump.
pub fn score_text(text: &str) -> f64 {
    score_text_impl(text)
}

/// Score a batch of texts in parallel via [`rayon`].
pub fn score_batch(texts: &[String]) -> Vec<f64> {
    texts.par_iter().map(|t| score_text_impl(t)).collect()
}

// ── Public API (positional + structural surface) ────────────────────────────
//
// Every function below is a thin pure-Rust wrapper over a module in
// [`positioned`], [`structure`], or [`search`]. No C dependencies.

/// Extract text per page with bounding boxes. Returns one
/// [`PositionedPage`] per page in document order; each carries a
/// re-assembled plain-text string and the underlying [`TextSpan`]s
/// with PDF user-space rects. Pass `page` (1-indexed) for one page only.
pub fn extract_text_positioned(
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> Result<Vec<PositionedPage>, ExtractError> {
    positioned::extract_text_positioned_impl(pdf_bytes, page)
}

/// Extract words with bounding boxes. Each word carries
/// `(block_no, line_no, word_no)` so callers can reconstruct paragraph
/// structure cheaply.
pub fn extract_words(
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> Result<Vec<Word>, ExtractError> {
    positioned::extract_words_impl(pdf_bytes, page)
}

/// Extract text blocks (paragraph-sized clusters) with bounding boxes.
pub fn extract_blocks(
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> Result<Vec<TextBlock>, ExtractError> {
    positioned::extract_blocks_impl(pdf_bytes, page)
}

/// Search for a string and return one rect per match.
/// Case-insensitive and flexible-whitespace by default; see [`SearchOptions`].
pub fn search(
    pdf_bytes: &[u8],
    query: &str,
    page: Option<u32>,
    options: Option<SearchOptions>,
) -> Result<Vec<SearchHit>, ExtractError> {
    search::search_impl(pdf_bytes, query, page, options.unwrap_or_default())
}

/// Extract the document outline (bookmarks / TOC). Returns an empty
/// Vec for outline-less PDFs.
pub fn extract_toc(pdf_bytes: &[u8]) -> Result<Vec<TocEntry>, ExtractError> {
    structure::extract_toc_impl(pdf_bytes)
}

/// Extract hyperlinks. Pass `page` to filter to one page (1-indexed).
pub fn extract_links(
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> Result<Vec<Link>, ExtractError> {
    structure::extract_links_impl(pdf_bytes, page)
}

/// Extract annotations (read-only).
pub fn extract_annotations(
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> Result<Vec<Annotation>, ExtractError> {
    structure::extract_annotations_impl(pdf_bytes, page)
}

/// Inventory of embedded images (size, colorspace, filter chain) —
/// equivalent. Image bytes are **not**
/// decoded; the inventory itself covers routing and observability.
pub fn extract_images(
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> Result<Vec<ImageInfo>, ExtractError> {
    structure::extract_images_impl(pdf_bytes, page)
}

/// Per-page geometry: mediabox, cropbox, rotation, width, height,
/// rolled into one cheap walk over the page tree.
pub fn extract_page_info(
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> Result<Vec<PageInfo>, ExtractError> {
    structure::extract_page_info_impl(pdf_bytes, page)
}

/// Document-level summary: page count, PDF version, encryption +
/// linearization flags, xref size, trailer ID. Cheap; doesn't decode any
/// content streams.
pub fn extract_document_info(pdf_bytes: &[u8]) -> Result<DocumentInfo, ExtractError> {
    structure::extract_document_info_impl(pdf_bytes)
}

// ── Helpers ──────────────────────────────────────────────────────────────────

fn normalize_text(text: &str) -> String {
    static BLANK_LINES_RE: OnceLock<Regex> = OnceLock::new();
    let re = BLANK_LINES_RE.get_or_init(|| {
        Regex::new(r"\n{3,}")
            .expect("blank-line regex literal compiles — change requires updating this expect")
    });
    re.replace_all(text, "\n\n").trim().to_string()
}