spectre_pdf 1.0.0

//! Python bindings — only compiled with `--features python`.
//!
//! Each `#[pyfunction]` is a thin wrapper over the pure-Rust API in
//! [`crate`]. `ExtractError` converts to `PyValueError`.

use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use rayon::prelude::*;
use std::collections::HashMap;

use crate::{
    extract_annotations as rs_extract_annotations, extract_blocks as rs_extract_blocks,
    extract_document_info as rs_extract_document_info, extract_images as rs_extract_images,
    extract_links as rs_extract_links, extract_metadata as rs_extract_metadata,
    extract_page_info as rs_extract_page_info, extract_pages as rs_extract_pages,
    extract_tables as rs_extract_tables, extract_text as rs_extract_text,
    extract_text_positioned as rs_extract_text_positioned, extract_toc as rs_extract_toc,
    extract_words as rs_extract_words, geom::Rect as RsRect, score_text_impl, search as rs_search,
    Annotation as RsAnnotation, DecodedImage as RsDecodedImage, Document as RsDocument,
    DocumentInfo as RsDocumentInfo, ExtractError, ImageContainer as RsImageContainer,
    ImageInfo as RsImageInfo, Link as RsLink, PageInfo as RsPageInfo,
    PositionedPage as RsPositionedPage, SearchHit as RsSearchHit, SearchOptions as RsSearchOptions,
    TextBlock as RsTextBlock, TextLine as RsTextLine, TextSpan as RsTextSpan, TocEntry as RsTocEntry,
    Widget as RsWidget, Word as RsWord,
};

impl From<ExtractError> for PyErr {
    fn from(e: ExtractError) -> Self {
        PyValueError::new_err(e.to_string())
    }
}

// ── Wrapper #[pyclass]es for return-shape types ─────────────────────────────
// Read-only: these are extraction snapshots, not editable objects.

#[pyclass(name = "Rect", get_all)]
#[derive(Clone)]
pub struct PyRect {
    pub x0: f32,
    pub y0: f32,
    pub x1: f32,
    pub y1: f32,
}

#[pymethods]
impl PyRect {
    fn __repr__(&self) -> String {
        format!(
            "Rect(x0={:.2}, y0={:.2}, x1={:.2}, y1={:.2})",
            self.x0, self.y0, self.x1, self.y1
        )
    }
    fn as_tuple(&self) -> (f32, f32, f32, f32) {
        (self.x0, self.y0, self.x1, self.y1)
    }
}

impl From<RsRect> for PyRect {
    fn from(r: RsRect) -> Self {
        PyRect {
            x0: r.x0,
            y0: r.y0,
            x1: r.x1,
            y1: r.y1,
        }
    }
}

#[pyclass(name = "Word", get_all)]
pub struct PyWord {
    pub text: String,
    pub bbox: PyRect,
    pub page: u32,
    pub block_no: u32,
    pub line_no: u32,
    pub word_no: u32,
}

#[pymethods]
impl PyWord {
    fn __repr__(&self) -> String {
        format!(
            "Word(page={}, text={:?}, bbox={})",
            self.page,
            self.text,
            self.bbox.__repr__()
        )
    }
    /// Returns `(x0, y0, x1, y1, text, block_no, line_no, word_no)`.
    fn as_tuple(&self) -> (f32, f32, f32, f32, String, u32, u32, u32) {
        (
            self.bbox.x0,
            self.bbox.y0,
            self.bbox.x1,
            self.bbox.y1,
            self.text.clone(),
            self.block_no,
            self.line_no,
            self.word_no,
        )
    }
}

impl From<RsWord> for PyWord {
    fn from(w: RsWord) -> Self {
        PyWord {
            text: w.text,
            bbox: w.bbox.into(),
            page: w.page,
            block_no: w.block_no,
            line_no: w.line_no,
            word_no: w.word_no,
        }
    }
}

#[pyclass(name = "TextSpan", get_all)]
#[derive(Clone)]
pub struct PyTextSpan {
    pub text: String,
    pub bbox: PyRect,
    pub page: u32,
    pub font: String,
    pub font_size: f32,
}

impl From<RsTextSpan> for PyTextSpan {
    fn from(s: RsTextSpan) -> Self {
        PyTextSpan {
            text: s.text,
            bbox: s.bbox.into(),
            page: s.page,
            font: s.font,
            font_size: s.font_size,
        }
    }
}

#[pyclass(name = "TextLine", get_all)]
#[derive(Clone)]
pub struct PyTextLine {
    pub text: String,
    pub bbox: PyRect,
    pub spans: Vec<PyTextSpan>,
}

impl From<RsTextLine> for PyTextLine {
    fn from(l: RsTextLine) -> Self {
        PyTextLine {
            text: l.text,
            bbox: l.bbox.into(),
            spans: l.spans.into_iter().map(Into::into).collect(),
        }
    }
}

#[pyclass(name = "TextBlock", get_all)]
pub struct PyTextBlock {
    pub text: String,
    pub bbox: PyRect,
    pub page: u32,
    pub block_no: u32,
    pub lines: Vec<PyTextLine>,
}

impl From<RsTextBlock> for PyTextBlock {
    fn from(b: RsTextBlock) -> Self {
        PyTextBlock {
            text: b.text,
            bbox: b.bbox.into(),
            page: b.page,
            block_no: b.block_no,
            lines: b.lines.into_iter().map(Into::into).collect(),
        }
    }
}

#[pyclass(name = "PositionedPage", get_all)]
pub struct PyPositionedPage {
    pub page: u32,
    pub text: String,
    pub spans: Vec<PyTextSpan>,
}

impl From<RsPositionedPage> for PyPositionedPage {
    fn from(p: RsPositionedPage) -> Self {
        PyPositionedPage {
            page: p.page,
            text: p.text,
            spans: p.spans.into_iter().map(Into::into).collect(),
        }
    }
}

#[pyclass(name = "SearchHit", get_all)]
pub struct PySearchHit {
    pub page: u32,
    pub bbox: PyRect,
    pub text: String,
}

impl From<RsSearchHit> for PySearchHit {
    fn from(h: RsSearchHit) -> Self {
        PySearchHit {
            page: h.page,
            bbox: h.bbox.into(),
            text: h.text,
        }
    }
}

#[pyclass(name = "TocEntry", get_all)]
pub struct PyTocEntry {
    pub level: usize,
    pub title: String,
    pub page: Option<u32>,
}

impl From<RsTocEntry> for PyTocEntry {
    fn from(t: RsTocEntry) -> Self {
        PyTocEntry {
            level: t.level,
            title: t.title,
            page: t.page,
        }
    }
}

#[pyclass(name = "Link", get_all)]
pub struct PyLink {
    pub page: u32,
    pub bbox: Option<PyRect>,
    pub uri: String,
    pub target_page: Option<u32>,
}

impl From<RsLink> for PyLink {
    fn from(l: RsLink) -> Self {
        PyLink {
            page: l.page,
            bbox: l.rect.map(Into::into),
            uri: l.uri,
            target_page: l.target_page,
        }
    }
}

#[pyclass(name = "Annotation", get_all)]
pub struct PyAnnotation {
    pub page: u32,
    pub subtype: String,
    pub bbox: Option<PyRect>,
    pub contents: String,
    pub author: String,
}

impl From<RsAnnotation> for PyAnnotation {
    fn from(a: RsAnnotation) -> Self {
        PyAnnotation {
            page: a.page,
            subtype: a.subtype,
            bbox: a.rect.map(Into::into),
            contents: a.contents,
            author: a.author,
        }
    }
}

#[pyclass(name = "ImageInfo", get_all)]
pub struct PyImageInfo {
    pub page: u32,
    pub xref: u32,
    pub width: u32,
    pub height: u32,
    pub color_space: Option<String>,
    pub bits_per_component: Option<u32>,
    pub filters: Vec<String>,
    pub size_bytes: usize,
}

impl From<RsImageInfo> for PyImageInfo {
    fn from(i: RsImageInfo) -> Self {
        PyImageInfo {
            page: i.page,
            xref: i.xref,
            width: i.width,
            height: i.height,
            color_space: i.color_space,
            bits_per_component: i.bits_per_component,
            filters: i.filters,
            size_bytes: i.size_bytes,
        }
    }
}

/// One AcroForm field.
#[pyclass(name = "Widget", get_all)]
pub struct PyWidget {
    pub name: String,
    pub value: String,
    pub default_value: String,
    pub field_type: String,
    pub flags: i64,
    pub rect: Option<PyRect>,
    pub page: Option<u32>,
}

impl From<RsWidget> for PyWidget {
    fn from(w: RsWidget) -> Self {
        PyWidget {
            name: w.name,
            value: w.value,
            default_value: w.default_value,
            field_type: w.field_type.as_str().to_string(),
            flags: w.flags,
            rect: w.rect.map(Into::into),
            page: w.page,
        }
    }
}

/// Decoded image bytes in a viewable container (JPEG / PNG / JP2 / JBIG2 /
/// TIFF) determined by `ext`.
#[pyclass(name = "DecodedImage", get_all)]
pub struct PyDecodedImage {
    pub ext: String,
    pub bytes: Vec<u8>,
}

impl From<RsDecodedImage> for PyDecodedImage {
    fn from(d: RsDecodedImage) -> Self {
        let ext = match d.container {
            RsImageContainer::Jpeg => "jpg",
            RsImageContainer::Jpeg2000 => "jp2",
            RsImageContainer::Png => "png",
            RsImageContainer::Jbig2 => "jb2",
            RsImageContainer::Ccitt => "tiff",
        }
        .to_string();
        PyDecodedImage {
            ext,
            bytes: d.bytes,
        }
    }
}

#[pyclass(name = "PageInfo", get_all)]
pub struct PyPageInfo {
    pub number: u32,
    pub width: f32,
    pub height: f32,
    pub rotation: i32,
    pub mediabox: PyRect,
    pub cropbox: PyRect,
}

impl From<RsPageInfo> for PyPageInfo {
    fn from(p: RsPageInfo) -> Self {
        PyPageInfo {
            number: p.number,
            width: p.width,
            height: p.height,
            rotation: p.rotation,
            mediabox: p.mediabox.into(),
            cropbox: p.cropbox.into(),
        }
    }
}

#[pyclass(name = "DocumentInfo", get_all)]
pub struct PyDocumentInfo {
    pub page_count: u32,
    pub pdf_version: String,
    pub is_encrypted: bool,
    pub is_linearized: bool,
    pub xref_count: u32,
    pub trailer_id: Option<String>,
}

impl From<RsDocumentInfo> for PyDocumentInfo {
    fn from(d: RsDocumentInfo) -> Self {
        PyDocumentInfo {
            page_count: d.page_count,
            pdf_version: d.pdf_version,
            is_encrypted: d.is_encrypted,
            is_linearized: d.is_linearized,
            xref_count: d.xref_count,
            trailer_id: d.trailer_id,
        }
    }
}

// ── Module functions ────────────────────────────────────────────────────────
//
// Each `#[pyfunction]` releases the GIL across the extraction call. The
// borrowed `&[u8]` from Python is only valid while the GIL is held, so we
// copy to an owned `Vec<u8>` before `allow_threads`.

#[pyfunction]
fn extract_text(py: Python<'_>, pdf_bytes: &[u8]) -> PyResult<String> {
    let owned = pdf_bytes.to_vec();
    py.allow_threads(|| rs_extract_text(&owned))
        .map_err(Into::into)
}

#[pyfunction]
fn extract_pages(py: Python<'_>, pdf_bytes: &[u8]) -> PyResult<Vec<String>> {
    let owned = pdf_bytes.to_vec();
    py.allow_threads(|| rs_extract_pages(&owned))
        .map_err(Into::into)
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, page=None))]
fn extract_tables(
    py: Python<'_>,
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> PyResult<Vec<Vec<Vec<String>>>> {
    let owned = pdf_bytes.to_vec();
    py.allow_threads(|| rs_extract_tables(&owned, page))
        .map_err(Into::into)
}

#[pyfunction]
fn extract_metadata(py: Python<'_>, pdf_bytes: &[u8]) -> PyResult<HashMap<String, String>> {
    let owned = pdf_bytes.to_vec();
    py.allow_threads(|| rs_extract_metadata(&owned))
        .map_err(Into::into)
}

#[pyfunction]
fn score_text(text: &str) -> f64 {
    score_text_impl(text)
}

#[pyfunction]
fn score_batch(py: Python<'_>, texts: Vec<String>) -> Vec<f64> {
    py.allow_threads(|| texts.par_iter().map(|t| score_text_impl(t)).collect())
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, page=None))]
fn extract_words(py: Python<'_>, pdf_bytes: &[u8], page: Option<u32>) -> PyResult<Vec<PyWord>> {
    let owned = pdf_bytes.to_vec();
    let words = py
        .allow_threads(|| rs_extract_words(&owned, page))
        .map_err(PyErr::from)?;
    Ok(words.into_iter().map(Into::into).collect())
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, page=None))]
fn extract_blocks(
    py: Python<'_>,
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> PyResult<Vec<PyTextBlock>> {
    let owned = pdf_bytes.to_vec();
    let blocks = py
        .allow_threads(|| rs_extract_blocks(&owned, page))
        .map_err(PyErr::from)?;
    Ok(blocks.into_iter().map(Into::into).collect())
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, page=None))]
fn extract_text_positioned(
    py: Python<'_>,
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> PyResult<Vec<PyPositionedPage>> {
    let owned = pdf_bytes.to_vec();
    let pages = py
        .allow_threads(|| rs_extract_text_positioned(&owned, page))
        .map_err(PyErr::from)?;
    Ok(pages.into_iter().map(Into::into).collect())
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, query, page=None, case_insensitive=true, flexible_whitespace=true))]
fn search(
    py: Python<'_>,
    pdf_bytes: &[u8],
    query: &str,
    page: Option<u32>,
    case_insensitive: bool,
    flexible_whitespace: bool,
) -> PyResult<Vec<PySearchHit>> {
    let owned = pdf_bytes.to_vec();
    let q = query.to_string();
    let opts = RsSearchOptions {
        case_insensitive,
        flexible_whitespace,
    };
    let hits = py
        .allow_threads(|| rs_search(&owned, &q, page, Some(opts)))
        .map_err(PyErr::from)?;
    Ok(hits.into_iter().map(Into::into).collect())
}

#[pyfunction]
fn extract_toc(py: Python<'_>, pdf_bytes: &[u8]) -> PyResult<Vec<PyTocEntry>> {
    let owned = pdf_bytes.to_vec();
    let toc = py
        .allow_threads(|| rs_extract_toc(&owned))
        .map_err(PyErr::from)?;
    Ok(toc.into_iter().map(Into::into).collect())
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, page=None))]
fn extract_links(py: Python<'_>, pdf_bytes: &[u8], page: Option<u32>) -> PyResult<Vec<PyLink>> {
    let owned = pdf_bytes.to_vec();
    let links = py
        .allow_threads(|| rs_extract_links(&owned, page))
        .map_err(PyErr::from)?;
    Ok(links.into_iter().map(Into::into).collect())
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, page=None))]
fn extract_annotations(
    py: Python<'_>,
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> PyResult<Vec<PyAnnotation>> {
    let owned = pdf_bytes.to_vec();
    let annots = py
        .allow_threads(|| rs_extract_annotations(&owned, page))
        .map_err(PyErr::from)?;
    Ok(annots.into_iter().map(Into::into).collect())
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, page=None))]
fn extract_images(
    py: Python<'_>,
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> PyResult<Vec<PyImageInfo>> {
    let owned = pdf_bytes.to_vec();
    let imgs = py
        .allow_threads(|| rs_extract_images(&owned, page))
        .map_err(PyErr::from)?;
    Ok(imgs.into_iter().map(Into::into).collect())
}

#[pyfunction]
#[pyo3(signature = (pdf_bytes, page=None))]
fn extract_page_info(
    py: Python<'_>,
    pdf_bytes: &[u8],
    page: Option<u32>,
) -> PyResult<Vec<PyPageInfo>> {
    let owned = pdf_bytes.to_vec();
    let pages = py
        .allow_threads(|| rs_extract_page_info(&owned, page))
        .map_err(PyErr::from)?;
    Ok(pages.into_iter().map(Into::into).collect())
}

#[pyfunction]
fn extract_document_info(py: Python<'_>, pdf_bytes: &[u8]) -> PyResult<PyDocumentInfo> {
    let owned = pdf_bytes.to_vec();
    let info = py
        .allow_threads(|| rs_extract_document_info(&owned))
        .map_err(PyErr::from)?;
    Ok(info.into())
}

/// Persistent PDF handle — parse once, query many. Preferred over the free
/// functions when more than one surface will be pulled from the same PDF.
#[pyclass(name = "Document")]
pub struct PyDocument {
    inner: RsDocument,
}

#[pymethods]
impl PyDocument {
    #[new]
    #[pyo3(signature = (pdf_bytes, password=None))]
    fn new(pdf_bytes: &[u8], password: Option<&[u8]>) -> PyResult<Self> {
        let owned = pdf_bytes.to_vec();
        let inner = match password {
            Some(p) => RsDocument::open_with_password(&owned, p).map_err(PyErr::from)?,
            None => RsDocument::open(&owned).map_err(PyErr::from)?,
        };
        Ok(Self { inner })
    }

    #[getter]
    fn page_count(&self) -> u32 {
        self.inner.page_count()
    }

    fn info(&self) -> PyDocumentInfo {
        self.inner.info().into()
    }

    #[pyo3(signature = (page=None))]
    fn pages(&self, page: Option<u32>) -> Vec<PyPageInfo> {
        self.inner.pages(page).into_iter().map(Into::into).collect()
    }

    fn toc(&self) -> PyResult<Vec<PyTocEntry>> {
        Ok(self
            .inner
            .toc()
            .map_err(PyErr::from)?
            .into_iter()
            .map(Into::into)
            .collect())
    }

    #[pyo3(signature = (page=None))]
    fn links(&self, page: Option<u32>) -> Vec<PyLink> {
        self.inner.links(page).into_iter().map(Into::into).collect()
    }

    #[pyo3(signature = (page=None))]
    fn annotations(&self, page: Option<u32>) -> Vec<PyAnnotation> {
        self.inner
            .annotations(page)
            .into_iter()
            .map(Into::into)
            .collect()
    }

    #[pyo3(signature = (page=None))]
    fn images(&self, page: Option<u32>) -> Vec<PyImageInfo> {
        self.inner.images(page).into_iter().map(Into::into).collect()
    }

    /// AcroForm fields across the document (or one page when `page` is set).
    #[pyo3(signature = (page=None))]
    fn widgets(&self, page: Option<u32>) -> Vec<PyWidget> {
        self.inner
            .widgets(page)
            .into_iter()
            .map(Into::into)
            .collect()
    }

    /// Decoded bytes for one image, keyed by the `xref` field on the
    /// `ImageInfo` rows from [`Self::images`]. Returns `None` for image
    /// formats we don't yet repackage.
    fn image_bytes(&self, xref: u32) -> PyResult<Option<PyDecodedImage>> {
        Ok(self.inner.image_bytes(xref).map(Into::into))
    }

    #[pyo3(signature = (page=None))]
    fn words(&self, page: Option<u32>) -> PyResult<Vec<PyWord>> {
        Ok(self
            .inner
            .words(page)
            .map_err(PyErr::from)?
            .into_iter()
            .map(Into::into)
            .collect())
    }

    #[pyo3(signature = (page=None))]
    fn blocks(&self, page: Option<u32>) -> PyResult<Vec<PyTextBlock>> {
        Ok(self
            .inner
            .blocks(page)
            .map_err(PyErr::from)?
            .into_iter()
            .map(Into::into)
            .collect())
    }

    #[pyo3(signature = (page=None))]
    fn text_positioned(&self, page: Option<u32>) -> PyResult<Vec<PyPositionedPage>> {
        Ok(self
            .inner
            .text_positioned(page)
            .map_err(PyErr::from)?
            .into_iter()
            .map(Into::into)
            .collect())
    }

    /// Nested per-page dicts/lists/tuples matching `pymupdf.get_text("dict")`:
    ///
    /// ```text
    /// [{
    ///   "width": float, "height": float, "number": int,
    ///   "blocks": [{
    ///     "type": 0,              # 0=text, 1=image (we emit text only)
    ///     "number": int, "bbox": (x0, y0, x1, y1),
    ///     "lines": [{
    ///       "wmode": 0, "dir": (1.0, 0.0), "bbox": (...),
    ///       "spans": [{
    ///         "size": float, "flags": int, "font": str,
    ///         "color": int, "text": str,
    ///         "origin": (x, y), "bbox": (...)
    ///       }]
    ///     }]
    ///   }]
    /// }, ...]
    /// ```
    ///
    /// `color` is hardcoded to 0 and `flags` to 0; surfacing them requires
    /// parsing `/FontDescriptor` font metrics, which we don't yet do.
    fn dict(&self, py: Python<'_>) -> PyResult<PyObject> {
        let pages = self.inner.pages(None);
        let blocks = self.inner.blocks(None).map_err(PyErr::from)?;
        // Group blocks by page so the per-page dict has its own block list.
        let mut blocks_by_page: std::collections::BTreeMap<u32, Vec<&crate::TextBlock>> =
            std::collections::BTreeMap::new();
        for b in &blocks {
            blocks_by_page.entry(b.page).or_default().push(b);
        }
        let out_list = pyo3::types::PyList::empty(py);
        for page_info in &pages {
            let page_dict = pyo3::types::PyDict::new(py);
            page_dict.set_item("width", page_info.width)?;
            page_dict.set_item("height", page_info.height)?;
            page_dict.set_item("number", page_info.number)?;
            let block_list = pyo3::types::PyList::empty(py);
            if let Some(blocks) = blocks_by_page.get(&page_info.number) {
                for (idx, block) in blocks.iter().enumerate() {
                    let block_dict = pyo3::types::PyDict::new(py);
                    block_dict.set_item("type", 0)?; // text block
                    block_dict.set_item("number", idx)?;
                    block_dict.set_item("bbox", rect_to_tuple(&block.bbox))?;
                    let line_list = pyo3::types::PyList::empty(py);
                    for line in &block.lines {
                        let line_dict = pyo3::types::PyDict::new(py);
                        line_dict.set_item("wmode", 0)?;
                        line_dict.set_item("dir", (1.0f32, 0.0f32))?;
                        line_dict.set_item("bbox", rect_to_tuple(&line.bbox))?;
                        let span_list = pyo3::types::PyList::empty(py);
                        for span in &line.spans {
                            let span_dict = pyo3::types::PyDict::new(py);
                            span_dict.set_item("size", span.font_size)?;
                            span_dict.set_item("flags", 0)?;
                            span_dict.set_item("font", span.font.as_str())?;
                            span_dict.set_item("color", 0)?;
                            span_dict.set_item("text", span.text.as_str())?;
                            span_dict.set_item("origin", (span.bbox.x0, span.bbox.y0))?;
                            span_dict.set_item("bbox", rect_to_tuple(&span.bbox))?;
                            span_list.append(span_dict)?;
                        }
                        line_dict.set_item("spans", span_list)?;
                        line_list.append(line_dict)?;
                    }
                    block_dict.set_item("lines", line_list)?;
                    block_list.append(block_dict)?;
                }
            }
            page_dict.set_item("blocks", block_list)?;
            out_list.append(page_dict)?;
        }
        Ok(out_list.into())
    }

    /// Render the document to Markdown for LLM ingestion.
    ///
    /// Heading detection: pymupdf4llm-style `IdentifyHeaders` (global
    /// character-weighted font-size Counter with `body_limit=12` floor,
    /// top-6 cap), plus spatial bbox-merge of vertically-adjacent
    /// heading blocks, `/Outline` TOC backfill, and a three-signal
    /// bold-span TOC match with two-line lookback. Body styling is
    /// preserved as `**bold**` / `*italic*` runs.
    fn markdown(&self) -> PyResult<String> {
        let raw_blocks = self.inner.blocks(None).map_err(PyErr::from)?;
        // body_limit must be computed before merging so the merge
        // threshold matches the downstream heading-detection threshold.
        let (_, raw_body_limit) = build_heading_size_map(&raw_blocks);
        let blocks = merge_adjacent_heading_blocks(raw_blocks, raw_body_limit);
        let images = self.inner.images(None);
        let styles = self.inner.font_styles();
        let toc = self.inner.toc().unwrap_or_default();
        let (size_to_level, body_limit) = build_heading_size_map(&blocks);
        // TOC matching is global (every outline entry) rather than
        // page-targeted: TOC page numbers in real PDFs are off-by-one
        // or land on interior pages often enough that the page filter
        // loses more recall than it gains in precision.
        let toc_titles: std::collections::HashSet<String> = toc
            .iter()
            .map(|e| normalize_heading_lookup(&e.title))
            .filter(|s| !s.is_empty() && s.len() > 2)
            .collect();
        let mut out = String::with_capacity(blocks.len() * 64);
        let mut current_page: Option<u32> = None;
        let mut images_by_page: std::collections::BTreeMap<u32, Vec<&RsImageInfo>> =
            std::collections::BTreeMap::new();
        for img in &images {
            images_by_page.entry(img.page).or_default().push(img);
        }
        for block in &blocks {
            if current_page != Some(block.page) {
                current_page = Some(block.page);
                if let Some(imgs) = images_by_page.get(&block.page) {
                    for img in imgs {
                        if !out.is_empty() && !out.ends_with("\n\n") {
                            out.push_str("\n\n");
                        }
                        out.push_str(&format!(
                            "==> picture [{}×{}] omitted <==",
                            img.width, img.height
                        ));
                    }
                }
            }
            let mut consecutive_body_lines: Vec<String> = Vec::new();
            // Consecutive same-level heading lines get joined into one
            // heading. Without this, "# Filing\n# Requirements" stays
            // split and neither half matches a /Outline lookup of
            // "filing requirements".
            let mut last_heading_prefix: Option<&'static str> = None;
            // Bold runs from the immediately preceding line; combined
            // with this line's bold spans for two-line heading match
            // (e.g., "Recapture of Low-Income" + "Housing Credit").
            let mut prev_line_bold: String = String::new();
            let flush_body =
                |buf: &mut Vec<String>, out: &mut String| -> bool {
                    if buf.is_empty() {
                        return false;
                    }
                    if !out.is_empty() && !out.ends_with("\n\n") {
                        out.push_str("\n\n");
                    }
                    if line_starts_with_list_marker(buf[0].trim()) {
                        out.push_str("- ");
                    }
                    out.push_str(&buf.join("\n"));
                    buf.clear();
                    true
                };
            for line in &block.lines {
                // Per-line dominant size beats per-span: per-span (the
                // strict pymupdf4llm algorithm) over-fires when a line
                // mixes heading-sized punctuation with body text.
                let dom_size = line_dominant_size(line);
                let heading_level = size_to_level
                    .get(&dom_size)
                    .copied()
                    .filter(|_| dom_size > body_limit);
                let rendered = render_styled_line(&line.spans, block.page, &styles);
                let alpha_count = rendered.chars().filter(|c| c.is_alphabetic()).count();
                let has_real_text = alpha_count >= 3;
                // Mixed-style lines are body text with emphasis
                // ("**Note.** The Form..."), not headings.
                let all_bold = !line.spans.is_empty()
                    && line.spans.iter().all(|s| {
                        styles
                            .get(&(block.page, s.font.clone()))
                            .map(|(b, _)| *b)
                            .unwrap_or(false)
                    });
                let stands_alone = block.lines.len() == 1;
                if let Some(lvl) = heading_level {
                    if has_real_text {
                        let prefix = HEADING_PREFIXES[lvl.saturating_sub(1).min(5)];
                        let body_flushed =
                            flush_body(&mut consecutive_body_lines, &mut out);
                        if body_flushed {
                            last_heading_prefix = None;
                        }
                        if last_heading_prefix == Some(prefix)
                            && !out.is_empty()
                            && !out.ends_with("\n\n")
                        {
                            out.push(' ');
                            out.push_str(rendered.trim_start());
                        } else {
                            if !out.is_empty() && !out.ends_with("\n\n") {
                                out.push_str("\n\n");
                            }
                            out.push_str(prefix);
                            out.push_str(&rendered);
                        }
                        last_heading_prefix = Some(prefix);
                        continue;
                    }
                    consecutive_body_lines.push(rendered);
                    continue;
                }
                let trimmed = rendered.trim();
 // Structural-pattern detection (numbered prefix /
 // all-caps / bold-stands-alone) is currently disabled
 // because IRS-style forms contain many short numbered
 // body fragments (e.g., "1. Federal income tax")
 // that aren't headings. With the document's `/Outline`
 // available, TOC backfill alone outperforms a structural
 // cascade. The detector is retained for future use on
                // outline-less documents.
                if toc_titles.is_empty() {
                    if let Some(level) =
                        detect_structural_heading(trimmed, all_bold, stands_alone)
                    {
                        flush_body(&mut consecutive_body_lines, &mut out);
                        if !out.is_empty() && !out.ends_with("\n\n") {
                            out.push_str("\n\n");
                        }
                        out.push_str(HEADING_PREFIXES[level.saturating_sub(1).min(5)]);
                        out.push_str(&rendered);
                        continue;
                    }
                }
                // Bold-span TOC backfill, three stacked signals:
                //   1. per-contiguous-bold-run → TOC
                //   2. full-line bold concat → TOC
                //   3. prev_line_bold + this_line_bold → TOC
                // Each catches a different segmentation failure where
                // the heading text isn't whole on one line.
                let current_line_bold =
                    collect_bold_text(&line.spans, &styles, block.page);
                let mut emitted_from_bold_span = false;
                if !toc_titles.is_empty() {
                    let mut run_match: Option<String> = None;
                    let mut current_run: Vec<String> = Vec::new();
                    for span in &line.spans {
                        let is_bold = styles
                            .get(&(block.page, span.font.clone()))
                            .map(|(b, _)| *b)
                            .unwrap_or(false);
                        if is_bold {
                            current_run.push(span.text.clone());
                        } else if !current_run.is_empty() {
                            let joined = current_run.join("");
                            let lookup = normalize_heading_lookup(&joined);
                            if lookup.len() > 3 && toc_titles.contains(&lookup) {
                                run_match = Some(joined.trim().to_string());
                                break;
                            }
                            current_run.clear();
                        }
                    }
                    if run_match.is_none() && !current_run.is_empty() {
                        let joined = current_run.join("");
                        let lookup = normalize_heading_lookup(&joined);
                        if lookup.len() > 3 && toc_titles.contains(&lookup) {
                            run_match = Some(joined.trim().to_string());
                        }
                    }
                    let two_concat = if !prev_line_bold.is_empty()
                        && !current_line_bold.is_empty()
                    {
                        format!(
                            "{} {}",
                            prev_line_bold.trim(),
                            current_line_bold.trim()
                        )
                    } else {
                        String::new()
                    };
                    let two_lookup = normalize_heading_lookup(&two_concat);
                    let two_hit = two_lookup.len() > 5
                        && toc_titles.contains(&two_lookup);
                    let full_lookup = normalize_heading_lookup(&current_line_bold);
                    let full_hit = run_match.is_none()
                        && full_lookup.len() > 3
                        && toc_titles.contains(&full_lookup);
                    if run_match.is_some() || full_hit || two_hit {
                        let emit_text = if two_hit {
                            two_concat.trim().to_string()
                        } else if let Some(t) = run_match {
                            t
                        } else {
                            current_line_bold.trim().to_string()
                        };
                        let prefix = "## ";
                        let body_flushed =
                            flush_body(&mut consecutive_body_lines, &mut out);
                        if body_flushed {
                            last_heading_prefix = None;
                        }
                        if last_heading_prefix == Some(prefix)
                            && !out.is_empty()
                            && !out.ends_with("\n\n")
                        {
                            out.push(' ');
                            out.push_str(&emit_text);
                        } else {
                            if !out.is_empty() && !out.ends_with("\n\n") {
                                out.push_str("\n\n");
                            }
                            out.push_str(prefix);
                            out.push_str("**");
                            out.push_str(&emit_text);
                            out.push_str("**");
                        }
                        last_heading_prefix = Some(prefix);
                        emitted_from_bold_span = true;
                    }
                }
                // Update prev_line_bold for next iteration's two-line check.
                // Clear it after a successful emission so the next line
                // doesn't double-emit by concatenating the just-used
                // bold text with new bold text from the following line.
                if emitted_from_bold_span {
                    prev_line_bold.clear();
                    continue;
                }
                prev_line_bold = current_line_bold.clone();
 // /Outline backfill: exact normalized match against
                // any TOC entry title across the document.
                if !toc_titles.is_empty() {
                    let lookup = normalize_heading_lookup(trimmed);
                    if toc_titles.contains(&lookup) && lookup.len() > 3 {
                        let prefix = "## ";
                        let body_flushed =
                            flush_body(&mut consecutive_body_lines, &mut out);
                        if body_flushed {
                            last_heading_prefix = None;
                        }
                        if last_heading_prefix == Some(prefix)
                            && !out.is_empty()
                            && !out.ends_with("\n\n")
                        {
                            out.push(' ');
                            out.push_str(rendered.trim_start());
                        } else {
                            if !out.is_empty() && !out.ends_with("\n\n") {
                                out.push_str("\n\n");
                            }
                            out.push_str(prefix);
                            out.push_str(&rendered);
                        }
                        last_heading_prefix = Some(prefix);
                        continue;
                    }
                }
                consecutive_body_lines.push(rendered);
            }
            flush_body(&mut consecutive_body_lines, &mut out);
        }
        Ok(out)
    }

    #[pyo3(signature = (query, page=None, case_insensitive=true, flexible_whitespace=true))]
    fn search(
        &self,
        query: &str,
        page: Option<u32>,
        case_insensitive: bool,
        flexible_whitespace: bool,
    ) -> PyResult<Vec<PySearchHit>> {
        let opts = RsSearchOptions {
            case_insensitive,
            flexible_whitespace,
        };
        Ok(self
            .inner
            .search(query, page, Some(opts))
            .map_err(PyErr::from)?
            .into_iter()
            .map(Into::into)
            .collect())
    }

    /// Plain-text extraction (strict mode).
    fn text(&self) -> PyResult<String> {
        self.inner.text().map_err(PyErr::from)
    }

    /// Lenient text extraction — pages that fail come back empty.
    fn text_lenient(&self) -> PyResult<String> {
        self.inner.text_lenient().map_err(PyErr::from)
    }
}

/// Backward-compat shim for pre-v0.3 callers. Prefer module-level
/// `score_text` / `score_batch`.
#[pyclass]
pub struct RustValidator;

#[pymethods]
impl RustValidator {
    #[new]
    fn new() -> Self {
        RustValidator
    }

    fn score_text(&self, text: &str) -> f64 {
        score_text_impl(text)
    }

    fn score_batch(&self, texts: Vec<String>) -> Vec<f64> {
        texts.par_iter().map(|t| score_text_impl(t)).collect()
    }

    #[pyo3(signature = (text, threshold=None))]
    fn is_garbage(&self, text: &str, threshold: Option<f64>) -> bool {
        score_text_impl(text) < threshold.unwrap_or(0.35)
    }

    #[pyo3(signature = (texts, threshold=None))]
    fn partition_batch(
        &self,
        texts: Vec<String>,
        threshold: Option<f64>,
    ) -> (Vec<String>, Vec<String>) {
        let thresh = threshold.unwrap_or(0.35);
        let scored: Vec<(String, f64)> = texts
            .into_par_iter()
            .map(|t| {
                let s = score_text_impl(&t);
                (t, s)
            })
            .collect();

        let mut clean = Vec::new();
        let mut garbage = Vec::new();
        for (text, score) in scored {
            if score >= thresh {
                clean.push(text);
            } else {
                garbage.push(text);
            }
        }
        (clean, garbage)
    }
}

// ── Helpers used by Document.dict() and .markdown() ────────────────────────

use crate::markdown::{
    build_heading_size_map, collect_bold_text, detect_structural_heading, line_dominant_size,
    merge_adjacent_heading_blocks, normalize_heading_lookup, render_styled_line,
    HEADING_PREFIXES,
};

fn rect_to_tuple(r: &RsRect) -> (f32, f32, f32, f32) {
    (r.x0, r.y0, r.x1, r.y1)
}


/// True when the first line of `text` begins with a list-item marker
/// (bullet glyph, `(a)`, `1.`, `i)`, etc.). Block-level counterpart to
/// `positioned::line_starts_list_item`.
fn line_starts_with_list_marker(text: &str) -> bool {
    let first_line = text.lines().next().unwrap_or("");
    let trimmed = first_line.trim_start();
    if trimmed.is_empty() {
        return false;
    }
    let mut chars = trimmed.chars();
    let first = match chars.next() {
        Some(c) => c,
        None => return false,
    };
    if matches!(
        first,
        '\u{2022}'
            | '\u{25E6}'
            | '\u{2023}'
            | '\u{25A0}'
            | '\u{25CF}'
            | '\u{2043}'
            | '\u{2219}'
    ) {
        return matches!(chars.next(), Some(c) if c.is_whitespace());
    }
    // `(a)` / `1.` / `i)` etc.
    if first == '(' {
        let inner: String = chars.clone().take(5).take_while(|c| *c != ')').collect();
        if !inner.is_empty() && inner.chars().all(|c| c.is_alphanumeric()) {
            return true;
        }
    }
    if first.is_alphanumeric() {
        let rest: String = chars.clone().take(4).collect();
        return rest.starts_with(|c: char| c == '.' || c == ')');
    }
    false
}

#[pymodule]
fn spectre_rs(m: &Bound<'_, PyModule>) -> PyResult<()> {
    m.add("__version__", env!("CARGO_PKG_VERSION"))?;
    m.add_function(wrap_pyfunction!(extract_text, m)?)?;
    m.add_function(wrap_pyfunction!(extract_pages, m)?)?;
    m.add_function(wrap_pyfunction!(extract_tables, m)?)?;
    m.add_function(wrap_pyfunction!(extract_metadata, m)?)?;
    m.add_function(wrap_pyfunction!(score_text, m)?)?;
    m.add_function(wrap_pyfunction!(score_batch, m)?)?;
    m.add_class::<RustValidator>()?;

    m.add_function(wrap_pyfunction!(extract_words, m)?)?;
    m.add_function(wrap_pyfunction!(extract_blocks, m)?)?;
    m.add_function(wrap_pyfunction!(extract_text_positioned, m)?)?;
    m.add_function(wrap_pyfunction!(search, m)?)?;
    m.add_function(wrap_pyfunction!(extract_toc, m)?)?;
    m.add_function(wrap_pyfunction!(extract_links, m)?)?;
    m.add_function(wrap_pyfunction!(extract_annotations, m)?)?;
    m.add_function(wrap_pyfunction!(extract_images, m)?)?;
    m.add_function(wrap_pyfunction!(extract_page_info, m)?)?;
    m.add_function(wrap_pyfunction!(extract_document_info, m)?)?;

    m.add_class::<PyRect>()?;
    m.add_class::<PyWord>()?;
    m.add_class::<PyTextSpan>()?;
    m.add_class::<PyTextLine>()?;
    m.add_class::<PyTextBlock>()?;
    m.add_class::<PyPositionedPage>()?;
    m.add_class::<PySearchHit>()?;
    m.add_class::<PyTocEntry>()?;
    m.add_class::<PyLink>()?;
    m.add_class::<PyAnnotation>()?;
    m.add_class::<PyImageInfo>()?;
    m.add_class::<PyWidget>()?;
    m.add_class::<PyDecodedImage>()?;
    m.add_class::<PyPageInfo>()?;
    m.add_class::<PyDocumentInfo>()?;
    m.add_class::<PyDocument>()?;

    Ok(())
}