pdf-untangle 0.3.0

Untangle garbled PDF text: layout-aware extraction that reconstructs reading order, strips page furniture, and splits documents into structured, citable sections
Documentation
//! Pure-Rust extraction backend (`backend-pure` feature).
//!
//! Built on the [`pdf-extract`] crate (which parses PDFs with `lopdf`).
//! No native dependencies: `cargo build` is all you need.
//!
//! # Fidelity and limitations (honest accounting)
//!
//! `pdf-extract` drives a character-level output device, so this backend
//! reconstructs runs itself from per-glyph positions:
//!
//! - **Positions** are real: x comes from the text rendering matrix, and run
//!   x-extents track glyph advances. **Bounding-box height is approximated**
//!   as one em above the baseline (PDF does not require ascent metrics to be
//!   consulted for this, and `pdf-extract` does not expose them).
//! - **Font size** is the effective (transformed) size, which equals the
//!   nominal size for the common untransformed-text case.
//! - **Style flags are not available**: the `pdf-extract` device callbacks
//!   carry no font identity, so [`TextRun::bold`]/[`TextRun::italic`] are
//!   always `false` and [`TextRun::font`] is always `None`. Downstream style
//!   heuristics must not rely on them under this backend.
//!
//! Runs are split when the baseline moves, the effective font size changes,
//! or a horizontal gap larger than typical word spacing appears (column
//! jumps, tab stops). Word gaps within a run become single spaces, mirroring
//! `pdf-extract`'s own plain-text heuristics.
//!
//! [`pdf-extract`]: https://crates.io/crates/pdf-extract

use anyhow::{Context, Result};
use pdf_extract::{MediaBox, OutputDev, OutputError, Transform};

use super::{PageRuns, TextRun, TextSource};

/// The pure-Rust [`TextSource`]. Stateless; construct freely.
pub struct PureSource;

impl TextSource for PureSource {
    fn pages(&self, pdf_bytes: &[u8]) -> Result<Vec<PageRuns>> {
        let mut doc =
            pdf_extract::Document::load_mem(pdf_bytes).context("lopdf could not parse the PDF")?;
        if doc.is_encrypted() {
            // Many "encrypted" PDFs in the wild use an empty user password.
            doc.decrypt("")
                .context("PDF is encrypted with a non-empty password")?;
        }
        let mut collector = RunCollector::default();
        pdf_extract::output_doc(&doc, &mut collector)
            .map_err(|e| anyhow::anyhow!("pdf-extract failed to walk the document: {e}"))?;
        Ok(collector.pages)
    }
}

/// Gap (in ems) treated as a word space within a run.
const WORD_GAP_EMS: f32 = 0.1;
/// Gap (in ems) that splits a run (tab stop / column jump).
const RUN_BREAK_EMS: f32 = 2.0;
/// Baseline shift (in ems) that splits a run.
const BASELINE_SHIFT_EMS: f32 = 0.25;

/// An `OutputDev` that assembles positioned characters into [`TextRun`]s.
#[derive(Default)]
struct RunCollector {
    pages: Vec<PageRuns>,
    /// Run currently being assembled, if any.
    current: Option<TextRun>,
    /// (llx, lly) of the current page's media box, so run coordinates are
    /// normalized to a bottom-left page origin.
    origin: (f32, f32),
}

impl RunCollector {
    fn flush(&mut self) {
        if let Some(run) = self.current.take() {
            if !run.text.trim().is_empty() {
                if let Some(page) = self.pages.last_mut() {
                    page.runs.push(run);
                }
            }
        }
    }
}

impl OutputDev for RunCollector {
    fn begin_page(
        &mut self,
        _page_num: u32,
        media_box: &MediaBox,
        _art_box: Option<(f64, f64, f64, f64)>,
    ) -> std::result::Result<(), OutputError> {
        self.flush();
        self.origin = (media_box.llx as f32, media_box.lly as f32);
        self.pages.push(PageRuns {
            width: (media_box.urx - media_box.llx) as f32,
            height: (media_box.ury - media_box.lly) as f32,
            runs: Vec::new(),
        });
        Ok(())
    }

    fn end_page(&mut self) -> std::result::Result<(), OutputError> {
        self.flush();
        Ok(())
    }

    fn output_character(
        &mut self,
        trm: &Transform,
        width: f64,
        _spacing: f64,
        font_size: f64,
        char: &str,
    ) -> std::result::Result<(), OutputError> {
        // Effective (transformed) font size: geometric mean of the em vector,
        // exactly as pdf-extract's own plain-text device computes it.
        let em_x = (trm.m11 + trm.m21) * font_size;
        let em_y = (trm.m12 + trm.m22) * font_size;
        let size = ((em_x * em_y).abs().sqrt()) as f32;
        if size <= 0.0 {
            return Ok(()); // degenerate/invisible text
        }
        let x = trm.m31 as f32 - self.origin.0;
        let y = trm.m32 as f32 - self.origin.1;
        // pdf-extract expresses glyph width as a fraction of the em square.
        let advance = width as f32 * size;
        let page = self.pages.len().saturating_sub(1);

        let mut split = true;
        let mut word_gap = false;
        if let Some(run) = &self.current {
            let same_page = run.page == page;
            let same_baseline = (y - run.bbox.1).abs() <= BASELINE_SHIFT_EMS * size;
            let same_size = (size - run.font_size).abs() <= 0.1;
            let gap = x - run.bbox.2;
            if same_page && same_baseline && same_size && gap <= RUN_BREAK_EMS * size {
                split = false;
                word_gap = gap > WORD_GAP_EMS * size;
            }
        }
        if split {
            self.flush();
            self.current = Some(TextRun {
                text: String::new(),
                bbox: (x, y, x, y + size),
                font_size: size,
                bold: false,
                italic: false,
                font: None,
                page,
            });
        }
        let run = self.current.as_mut().expect("run was just ensured");
        if word_gap && !run.text.ends_with(' ') && !char.starts_with(' ') {
            run.text.push(' ');
        }
        run.text.push_str(char);
        run.bbox.2 = (x + advance).max(run.bbox.2);
        run.bbox.3 = run.bbox.3.max(y + size);
        Ok(())
    }

    fn begin_word(&mut self) -> std::result::Result<(), OutputError> {
        Ok(())
    }

    fn end_word(&mut self) -> std::result::Result<(), OutputError> {
        Ok(())
    }

    fn end_line(&mut self) -> std::result::Result<(), OutputError> {
        Ok(())
    }
}