pdf-untangle 0.3.0

Untangle garbled PDF text: layout-aware extraction that reconstructs reading order, strips page furniture, and splits documents into structured, citable sections
Documentation
//! Layer 2 — positioned runs → ordered logical lines and blocks.
//!
//! The backend-agnostic hard part of PDF untangling. Given the raw
//! [`PageRuns`] from Layer 1 (in whatever order the PDF emitted them), this
//! module rebuilds the logical document:
//!
//! 1. **Line assembly** ([`rows`]) — runs are clustered onto shared baselines
//!    (tolerance scaled by font size, so small-caps and superscript runs land
//!    on their line) and joined left-to-right with gap-derived spacing.
//! 2. **Column detection** ([`columns`]) — a page is split into two columns
//!    only when a persistent vertical gutter exists across many lines.
//!    Detection keys on line-start x0 clusters, never on run end-x: the pure
//!    backend's x1 drifts (both directions) on fonts with missing width
//!    tables, so widths are estimated from character counts when the reported
//!    extent is implausible (see [`rows::width_est`]).
//! 3. **Reading order** — lines top-to-bottom within a column, columns
//!    left-to-right, pages in sequence.
//! 4. **Page-furniture stripping** ([`furniture`]) — repeated running
//!    headers/footers and page numbers are identified by cross-page
//!    repetition of digit-masked text within the top/bottom position bands
//!    and removed from the flow — but recorded on [`Layout::furniture`], not
//!    silently dropped (Layer 3's coverage invariant accounts for them).
//! 5. **Flow repair** ([`flow`]) — lines are grouped into paragraph-ish
//!    [`Block`]s, end-of-line hyphenation is repaired conservatively, and a
//!    paragraph interrupted by a page or column break is merged back together
//!    when indentation and punctuation say the sentence continues.
//! 6. **Indentation levels** — line-start x0 is quantized into per-page,
//!    per-column indent levels (subdivision structure like (a)(1)(A) carries
//!    meaning in rules).
//!
//! Style flags degrade gracefully: [`Line::bold`] is real under the PDFium
//! backend and always `false` under the pure backend, while
//! [`Line::centered`], [`Line::font_size`], and all positional facts work
//! under both.

use serde::Serialize;

use crate::extract::PageRuns;

mod columns;
mod flow;
mod furniture;
mod rows;

/// A logical line in reading order, with the layout facts Layer 3 needs.
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Line {
    /// The line's text, spacing repaired, reading left to right.
    pub text: String,
    /// Line-start x in page points — the one horizontal fact that is reliable
    /// under every backend (run end-x can drift; see the module docs).
    pub x0: f32,
    /// Baseline y in page points (origin bottom-left).
    pub baseline: f32,
    /// Indentation level, quantized from `x0` per page/column (0 = flush
    /// with the leftmost text of its column).
    pub indent: u8,
    /// Dominant font size on the line (weighted by character count).
    pub font_size: f32,
    /// Whether most of the line's characters come from bold runs.
    /// Backend-dependent: always `false` under the pure backend.
    pub bold: bool,
    /// Whether the line is horizontally centered in its column (estimated
    /// width well short of the column, midpoint near the column midpoint).
    pub centered: bool,
    /// Column index on its page (0 = leftmost; pages are 1–2 columns).
    pub column: u8,
    /// Pages this line touches (0-based; usually one, two when a hyphenated
    /// word was pulled across a page break).
    pub pages: Vec<usize>,
}

/// A contiguous block of lines (paragraph-ish unit) in reading order.
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Block {
    /// The block's lines, in reading order.
    pub lines: Vec<Line>,
}

/// The reconstructed document layout: content blocks plus accounting for
/// everything that was stripped, so no text is ever silently dropped.
#[derive(Debug, Clone, Serialize)]
pub struct Layout {
    /// Content blocks in reading order.
    pub blocks: Vec<Block>,
    /// Page-furniture lines (running headers/footers, page numbers) that were
    /// stripped from the flow, in page order. Empty when furniture stripping
    /// is disabled via [`LayoutOptions::strip_furniture`].
    pub furniture: Vec<Line>,
}

impl Layout {
    /// Render the layout as clean reading-ordered text: one line per
    /// [`Line`], a blank line between blocks. Hyphenation and page-flow
    /// repairs have already been applied to the line texts.
    pub fn text(&self) -> String {
        let mut out = String::new();
        for (i, block) in self.blocks.iter().enumerate() {
            if i > 0 {
                out.push('\n');
            }
            for line in &block.lines {
                out.push_str(line.text.trim_end());
                out.push('\n');
            }
        }
        out
    }
}

/// Options for [`reconstruct_with`].
#[derive(Debug, Clone)]
pub struct LayoutOptions {
    /// Strip repeated page furniture (headers/footers/page numbers) from the
    /// flow, recording it on [`Layout::furniture`]. Defaults to `true`; when
    /// `false`, furniture lines stay inline in reading position.
    pub strip_furniture: bool,
}

impl Default for LayoutOptions {
    fn default() -> Self {
        Self {
            strip_furniture: true,
        }
    }
}

/// Reconstruct the logical layout from per-page runs with default options
/// (furniture stripped). See the [module docs](self) for the pipeline.
pub fn reconstruct(pages: &[PageRuns]) -> Layout {
    reconstruct_with(pages, &LayoutOptions::default())
}

/// Reconstruct the logical layout from per-page runs.
pub fn reconstruct_with(pages: &[PageRuns], options: &LayoutOptions) -> Layout {
    // Per page: baseline rows -> columns -> ordered lines with style facts.
    let mut lines: Vec<Line> = Vec::new();
    for (page_index, page) in pages.iter().enumerate() {
        let page_rows = rows::assemble(page, page_index);
        lines.extend(columns::lines_in_reading_order(page_rows, page, page_index));
    }

    // Cross-page furniture detection, then strip (with accounting).
    let stripped = if options.strip_furniture {
        furniture::strip(&mut lines, pages)
    } else {
        Vec::new()
    };

    // Indent quantization, then paragraph flow (blocks, hyphens, page merge).
    flow::quantize_indents(&mut lines);
    let blocks = flow::into_blocks(lines);

    Layout {
        blocks,
        furniture: stripped,
    }
}

#[cfg(test)]
pub(crate) mod testutil {
    use crate::extract::{PageRuns, TextRun};

    /// A synthetic run: `size`-pt text at (x, baseline) on `page`.
    /// The reported x1 is a plausible width estimate, mimicking a
    /// well-behaved backend.
    pub fn run(text: &str, x: f32, baseline: f32, size: f32, page: usize) -> TextRun {
        TextRun {
            text: text.into(),
            bbox: (
                x,
                baseline,
                x + text.chars().count() as f32 * 0.5 * size,
                baseline + size,
            ),
            font_size: size,
            bold: false,
            italic: false,
            font: None,
            page,
        }
    }

    /// A US-letter page holding `runs`.
    pub fn page(runs: Vec<TextRun>) -> PageRuns {
        PageRuns {
            width: 612.0,
            height: 792.0,
            runs,
        }
    }

    /// Lay `text` out as one full-width run per line from `top` down,
    /// stepping `leading` points per line.
    pub fn body_lines(lines: &[&str], x: f32, top: f32, leading: f32, page: usize) -> Vec<TextRun> {
        lines
            .iter()
            .enumerate()
            .map(|(i, t)| run(t, x, top - i as f32 * leading, 10.0, page))
            .collect()
    }
}

#[cfg(test)]
mod tests {
    use super::testutil::{body_lines, page, run};
    use super::*;

    #[test]
    fn empty_input_yields_empty_layout() {
        let layout = reconstruct(&[]);
        assert!(layout.blocks.is_empty());
        assert!(layout.furniture.is_empty());
        assert_eq!(layout.text(), "");
    }

    #[test]
    fn single_paragraph_reads_top_to_bottom() {
        let runs = body_lines(
            &["first line of text here", "second line follows it"],
            72.0,
            700.0,
            12.0,
            0,
        );
        let layout = reconstruct(&[page(runs)]);
        assert_eq!(layout.blocks.len(), 1);
        assert_eq!(
            layout.text(),
            "first line of text here\nsecond line follows it\n"
        );
    }

    #[test]
    fn same_baseline_runs_join_with_gap_spacing() {
        // Small-caps style: several runs of different sizes on one baseline,
        // abutting except for real word gaps (before "1:" and "G").
        let mut runs = Vec::new();
        let mut x = 100.0;
        for (text, size, gap) in [
            ("T", 9.0, 0.0),
            ("ITLE ", 7.0, 0.0),
            ("1:", 9.0, 2.0),
            ("G", 9.0, 2.5),
            ("OVERNMENT", 7.0, 0.0),
        ] {
            x += gap;
            let r = run(text, x, 700.0, size, 0);
            x += rows::width_est(&r);
            runs.push(r);
        }
        let layout = reconstruct(&[page(runs)]);
        let text = layout.text();
        assert_eq!(layout.blocks.len(), 1);
        assert_eq!(layout.blocks[0].lines.len(), 1, "one baseline = one line");
        assert!(
            text.contains("TITLE 1: GOVERNMENT"),
            "expected joined small-caps line, got {text:?}"
        );
    }

    #[test]
    fn layout_serializes_for_json_output() {
        let runs = body_lines(&["hello world"], 72.0, 700.0, 12.0, 0);
        let layout = reconstruct(&[page(runs)]);
        let v: serde_json::Value = serde_json::to_value(&layout).unwrap();
        let line = &v["blocks"][0]["lines"][0];
        assert_eq!(line["text"], "hello world");
        assert_eq!(line["x0"], 72.0);
        assert_eq!(line["indent"], 0);
        assert_eq!(line["column"], 0);
        assert_eq!(line["pages"][0], 0);
    }

    #[test]
    fn keep_furniture_leaves_lines_inline() {
        let mut pages_vec = Vec::new();
        for p in 0..4 {
            let mut runs = vec![run("RUNNING HEAD", 250.0, 770.0, 9.0, p)];
            let second = format!("is never furniture, page {p}");
            runs.extend(body_lines(
                &["body text well below the band so it", &second],
                72.0,
                620.0,
                12.0,
                p,
            ));
            pages_vec.push(page(runs));
        }
        let stripped = reconstruct(&pages_vec);
        assert_eq!(stripped.furniture.len(), 4);
        assert!(!stripped.text().contains("RUNNING HEAD"));

        let kept = reconstruct_with(
            &pages_vec,
            &LayoutOptions {
                strip_furniture: false,
            },
        );
        assert!(kept.furniture.is_empty());
        assert!(kept.text().contains("RUNNING HEAD"));
    }
}