pdf-untangle 0.3.0

Untangle garbled PDF text: layout-aware extraction that reconstructs reading order, strips page furniture, and splits documents into structured, citable sections
Documentation
//! Page-furniture detection: running headers, footers, page numbers.
//!
//! A line is furniture when it sits in the top or bottom position band of its
//! page and either (a) its digit-masked text repeats across enough pages
//! ("Page 12" ≈ "Page 13" ≈ "page #"), or (b) it is a standalone page number
//! (arabic or roman, with optional dashes). Repetition is required for
//! anything wordier than a page number, so body text that merely reaches the
//! band on a full page is never stripped.
//!
//! Stripped lines are returned to the caller for accounting — Layer 3's
//! coverage invariant counts them as deliberately removed, never lost.

use std::collections::HashMap;
use std::sync::OnceLock;

use regex::Regex;

use crate::extract::PageRuns;

use super::Line;

/// Top band: baselines above this fraction of the page height. Running heads
/// in the corpus sit as low as 89% (CNMI's two-line head, CT at 90%).
const TOP_BAND: f32 = 0.88;
/// Bottom band: baselines below this fraction of the page height. Kept
/// tighter than the top band because dense body pages run text down to ~11%.
const BOTTOM_BAND: f32 = 0.10;
/// Minimum distinct pages a masked text must appear on to count as repeated
/// furniture (relaxed to 2 for documents of at most 3 pages).
const MIN_REPEAT_PAGES: usize = 3;

#[derive(PartialEq, Eq, Hash, Clone, Copy)]
enum Band {
    Top,
    Bottom,
}

/// Remove furniture lines from `lines`, returning them (in reading order).
pub fn strip(lines: &mut Vec<Line>, pages: &[PageRuns]) -> Vec<Line> {
    let n_pages = pages.len();
    let min_repeat = if n_pages <= 3 { 2 } else { MIN_REPEAT_PAGES };

    let band_of = |line: &Line| -> Option<Band> {
        let page = line.pages.first().copied()?;
        let height = pages.get(page)?.height;
        if height <= 0.0 {
            return None;
        }
        let frac = line.baseline / height;
        if frac >= TOP_BAND {
            Some(Band::Top)
        } else if frac <= BOTTOM_BAND {
            Some(Band::Bottom)
        } else {
            None
        }
    };

    // Pass 1: how many distinct pages does each banded masked text occur on?
    let mut seen: HashMap<(Band, String), Vec<usize>> = HashMap::new();
    for line in lines.iter() {
        if let Some(band) = band_of(line) {
            let key = (band, mask(&line.text));
            let page = line.pages[0];
            let entry = seen.entry(key).or_default();
            if entry.last() != Some(&page) {
                entry.push(page);
            }
        }
    }

    // Pass 2: strip repeated banded lines and standalone page numbers.
    let mut stripped = Vec::new();
    lines.retain(|line| {
        let Some(band) = band_of(line) else {
            return true;
        };
        let masked = mask(&line.text);
        let repeats = seen
            .get(&(band, masked.clone()))
            .map(|pages| pages.len())
            .unwrap_or(0);
        if repeats >= min_repeat || is_page_number(&masked) {
            stripped.push(line.clone());
            false
        } else {
            true
        }
    });
    stripped
}

/// Normalize text for cross-page comparison: lowercase, every digit run —
/// including an attached letter suffix ("4A", "12b") — masked to `#`,
/// whitespace collapsed.
pub fn mask(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    let mut in_digits = false;
    let mut in_space = false;
    for c in text.trim().chars() {
        if c.is_ascii_digit() || (in_digits && c.is_ascii_alphabetic()) {
            if !in_digits {
                out.push('#');
            }
            in_digits = true;
            in_space = false;
        } else if c.is_whitespace() {
            if !in_space && !out.is_empty() {
                out.push(' ');
            }
            in_space = true;
            in_digits = false;
        } else {
            out.extend(c.to_lowercase());
            in_digits = false;
            in_space = false;
        }
    }
    out.trim_end().to_string()
}

/// A standalone page number: masked digits or a well-formed roman numeral,
/// optionally decorated ("#", "- # -", "page #", "xiv").
fn is_page_number(masked: &str) -> bool {
    static RE: OnceLock<Regex> = OnceLock::new();
    let re = RE.get_or_init(|| {
        Regex::new(
            r"^[-–—.\s]*(?:page\s+)?(#|m{0,4}(?:cm|cd|d?c{0,3})(?:xc|xl|l?x{0,3})(?:ix|iv|v?i{0,3}))[-–—.\s]*$",
        )
        .expect("static regex")
    });
    match re.captures(masked) {
        Some(caps) => !caps[1].is_empty(), // the roman branch can match ""
        None => false,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::layout::reconstruct;
    use crate::layout::testutil::{body_lines, page, run};

    #[test]
    fn masking_equates_page_numbers() {
        assert_eq!(mask("Page 12"), mask("Page 13"));
        assert_eq!(mask("TITLE 1:  GOVERNMENT"), "title #: government");
        assert_ne!(mask("Rule 1.8"), mask("Rule 12"));
        assert_eq!(mask("Rule 1.8"), mask("Rule 2.3"));
    }

    #[test]
    fn standalone_page_numbers_detected() {
        assert!(is_page_number(&mask("12")));
        assert!(is_page_number(&mask("- 12 -")));
        assert!(is_page_number(&mask("Page 7")));
        assert!(is_page_number(&mask("xiv")));
        assert!(!is_page_number(&mask("Rule 12 continued")));
        assert!(!is_page_number(&mask("mild"))); // roman-lookalike chars, not a numeral
        assert!(!is_page_number(&mask(""))); // the roman branch must not match empty
    }

    #[test]
    fn repeated_headers_and_page_numbers_stripped_with_accounting() {
        let mut pages_vec = Vec::new();
        for p in 0..4 {
            let mut runs = vec![
                run("MY RUNNING HEAD", 240.0, 772.0, 9.0, p),
                run(&format!("{}", p + 1), 300.0, 40.0, 10.0, p),
            ];
            let second = format!("and differs on page {p} entirely");
            runs.extend(body_lines(
                &["unique body sentence that continues along", &second],
                72.0,
                620.0,
                12.0,
                p,
            ));
            pages_vec.push(page(runs));
        }
        let layout = reconstruct(&pages_vec);
        let text = layout.text();
        assert!(!text.contains("MY RUNNING HEAD"), "header must be stripped");
        assert_eq!(
            layout.furniture.len(),
            8,
            "4 headers + 4 page numbers recorded"
        );
        assert!(text.contains("unique body sentence"));
    }

    #[test]
    fn body_text_in_band_is_kept_without_repetition() {
        // A dense page whose last body line dips into the bottom band.
        let mut runs = body_lines(&["body body body"], 72.0, 700.0, 12.0, 0);
        runs.push(run("reinstated in its entirety.", 72.0, 60.0, 10.0, 0));
        let layout = reconstruct(&[page(runs)]);
        assert!(layout.text().contains("reinstated in its entirety."));
        assert!(layout.furniture.is_empty());
    }
}