pdf-untangle 0.3.0

Untangle garbled PDF text: layout-aware extraction that reconstructs reading order, strips page furniture, and splits documents into structured, citable sections
Documentation
//! Date scanning for history lines and adoption-date detection.
//!
//! A profile declares `date_formats` as strftime-style patterns (the same
//! syntax `chrono` parses). For each format this module derives a *finder*
//! regex, locates candidate substrings in a line, and hands the exact match
//! to chrono — so profiles stay purely declarative and never need their own
//! date regexes.
//!
//! Month names are normalized through a small data table before matching:
//! Spanish month names (for the Puerto Rico profiles) are rewritten to their
//! English equivalents, so `"%d de %B de %Y"` parses `"9 de enero de 2019"`
//! without any Spanish-specific code path. Adding a language is adding table
//! rows.

use anyhow::{Context, Result};
use chrono::NaiveDate;
use regex::Regex;

/// Month-name normalization table: `(foreign, english)`, matched
/// case-insensitively on word boundaries. Data, not code: extend this table
/// to support another language's month names.
const MONTH_NAMES: &[(&str, &str)] = &[
    ("enero", "January"),
    ("febrero", "February"),
    ("marzo", "March"),
    ("abril", "April"),
    ("mayo", "May"),
    ("junio", "June"),
    ("julio", "July"),
    ("agosto", "August"),
    ("septiembre", "September"),
    ("setiembre", "September"), // common variant spelling
    // Bluebook month abbreviation: "Sept." is the one whose stem is not the
    // 3-letter form chrono's %b expects (Jan./Feb./…/Dec. all are). Legal
    // history credits use it pervasively ("Amended, effective Sept. 25,
    // 2023." — Delaware court rules et al.). \b keeps it out of the full
    // word: "September" has a word character after the t.
    ("sept", "Sep"),
    ("octubre", "October"),
    ("noviembre", "November"),
    ("diciembre", "December"),
];

const MONTHS_FULL: &str =
    "January|February|March|April|May|June|July|August|September|October|November|December";
const MONTHS_ABBR: &str = "Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec";

/// Compiled date scanner for one profile's `date_formats`.
#[derive(Debug)]
pub struct DateScanner {
    /// `(finder, strftime format)` per profile format, in profile order.
    finders: Vec<(Regex, String)>,
    /// One pass of the month-name table, applied before matching.
    months: Regex,
}

impl DateScanner {
    /// Build a scanner from strftime-style formats. Errors on specifiers the
    /// finder derivation does not support (kept to the date-only subset a
    /// legal history line can carry).
    pub fn new(formats: &[String]) -> Result<Self> {
        let mut finders = Vec::with_capacity(formats.len());
        for fmt in formats {
            let pattern =
                finder_pattern(fmt).with_context(|| format!("unsupported date format {fmt:?}"))?;
            let re = Regex::new(&pattern)
                .with_context(|| format!("date format {fmt:?} derived an invalid regex"))?;
            finders.push((re, fmt.clone()));
        }
        let alternation: Vec<&str> = MONTH_NAMES.iter().map(|(m, _)| *m).collect();
        let months = Regex::new(&format!(r"(?i)\b(?:{})\b", alternation.join("|")))
            .expect("static month table regex");
        Ok(Self { finders, months })
    }

    /// Rewrite foreign month names to English using the data table.
    pub fn normalize_months(&self, text: &str) -> String {
        self.months
            .replace_all(text, |caps: &regex::Captures<'_>| {
                let hit = caps[0].to_lowercase();
                MONTH_NAMES
                    .iter()
                    .find(|(m, _)| *m == hit)
                    .map(|(_, en)| *en)
                    .unwrap_or("")
                    .to_string()
            })
            .into_owned()
    }

    /// Find every parseable date in `text`, in text order (dates from an
    /// earlier format are merged by position, not appended).
    pub fn scan(&self, text: &str) -> Vec<NaiveDate> {
        let text = self.normalize_months(text);
        let mut hits: Vec<(usize, NaiveDate)> = Vec::new();
        for (re, fmt) in &self.finders {
            for m in re.find_iter(&text) {
                // Collapse runs of whitespace so "January  9,  2019" parses
                // with a single-space format string.
                let compact = m.as_str().split_whitespace().collect::<Vec<_>>().join(" ");
                if let Ok(date) = NaiveDate::parse_from_str(&compact, fmt) {
                    if !hits.iter().any(|(pos, _)| *pos == m.start()) {
                        hits.push((m.start(), date));
                    }
                }
            }
        }
        hits.sort_by_key(|(pos, _)| *pos);
        hits.into_iter().map(|(_, d)| d).collect()
    }
}

/// Derive a finder regex from a strftime-style format string.
fn finder_pattern(fmt: &str) -> Result<String> {
    let mut out = String::new();
    let mut chars = fmt.chars();
    while let Some(c) = chars.next() {
        if c == '%' {
            let spec = chars
                .next()
                .context("dangling % at the end of the format")?;
            match spec {
                'B' => out.push_str(&format!("(?:{MONTHS_FULL})")),
                'b' => out.push_str(&format!("(?:{MONTHS_ABBR})")),
                'd' | 'e' => out.push_str(r"\d{1,2}"),
                'm' => out.push_str(r"\d{1,2}"),
                'Y' => out.push_str(r"\d{4}"),
                'y' => out.push_str(r"\d{2}"),
                '%' => out.push('%'),
                other => anyhow::bail!("unsupported strftime specifier %{other}"),
            }
        } else if c.is_whitespace() {
            // Corpus PDFs carry doubled spaces; match any whitespace run.
            out.push_str(r"\s+");
        } else {
            out.push_str(&regex::escape(&c.to_string()));
        }
    }
    Ok(out)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn scanner(formats: &[&str]) -> DateScanner {
        let owned: Vec<String> = formats.iter().map(|s| s.to_string()).collect();
        DateScanner::new(&owned).unwrap()
    }

    fn ymd(y: i32, m: u32, d: u32) -> NaiveDate {
        NaiveDate::from_ymd_opt(y, m, d).unwrap()
    }

    #[test]
    fn finds_english_month_dates_in_prose() {
        let s = scanner(&["%B %d, %Y"]);
        let dates = s.scan("adopted as permanent on January 9, 2019. See In re");
        assert_eq!(dates, vec![ymd(2019, 1, 9)]);
    }

    #[test]
    fn doubled_spaces_still_parse() {
        let s = scanner(&["%B %d, %Y"]);
        assert_eq!(
            s.scan("Effective  February 13,  2002   [rev]"),
            vec![ymd(2002, 2, 13)]
        );
    }

    #[test]
    fn bluebook_sept_abbreviation_normalizes_via_the_table() {
        let s = scanner(&["%b. %d, %Y"]);
        assert_eq!(
            s.scan("Amended, effective Sept. 25, 2023."),
            vec![ymd(2023, 9, 25)]
        );
        // The word boundary keeps the rewrite out of the full month name.
        let s = scanner(&["%B %d, %Y"]);
        assert_eq!(
            s.scan("effective September 25, 2023"),
            vec![ymd(2023, 9, 25)]
        );
    }

    #[test]
    fn multiple_dates_come_back_in_text_order() {
        let s = scanner(&["%B %d, %Y", "%m/%d/%Y"]);
        let dates = s.scan("adopted 08/01/1998; amended effective March 2, 2004");
        assert_eq!(dates, vec![ymd(1998, 8, 1), ymd(2004, 3, 2)]);
    }

    #[test]
    fn spanish_month_names_normalize_via_the_table() {
        let s = scanner(&["%d de %B de %Y"]);
        assert_eq!(
            s.scan("enmendada el 9 de enero de 2019"),
            vec![ymd(2019, 1, 9)]
        );
        assert_eq!(
            s.scan("efectivo el 1 de Septiembre de 2020"),
            vec![ymd(2020, 9, 1)]
        );
    }

    #[test]
    fn abbreviated_months_and_literal_dots() {
        let s = scanner(&["%b. %d, %Y"]);
        assert_eq!(s.scan("amended Feb. 25, 2019"), vec![ymd(2019, 2, 25)]);
    }

    #[test]
    fn nonsense_numbers_do_not_parse() {
        let s = scanner(&["%m/%d/%Y"]);
        assert!(s.scan("case no. 80/99/0012 is not a date").is_empty());
    }

    #[test]
    fn unsupported_specifier_is_rejected() {
        let err = DateScanner::new(&["%H:%M".to_string()]).unwrap_err();
        assert!(format!("{err:#}").contains("unsupported strftime specifier"));
    }
}