pdf-untangle 0.3.0

Untangle garbled PDF text: layout-aware extraction that reconstructs reading order, strips page furniture, and splits documents into structured, citable sections
Documentation
//! Output model: structured provisions plus document-level accounting.
//!
//! Field semantics locked with the consuming resolution engine (docs/DESIGN.md
//! §3): `jurisdiction`/`kind` are routing keys copied from the profile;
//! `effective_date` is NEVER null (parsed date → document adoption date →
//! the `1800-01-01` sentinel, which resolves as "in force now");
//! `confidence`/`confidence_basis` let shakier PDF extractions self-report
//! instead of masquerading as clean data.

use std::io::Write;

use anyhow::Result;
use serde::{Deserialize, Serialize};

/// The corpus-standard "no date available" sentinel (in force now).
pub const SENTINEL_DATE: &str = "1800-01-01";

/// One extracted provision (rule or statute section) — one JSONL line.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Provision {
    /// Routing key, from the profile (e.g. "northern_mariana_islands").
    pub jurisdiction: String,
    /// Routing key, from the profile: "rule" or "statute".
    pub kind: String,
    /// Full citation, rendered from the profile's citation_template.
    pub citation: String,
    /// The bare provision number (e.g. "12" or "15-6-56").
    pub number: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub heading: Option<String>,
    pub text: String,
    /// The history/credits line, when the document carries one.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub history: Option<String>,
    /// YYYY-MM-DD; never empty — falls back to [`SENTINEL_DATE`].
    pub effective_date: String,
    /// BCP-47-ish language tag of the extracted text (e.g. "en", "es").
    pub language: String,
    /// 0.0–1.0; derived from coverage %, clean citation parse, clean date parse.
    pub confidence: f32,
    pub confidence_basis: String,
    pub source_file: String,
    /// 1-based page span [first, last] in the source PDF.
    pub pages: [u32; 2],
    /// Name of the profile that produced this record.
    pub profile: String,
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub warnings: Vec<String>,
}

/// A fully processed document: its provisions plus extraction accounting.
#[derive(Debug, Clone, Serialize)]
pub struct Document {
    pub provisions: Vec<Provision>,
    /// Fraction of input lines accounted for (silent-loss guard): consumed
    /// into a provision, skipped by a profile rule, or stripped as page
    /// furniture. Equals `1 − unconsumed/total`; `pdf-untangle check` fails
    /// below 0.98.
    pub line_coverage: f32,
    /// Lines that were neither consumed nor deliberately skipped.
    pub unconsumed_lines: Vec<String>,
    pub warnings: Vec<String>,
    /// Total input lines (content lines plus stripped page furniture).
    pub lines_total: usize,
    /// Lines consumed into provisions (headings, text, history).
    pub lines_consumed: usize,
    /// Lines inside profile-declared skip regions (recorded, never lost).
    pub lines_skipped: usize,
    /// Lines Layer 2 stripped as repeated page furniture.
    pub lines_furniture: usize,
    /// Provision numbers missing from the sequence (profile `expected_gaps`
    /// already excluded) — structure-integrity warnings.
    pub sequence_gaps: Vec<String>,
    /// Provision numbers seen more than once — structure-integrity failures.
    pub sequence_duplicates: Vec<String>,
    /// The line that identified this as a recognized intentionally-empty
    /// document (a whole-chapter disposition stub matching the profile's
    /// `empty_doc_patterns`, e.g. "[Repealed by S.L. 1947, ch. 1, § 18]").
    /// `None` for every document with provisions; zero provisions WITHOUT a
    /// marker remains a `check` failure.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub empty_doc_marker: Option<String>,
}

/// Write provisions as JSONL (one provision per line).
pub fn write_jsonl<W: Write>(mut w: W, provisions: &[Provision]) -> Result<()> {
    for p in provisions {
        serde_json::to_writer(&mut w, p)?;
        w.write_all(b"\n")?;
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    fn sample() -> Provision {
        Provision {
            jurisdiction: "northern_mariana_islands".into(),
            kind: "rule".into(),
            citation: "NMI R. Civ. P. 12".into(),
            number: "12".into(),
            heading: Some("Defenses and Objections".into()),
            text: "…".into(),
            history: None,
            effective_date: SENTINEL_DATE.into(),
            language: "en".into(),
            confidence: 0.93,
            confidence_basis: "coverage=99.4%; citation+date parsed clean".into(),
            source_file: "R05.pdf".into(),
            pages: [11, 14],
            profile: "cnmi-rules".into(),
            warnings: vec![],
        }
    }

    #[test]
    fn jsonl_round_trips() {
        let mut buf = Vec::new();
        write_jsonl(&mut buf, &[sample()]).unwrap();
        let line = String::from_utf8(buf).unwrap();
        let back: Provision = serde_json::from_str(line.trim()).unwrap();
        assert_eq!(back.citation, "NMI R. Civ. P. 12");
        assert_eq!(back.effective_date, SENTINEL_DATE);
        assert_eq!(back.jurisdiction, "northern_mariana_islands");
    }

    #[test]
    fn empty_optionals_are_omitted() {
        let mut p = sample();
        p.heading = None;
        let s = serde_json::to_string(&p).unwrap();
        assert!(!s.contains("\"heading\""));
        assert!(!s.contains("\"warnings\""));
    }
}