pub(crate) mod dates;
use anyhow::{Context, Result};
use chrono::NaiveDate;
use regex::Regex;
use crate::layout::{Layout, Line};
use crate::output::{Document, Provision, SENTINEL_DATE};
use crate::profile::Profile;
use dates::DateScanner;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DateSource {
Parsed,
Adoption,
Sentinel,
}
struct Candidate {
num: String,
heading: Option<String>,
title: Option<String>,
text_head: Option<String>,
}
struct Builder {
num: String,
heading: Option<String>,
title: Option<String>,
text_lines: Vec<String>,
last_text_block: Option<usize>,
history_lines: Vec<String>,
history_block: Option<usize>,
dates: Vec<NaiveDate>,
first_page: usize,
last_page: usize,
warnings: Vec<String>,
}
impl Builder {
fn new(cand: Candidate, page: usize, block: usize) -> Self {
let (text_lines, last_text_block) = match &cand.text_head {
Some(head) => (vec![head.clone()], Some(block)),
None => (Vec::new(), None),
};
Self {
num: cand.num,
heading: cand.heading,
title: cand.title,
text_lines,
last_text_block,
history_lines: Vec::new(),
history_block: None,
dates: Vec::new(),
first_page: page,
last_page: page,
warnings: Vec::new(),
}
}
fn touch_page(&mut self, line: &Line) {
for &p in &line.pages {
self.first_page = self.first_page.min(p);
self.last_page = self.last_page.max(p);
}
}
fn push_text(&mut self, line: &Line, block: usize) {
if self.last_text_block != Some(block) && !self.text_lines.is_empty() {
self.text_lines.push(String::new()); }
self.text_lines.push(line.text.trim_end().to_string());
self.last_text_block = Some(block);
}
}
struct Engine {
heading: Vec<Regex>,
history: Vec<Regex>,
skip: Vec<Regex>,
skip_block: Vec<Regex>,
stop: Vec<Regex>,
empty_doc: Vec<Regex>,
adoption: Vec<Regex>,
title: Vec<Regex>,
scanner: DateScanner,
}
impl Engine {
fn new(profile: &Profile) -> Result<Self> {
let compile = |pats: &[String]| -> Result<Vec<Regex>> {
pats.iter()
.map(|p| Regex::new(p).with_context(|| format!("profile regex {p:?}")))
.collect()
};
Ok(Self {
heading: compile(&profile.heading_patterns)?,
history: compile(&profile.history_patterns)?,
skip: compile(&profile.skip_patterns)?,
skip_block: compile(&profile.skip_block_patterns)?,
stop: compile(&profile.stop_patterns)?,
empty_doc: compile(&profile.empty_doc_patterns)?,
adoption: compile(&profile.adoption_patterns)?,
title: compile(&profile.title_patterns)?,
scanner: DateScanner::new(&profile.date_formats)?,
})
}
fn candidate(&self, text: &str) -> Option<Candidate> {
let text = text.trim();
for re in &self.heading {
if let Some(caps) = re.captures(text) {
let num = caps.name("num")?.as_str().to_string();
let grab = |name: &str| {
caps.name(name)
.map(|m| m.as_str().trim().to_string())
.filter(|s| !s.is_empty())
};
let text_head = caps
.get(0)
.map(|m| text[m.end()..].trim().to_string())
.filter(|s| !s.is_empty());
return Some(Candidate {
num,
heading: grab("heading"),
title: grab("title"),
text_head,
});
}
}
None
}
fn is_history(&self, text: &str) -> bool {
let text = text.trim();
self.history.iter().any(|re| re.is_match(text))
}
fn is_skip(&self, text: &str) -> bool {
let text = text.trim();
self.skip.iter().any(|re| re.is_match(text))
}
fn is_skip_block(&self, first_line: &str) -> bool {
let text = first_line.trim();
self.skip_block.iter().any(|re| re.is_match(text))
}
fn is_stop(&self, text: &str) -> bool {
let text = text.trim();
self.stop.iter().any(|re| re.is_match(text))
}
}
fn num_key(num: &str) -> (Vec<(u64, String)>, String) {
let mut segs: Vec<(u64, String)> = Vec::new();
let mut suffix = String::new();
for part in num.split(['.', '-']) {
let digits: String = part.chars().take_while(char::is_ascii_digit).collect();
let rest: &str = &part[digits.len()..];
match digits.parse::<u64>() {
Ok(n) => segs.push((n, rest.to_string())),
Err(_) => suffix.push_str(part),
}
}
(segs, suffix)
}
fn x0_cluster(cands: &[(f32, f32)]) -> Vec<bool> {
if cands.len() < 3 {
return vec![true; cands.len()];
}
let mut sizes: Vec<f32> = cands.iter().map(|&(_, size)| size).collect();
sizes.sort_by(|a, b| a.total_cmp(b));
let tol = 1.25 * sizes[sizes.len() / 2];
let mut best_center = cands[0].0;
let mut best_count = 0usize;
for &(center, _) in cands {
let count = cands
.iter()
.filter(|&&(x, _)| (x - center).abs() <= tol)
.count();
if count > best_count || (count == best_count && center < best_center) {
best_center = center;
best_count = count;
}
}
cands
.iter()
.map(|&(x, _)| (x - best_center).abs() <= tol)
.collect()
}
pub fn detect(layout: &Layout, profile: &Profile, source_file: &str) -> Result<Document> {
let engine = Engine::new(profile)?;
let flat: Vec<(usize, &Line)> = layout
.blocks
.iter()
.enumerate()
.flat_map(|(bi, b)| b.lines.iter().map(move |l| (bi, l)))
.collect();
let block_skipped: Vec<bool> = layout
.blocks
.iter()
.map(|b| {
b.lines
.first()
.is_some_and(|l| engine.is_skip_block(&l.text))
})
.collect();
let doc_has_bold = flat.iter().any(|(_, l)| l.bold);
let mut candidates: Vec<Option<Candidate>> = flat
.iter()
.map(|(bi, l)| {
if block_skipped[*bi] {
None
} else {
engine.candidate(&l.text)
}
})
.collect();
if profile.heading_x0_cluster {
let feats: Vec<(f32, f32)> = flat
.iter()
.zip(&candidates)
.filter(|(_, c)| c.is_some())
.map(|((_, l), _)| (l.x0, l.font_size))
.collect();
let keep = x0_cluster(&feats);
let mut k = keep.iter();
for cand in candidates.iter_mut() {
if cand.is_some() && !*k.next().expect("one flag per candidate") {
*cand = None;
}
}
}
if let Some(style) = &profile.heading_style {
for ((_, line), cand) in flat.iter().zip(candidates.iter_mut()) {
if cand.is_none() {
continue;
}
let bold_ok = match style.bold {
Some(want) if doc_has_bold => line.bold == want,
_ => true,
};
let centered_ok = style.centered.map_or(true, |want| line.centered == want);
let min_ok = style.min_size.map_or(true, |s| line.font_size >= s);
let max_ok = style.max_size.map_or(true, |s| line.font_size <= s);
if !(bold_ok && centered_ok && min_ok && max_ok) {
*cand = None;
}
}
}
let mut builders: Vec<Builder> = Vec::new();
let mut current: Option<Builder> = None;
let mut skipping = false;
let mut consumed = 0usize;
let mut skipped = 0usize;
let mut unconsumed_lines: Vec<String> = Vec::new();
let mut first_key: Option<(Vec<(u64, String)>, String)> = None;
let mut restart_note: Option<String> = None;
let mut stopped = false;
for ((block, line), cand) in flat.iter().zip(candidates) {
if stopped {
skipped += 1;
continue;
}
if engine.is_stop(&line.text) {
stopped = true;
if let Some(done) = current.take() {
builders.push(done);
}
skipped += 1;
continue;
}
if block_skipped[*block] {
if let Some(b) = current.as_mut() {
b.touch_page(line);
if engine.is_history(&line.text) {
b.dates.extend(engine.scanner.scan(&line.text));
}
}
skipped += 1;
continue;
}
let mut accepted = None;
if let Some(c) = cand {
let monotonic_ok = !profile.monotonic_numbers
|| current
.as_ref()
.map(|b| num_key(&c.num) > num_key(&b.num))
.unwrap_or(true);
if monotonic_ok {
accepted = Some(c);
} else if profile.toc_restart
&& restart_note.is_none()
&& first_key.as_ref() == Some(&num_key(&c.num))
{
let discarded = builders.len() + usize::from(current.is_some());
restart_note = Some(format!(
"table-of-contents pass discarded: {discarded} provision(s) \
reclassified as front matter; detection restarted at {}",
c.num
));
builders.clear();
current = None;
skipped += consumed;
consumed = 0;
accepted = Some(c);
}
}
if let Some(cand) = accepted {
if first_key.is_none() {
first_key = Some(num_key(&cand.num));
}
if let Some(done) = current.take() {
builders.push(done);
}
let mut b = Builder::new(cand, line.pages.first().copied().unwrap_or(0), *block);
b.touch_page(line);
current = Some(b);
skipping = false;
consumed += 1;
continue;
}
if engine.is_skip(&line.text) {
skipping = true;
}
if let Some(b) = current.as_mut() {
b.touch_page(line);
let history_hit = engine.is_history(&line.text);
if history_hit {
b.dates.extend(engine.scanner.scan(&line.text));
}
if skipping {
skipped += 1;
continue;
}
if history_hit {
b.history_lines.push(line.text.trim().to_string());
b.history_block = Some(*block);
} else if b.history_block == Some(*block) {
b.history_lines.push(line.text.trim().to_string());
b.dates.extend(engine.scanner.scan(&line.text));
} else {
b.history_block = None;
b.push_text(line, *block);
}
consumed += 1;
} else if skipping {
skipped += 1;
} else {
unconsumed_lines.push(line.text.trim_end().to_string());
}
}
if let Some(done) = current.take() {
builders.push(done);
}
let furniture = layout.furniture.len();
let total = flat.len() + furniture;
let line_coverage = if total == 0 {
1.0
} else {
1.0 - unconsumed_lines.len() as f32 / total as f32
};
let all_lines = flat.iter().map(|(_, l)| *l).chain(layout.furniture.iter());
let mut adoption: Option<NaiveDate> = None;
let mut doc_title: Option<String> = None;
for line in all_lines {
let text = line.text.trim();
if adoption.is_none() {
for re in &engine.adoption {
if let Some(caps) = re.captures(text) {
let target = caps.name("date").map_or(text, |m| m.as_str());
if let Some(d) = engine.scanner.scan(target).into_iter().next() {
adoption = Some(d);
break;
}
}
}
}
if doc_title.is_none() {
for re in &engine.title {
if let Some(m) = re.captures(text).and_then(|c| c.name("title")) {
doc_title = Some(m.as_str().to_string());
break;
}
}
}
if adoption.is_some() && doc_title.is_some() {
break;
}
}
if adoption.is_none() {
adoption = profile
.adoption_date
.as_deref()
.and_then(|d| NaiveDate::parse_from_str(d, "%Y-%m-%d").ok());
}
let mut warnings: Vec<String> = Vec::new();
if let Some(note) = restart_note {
warnings.push(note);
}
let mut sequence_gaps: Vec<String> = Vec::new();
let mut sequence_duplicates: Vec<String> = Vec::new();
let mut seen = std::collections::HashSet::new();
for b in &builders {
if !seen.insert(num_key(&b.num)) {
sequence_duplicates.push(b.num.clone());
warnings.push(format!("duplicate provision number {}", b.num));
}
}
for pair in builders.windows(2) {
let (prev, next) = (&pair[0], &pair[1]);
let (pk, nk) = (num_key(&prev.num), num_key(&next.num));
if nk < pk {
warnings.push(format!(
"provisions out of order: {} follows {}",
next.num, prev.num
));
continue;
}
if let (Some(p), Some(n)) = (pk.0.first(), nk.0.first()) {
for missing in (p.0 + 1)..n.0 {
let m = missing.to_string();
if !profile.expected_gaps.contains(&m) {
warnings.push(format!(
"sequence gap: {m} missing between {} and {}",
prev.num, next.num
));
sequence_gaps.push(m);
}
}
}
}
let empty_doc_marker = if builders.is_empty() {
flat.iter()
.map(|(_, l)| *l)
.chain(layout.furniture.iter())
.map(|l| l.text.trim())
.find(|t| engine.empty_doc.iter().any(|re| re.is_match(t)))
.map(str::to_string)
} else {
None
};
if builders.is_empty() {
match &empty_doc_marker {
Some(m) => warnings.push(format!("recognized empty document: {m}")),
None => warnings.push("no provisions detected".to_string()),
}
}
let wants_title = profile.citation_template.contains("{title}");
let provisions = builders
.into_iter()
.map(|b| {
let mut prov_warnings = b.warnings;
let mut citation = profile.citation_template.replace("{num}", &b.num);
let mut citation_clean = true;
if wants_title {
match b.title.clone().or_else(|| doc_title.clone()) {
Some(t) => citation = citation.replace("{title}", &t),
None => {
citation_clean = false;
prov_warnings.push(
"citation {title} unresolved: no title capture matched".to_string(),
);
}
}
}
let (date, source) = match (b.dates.last(), adoption) {
(Some(&d), _) => (d, DateSource::Parsed),
(None, Some(d)) => (d, DateSource::Adoption),
(None, None) => (
NaiveDate::parse_from_str(SENTINEL_DATE, "%Y-%m-%d").expect("sentinel"),
DateSource::Sentinel,
),
};
let (confidence, confidence_basis) = confidence(line_coverage, citation_clean, source);
let history = if b.history_lines.is_empty() {
None
} else {
Some(b.history_lines.join(" "))
};
Provision {
jurisdiction: profile.jurisdiction.clone(),
kind: profile.kind.clone(),
citation,
number: b.num,
heading: b.heading,
text: b.text_lines.join("\n").trim().to_string(),
history,
effective_date: date.format("%Y-%m-%d").to_string(),
language: profile.language.clone(),
confidence,
confidence_basis,
source_file: source_file.to_string(),
pages: [b.first_page as u32 + 1, b.last_page as u32 + 1],
profile: profile.name.clone(),
warnings: prov_warnings,
}
})
.collect();
Ok(Document {
provisions,
line_coverage,
unconsumed_lines,
warnings,
lines_total: total,
lines_consumed: consumed,
lines_skipped: skipped,
lines_furniture: furniture,
sequence_gaps,
sequence_duplicates,
empty_doc_marker,
})
}
fn confidence(coverage: f32, citation_clean: bool, source: DateSource) -> (f32, String) {
let citation_factor: f64 = if citation_clean { 1.0 } else { 0.85 };
let (date_factor, date_note): (f64, &str) = match source {
DateSource::Parsed => (1.0, "date parsed from history"),
DateSource::Adoption => (0.97, "date from document adoption"),
DateSource::Sentinel => (0.9, "date sentinel (none found)"),
};
let raw = f64::from(coverage.clamp(0.0, 1.0)) * citation_factor * date_factor;
let rounded = ((raw * 100.0).round() / 100.0) as f32;
let basis = format!(
"coverage={:.1}%; citation {}; {}",
coverage * 100.0,
if citation_clean {
"clean"
} else {
"unresolved"
},
date_note
);
(rounded.max(0.05), basis)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::{Block, Line};
fn line(text: &str, x0: f32, page: usize) -> Line {
Line {
text: text.to_string(),
x0,
baseline: 0.0,
indent: 0,
font_size: 11.0,
bold: false,
centered: false,
column: 0,
pages: vec![page],
}
}
fn block(lines: Vec<Line>) -> Block {
Block { lines }
}
fn layout(blocks: Vec<Block>) -> Layout {
Layout {
blocks,
furniture: Vec::new(),
}
}
fn profile(extra: &str) -> Profile {
let src = format!(
r#"
name = "test"
jurisdiction = "guam"
kind = "rule"
citation_template = "G. R. {{num}}"
heading_patterns = ['^Rule\s+(?P<num>\d+(?:\.\d+)?)\.?\s+(?P<heading>\S.*)$']
date_formats = ["%B %d, %Y"]
{extra}
"#
);
Profile::from_toml(&src).expect("test profile")
}
#[test]
fn splits_provisions_and_renders_citations() {
let l = layout(vec![
block(vec![line("Rule 1. Scope.", 72.0, 0)]),
block(vec![line("These rules govern.", 72.0, 0)]),
block(vec![line("Rule 2. One Form of Action.", 72.0, 1)]),
block(vec![line("There is one form of action.", 72.0, 1)]),
]);
let doc = detect(&l, &profile(""), "x.pdf").unwrap();
assert_eq!(doc.provisions.len(), 2);
let p = &doc.provisions[0];
assert_eq!(p.citation, "G. R. 1");
assert_eq!(p.number, "1");
assert_eq!(p.heading.as_deref(), Some("Scope."));
assert_eq!(p.text, "These rules govern.");
assert_eq!(p.pages, [1, 1]);
assert_eq!(doc.provisions[1].pages, [2, 2]);
assert!((doc.line_coverage - 1.0).abs() < 1e-6);
assert!(doc.unconsumed_lines.is_empty());
assert_eq!(doc.lines_consumed, 4);
}
#[test]
fn heading_line_remainder_after_match_becomes_body_text() {
let ri_profile = {
let src = r#"
name = "test-ri"
jurisdiction = "rhode_island"
kind = "rule"
citation_template = "R.I. Super. R. Civ. P. {num}"
heading_patterns = ['^(?P<num>\d+\.\d+)\.\s+(?P<heading>[^—]+)—\s*']
date_formats = ["%B %d, %Y"]
"#;
Profile::from_toml(src).expect("test profile")
};
let l = layout(vec![block(vec![
line(
"23.1. Derivative actions by shareholders — In a derivative action brought by",
72.0,
0,
),
line(
"one (1) or more shareholders to enforce a right of the corporation.",
72.0,
0,
),
])]);
let doc = detect(&l, &ri_profile, "x.pdf").unwrap();
assert_eq!(doc.provisions.len(), 1);
let p = &doc.provisions[0];
assert_eq!(
p.heading.as_deref(),
Some("Derivative actions by shareholders")
);
assert_eq!(
p.text,
"In a derivative action brought by\n\
one (1) or more shareholders to enforce a right of the corporation."
);
}
#[test]
fn text_keeps_paragraph_breaks_between_blocks() {
let l = layout(vec![
block(vec![line("Rule 1. Scope.", 72.0, 0)]),
block(vec![line("First paragraph.", 72.0, 0)]),
block(vec![line("Second paragraph.", 72.0, 0)]),
]);
let doc = detect(&l, &profile(""), "x.pdf").unwrap();
assert_eq!(
doc.provisions[0].text,
"First paragraph.\n\nSecond paragraph."
);
}
#[test]
fn preamble_without_skip_rule_is_unconsumed_and_hurts_coverage() {
let l = layout(vec![
block(vec![line("SOME COVER PAGE", 72.0, 0)]),
block(vec![line("Rule 1. Scope.", 72.0, 1)]),
block(vec![line("Body.", 72.0, 1)]),
]);
let doc = detect(&l, &profile(""), "x.pdf").unwrap();
assert_eq!(doc.unconsumed_lines, vec!["SOME COVER PAGE".to_string()]);
assert!(doc.line_coverage < 1.0);
}
#[test]
fn skip_region_runs_until_the_next_accepted_heading() {
let l = layout(vec![
block(vec![line("E-FILED", 72.0, 0)]),
block(vec![line("TABLE OF CONTENTS", 72.0, 0)]),
block(vec![line("Rule 1. Scope.", 72.0, 1)]),
block(vec![
line("Body.", 72.0, 1),
line("Notes", 72.0, 1),
line("[1] A committee note.", 72.0, 1),
]),
block(vec![line("Rule 2. Next.", 72.0, 2)]),
block(vec![line("More body.", 72.0, 2)]),
]);
let doc = detect(
&l,
&profile("skip_patterns = ['^E-FILED$', '^Notes$']"),
"x.pdf",
)
.unwrap();
assert_eq!(doc.provisions.len(), 2);
assert_eq!(doc.provisions[0].text, "Body.");
assert_eq!(doc.provisions[1].text, "More body.");
assert!(
doc.unconsumed_lines.is_empty(),
"{:?}",
doc.unconsumed_lines
);
assert_eq!(doc.lines_skipped, 4); assert!((doc.line_coverage - 1.0).abs() < 1e-6);
}
#[test]
fn monotonic_guard_rejects_backward_and_equal_numbers() {
let l = layout(vec![
block(vec![line("Rule 23.1. Derivative Actions.", 72.0, 0)]),
block(vec![line(
"Rule 23.1 applies when a shareholder sues.",
72.0,
0,
)]),
block(vec![line("Rule 24. Intervention.", 72.0, 0)]),
block(vec![line(
"Rule 12. A cross-reference, not a rule.",
72.0,
0,
)]),
]);
let p = profile("heading_x0_cluster = false");
let doc = detect(&l, &p, "x.pdf").unwrap();
let nums: Vec<&str> = doc.provisions.iter().map(|p| p.number.as_str()).collect();
assert_eq!(nums, vec!["23.1", "24"]);
assert!(doc.provisions[0]
.text
.contains("applies when a shareholder"));
assert!(doc.provisions[1].text.contains("cross-reference"));
}
#[test]
fn x0_cluster_rejects_candidates_at_body_indents() {
let l = layout(vec![
block(vec![line("Rule 1. Scope.", 144.0, 0)]),
block(vec![line("Body text.", 208.8, 0)]),
block(vec![line("Rule 2. Service.", 144.0, 0)]),
block(vec![line(
"Rule 3. By order of the court, service may",
208.8,
0,
)]),
block(vec![line("Rule 3. Commencement.", 144.0, 0)]),
block(vec![line("Rule 4. Summons.", 144.0, 0)]),
]);
let doc = detect(&l, &profile(""), "x.pdf").unwrap();
let nums: Vec<&str> = doc.provisions.iter().map(|p| p.number.as_str()).collect();
assert_eq!(nums, vec!["1", "2", "3", "4"]);
assert!(doc.provisions[1].text.contains("By order of the court"));
}
#[test]
fn slightly_indented_heading_stays_in_the_cluster() {
let l = layout(vec![
block(vec![line("Rule 7. Pleadings.", 144.0, 0)]),
block(vec![line("Rule 7.1 Disclosure Statement.", 153.1, 0)]),
block(vec![line("Rule 8. General Rules.", 144.0, 0)]),
]);
let doc = detect(&l, &profile(""), "x.pdf").unwrap();
let nums: Vec<&str> = doc.provisions.iter().map(|p| p.number.as_str()).collect();
assert_eq!(nums, vec!["7", "7.1", "8"]);
}
#[test]
fn history_block_is_captured_and_excluded_from_text() {
let l = layout(vec![
block(vec![line("Rule 5. Service.", 72.0, 0)]),
block(vec![line("Body of the rule.", 72.0, 0)]),
block(vec![
line("Source: PL 1-8, tit. 1, § 1; amended by PL", 72.0, 0),
line("13-1, § 4(a), effective March 2, 2004.", 72.0, 0),
]),
]);
let doc = detect(&l, &profile("history_patterns = ['^Source:']"), "x.pdf").unwrap();
let p = &doc.provisions[0];
assert_eq!(p.text, "Body of the rule.");
assert_eq!(
p.history.as_deref(),
Some(
"Source: PL 1-8, tit. 1, § 1; amended by PL 13-1, § 4(a), effective March 2, 2004."
)
);
assert_eq!(p.effective_date, "2004-03-02");
assert!(p.confidence_basis.contains("date parsed from history"));
}
#[test]
fn last_history_date_wins() {
let l = layout(vec![
block(vec![line("Rule 5. Service.", 72.0, 0)]),
block(vec![line(
"Adopted January 9, 1998; amended effective March 2, 2004.",
72.0,
0,
)]),
]);
let doc = detect(
&l,
&profile("history_patterns = ['(?i)adopted|amended']"),
"x.pdf",
)
.unwrap();
assert_eq!(doc.provisions[0].effective_date, "2004-03-02");
}
#[test]
fn history_dates_inside_skipped_regions_still_count() {
let l = layout(vec![
block(vec![line("Rule 5. Service.", 72.0, 0)]),
block(vec![line("Body.", 72.0, 0)]),
block(vec![line("Commission Comment: history follows.", 72.0, 0)]),
block(vec![line(
"PL 13-1 took effect February 13, 2002.",
72.0,
0,
)]),
]);
let doc = detect(
&l,
&profile(
"history_patterns = ['(?i)took effect']\nskip_patterns = ['^Commission Comment:']",
),
"x.pdf",
)
.unwrap();
let p = &doc.provisions[0];
assert_eq!(p.effective_date, "2002-02-13");
assert_eq!(p.text, "Body.", "skipped lines stay out of the text");
assert!(
p.history.is_none(),
"skipped lines stay out of the history field"
);
assert_eq!(doc.lines_skipped, 2);
}
#[test]
fn adoption_date_chain_pattern_then_static_then_sentinel() {
let mk = || {
layout(vec![
block(vec![line("Effective January 9, 2019", 72.0, 0)]),
block(vec![line("Rule 1. Scope.", 72.0, 0)]),
block(vec![line("Body.", 72.0, 0)]),
])
};
let doc = detect(
&mk(),
&profile("adoption_patterns = ['^Effective\\s+(?P<date>.+)$']"),
"x.pdf",
)
.unwrap();
assert_eq!(doc.provisions[0].effective_date, "2019-01-09");
assert!(doc.provisions[0]
.confidence_basis
.contains("date from document adoption"));
let doc = detect(&mk(), &profile("adoption_date = \"2002-02-13\""), "x.pdf").unwrap();
assert_eq!(doc.provisions[0].effective_date, "2002-02-13");
let doc = detect(&mk(), &profile(""), "x.pdf").unwrap();
assert_eq!(doc.provisions[0].effective_date, SENTINEL_DATE);
assert!(doc.provisions[0].confidence_basis.contains("sentinel"));
}
#[test]
fn title_placeholder_resolves_from_furniture() {
let mut l = layout(vec![
block(vec![line("Rule 1. Scope.", 72.0, 0)]),
block(vec![line("Body.", 72.0, 0)]),
]);
l.furniture.push(line("TITLE 1: GOVERNMENT", 200.0, 0));
let p = Profile::from_toml(
r#"
name = "t"
jurisdiction = "northern_mariana_islands"
kind = "statute"
citation_template = "{title} CMC § {num}"
heading_patterns = ['^Rule\s+(?P<num>\d+)\.\s+(?P<heading>.+)$']
title_patterns = ['^TITLE\s+(?P<title>\d+)\b']
"#,
)
.unwrap();
let doc = detect(&l, &p, "x.pdf").unwrap();
assert_eq!(doc.provisions[0].citation, "1 CMC § 1");
assert!(doc.provisions[0].warnings.is_empty());
l.furniture.clear();
let doc = detect(&l, &p, "x.pdf").unwrap();
let prov = &doc.provisions[0];
assert_eq!(prov.citation, "{title} CMC § 1");
assert!(prov.warnings[0].contains("{title} unresolved"));
assert!(prov.confidence_basis.contains("citation unresolved"));
}
#[test]
fn sequence_gaps_and_expected_gaps() {
let l = layout(vec![
block(vec![line("Rule 1. A.", 72.0, 0)]),
block(vec![line("Rule 2. B.", 72.0, 0)]),
block(vec![line("Rule 5. C.", 72.0, 0)]),
]);
let doc = detect(&l, &profile(""), "x.pdf").unwrap();
assert_eq!(doc.sequence_gaps, vec!["3", "4"]);
assert!(doc.warnings.iter().any(|w| w.contains("sequence gap: 3")));
let doc = detect(&l, &profile("expected_gaps = ['3', '4']"), "x.pdf").unwrap();
assert!(doc.sequence_gaps.is_empty());
assert!(doc.warnings.is_empty());
}
#[test]
fn duplicates_flagged_when_monotonic_is_off() {
let l = layout(vec![
block(vec![line("Rule 1. A.", 72.0, 0)]),
block(vec![line("Rule 1. A again.", 72.0, 0)]),
]);
let doc = detect(&l, &profile("monotonic_numbers = false"), "x.pdf").unwrap();
assert_eq!(doc.provisions.len(), 2);
assert_eq!(doc.sequence_duplicates, vec!["1"]);
assert!(doc.warnings.iter().any(|w| w.contains("duplicate")));
}
#[test]
fn bold_style_hint_degrades_without_backend_support() {
let mut heading = line("Rule 1. Scope.", 72.0, 0);
heading.bold = false; let l = layout(vec![
block(vec![heading]),
block(vec![line("Body.", 72.0, 0)]),
]);
let p = profile("[heading_style]\nbold = true");
let doc = detect(&l, &p, "x.pdf").unwrap();
assert_eq!(
doc.provisions.len(),
1,
"hint must not fire without bold data"
);
let mut bold_body = line("Rule 2. Not a heading, just bold-less.", 72.0, 0);
bold_body.bold = false;
let mut bold_somewhere = line("Something bold.", 72.0, 0);
bold_somewhere.bold = true;
let l = layout(vec![block(vec![bold_somewhere]), block(vec![bold_body])]);
let doc = detect(&l, &p, "x.pdf").unwrap();
assert!(doc.provisions.is_empty(), "non-bold heading rejected");
}
#[test]
fn confidence_formula_is_the_documented_product() {
let (c, basis) = confidence(1.0, true, DateSource::Parsed);
assert_eq!(c, 1.0);
assert!(basis.contains("coverage=100.0%"));
let (c, _) = confidence(0.99, true, DateSource::Adoption);
assert_eq!(c, 0.96); let (c, _) = confidence(1.0, false, DateSource::Sentinel);
assert_eq!(c, 0.77); }
#[test]
fn num_key_orders_dotted_and_suffixed_numbers() {
assert!(num_key("7.1") > num_key("7"));
assert!(num_key("8") > num_key("7.1"));
assert!(num_key("23.2") > num_key("23.1"));
assert!(num_key("24") > num_key("23.2"));
assert!(num_key("12a") > num_key("12"));
assert_eq!(
num_key("23.1"),
(vec![(23, String::new()), (1, String::new())], String::new())
);
}
#[test]
fn num_key_orders_lettered_and_chapter_section_numbers() {
assert!(num_key("4") < num_key("4A"));
assert!(num_key("4A") < num_key("4B"));
assert!(num_key("4C") < num_key("5"));
assert!(num_key("1-9") < num_key("1-9A"));
assert!(num_key("1-9A") < num_key("1-10"));
assert!(num_key("1-25") < num_key("2-1"));
assert!(num_key("32-12") < num_key("32a-1"));
assert!(num_key("32a-1") < num_key("33-1"));
assert!(num_key("34a-20") < num_key("35-1"));
assert!(num_key("362.150") < num_key("362.155"));
}
#[test]
fn toc_restart_reclassifies_the_first_pass_as_front_matter() {
let l = layout(vec![
block(vec![
line("Rule 1. Scope.", 72.0, 0),
line("Rule 2. One Form of Action.", 72.0, 0),
line("Rule 3. Commencement.", 72.0, 0),
]),
block(vec![line("Rule 1. Scope.", 72.0, 1)]),
block(vec![line("These rules govern.", 72.0, 1)]),
block(vec![line("Rule 2. One Form of Action.", 72.0, 1)]),
block(vec![line("There is one form of action.", 72.0, 1)]),
block(vec![line("Rule 3. Commencement.", 72.0, 2)]),
block(vec![line("An action commences by filing.", 72.0, 2)]),
]);
let doc = detect(&l, &profile("toc_restart = true"), "x.pdf").unwrap();
let nums: Vec<&str> = doc.provisions.iter().map(|p| p.number.as_str()).collect();
assert_eq!(nums, vec!["1", "2", "3"]);
assert_eq!(doc.provisions[0].text, "These rules govern.");
assert_eq!(doc.provisions[0].pages, [2, 2], "TOC page discarded");
assert_eq!(doc.lines_skipped, 3, "the TOC pass is skipped, not lost");
assert!(doc.unconsumed_lines.is_empty());
assert!((doc.line_coverage - 1.0).abs() < 1e-6);
assert!(doc
.warnings
.iter()
.any(|w| w.contains("table-of-contents pass discarded: 3 provision(s)")));
let doc = detect(&l, &profile(""), "x.pdf").unwrap();
assert_eq!(doc.provisions.len(), 3);
assert!(doc.provisions[2].text.contains("These rules govern."));
}
#[test]
fn toc_restart_fires_at_most_once() {
let l = layout(vec![
block(vec![
line("Rule 1. Scope.", 72.0, 0),
line("Rule 2. One Form.", 72.0, 0),
]),
block(vec![line("Rule 1. Scope.", 72.0, 1)]),
block(vec![line("Body one.", 72.0, 1)]),
block(vec![line("Rule 2. One Form.", 72.0, 1)]),
block(vec![line("Rule 1. Scope. governs this rule too.", 72.0, 1)]),
block(vec![line("Rule 3. Commencement.", 72.0, 2)]),
block(vec![line("Body three.", 72.0, 2)]),
]);
let p = profile("toc_restart = true\nheading_x0_cluster = false");
let doc = detect(&l, &p, "x.pdf").unwrap();
let nums: Vec<&str> = doc.provisions.iter().map(|p| p.number.as_str()).collect();
assert_eq!(nums, vec!["1", "2", "3"]);
assert!(doc.provisions[1].text.contains("governs this rule too"));
}
#[test]
fn skip_block_patterns_skip_exactly_the_matching_block() {
let l = layout(vec![
block(vec![line("Rule 1. Scope.", 72.0, 0)]),
block(vec![line("Body starts here and", 72.0, 0)]),
block(vec![
line("Sec.", 313.0, 0),
line("1-12. Court Opening", 313.0, 0),
line("1-13. Recess and Adjournment", 313.0, 0),
]),
block(vec![line("continues after the index.", 313.0, 0)]),
]);
let doc = detect(
&l,
&profile(r#"skip_block_patterns = ['^Sec\.$']"#),
"x.pdf",
)
.unwrap();
assert_eq!(doc.provisions.len(), 1);
assert_eq!(
doc.provisions[0].text,
"Body starts here and\n\ncontinues after the index."
);
assert_eq!(doc.lines_skipped, 3);
assert!(doc.unconsumed_lines.is_empty());
}
#[test]
fn stop_marker_ends_detection_and_date_feeding() {
let l = layout(vec![
block(vec![line("Rule 1. Scope.", 72.0, 0)]),
block(vec![line("Body.", 72.0, 0)]),
block(vec![line("(Amended June 20, 2011.)", 72.0, 0)]),
block(vec![line("APPENDIX OF FORMS", 72.0, 1)]),
block(vec![line("(Amended June 12, 2025.)", 72.0, 1)]),
block(vec![line("Rule 9. A form caption, not a rule.", 72.0, 1)]),
]);
let doc = detect(
&l,
&profile(
r#"history_patterns = ['^\(Amended']
stop_patterns = ['^APPENDIX OF FORMS$']"#,
),
"x.pdf",
)
.unwrap();
assert_eq!(doc.provisions.len(), 1);
let p = &doc.provisions[0];
assert_eq!(p.effective_date, "2011-06-20");
assert_eq!(p.text, "Body.");
assert_eq!(doc.lines_skipped, 3);
assert!(doc.unconsumed_lines.is_empty());
}
#[test]
fn history_reopens_so_trailing_credits_stay_out_of_text() {
let l = layout(vec![
block(vec![line("Rule 1. Scope.", 72.0, 0)]),
block(vec![line(
"(Amended June 20, 2011, to take effect January 1, 2012.)",
72.0,
0,
)]),
block(vec![line("The rules govern practice.", 72.0, 0)]),
block(vec![
line(
"(P.B. 1978-1997, Sec. 1.) (Amended June 26, 2000, to take",
72.0,
0,
),
line("effect January 1, 2001.)", 72.0, 0),
]),
]);
let doc = detect(
&l,
&profile(r#"history_patterns = ['^\(P\.B\.', '^\(Amended']"#),
"x.pdf",
)
.unwrap();
let p = &doc.provisions[0];
assert_eq!(p.text, "The rules govern practice.");
let history = p.history.as_deref().unwrap();
assert!(history.starts_with("(Amended June 20, 2011"));
assert!(history.ends_with("effect January 1, 2001.)"));
assert_eq!(p.effective_date, "2001-01-01");
}
#[test]
fn empty_layout_yields_empty_document_with_warning() {
let doc = detect(&layout(vec![]), &profile(""), "x.pdf").unwrap();
assert!(doc.provisions.is_empty());
assert!((doc.line_coverage - 1.0).abs() < 1e-6);
assert!(doc.warnings.iter().any(|w| w.contains("no provisions")));
assert!(doc.empty_doc_marker.is_none());
}
#[test]
fn declared_empty_doc_marker_recognizes_repealed_stub() {
let p = profile(
r#"skip_patterns = ['^CHAPTER\s']
empty_doc_patterns = ['^\[(Repealed|Superseded)\b']"#,
);
let l = layout(vec![
block(vec![line("CHAPTER 2-01", 200.0, 0)]),
block(vec![line("REGULATION", 220.0, 0)]),
block(vec![line("[Repealed by S.L. 1947, ch. 1, § 18]", 72.0, 0)]),
]);
let doc = detect(&l, &p, "t02c01.pdf").unwrap();
assert!(doc.provisions.is_empty());
assert_eq!(
doc.empty_doc_marker.as_deref(),
Some("[Repealed by S.L. 1947, ch. 1, § 18]")
);
assert!(
doc.warnings
.iter()
.any(|w| w.contains("recognized empty document")),
"warnings: {:?}",
doc.warnings
);
assert!(
!doc.warnings.iter().any(|w| w.contains("no provisions")),
"a recognized empty document must not also warn as a failure"
);
assert!((doc.line_coverage - 1.0).abs() < 1e-6);
}
#[test]
fn zero_provisions_without_marker_still_warns() {
let p = profile(r#"empty_doc_patterns = ['^\[Repealed\b']"#);
let l = layout(vec![block(vec![line("Some stray content.", 72.0, 0)])]);
let doc = detect(&l, &p, "x.pdf").unwrap();
assert!(doc.provisions.is_empty());
assert!(doc.empty_doc_marker.is_none());
assert!(doc.warnings.iter().any(|w| w.contains("no provisions")));
}
#[test]
fn marker_line_in_a_document_with_provisions_is_ignored() {
let p = profile(r#"empty_doc_patterns = ['^\[Repealed\b']"#);
let l = layout(vec![
block(vec![line("Rule 1. Scope.", 72.0, 0)]),
block(vec![line("[Repealed by S.L. 1947, ch. 1, § 18]", 72.0, 0)]),
]);
let doc = detect(&l, &p, "x.pdf").unwrap();
assert_eq!(doc.provisions.len(), 1);
assert!(
doc.empty_doc_marker.is_none(),
"empty_doc_patterns must only apply to provision-less documents"
);
}
}