use std::collections::HashMap;
use std::sync::OnceLock;
use regex::Regex;
use crate::extract::PageRuns;
use super::Line;
const TOP_BAND: f32 = 0.88;
const BOTTOM_BAND: f32 = 0.10;
const MIN_REPEAT_PAGES: usize = 3;
#[derive(PartialEq, Eq, Hash, Clone, Copy)]
enum Band {
Top,
Bottom,
}
pub fn strip(lines: &mut Vec<Line>, pages: &[PageRuns]) -> Vec<Line> {
let n_pages = pages.len();
let min_repeat = if n_pages <= 3 { 2 } else { MIN_REPEAT_PAGES };
let band_of = |line: &Line| -> Option<Band> {
let page = line.pages.first().copied()?;
let height = pages.get(page)?.height;
if height <= 0.0 {
return None;
}
let frac = line.baseline / height;
if frac >= TOP_BAND {
Some(Band::Top)
} else if frac <= BOTTOM_BAND {
Some(Band::Bottom)
} else {
None
}
};
let mut seen: HashMap<(Band, String), Vec<usize>> = HashMap::new();
for line in lines.iter() {
if let Some(band) = band_of(line) {
let key = (band, mask(&line.text));
let page = line.pages[0];
let entry = seen.entry(key).or_default();
if entry.last() != Some(&page) {
entry.push(page);
}
}
}
let mut stripped = Vec::new();
lines.retain(|line| {
let Some(band) = band_of(line) else {
return true;
};
let masked = mask(&line.text);
let repeats = seen
.get(&(band, masked.clone()))
.map(|pages| pages.len())
.unwrap_or(0);
if repeats >= min_repeat || is_page_number(&masked) {
stripped.push(line.clone());
false
} else {
true
}
});
stripped
}
pub fn mask(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut in_digits = false;
let mut in_space = false;
for c in text.trim().chars() {
if c.is_ascii_digit() || (in_digits && c.is_ascii_alphabetic()) {
if !in_digits {
out.push('#');
}
in_digits = true;
in_space = false;
} else if c.is_whitespace() {
if !in_space && !out.is_empty() {
out.push(' ');
}
in_space = true;
in_digits = false;
} else {
out.extend(c.to_lowercase());
in_digits = false;
in_space = false;
}
}
out.trim_end().to_string()
}
fn is_page_number(masked: &str) -> bool {
static RE: OnceLock<Regex> = OnceLock::new();
let re = RE.get_or_init(|| {
Regex::new(
r"^[-–—.\s]*(?:page\s+)?(#|m{0,4}(?:cm|cd|d?c{0,3})(?:xc|xl|l?x{0,3})(?:ix|iv|v?i{0,3}))[-–—.\s]*$",
)
.expect("static regex")
});
match re.captures(masked) {
Some(caps) => !caps[1].is_empty(), None => false,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::reconstruct;
use crate::layout::testutil::{body_lines, page, run};
#[test]
fn masking_equates_page_numbers() {
assert_eq!(mask("Page 12"), mask("Page 13"));
assert_eq!(mask("TITLE 1: GOVERNMENT"), "title #: government");
assert_ne!(mask("Rule 1.8"), mask("Rule 12"));
assert_eq!(mask("Rule 1.8"), mask("Rule 2.3"));
}
#[test]
fn standalone_page_numbers_detected() {
assert!(is_page_number(&mask("12")));
assert!(is_page_number(&mask("- 12 -")));
assert!(is_page_number(&mask("Page 7")));
assert!(is_page_number(&mask("xiv")));
assert!(!is_page_number(&mask("Rule 12 continued")));
assert!(!is_page_number(&mask("mild"))); assert!(!is_page_number(&mask(""))); }
#[test]
fn repeated_headers_and_page_numbers_stripped_with_accounting() {
let mut pages_vec = Vec::new();
for p in 0..4 {
let mut runs = vec![
run("MY RUNNING HEAD", 240.0, 772.0, 9.0, p),
run(&format!("{}", p + 1), 300.0, 40.0, 10.0, p),
];
let second = format!("and differs on page {p} entirely");
runs.extend(body_lines(
&["unique body sentence that continues along", &second],
72.0,
620.0,
12.0,
p,
));
pages_vec.push(page(runs));
}
let layout = reconstruct(&pages_vec);
let text = layout.text();
assert!(!text.contains("MY RUNNING HEAD"), "header must be stripped");
assert_eq!(
layout.furniture.len(),
8,
"4 headers + 4 page numbers recorded"
);
assert!(text.contains("unique body sentence"));
}
#[test]
fn body_text_in_band_is_kept_without_repetition() {
let mut runs = body_lines(&["body body body"], 72.0, 700.0, 12.0, 0);
runs.push(run("reinstated in its entirety.", 72.0, 60.0, 10.0, 0));
let layout = reconstruct(&[page(runs)]);
assert!(layout.text().contains("reinstated in its entirety."));
assert!(layout.furniture.is_empty());
}
}