use std::collections::HashSet;
use super::Line;
const INDENT_TOL: f32 = 3.5;
const BLOCK_GAP_FACTOR: f32 = 1.7;
const SIZE_BREAK: f32 = 1.5;
const COMPOUND_PREFIXES: &[&str] = &[
"self", "non", "cross", "well", "quasi", "anti", "semi", "ex", "half", "all", "co",
];
pub fn quantize_indents(lines: &mut [Line]) {
let mut keys: Vec<(usize, u8)> = lines
.iter()
.map(|l| (l.pages[0], l.column))
.collect::<HashSet<_>>()
.into_iter()
.collect();
keys.sort_unstable();
for key in keys {
let mut xs: Vec<f32> = lines
.iter()
.filter(|l| (l.pages[0], l.column) == key)
.map(|l| l.x0)
.collect();
xs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let mut levels: Vec<f32> = Vec::new();
for x in xs {
match levels.last() {
Some(&last) if x - last <= INDENT_TOL => {}
_ => levels.push(x),
}
}
for line in lines.iter_mut().filter(|l| (l.pages[0], l.column) == key) {
let level = levels
.iter()
.rposition(|&edge| line.x0 >= edge - 0.01)
.unwrap_or(0);
line.indent = level.min(u8::MAX as usize) as u8;
}
}
}
pub fn into_blocks(lines: Vec<Line>) -> Vec<super::Block> {
let vocab = Vocab::build(&lines);
let mut blocks: Vec<super::Block> = Vec::new();
let mut leading = LeadingEstimate::default();
for line in lines {
let start_new = match blocks.last().and_then(|b| b.lines.last()) {
None => true,
Some(prev) => is_block_break(prev, &line, &mut leading),
};
if start_new {
blocks.push(super::Block { lines: vec![line] });
} else {
let block = blocks.last_mut().expect("non-empty on merge");
block.lines.push(line);
let n = block.lines.len();
dehyphenate_pair(&mut block.lines, n - 2, &vocab);
}
}
blocks.retain(|b| !b.lines.is_empty());
blocks
}
#[derive(Default)]
struct LeadingEstimate {
key: Option<(usize, u8)>,
value: Option<f32>,
}
impl LeadingEstimate {
fn observe(&mut self, prev: &Line, cur: &Line) -> f32 {
let key = (cur.pages[0], cur.column);
if self.key != Some(key) {
self.key = Some(key);
self.value = None;
}
let gap = prev.baseline - cur.baseline;
let fallback = 1.25 * cur.font_size.max(1.0);
let current = self.value.unwrap_or(fallback);
if gap > 0.0 && gap <= BLOCK_GAP_FACTOR * current {
let updated = if self.value.is_some() {
0.7 * current + 0.3 * gap
} else {
gap
};
self.value = Some(updated);
}
self.value.unwrap_or(fallback)
}
}
fn is_block_break(prev: &Line, cur: &Line, leading: &mut LeadingEstimate) -> bool {
let prev_page = prev.pages.last().copied().unwrap_or(0);
let same_flow = prev_page == cur.pages[0] && prev.column == cur.column;
if !same_flow {
let continues = !ends_terminal(&prev.text)
&& !cur.centered
&& cur.indent == 0
&& (cur.font_size - prev.font_size).abs() <= 0.7;
return !continues;
}
if (cur.font_size - prev.font_size).abs() > SIZE_BREAK {
return true;
}
if cur.centered != prev.centered {
return true;
}
if trailing_hyphen(&prev.text).is_some() {
return false;
}
if cur.x0 > prev.x0 + INDENT_TOL && ends_terminal(&prev.text) {
return true;
}
let expected = leading.observe(prev, cur);
let gap = prev.baseline - cur.baseline;
gap > BLOCK_GAP_FACTOR * expected
}
fn ends_terminal(text: &str) -> bool {
let trimmed = text.trim_end();
let Some(last) = trimmed
.chars()
.rev()
.find(|c| !matches!(c, '"' | '\'' | '\u{2019}' | '\u{201d}' | ')' | ']' | '}'))
else {
return true; };
matches!(last, '.' | '?' | '!' | ':' | ';')
}
struct Vocab {
words: HashSet<String>,
hyphenated: HashSet<String>,
}
impl Vocab {
fn build(lines: &[Line]) -> Self {
let mut words = HashSet::new();
let mut hyphenated = HashSet::new();
for line in lines {
for (i, token) in line.text.split_whitespace().enumerate() {
let is_last = line.text.split_whitespace().count() == i + 1;
let cleaned: String = token
.trim_matches(|c: char| !c.is_alphabetic() && c != '-')
.to_lowercase();
if cleaned.is_empty() {
continue;
}
if let Some((a, b)) = cleaned.split_once('-') {
if !is_last
&& !a.is_empty()
&& !b.is_empty()
&& a.chars().all(char::is_alphabetic)
&& b.chars().all(char::is_alphabetic)
{
hyphenated.insert(cleaned.clone());
}
} else if cleaned.chars().all(char::is_alphabetic) && cleaned.len() >= 3 {
words.insert(cleaned);
}
}
}
Self { words, hyphenated }
}
}
fn trailing_hyphen(text: &str) -> Option<(usize, char)> {
let trimmed = text.trim_end();
let c = trimmed.chars().last()?;
if matches!(c, '-' | '\u{2010}' | '\u{ad}') {
Some((trimmed.len() - c.len_utf8(), c))
} else {
None
}
}
fn dehyphenate_pair(lines: &mut Vec<Line>, i: usize, vocab: &Vocab) {
let (head_line, tail_line) = (&lines[i], &lines[i + 1]);
let Some((hyphen_at, hyphen_char)) = trailing_hyphen(&head_line.text) else {
return;
};
let head: String = head_line.text[..hyphen_at]
.chars()
.rev()
.take_while(|c| c.is_alphabetic())
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect();
let Some(first_token) = tail_line.text.split_whitespace().next().map(String::from) else {
return;
};
let tail: String = first_token
.chars()
.take_while(|c| c.is_alphabetic())
.collect();
let rest = tail_line
.text
.trim_start()
.strip_prefix(&first_token)
.unwrap_or("")
.trim_start()
.to_string();
let tail_page = tail_line.pages[0];
let soft = hyphen_char == '\u{ad}';
if !soft && (head.chars().count() < 2 || tail.chars().count() < 2) {
return;
}
if head.is_empty() || tail.is_empty() {
return;
}
let fused = format!("{}{}", head.to_lowercase(), tail.to_lowercase());
let compound = format!("{}-{}", head.to_lowercase(), tail.to_lowercase());
let keep_hyphen = if soft || vocab.words.contains(&fused) {
false
} else {
vocab.hyphenated.contains(&compound)
|| tail.chars().next().is_some_and(char::is_uppercase)
|| COMPOUND_PREFIXES.contains(&head.to_lowercase().as_str())
};
let mut new_head = lines[i].text[..hyphen_at].to_string();
if keep_hyphen {
new_head.push('-');
}
new_head.push_str(&first_token);
lines[i].text = new_head;
if !lines[i].pages.contains(&tail_page) {
lines[i].pages.push(tail_page);
}
if rest.is_empty() {
lines.remove(i + 1);
} else {
lines[i + 1].text = rest;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::reconstruct;
use crate::layout::testutil::{body_lines, page, run};
fn all_lines(layout: &crate::layout::Layout) -> Vec<&Line> {
layout.blocks.iter().flat_map(|b| &b.lines).collect()
}
#[test]
fn indents_quantize_per_column() {
let runs = vec![
run("flush line", 72.0, 700.0, 10.0, 0),
run("flush again with wobble", 72.3, 688.0, 10.0, 0),
run("one level in", 90.0, 676.0, 10.0, 0),
run("two levels in", 108.0, 664.0, 10.0, 0),
];
let layout = reconstruct(&[page(runs)]);
let indents: Vec<u8> = all_lines(&layout).iter().map(|l| l.indent).collect();
assert_eq!(indents, vec![0, 0, 1, 2]);
}
#[test]
fn default_hyphen_break_is_repaired() {
let runs = body_lines(
&[
"the office of the Governor, com-",
"posed of the Governor and offices",
],
72.0,
700.0,
12.0,
0,
);
let layout = reconstruct(&[page(runs)]);
let text = layout.text();
assert!(
text.contains("composed"),
"soft line-break hyphen must be repaired: {text}"
);
assert!(text.contains("composed\nof the Governor"));
}
#[test]
fn known_compound_keeps_its_hyphen() {
let runs = body_lines(
&[
"continue the client-lawyer relationship after",
"retention and also the client-",
"lawyer privilege discussion here",
],
72.0,
700.0,
12.0,
0,
);
let layout = reconstruct(&[page(runs)]);
let text = layout.text();
assert!(
text.contains("the client-lawyer\nprivilege"),
"compound seen intact elsewhere keeps its hyphen: {text}"
);
}
#[test]
fn fused_word_seen_elsewhere_drops_the_hyphen() {
let runs = body_lines(
&[
"the commission met yesterday and the com-",
"mission adjourned quickly",
],
72.0,
700.0,
12.0,
0,
);
let layout = reconstruct(&[page(runs)]);
assert!(layout.text().contains("the commission\nadjourned"));
}
#[test]
fn capitalized_tail_keeps_hyphen() {
let runs = body_lines(
&["a treaty between the Anglo-", "American parties was signed"],
72.0,
700.0,
12.0,
0,
);
let layout = reconstruct(&[page(runs)]);
assert!(layout.text().contains("Anglo-American"));
}
#[test]
fn compound_prefix_keeps_hyphen() {
let runs = body_lines(
&["this was a non-", "party subpoena served on the firm"],
72.0,
700.0,
12.0,
0,
);
let layout = reconstruct(&[page(runs)]);
assert!(layout.text().contains("non-party"));
}
#[test]
fn implausible_join_is_left_alone() {
let runs = body_lines(
&["see sections 12-", "14 of the code for details"],
72.0,
700.0,
12.0,
0,
);
let layout = reconstruct(&[page(runs)]);
assert!(
layout.text().contains("12-\n14"),
"numeric ranges must never be fused: {}",
layout.text()
);
}
#[test]
fn paragraph_merges_across_pages_when_sentence_continues() {
let p1 = page(body_lines(
&[
"the powers of the office extend to",
"matters concerning the",
],
72.0,
120.0,
12.0,
0,
));
let p2 = page(body_lines(
&[
"administration of justice generally.",
"A new paragraph starts.",
],
72.0,
700.0,
12.0,
1,
));
let layout = reconstruct(&[p1, p2]);
assert!(
layout.blocks[0]
.lines
.iter()
.any(|l| l.text.contains("administration of justice")),
"continuation must merge into the same block"
);
}
#[test]
fn terminal_punctuation_blocks_the_cross_page_merge() {
let p1 = page(body_lines(
&["the previous provision ends here."],
72.0,
120.0,
12.0,
0,
));
let p2 = page(body_lines(
&["The next provision begins now"],
72.0,
700.0,
12.0,
1,
));
let layout = reconstruct(&[p1, p2]);
assert_eq!(layout.blocks.len(), 2, "terminal stop = separate blocks");
}
#[test]
fn hyphen_pull_across_pages_records_both_pages() {
let p1 = page(body_lines(
&["this provision was under con-"],
72.0,
120.0,
12.0,
0,
));
let p2 = page(body_lines(
&["sideration by the committee today"],
72.0,
700.0,
12.0,
1,
));
let layout = reconstruct(&[p1, p2]);
let text = layout.text();
assert!(text.contains("consideration"), "got: {text}");
let first = &layout.blocks[0].lines[0];
assert_eq!(first.pages, vec![0, 1], "merged line spans both pages");
}
#[test]
fn indent_jump_starts_a_new_block() {
let runs = vec![
run(
"Section 101. Something is stated here",
146.0,
700.0,
10.0,
0,
),
run(
"and continues at the flush margin left.",
134.0,
688.0,
10.0,
0,
),
run(
"Section 102. A new subdivision starts",
146.0,
676.0,
10.0,
0,
),
];
let layout = reconstruct(&[page(runs)]);
assert_eq!(layout.blocks.len(), 2, "indent jump right = new block");
}
#[test]
fn hanging_indent_continuation_stays_in_its_block() {
let runs = vec![
run(
"Chapter 33 Hearings concerning Termina-",
80.0,
700.0,
10.0,
0,
),
run("tion of Parental Rights . . . 356", 100.0, 688.0, 10.0, 0),
];
let layout = reconstruct(&[page(runs)]);
assert_eq!(layout.blocks.len(), 1, "hanging indent is a continuation");
assert!(
layout.text().contains("Termination"),
"hyphen repaired across the hanging indent: {}",
layout.text()
);
}
#[test]
fn vertical_gap_starts_a_new_block() {
let runs = vec![
run("first paragraph line one here", 72.0, 700.0, 10.0, 0),
run("first paragraph line two here", 72.0, 688.0, 10.0, 0),
run("second paragraph after a gap", 72.0, 652.0, 10.0, 0),
];
let layout = reconstruct(&[page(runs)]);
assert_eq!(layout.blocks.len(), 2);
}
#[test]
fn word_stream_round_trips_through_layout() {
let words: Vec<String> = (0..600).map(|i| format!("word{i}")).collect();
let mut rng = 0x2545f4914f6cdd1du64;
let mut next = |bound: usize| {
rng ^= rng << 13;
rng ^= rng >> 7;
rng ^= rng << 17;
(rng % bound as u64) as usize
};
for _trial in 0..5 {
let mut pages_vec = Vec::new();
let mut remaining = words.clone();
let mut page_index = 0;
while !remaining.is_empty() {
let mut runs = vec![run("REPEATED HEADER", 220.0, 772.0, 9.0, page_index)];
let mut y = 690.0; while !remaining.is_empty() && y > 100.0 {
let take = (3 + next(6)).min(remaining.len());
let line: Vec<String> = remaining.drain(..take).collect();
runs.push(run(&line.join(" "), 72.0, y, 10.0, page_index));
y -= 12.0;
}
runs.push(run(
&format!("{}", page_index + 1),
300.0,
40.0,
10.0,
page_index,
));
pages_vec.push(page(runs));
page_index += 1;
}
let layout = reconstruct(&pages_vec);
let text = layout.text();
let got: Vec<&str> = text.split_whitespace().collect();
let want: Vec<&str> = words.iter().map(String::as_str).collect();
assert_eq!(got, want, "word stream must survive the round trip");
assert_eq!(
layout.furniture.len(),
2 * pages_vec.len(),
"header + page number accounted per page"
);
}
}
}