use crate::rules::Rule;
use pdf_extract::TextBlock;
const LINE_Y_TOLERANCE: f64 = 2.0;
const PARAGRAPH_GAP_FACTOR: f64 = 1.5;
const STYLE_SIZE_STEP: f64 = 1.0;
const TABLE_X_TOLERANCE: f64 = 5.0;
#[derive(Debug, Clone)]
pub struct Run {
pub text: String,
pub font_name: String,
pub font_size: f64,
pub bold: bool,
pub italic: bool,
}
#[derive(Debug, Clone)]
pub struct Paragraph {
pub runs: Vec<Run>,
}
#[derive(Debug, Clone)]
pub struct Table {
pub rows: Vec<Vec<String>>,
pub col_count: usize,
}
#[derive(Debug, Clone)]
pub struct DocxImage {
pub data: Vec<u8>,
pub width: u32,
pub height: u32,
pub content_type: String,
pub id: String,
}
#[derive(Debug, Clone)]
pub enum PageElement {
Para(Paragraph),
Tbl(Table),
Img(DocxImage),
}
#[derive(Debug)]
struct Line {
y: f64,
font_size: f64,
blocks: Vec<TextBlock>,
}
pub fn analyze_page(blocks: &[TextBlock]) -> Vec<PageElement> {
analyze_page_with_rules(blocks, &[])
}
pub fn analyze_page_with_rules(blocks: &[TextBlock], rules: &[Rule]) -> Vec<PageElement> {
if blocks.is_empty() {
return Vec::new();
}
let grids = ruled_grids(rules);
if grids.is_empty() {
let lines = group_into_lines(blocks);
if let Some(tbl) = try_detect_table(&lines) {
return vec![PageElement::Tbl(tbl)];
}
return group_into_paragraphs(&lines)
.into_iter()
.map(|(_, e)| e)
.collect();
}
let mut inside: Vec<Vec<TextBlock>> = vec![Vec::new(); grids.len()];
let mut outside: Vec<TextBlock> = Vec::new();
for block in blocks {
let cx = (block.bbox[0] + block.bbox[2]) / 2.0;
let cy = (block.bbox[1] + block.bbox[3]) / 2.0;
match grids.iter().position(|g| g.contains(cx, cy)) {
Some(i) => inside[i].push(block.clone()),
None => outside.push(block.clone()),
}
}
let mut items: Vec<(f64, PageElement)> = Vec::new();
for (grid, cell_blocks) in grids.iter().zip(&inside) {
items.push((grid.y1, PageElement::Tbl(grid_table(grid, cell_blocks))));
}
let lines = group_into_lines(&outside);
let section = |y: f64| grids.iter().filter(|g| g.y0 > y).count();
let mut start = 0;
while start < lines.len() {
let id = section(lines[start].y);
let mut end = start + 1;
while end < lines.len() && section(lines[end].y) == id {
end += 1;
}
items.extend(group_into_paragraphs(&lines[start..end]));
start = end;
}
items.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
items.into_iter().map(|(_, e)| e).collect()
}
#[derive(Debug)]
struct Grid {
x0: f64,
y0: f64,
x1: f64,
y1: f64,
xs: Vec<f64>,
ys: Vec<f64>,
}
impl Grid {
fn contains(&self, x: f64, y: f64) -> bool {
x >= self.x0 - 1.0 && x <= self.x1 + 1.0 && y >= self.y0 - 1.0 && y <= self.y1 + 1.0
}
}
const RULE_TOLERANCE: f64 = 2.0;
fn cluster(mut values: Vec<f64>) -> Vec<f64> {
values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let mut out: Vec<(f64, usize)> = Vec::new();
for v in values {
match out.last_mut() {
Some((mean, n)) if (v - *mean).abs() <= RULE_TOLERANCE => {
*mean = (*mean * *n as f64 + v) / (*n as f64 + 1.0);
*n += 1;
}
_ => out.push((v, 1)),
}
}
out.into_iter().map(|(m, _)| m).collect()
}
fn find(parent: &mut [usize], mut i: usize) -> usize {
while parent[i] != i {
parent[i] = parent[parent[i]];
i = parent[i];
}
i
}
fn ruled_grids(rules: &[Rule]) -> Vec<Grid> {
let n = rules.len();
if n < 4 {
return Vec::new();
}
let mut parent: Vec<usize> = (0..n).collect();
for i in 0..n {
for j in (i + 1)..n {
let (a, b) = (&rules[i], &rules[j]);
let touch = a.x0 - RULE_TOLERANCE <= b.x1
&& b.x0 - RULE_TOLERANCE <= a.x1
&& a.y0 - RULE_TOLERANCE <= b.y1
&& b.y0 - RULE_TOLERANCE <= a.y1;
if touch {
let (ri, rj) = (find(&mut parent, i), find(&mut parent, j));
parent[ri] = rj;
}
}
}
let mut groups: std::collections::BTreeMap<usize, Vec<usize>> = Default::default();
for i in 0..n {
let root = find(&mut parent, i);
groups.entry(root).or_default().push(i);
}
let mut grids = Vec::new();
for members in groups.values() {
let mut xs = Vec::new();
let mut ys = Vec::new();
let (mut x0, mut y0, mut x1, mut y1) = (f64::MAX, f64::MAX, f64::MIN, f64::MIN);
for &m in members {
let r = &rules[m];
if r.is_horizontal() {
ys.push((r.y0 + r.y1) / 2.0);
} else {
xs.push((r.x0 + r.x1) / 2.0);
}
x0 = x0.min(r.x0);
y0 = y0.min(r.y0);
x1 = x1.max(r.x1);
y1 = y1.max(r.y1);
}
let xs = cluster(xs);
let mut ys = cluster(ys);
if xs.len() < 3 || ys.len() < 3 {
continue;
}
ys.reverse();
grids.push(Grid {
x0,
y0,
x1,
y1,
xs,
ys,
});
}
grids
}
fn band(bounds: &[f64], v: f64, descending: bool) -> usize {
let cells = bounds.len() - 1;
for i in 0..cells {
let inside = if descending {
v <= bounds[i] && v > bounds[i + 1]
} else {
v >= bounds[i] && v < bounds[i + 1]
};
if inside {
return i;
}
}
let first = bounds[0];
let past_start = if descending { v > first } else { v < first };
if past_start {
0
} else {
cells - 1
}
}
fn grid_table(grid: &Grid, blocks: &[TextBlock]) -> Table {
let cols = grid.xs.len() - 1;
let rows = grid.ys.len() - 1;
let mut cells: Vec<Vec<Vec<TextBlock>>> = vec![vec![Vec::new(); cols]; rows];
for block in blocks {
let cx = (block.bbox[0] + block.bbox[2]) / 2.0;
let cy = (block.bbox[1] + block.bbox[3]) / 2.0;
let c = band(&grid.xs, cx, false);
let r = band(&grid.ys, cy, true);
cells[r][c].push(block.clone());
}
let rows = cells
.into_iter()
.map(|row| {
row.into_iter()
.map(|cell| {
group_into_lines(&cell)
.iter()
.map(line_text)
.collect::<Vec<_>>()
.join(" ")
})
.collect()
})
.collect();
Table {
rows,
col_count: cols,
}
}
fn line_text(line: &Line) -> String {
line.blocks
.iter()
.map(|b| b.text.as_str())
.collect::<Vec<_>>()
.join(" ")
}
fn group_into_lines(blocks: &[TextBlock]) -> Vec<Line> {
let mut sorted: Vec<&TextBlock> = blocks.iter().collect();
sorted.sort_by(|a, b| {
let y_cmp = b.bbox[1]
.partial_cmp(&a.bbox[1])
.unwrap_or(std::cmp::Ordering::Equal);
if y_cmp == std::cmp::Ordering::Equal {
a.bbox[0]
.partial_cmp(&b.bbox[0])
.unwrap_or(std::cmp::Ordering::Equal)
} else {
y_cmp
}
});
let mut lines: Vec<Line> = Vec::new();
for block in sorted {
let y = block.bbox[1];
let matched = lines
.iter_mut()
.find(|line| (line.y - y).abs() < LINE_Y_TOLERANCE);
if let Some(line) = matched {
line.blocks.push(block.clone());
} else {
lines.push(Line {
y,
font_size: block.font_size,
blocks: vec![block.clone()],
});
}
}
for line in &mut lines {
line.blocks.sort_by(|a, b| {
a.bbox[0]
.partial_cmp(&b.bbox[0])
.unwrap_or(std::cmp::Ordering::Equal)
});
line.blocks = merge_glyph_runs(std::mem::take(&mut line.blocks));
}
lines
}
const GLYPH_JOIN_GAP: f64 = 0.15;
fn merge_glyph_runs(blocks: Vec<TextBlock>) -> Vec<TextBlock> {
let mut out: Vec<TextBlock> = Vec::with_capacity(blocks.len());
let mut prev_was_glyph = false;
for block in blocks {
let is_glyph = block.text.chars().count() == 1;
if let Some(last) = out.last_mut() {
let gap = block.bbox[0] - last.bbox[2];
if prev_was_glyph && is_glyph && gap < GLYPH_JOIN_GAP * last.font_size.max(1.0) {
last.text.push_str(&block.text);
last.bbox[2] = last.bbox[2].max(block.bbox[2]);
last.bbox[3] = last.bbox[3].max(block.bbox[3]);
prev_was_glyph = is_glyph;
continue;
}
}
prev_was_glyph = is_glyph;
out.push(block);
}
out
}
const MIN_COLUMN_LINES: usize = 2;
const MIN_COLUMN_GAP_EM: f64 = 1.0;
const MIN_COLUMN_SUPPORT_PERCENT: usize = 75;
fn column_positions(lines: &[Line]) -> Vec<f64> {
let mut candidates: Vec<(f64, usize)> = Vec::new();
for line in lines {
let mut counted: Vec<usize> = Vec::new();
for (i, block) in line.blocks.iter().enumerate() {
if i > 0
&& block.bbox[0] - line.blocks[i - 1].bbox[2] < MIN_COLUMN_GAP_EM * line.font_size
{
continue;
}
let x = block.bbox[0];
match candidates
.iter()
.position(|&(px, _)| (px - x).abs() < TABLE_X_TOLERANCE)
{
Some(i) => {
if !counted.contains(&i) {
candidates[i].1 += 1;
counted.push(i);
}
}
None => {
candidates.push((x, 1));
counted.push(candidates.len() - 1);
}
}
}
}
let required = (lines.len() * MIN_COLUMN_SUPPORT_PERCENT).div_ceil(100);
let threshold = std::cmp::max(MIN_COLUMN_LINES, required);
let mut positions: Vec<f64> = candidates
.into_iter()
.filter(|&(_, support)| support >= threshold)
.map(|(x, _)| x)
.collect();
positions.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
positions
}
fn column_for(x: f64, positions: &[f64]) -> usize {
let mut best = 0;
for (i, &px) in positions.iter().enumerate() {
if px <= x + TABLE_X_TOLERANCE {
best = i;
} else {
break;
}
}
best
}
fn try_detect_table(lines: &[Line]) -> Option<Table> {
if lines.len() < 2 {
return None;
}
let x_positions = column_positions(lines);
if x_positions.len() < 2 {
return None;
}
let multi_col_lines = lines
.iter()
.filter(|line| {
line.blocks
.iter()
.map(|b| column_for(b.bbox[0], &x_positions))
.collect::<std::collections::HashSet<_>>()
.len()
>= 2
})
.count();
if multi_col_lines * 100 / lines.len() < 60 {
return None;
}
let col_count = x_positions.len();
let mut rows = Vec::new();
for line in lines {
let mut row = vec![String::new(); col_count];
for block in &line.blocks {
let col_idx = column_for(block.bbox[0], &x_positions);
if !row[col_idx].is_empty() {
row[col_idx].push(' ');
}
row[col_idx].push_str(&block.text);
}
rows.push(row);
}
Some(Table { rows, col_count })
}
fn group_into_paragraphs(lines: &[Line]) -> Vec<(f64, PageElement)> {
let mut elements = Vec::new();
let mut current_runs: Vec<Run> = Vec::new();
let mut current_y = 0.0;
let mut prev_y: Option<f64> = None;
let mut prev_font_size: f64 = 12.0;
let mut prev_bold = false;
for line in lines {
let mut line_text = line_text(line);
if line_text.trim().is_empty() {
continue;
}
let font_name = line
.blocks
.first()
.map(|b| b.font_name.clone())
.unwrap_or_default();
let font_size = line.font_size;
let bold = font_name.contains("Bold")
|| font_name.contains("bold")
|| line.blocks.first().is_some_and(|b| b.is_bold);
let italic = font_name.contains("Italic")
|| font_name.contains("italic")
|| font_name.contains("Oblique")
|| line.blocks.first().is_some_and(|b| b.is_italic);
let is_new_paragraph = match prev_y {
Some(py) => {
(py - line.y).abs() > prev_font_size * PARAGRAPH_GAP_FACTOR
|| (font_size - prev_font_size).abs() > STYLE_SIZE_STEP
|| bold != prev_bold
}
None => false,
};
if is_new_paragraph && !current_runs.is_empty() {
elements.push((
current_y,
PageElement::Para(Paragraph {
runs: std::mem::take(&mut current_runs),
}),
));
}
if current_runs.is_empty() {
current_y = line.y;
} else {
line_text.insert(0, ' ');
}
current_runs.push(Run {
text: line_text,
font_name,
font_size,
bold,
italic,
});
prev_y = Some(line.y);
prev_font_size = font_size;
prev_bold = bold;
}
if !current_runs.is_empty() {
elements.push((
current_y,
PageElement::Para(Paragraph { runs: current_runs }),
));
}
elements
}
pub fn map_font_name(pdf_font: &str) -> &str {
let name = if let Some(pos) = pdf_font.find('+') {
&pdf_font[pos + 1..]
} else {
pdf_font
};
if name.contains("Times") || name.contains("Serif") {
"Times New Roman"
} else if name.contains("Arial") || name.contains("Helvetica") || name.contains("Sans") {
"Arial"
} else if name.contains("Courier") || name.contains("Mono") {
"Courier New"
} else if name.contains("Symbol") {
"Symbol"
} else {
"Calibri"
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_block(text: &str, x: f64, y: f64, font_size: f64) -> TextBlock {
TextBlock {
text: text.to_string(),
page: 1,
bbox: [x, y, x + text.len() as f64 * font_size * 0.5, y + font_size],
font_name: "F1".to_string(),
font_size,
actual_text: None,
base_font: None,
is_bold: false,
is_italic: false,
color: None,
width_source: Default::default(),
char_bounds: vec![],
}
}
#[test]
fn single_line_becomes_paragraph() {
let blocks = vec![make_block("Hello World", 72.0, 720.0, 12.0)];
let elements = analyze_page(&blocks);
assert_eq!(elements.len(), 1);
assert!(matches!(elements[0], PageElement::Para(_)));
}
#[test]
fn two_close_lines_same_paragraph() {
let blocks = vec![
make_block("Line 1", 72.0, 720.0, 12.0),
make_block("Line 2", 72.0, 706.0, 12.0), ];
let elements = analyze_page(&blocks);
assert_eq!(elements.len(), 1);
}
#[test]
fn two_distant_lines_different_paragraphs() {
let blocks = vec![
make_block("Para 1", 72.0, 720.0, 12.0),
make_block("Para 2", 72.0, 680.0, 12.0), ];
let elements = analyze_page(&blocks);
assert_eq!(elements.len(), 2);
}
#[test]
fn table_detection() {
let blocks = vec![
make_block("Name", 72.0, 700.0, 12.0),
make_block("Age", 200.0, 700.0, 12.0),
make_block("Alice", 72.0, 684.0, 12.0),
make_block("30", 200.0, 684.0, 12.0),
make_block("Bob", 72.0, 668.0, 12.0),
make_block("25", 200.0, 668.0, 12.0),
];
let elements = analyze_page(&blocks);
assert_eq!(elements.len(), 1);
assert!(matches!(elements[0], PageElement::Tbl(_)));
if let PageElement::Tbl(ref tbl) = elements[0] {
assert_eq!(tbl.rows.len(), 3);
assert_eq!(tbl.col_count, 2);
}
}
#[test]
fn empty_blocks_returns_empty() {
let elements = analyze_page(&[]);
assert!(elements.is_empty());
}
#[test]
fn a_sentence_does_not_become_a_table_with_a_column_per_word() {
let mut blocks = Vec::new();
for (row, y) in [(0, 700.0), (1, 684.0), (2, 668.0)] {
for w in 0..8 {
let x = 72.0 + (w as f64) * 37.0 + (row as f64) * 11.0;
blocks.push(make_block("word", x, y, 12.0));
}
}
let elements = analyze_page(&blocks);
for el in &elements {
if let PageElement::Tbl(t) = el {
assert!(
t.col_count <= 4,
"flowing text became a table of {} columns; that is the \
symptom from #161",
t.col_count
);
}
}
}
#[test]
fn a_real_table_keeps_its_columns() {
let mut blocks = Vec::new();
for y in [700.0_f64, 684.0, 668.0, 652.0] {
blocks.push(make_block("left", 72.0, y, 12.0));
blocks.push(make_block("middle", 200.0, y, 12.0));
blocks.push(make_block("right", 340.0, y, 12.0));
}
let elements = analyze_page(&blocks);
let table = elements.iter().find_map(|e| match e {
PageElement::Tbl(t) => Some(t),
_ => None,
});
let t = table.expect("an aligned three-column table was not detected");
assert_eq!(t.col_count, 3, "columns: {:?}", t.rows.first());
assert_eq!(t.rows.len(), 4);
}
#[test]
fn text_without_a_column_of_its_own_does_not_land_in_the_first_cell() {
let mut blocks = Vec::new();
for y in [700.0_f64, 684.0, 668.0] {
blocks.push(make_block("A", 72.0, y, 12.0));
blocks.push(make_block("B", 300.0, y, 12.0));
}
blocks.push(make_block("loose", 310.0, 652.0, 12.0));
blocks.push(make_block("A", 72.0, 652.0, 12.0));
let elements = analyze_page(&blocks);
if let Some(PageElement::Tbl(t)) =
elements.iter().find(|e| matches!(e, PageElement::Tbl(_)))
{
let last = t.rows.last().expect("no rows");
assert!(
!last[0].contains("loose"),
"unaligned text was dumped into the first cell: {last:?}"
);
}
}
}