1use crate::rules::Rule;
10use pdf_extract::TextBlock;
11
12const LINE_Y_TOLERANCE: f64 = 2.0;
14
15const PARAGRAPH_GAP_FACTOR: f64 = 1.5;
17
18const STYLE_SIZE_STEP: f64 = 1.0;
20
21const TABLE_X_TOLERANCE: f64 = 5.0;
23
24#[derive(Debug, Clone)]
32pub struct Run {
33 pub text: String,
36 pub font_name: String,
40 pub font_size: f64,
43 pub bold: bool,
46 pub italic: bool,
49}
50
51#[derive(Debug, Clone)]
57pub struct Paragraph {
58 pub runs: Vec<Run>,
61}
62
63#[derive(Debug, Clone)]
70pub struct Table {
71 pub rows: Vec<Vec<String>>,
74 pub col_count: usize,
77}
78
79#[derive(Debug, Clone)]
86pub struct DocxImage {
87 pub data: Vec<u8>,
90 pub width: u32,
92 pub height: u32,
94 pub content_type: String,
97 pub id: String,
100}
101
102#[derive(Debug, Clone)]
108pub enum PageElement {
109 Para(Paragraph),
111 Tbl(Table),
113 Img(DocxImage),
115}
116
117#[derive(Debug)]
119struct Line {
120 y: f64,
121 font_size: f64,
122 blocks: Vec<TextBlock>,
123}
124
125pub fn analyze_page(blocks: &[TextBlock]) -> Vec<PageElement> {
130 analyze_page_with_rules(blocks, &[])
131}
132
133pub fn analyze_page_with_rules(blocks: &[TextBlock], rules: &[Rule]) -> Vec<PageElement> {
143 if blocks.is_empty() {
144 return Vec::new();
145 }
146
147 let grids = ruled_grids(rules);
148 if grids.is_empty() {
149 let lines = group_into_lines(blocks);
150 if let Some(tbl) = try_detect_table(&lines) {
151 return vec![PageElement::Tbl(tbl)];
152 }
153 return group_into_paragraphs(&lines)
154 .into_iter()
155 .map(|(_, e)| e)
156 .collect();
157 }
158
159 let mut inside: Vec<Vec<TextBlock>> = vec![Vec::new(); grids.len()];
160 let mut outside: Vec<TextBlock> = Vec::new();
161 for block in blocks {
162 let cx = (block.bbox[0] + block.bbox[2]) / 2.0;
163 let cy = (block.bbox[1] + block.bbox[3]) / 2.0;
164 match grids.iter().position(|g| g.contains(cx, cy)) {
165 Some(i) => inside[i].push(block.clone()),
166 None => outside.push(block.clone()),
167 }
168 }
169
170 let mut items: Vec<(f64, PageElement)> = Vec::new();
171 for (grid, cell_blocks) in grids.iter().zip(&inside) {
172 items.push((grid.y1, PageElement::Tbl(grid_table(grid, cell_blocks))));
173 }
174
175 let lines = group_into_lines(&outside);
177 let section = |y: f64| grids.iter().filter(|g| g.y0 > y).count();
178 let mut start = 0;
179 while start < lines.len() {
180 let id = section(lines[start].y);
181 let mut end = start + 1;
182 while end < lines.len() && section(lines[end].y) == id {
183 end += 1;
184 }
185 items.extend(group_into_paragraphs(&lines[start..end]));
186 start = end;
187 }
188
189 items.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
190 items.into_iter().map(|(_, e)| e).collect()
191}
192
193#[derive(Debug)]
195struct Grid {
196 x0: f64,
197 y0: f64,
198 x1: f64,
199 y1: f64,
200 xs: Vec<f64>,
202 ys: Vec<f64>,
204}
205
206impl Grid {
207 fn contains(&self, x: f64, y: f64) -> bool {
208 x >= self.x0 - 1.0 && x <= self.x1 + 1.0 && y >= self.y0 - 1.0 && y <= self.y1 + 1.0
209 }
210}
211
212const RULE_TOLERANCE: f64 = 2.0;
215
216fn cluster(mut values: Vec<f64>) -> Vec<f64> {
217 values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
218 let mut out: Vec<(f64, usize)> = Vec::new();
219 for v in values {
220 match out.last_mut() {
221 Some((mean, n)) if (v - *mean).abs() <= RULE_TOLERANCE => {
222 *mean = (*mean * *n as f64 + v) / (*n as f64 + 1.0);
223 *n += 1;
224 }
225 _ => out.push((v, 1)),
226 }
227 }
228 out.into_iter().map(|(m, _)| m).collect()
229}
230
231fn find(parent: &mut [usize], mut i: usize) -> usize {
232 while parent[i] != i {
233 parent[i] = parent[parent[i]];
234 i = parent[i];
235 }
236 i
237}
238
239fn ruled_grids(rules: &[Rule]) -> Vec<Grid> {
241 let n = rules.len();
242 if n < 4 {
243 return Vec::new();
244 }
245 let mut parent: Vec<usize> = (0..n).collect();
246 for i in 0..n {
247 for j in (i + 1)..n {
248 let (a, b) = (&rules[i], &rules[j]);
249 let touch = a.x0 - RULE_TOLERANCE <= b.x1
250 && b.x0 - RULE_TOLERANCE <= a.x1
251 && a.y0 - RULE_TOLERANCE <= b.y1
252 && b.y0 - RULE_TOLERANCE <= a.y1;
253 if touch {
254 let (ri, rj) = (find(&mut parent, i), find(&mut parent, j));
255 parent[ri] = rj;
256 }
257 }
258 }
259
260 let mut groups: std::collections::BTreeMap<usize, Vec<usize>> = Default::default();
261 for i in 0..n {
262 let root = find(&mut parent, i);
263 groups.entry(root).or_default().push(i);
264 }
265
266 let mut grids = Vec::new();
267 for members in groups.values() {
268 let mut xs = Vec::new();
269 let mut ys = Vec::new();
270 let (mut x0, mut y0, mut x1, mut y1) = (f64::MAX, f64::MAX, f64::MIN, f64::MIN);
271 for &m in members {
272 let r = &rules[m];
273 if r.is_horizontal() {
274 ys.push((r.y0 + r.y1) / 2.0);
275 } else {
276 xs.push((r.x0 + r.x1) / 2.0);
277 }
278 x0 = x0.min(r.x0);
279 y0 = y0.min(r.y0);
280 x1 = x1.max(r.x1);
281 y1 = y1.max(r.y1);
282 }
283 let xs = cluster(xs);
284 let mut ys = cluster(ys);
285 if xs.len() < 3 || ys.len() < 3 {
286 continue;
287 }
288 ys.reverse();
289 grids.push(Grid {
290 x0,
291 y0,
292 x1,
293 y1,
294 xs,
295 ys,
296 });
297 }
298 grids
299}
300
301fn band(bounds: &[f64], v: f64, descending: bool) -> usize {
303 let cells = bounds.len() - 1;
304 for i in 0..cells {
305 let inside = if descending {
306 v <= bounds[i] && v > bounds[i + 1]
307 } else {
308 v >= bounds[i] && v < bounds[i + 1]
309 };
310 if inside {
311 return i;
312 }
313 }
314 let first = bounds[0];
315 let past_start = if descending { v > first } else { v < first };
316 if past_start {
317 0
318 } else {
319 cells - 1
320 }
321}
322
323fn grid_table(grid: &Grid, blocks: &[TextBlock]) -> Table {
324 let cols = grid.xs.len() - 1;
325 let rows = grid.ys.len() - 1;
326 let mut cells: Vec<Vec<Vec<TextBlock>>> = vec![vec![Vec::new(); cols]; rows];
327 for block in blocks {
328 let cx = (block.bbox[0] + block.bbox[2]) / 2.0;
329 let cy = (block.bbox[1] + block.bbox[3]) / 2.0;
330 let c = band(&grid.xs, cx, false);
331 let r = band(&grid.ys, cy, true);
332 cells[r][c].push(block.clone());
333 }
334 let rows = cells
335 .into_iter()
336 .map(|row| {
337 row.into_iter()
338 .map(|cell| {
339 group_into_lines(&cell)
340 .iter()
341 .map(line_text)
342 .collect::<Vec<_>>()
343 .join(" ")
344 })
345 .collect()
346 })
347 .collect();
348 Table {
349 rows,
350 col_count: cols,
351 }
352}
353
354fn line_text(line: &Line) -> String {
355 line.blocks
356 .iter()
357 .map(|b| b.text.as_str())
358 .collect::<Vec<_>>()
359 .join(" ")
360}
361
362fn group_into_lines(blocks: &[TextBlock]) -> Vec<Line> {
364 let mut sorted: Vec<&TextBlock> = blocks.iter().collect();
365 sorted.sort_by(|a, b| {
367 let y_cmp = b.bbox[1]
368 .partial_cmp(&a.bbox[1])
369 .unwrap_or(std::cmp::Ordering::Equal);
370 if y_cmp == std::cmp::Ordering::Equal {
371 a.bbox[0]
372 .partial_cmp(&b.bbox[0])
373 .unwrap_or(std::cmp::Ordering::Equal)
374 } else {
375 y_cmp
376 }
377 });
378
379 let mut lines: Vec<Line> = Vec::new();
380
381 for block in sorted {
382 let y = block.bbox[1];
383 let matched = lines
384 .iter_mut()
385 .find(|line| (line.y - y).abs() < LINE_Y_TOLERANCE);
386
387 if let Some(line) = matched {
388 line.blocks.push(block.clone());
389 } else {
390 lines.push(Line {
391 y,
392 font_size: block.font_size,
393 blocks: vec![block.clone()],
394 });
395 }
396 }
397
398 for line in &mut lines {
400 line.blocks.sort_by(|a, b| {
401 a.bbox[0]
402 .partial_cmp(&b.bbox[0])
403 .unwrap_or(std::cmp::Ordering::Equal)
404 });
405 line.blocks = merge_glyph_runs(std::mem::take(&mut line.blocks));
406 }
407
408 lines
409}
410
411const GLYPH_JOIN_GAP: f64 = 0.15;
414
415fn merge_glyph_runs(blocks: Vec<TextBlock>) -> Vec<TextBlock> {
423 let mut out: Vec<TextBlock> = Vec::with_capacity(blocks.len());
424 let mut prev_was_glyph = false;
425 for block in blocks {
426 let is_glyph = block.text.chars().count() == 1;
427 if let Some(last) = out.last_mut() {
428 let gap = block.bbox[0] - last.bbox[2];
429 if prev_was_glyph && is_glyph && gap < GLYPH_JOIN_GAP * last.font_size.max(1.0) {
430 last.text.push_str(&block.text);
431 last.bbox[2] = last.bbox[2].max(block.bbox[2]);
432 last.bbox[3] = last.bbox[3].max(block.bbox[3]);
433 prev_was_glyph = is_glyph;
434 continue;
435 }
436 }
437 prev_was_glyph = is_glyph;
438 out.push(block);
439 }
440 out
441}
442
443const MIN_COLUMN_LINES: usize = 2;
466const MIN_COLUMN_GAP_EM: f64 = 1.0;
469const MIN_COLUMN_SUPPORT_PERCENT: usize = 75;
470
471fn column_positions(lines: &[Line]) -> Vec<f64> {
473 let mut candidates: Vec<(f64, usize)> = Vec::new();
475 for line in lines {
476 let mut counted: Vec<usize> = Vec::new();
477 for (i, block) in line.blocks.iter().enumerate() {
478 if i > 0
484 && block.bbox[0] - line.blocks[i - 1].bbox[2] < MIN_COLUMN_GAP_EM * line.font_size
485 {
486 continue;
487 }
488 let x = block.bbox[0];
489 match candidates
490 .iter()
491 .position(|&(px, _)| (px - x).abs() < TABLE_X_TOLERANCE)
492 {
493 Some(i) => {
494 if !counted.contains(&i) {
498 candidates[i].1 += 1;
499 counted.push(i);
500 }
501 }
502 None => {
503 candidates.push((x, 1));
504 counted.push(candidates.len() - 1);
505 }
506 }
507 }
508 }
509
510 let required = (lines.len() * MIN_COLUMN_SUPPORT_PERCENT).div_ceil(100);
517 let threshold = std::cmp::max(MIN_COLUMN_LINES, required);
518 let mut positions: Vec<f64> = candidates
519 .into_iter()
520 .filter(|&(_, support)| support >= threshold)
521 .map(|(x, _)| x)
522 .collect();
523 positions.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
524 positions
525}
526
527fn column_for(x: f64, positions: &[f64]) -> usize {
534 let mut best = 0;
535 for (i, &px) in positions.iter().enumerate() {
536 if px <= x + TABLE_X_TOLERANCE {
537 best = i;
538 } else {
539 break;
540 }
541 }
542 best
543}
544
545fn try_detect_table(lines: &[Line]) -> Option<Table> {
550 if lines.len() < 2 {
551 return None;
552 }
553
554 let x_positions = column_positions(lines);
555 if x_positions.len() < 2 {
556 return None;
557 }
558
559 let multi_col_lines = lines
561 .iter()
562 .filter(|line| {
563 line.blocks
564 .iter()
565 .map(|b| column_for(b.bbox[0], &x_positions))
566 .collect::<std::collections::HashSet<_>>()
567 .len()
568 >= 2
569 })
570 .count();
571
572 if multi_col_lines * 100 / lines.len() < 60 {
574 return None;
575 }
576
577 let col_count = x_positions.len();
578 let mut rows = Vec::new();
579
580 for line in lines {
581 let mut row = vec![String::new(); col_count];
582 for block in &line.blocks {
583 let col_idx = column_for(block.bbox[0], &x_positions);
584 if !row[col_idx].is_empty() {
585 row[col_idx].push(' ');
586 }
587 row[col_idx].push_str(&block.text);
588 }
589 rows.push(row);
590 }
591
592 Some(Table { rows, col_count })
593}
594
595fn group_into_paragraphs(lines: &[Line]) -> Vec<(f64, PageElement)> {
602 let mut elements = Vec::new();
603 let mut current_runs: Vec<Run> = Vec::new();
604 let mut current_y = 0.0;
605 let mut prev_y: Option<f64> = None;
606 let mut prev_font_size: f64 = 12.0;
607 let mut prev_bold = false;
608
609 for line in lines {
610 let mut line_text = line_text(line);
611 if line_text.trim().is_empty() {
612 continue;
613 }
614
615 let font_name = line
616 .blocks
617 .first()
618 .map(|b| b.font_name.clone())
619 .unwrap_or_default();
620 let font_size = line.font_size;
621
622 let bold = font_name.contains("Bold")
623 || font_name.contains("bold")
624 || line.blocks.first().is_some_and(|b| b.is_bold);
625 let italic = font_name.contains("Italic")
626 || font_name.contains("italic")
627 || font_name.contains("Oblique")
628 || line.blocks.first().is_some_and(|b| b.is_italic);
629
630 let is_new_paragraph = match prev_y {
631 Some(py) => {
632 (py - line.y).abs() > prev_font_size * PARAGRAPH_GAP_FACTOR
633 || (font_size - prev_font_size).abs() > STYLE_SIZE_STEP
634 || bold != prev_bold
635 }
636 None => false,
637 };
638
639 if is_new_paragraph && !current_runs.is_empty() {
640 elements.push((
641 current_y,
642 PageElement::Para(Paragraph {
643 runs: std::mem::take(&mut current_runs),
644 }),
645 ));
646 }
647 if current_runs.is_empty() {
648 current_y = line.y;
649 } else {
650 line_text.insert(0, ' ');
651 }
652
653 current_runs.push(Run {
654 text: line_text,
655 font_name,
656 font_size,
657 bold,
658 italic,
659 });
660
661 prev_y = Some(line.y);
662 prev_font_size = font_size;
663 prev_bold = bold;
664 }
665
666 if !current_runs.is_empty() {
667 elements.push((
668 current_y,
669 PageElement::Para(Paragraph { runs: current_runs }),
670 ));
671 }
672
673 elements
674}
675
676pub fn map_font_name(pdf_font: &str) -> &str {
678 let name = if let Some(pos) = pdf_font.find('+') {
680 &pdf_font[pos + 1..]
681 } else {
682 pdf_font
683 };
684
685 if name.contains("Times") || name.contains("Serif") {
687 "Times New Roman"
688 } else if name.contains("Arial") || name.contains("Helvetica") || name.contains("Sans") {
689 "Arial"
690 } else if name.contains("Courier") || name.contains("Mono") {
691 "Courier New"
692 } else if name.contains("Symbol") {
693 "Symbol"
694 } else {
695 "Calibri"
696 }
697}
698
699#[cfg(test)]
700mod tests {
701 use super::*;
702
703 fn make_block(text: &str, x: f64, y: f64, font_size: f64) -> TextBlock {
704 TextBlock {
705 text: text.to_string(),
706 page: 1,
707 bbox: [x, y, x + text.len() as f64 * font_size * 0.5, y + font_size],
708 font_name: "F1".to_string(),
709 font_size,
710 actual_text: None,
711 base_font: None,
712 is_bold: false,
713 is_italic: false,
714 color: None,
715 width_source: Default::default(),
716 char_bounds: vec![],
717 }
718 }
719
720 #[test]
721 fn single_line_becomes_paragraph() {
722 let blocks = vec![make_block("Hello World", 72.0, 720.0, 12.0)];
723 let elements = analyze_page(&blocks);
724 assert_eq!(elements.len(), 1);
725 assert!(matches!(elements[0], PageElement::Para(_)));
726 }
727
728 #[test]
729 fn two_close_lines_same_paragraph() {
730 let blocks = vec![
731 make_block("Line 1", 72.0, 720.0, 12.0),
732 make_block("Line 2", 72.0, 706.0, 12.0), ];
734 let elements = analyze_page(&blocks);
735 assert_eq!(elements.len(), 1);
736 }
737
738 #[test]
739 fn two_distant_lines_different_paragraphs() {
740 let blocks = vec![
741 make_block("Para 1", 72.0, 720.0, 12.0),
742 make_block("Para 2", 72.0, 680.0, 12.0), ];
744 let elements = analyze_page(&blocks);
745 assert_eq!(elements.len(), 2);
746 }
747
748 #[test]
749 fn table_detection() {
750 let blocks = vec![
751 make_block("Name", 72.0, 700.0, 12.0),
753 make_block("Age", 200.0, 700.0, 12.0),
754 make_block("Alice", 72.0, 684.0, 12.0),
756 make_block("30", 200.0, 684.0, 12.0),
757 make_block("Bob", 72.0, 668.0, 12.0),
759 make_block("25", 200.0, 668.0, 12.0),
760 ];
761 let elements = analyze_page(&blocks);
762 assert_eq!(elements.len(), 1);
763 assert!(matches!(elements[0], PageElement::Tbl(_)));
764 if let PageElement::Tbl(ref tbl) = elements[0] {
765 assert_eq!(tbl.rows.len(), 3);
766 assert_eq!(tbl.col_count, 2);
767 }
768 }
769
770 #[test]
771 fn empty_blocks_returns_empty() {
772 let elements = analyze_page(&[]);
773 assert!(elements.is_empty());
774 }
775
776 #[test]
779 fn a_sentence_does_not_become_a_table_with_a_column_per_word() {
780 let mut blocks = Vec::new();
785 for (row, y) in [(0, 700.0), (1, 684.0), (2, 668.0)] {
786 for w in 0..8 {
788 let x = 72.0 + (w as f64) * 37.0 + (row as f64) * 11.0;
789 blocks.push(make_block("word", x, y, 12.0));
790 }
791 }
792 let elements = analyze_page(&blocks);
793 for el in &elements {
794 if let PageElement::Tbl(t) = el {
795 assert!(
796 t.col_count <= 4,
797 "flowing text became a table of {} columns; that is the \
798 symptom from #161",
799 t.col_count
800 );
801 }
802 }
803 }
804
805 #[test]
806 fn a_real_table_keeps_its_columns() {
807 let mut blocks = Vec::new();
811 for y in [700.0_f64, 684.0, 668.0, 652.0] {
812 blocks.push(make_block("left", 72.0, y, 12.0));
813 blocks.push(make_block("middle", 200.0, y, 12.0));
814 blocks.push(make_block("right", 340.0, y, 12.0));
815 }
816 let elements = analyze_page(&blocks);
817 let table = elements.iter().find_map(|e| match e {
818 PageElement::Tbl(t) => Some(t),
819 _ => None,
820 });
821 let t = table.expect("an aligned three-column table was not detected");
822 assert_eq!(t.col_count, 3, "columns: {:?}", t.rows.first());
823 assert_eq!(t.rows.len(), 4);
824 }
825
826 #[test]
827 fn text_without_a_column_of_its_own_does_not_land_in_the_first_cell() {
828 let mut blocks = Vec::new();
832 for y in [700.0_f64, 684.0, 668.0] {
833 blocks.push(make_block("A", 72.0, y, 12.0));
834 blocks.push(make_block("B", 300.0, y, 12.0));
835 }
836 blocks.push(make_block("loose", 310.0, 652.0, 12.0));
838 blocks.push(make_block("A", 72.0, 652.0, 12.0));
839
840 let elements = analyze_page(&blocks);
841 if let Some(PageElement::Tbl(t)) =
842 elements.iter().find(|e| matches!(e, PageElement::Tbl(_)))
843 {
844 let last = t.rows.last().expect("no rows");
845 assert!(
846 !last[0].contains("loose"),
847 "unaligned text was dumped into the first cell: {last:?}"
848 );
849 }
850 }
851}