1use crate::error::PdfiumError;
8use crate::pdf::document::page::PdfPage;
9use crate::pdf::document::page::object::PdfPageObjectCommon;
10use crate::pdf::document::page::objects::common::PdfPageObjectsCommon;
11use crate::pdf::document::page::struct_element::{PdfStructElement, PdfStructElementType};
12use crate::pdf::font::PdfFontWeight;
13use crate::pdf::points::PdfPoints;
14use crate::pdf::rect::PdfRect;
15use std::collections::HashMap;
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum PageExtractionMethod {
20 StructureTree,
22 Heuristic,
24}
25
26#[derive(Debug, Clone, PartialEq)]
28pub enum ContentRole {
29 Heading { level: u8 },
31 Paragraph,
33 ListItem { label: Option<String> },
35 TableCell { row: usize, col: usize, is_header: bool },
37 Figure { alt_text: Option<String> },
39 Caption,
41 Code,
43 BlockQuote,
45 Link { url: Option<String> },
47 Other(String),
49}
50
51#[derive(Debug, Clone)]
53pub struct ExtractedBlock {
54 pub role: ContentRole,
56 pub text: String,
58 pub bounds: Option<PdfRect>,
60 pub font_size: Option<f32>,
62 pub is_bold: bool,
64 pub is_italic: bool,
66 pub is_monospace: bool,
68 pub children: Vec<ExtractedBlock>,
70}
71
72#[derive(Debug)]
74pub struct PageExtraction {
75 pub method: PageExtractionMethod,
77 pub blocks: Vec<ExtractedBlock>,
79}
80
81pub fn extract_page_content(page: &PdfPage<'_>) -> Result<PageExtraction, PdfiumError> {
87 if let Some(extraction) = extract_via_structure_tree(page)? {
88 return Ok(extraction);
89 }
90
91 extract_via_heuristics(page)
92}
93
94fn extract_via_structure_tree(page: &PdfPage<'_>) -> Result<Option<PageExtraction>, PdfiumError> {
97 let tree = match page.struct_tree() {
98 Some(tree) => tree,
99 None => return Ok(None),
100 };
101
102 if tree.children_count() == 0 {
103 return Ok(None);
104 }
105
106 let (mcid_text_map, mcid_style_map) = build_mcid_maps(page)?;
107
108 if mcid_text_map.is_empty() {
109 return Ok(None);
110 }
111
112 let mut blocks = Vec::new();
113 let mut resolved = false;
114
115 for child in tree.children() {
116 if let Some(block) = extract_element_block(&child, &mcid_text_map, &mcid_style_map)
117 && (!block.text.is_empty() || !block.children.is_empty())
118 {
119 resolved = true;
120 blocks.push(block);
121 }
122 }
123
124 if !resolved {
125 return Ok(None);
126 }
127
128 let blocks = flatten_structural_wrappers(blocks);
129
130 Ok(Some(PageExtraction {
131 method: PageExtractionMethod::StructureTree,
132 blocks,
133 }))
134}
135
136#[derive(Debug, Clone)]
138struct TextStyle {
139 font_size: f32,
140 is_bold: bool,
141 is_italic: bool,
142 is_monospace: bool,
143 bounds: Option<PdfRect>,
144}
145
146type McidMaps = (HashMap<i32, String>, HashMap<i32, TextStyle>);
151
152fn build_mcid_maps(page: &PdfPage<'_>) -> Result<McidMaps, PdfiumError> {
153 let objects = page.objects();
154 let text_page = page.text()?;
155
156 let mut text_map: HashMap<i32, String> = HashMap::new();
157 let mut style_map: HashMap<i32, TextStyle> = HashMap::new();
158
159 for i in 0..objects.len() {
160 let object = objects.get(i)?;
161
162 if let Some(text_obj) = object.as_text_object()
163 && let Some(mcid) = object.marked_content_id()
164 {
165 let text = text_page.for_object(text_obj);
166 if !text.is_empty() {
167 text_map
168 .entry(mcid)
169 .and_modify(|existing| existing.push_str(&text))
170 .or_insert(text);
171 }
172
173 style_map.entry(mcid).or_insert_with(|| {
174 let font = text_obj.font();
175 let is_bold = font.weight().ok().is_some_and(|w| {
176 matches!(
177 w,
178 PdfFontWeight::Weight700Bold | PdfFontWeight::Weight800 | PdfFontWeight::Weight900
179 )
180 }) || font.is_bold_reenforced()
181 || font.name().to_ascii_lowercase().contains("bold");
182 let is_italic = font.is_italic();
183 let is_monospace = font.is_fixed_pitch();
184 let font_size = text_obj.scaled_font_size().value;
185 let bounds = object.bounds().ok().map(|qp| qp.to_rect());
186 TextStyle {
187 font_size,
188 is_bold,
189 is_italic,
190 is_monospace,
191 bounds,
192 }
193 });
194 }
195 }
196
197 Ok((text_map, style_map))
198}
199
200fn extract_element_block(
202 element: &PdfStructElement<'_>,
203 mcid_text_map: &HashMap<i32, String>,
204 mcid_style_map: &HashMap<i32, TextStyle>,
205) -> Option<ExtractedBlock> {
206 let element_type = element.element_type();
207
208 let role = element_type_to_role(&element_type, element);
209
210 let mcids = element.all_marked_content_ids();
211 let mut text_parts: Vec<&str> = Vec::new();
212 let mut style: Option<&TextStyle> = None;
213
214 for mcid in &mcids {
215 if let Some(t) = mcid_text_map.get(mcid) {
216 text_parts.push(t);
217 }
218 if style.is_none() {
219 style = mcid_style_map.get(mcid);
220 }
221 }
222
223 let actual_text = element.actual_text();
224 let alt_text = element.alt_text();
225
226 let text = if !text_parts.is_empty() {
227 text_parts.join("")
228 } else if let Some(ref at) = actual_text {
229 at.clone()
230 } else if let Some(ref alt) = alt_text {
231 alt.clone()
232 } else {
233 String::new()
234 };
235
236 let children = extract_children_blocks(element, mcid_text_map, mcid_style_map);
237
238 if text.is_empty() && children.is_empty() {
239 return None;
240 }
241
242 Some(ExtractedBlock {
243 role,
244 text,
245 bounds: style.and_then(|s| s.bounds),
246 font_size: style.map(|s| s.font_size),
247 is_bold: style.is_some_and(|s| s.is_bold),
248 is_italic: style.is_some_and(|s| s.is_italic),
249 is_monospace: style.is_some_and(|s| s.is_monospace),
250 children,
251 })
252}
253
254fn extract_children_blocks(
256 element: &PdfStructElement<'_>,
257 mcid_text_map: &HashMap<i32, String>,
258 mcid_style_map: &HashMap<i32, TextStyle>,
259) -> Vec<ExtractedBlock> {
260 let mut children = Vec::new();
261 for child in element.children() {
262 if let Some(block) = extract_element_block(&child, mcid_text_map, mcid_style_map)
263 && (!block.text.is_empty() || !block.children.is_empty())
264 {
265 children.push(block);
266 }
267 }
268 children
269}
270
271fn element_type_to_role(element_type: &PdfStructElementType, element: &PdfStructElement<'_>) -> ContentRole {
273 match element_type {
274 PdfStructElementType::H => ContentRole::Heading { level: 1 },
275 PdfStructElementType::H1 => ContentRole::Heading { level: 1 },
276 PdfStructElementType::H2 => ContentRole::Heading { level: 2 },
277 PdfStructElementType::H3 => ContentRole::Heading { level: 3 },
278 PdfStructElementType::H4 => ContentRole::Heading { level: 4 },
279 PdfStructElementType::H5 => ContentRole::Heading { level: 5 },
280 PdfStructElementType::H6 => ContentRole::Heading { level: 6 },
281 PdfStructElementType::P | PdfStructElementType::Span => ContentRole::Paragraph,
282 PdfStructElementType::LI => {
283 let label = find_child_text_by_type(element, &PdfStructElementType::Lbl);
284 ContentRole::ListItem { label }
285 }
286 PdfStructElementType::Figure => {
287 let alt = element.alt_text();
288 ContentRole::Figure { alt_text: alt }
289 }
290 PdfStructElementType::Caption => ContentRole::Caption,
291 PdfStructElementType::Code => ContentRole::Code,
292 PdfStructElementType::BlockQuote => ContentRole::BlockQuote,
293 PdfStructElementType::Link => {
294 let url = element.string_attribute("O");
295 ContentRole::Link { url }
296 }
297 PdfStructElementType::TD => ContentRole::TableCell {
298 row: 0,
299 col: 0,
300 is_header: false,
301 },
302 PdfStructElementType::TH => ContentRole::TableCell {
303 row: 0,
304 col: 0,
305 is_header: true,
306 },
307 _ => {
308 let type_str = element.element_type_raw().unwrap_or_default();
309 ContentRole::Other(type_str)
310 }
311 }
312}
313
314fn find_child_text_by_type(element: &PdfStructElement<'_>, target_type: &PdfStructElementType) -> Option<String> {
316 for child in element.children() {
317 if child.element_type() == *target_type {
318 if let Some(text) = child.actual_text() {
319 return Some(text);
320 }
321 if let Some(alt) = child.alt_text() {
322 return Some(alt);
323 }
324 }
325 }
326 None
327}
328
329fn flatten_structural_wrappers(blocks: Vec<ExtractedBlock>) -> Vec<ExtractedBlock> {
332 let mut result = Vec::new();
333 for block in blocks {
334 if is_structural_wrapper(&block.role) && block.text.is_empty() {
335 let children = flatten_structural_wrappers(block.children);
336 result.extend(children);
337 } else {
338 let children = flatten_structural_wrappers(block.children);
339 result.push(ExtractedBlock { children, ..block });
340 }
341 }
342 result
343}
344
345fn is_structural_wrapper(role: &ContentRole) -> bool {
347 matches!(
348 role,
349 ContentRole::Other(s) if matches!(s.as_str(), "Document" | "Part" | "Div" | "Sect" | "Art" | "NonStruct" | "")
350 )
351}
352
353fn extract_via_heuristics(page: &PdfPage<'_>) -> Result<PageExtraction, PdfiumError> {
357 let objects = page.objects();
358 let text_page = page.text()?;
359 let mut text_entries: Vec<TextEntry> = Vec::new();
360
361 for i in 0..objects.len() {
362 let object = objects.get(i)?;
363
364 if let Some(text_obj) = object.as_text_object() {
365 let text = text_page.for_object(text_obj);
366 if text.is_empty() {
367 continue;
368 }
369
370 let font = text_obj.font();
371 let font_size = text_obj.scaled_font_size().value;
372 let is_bold = font.weight().ok().is_some_and(|w| {
373 matches!(
374 w,
375 PdfFontWeight::Weight700Bold | PdfFontWeight::Weight800 | PdfFontWeight::Weight900
376 )
377 }) || font.is_bold_reenforced()
378 || font.name().to_ascii_lowercase().contains("bold");
379 let is_italic = font.is_italic();
380 let is_monospace = font.is_fixed_pitch();
381
382 let bounds = object.bounds().ok().map(|qp| qp.to_rect());
383
384 text_entries.push(TextEntry {
385 text,
386 font_size,
387 is_bold,
388 is_italic,
389 is_monospace,
390 bounds,
391 });
392 }
393 }
394
395 if text_entries.is_empty() {
396 return Ok(PageExtraction {
397 method: PageExtractionMethod::Heuristic,
398 blocks: Vec::new(),
399 });
400 }
401
402 let body_font_size = find_body_font_size(&text_entries);
403
404 let blocks = group_text_into_blocks(text_entries, body_font_size, page.height());
405
406 Ok(PageExtraction {
407 method: PageExtractionMethod::Heuristic,
408 blocks,
409 })
410}
411
412struct TextEntry {
414 text: String,
415 font_size: f32,
416 is_bold: bool,
417 is_italic: bool,
418 is_monospace: bool,
419 bounds: Option<PdfRect>,
420}
421
422fn find_body_font_size(entries: &[TextEntry]) -> f32 {
424 let mut size_counts: HashMap<u32, usize> = HashMap::new();
425 for entry in entries {
426 let key = (entry.font_size * 2.0).round() as u32;
427 *size_counts.entry(key).or_insert(0) += 1;
428 }
429
430 size_counts
431 .into_iter()
432 .max_by_key(|(_, count)| *count)
433 .map(|(key, _)| key as f32 / 2.0)
434 .unwrap_or(12.0)
435}
436
437fn group_text_into_blocks(entries: Vec<TextEntry>, body_font_size: f32, page_height: PdfPoints) -> Vec<ExtractedBlock> {
439 if entries.is_empty() {
440 return Vec::new();
441 }
442
443 let mut sorted = entries;
444 sorted.sort_by(|a, b| {
445 let a_top = a
446 .bounds
447 .as_ref()
448 .map(|r| page_height.value - r.top().value)
449 .unwrap_or(0.0);
450 let b_top = b
451 .bounds
452 .as_ref()
453 .map(|r| page_height.value - r.top().value)
454 .unwrap_or(0.0);
455 a_top.total_cmp(&b_top).then_with(|| {
456 let a_left = a.bounds.as_ref().map(|r| r.left().value).unwrap_or(0.0);
457 let b_left = b.bounds.as_ref().map(|r| r.left().value).unwrap_or(0.0);
458 a_left.total_cmp(&b_left)
459 })
460 });
461
462 let mut blocks = Vec::new();
463 let mut current_group: Vec<TextEntry> = vec![sorted.remove(0)];
464
465 for entry in sorted {
466 let should_break = {
467 let last = current_group.last().unwrap();
468 let gap = vertical_gap(last, &entry, page_height);
469 gap > body_font_size * 1.2
470 };
471
472 if should_break {
473 blocks.push(finalize_block(current_group, body_font_size));
474 current_group = vec![entry];
475 } else {
476 current_group.push(entry);
477 }
478 }
479
480 if !current_group.is_empty() {
481 blocks.push(finalize_block(current_group, body_font_size));
482 }
483
484 blocks
485}
486
487fn vertical_gap(a: &TextEntry, b: &TextEntry, page_height: PdfPoints) -> f32 {
489 let a_bottom = a
490 .bounds
491 .as_ref()
492 .map(|r| page_height.value - r.bottom().value)
493 .unwrap_or(0.0);
494 let b_top = b
495 .bounds
496 .as_ref()
497 .map(|r| page_height.value - r.top().value)
498 .unwrap_or(0.0);
499 (b_top - a_bottom).abs()
500}
501
502fn finalize_block(group: Vec<TextEntry>, body_font_size: f32) -> ExtractedBlock {
504 let text: String = group.iter().map(|e| e.text.as_str()).collect::<Vec<_>>().join(" ");
505
506 let first = &group[0];
507 let font_size = first.font_size;
508 let is_bold = first.is_bold;
509 let is_italic = first.is_italic;
510 let is_monospace = first.is_monospace;
511
512 let role = if font_size > body_font_size * 1.3 {
513 let level = if font_size > body_font_size * 1.8 {
514 1
515 } else if font_size > body_font_size * 1.5 {
516 2
517 } else {
518 3
519 };
520 ContentRole::Heading { level }
521 } else {
522 ContentRole::Paragraph
523 };
524
525 let bounds = compute_union_bounds(&group);
526
527 ExtractedBlock {
528 role,
529 text,
530 bounds,
531 font_size: Some(font_size),
532 is_bold,
533 is_italic,
534 is_monospace,
535 children: Vec::new(),
536 }
537}
538
539fn compute_union_bounds(group: &[TextEntry]) -> Option<PdfRect> {
541 let mut result: Option<PdfRect> = None;
542 for entry in group {
543 if let Some(bounds) = &entry.bounds {
544 result = Some(match result {
545 None => *bounds,
546 Some(r) => union_rect(&r, bounds),
547 });
548 }
549 }
550 result
551}
552
553fn union_rect(a: &PdfRect, b: &PdfRect) -> PdfRect {
555 PdfRect::new(
556 PdfPoints::new(a.bottom().value.min(b.bottom().value)),
557 PdfPoints::new(a.left().value.min(b.left().value)),
558 PdfPoints::new(a.top().value.max(b.top().value)),
559 PdfPoints::new(a.right().value.max(b.right().value)),
560 )
561}
562
563#[cfg(test)]
564mod tests {
565 use super::*;
566
567 fn make_entry(text: &str, font_size: f32, y_top: f32, y_bottom: f32) -> TextEntry {
568 TextEntry {
569 text: text.to_string(),
570 font_size,
571 is_bold: false,
572 is_italic: false,
573 is_monospace: false,
574 bounds: Some(PdfRect::new(
575 PdfPoints::new(y_bottom),
576 PdfPoints::new(0.0),
577 PdfPoints::new(y_top),
578 PdfPoints::new(100.0),
579 )),
580 }
581 }
582
583 fn make_block(role: ContentRole, text: &str, children: Vec<ExtractedBlock>) -> ExtractedBlock {
584 ExtractedBlock {
585 role,
586 text: text.to_string(),
587 bounds: None,
588 font_size: Some(12.0),
589 is_bold: false,
590 is_italic: false,
591 is_monospace: false,
592 children,
593 }
594 }
595
596 #[test]
597 fn test_find_body_font_size_most_common() {
598 let entries = vec![
599 make_entry("a", 12.0, 100.0, 90.0),
600 make_entry("b", 12.0, 90.0, 80.0),
601 make_entry("c", 12.0, 80.0, 70.0),
602 make_entry("d", 24.0, 60.0, 50.0),
603 ];
604 assert_eq!(find_body_font_size(&entries), 12.0);
605 }
606
607 #[test]
608 fn test_find_body_font_size_single_entry() {
609 let entries = vec![make_entry("a", 14.0, 100.0, 90.0)];
610 assert_eq!(find_body_font_size(&entries), 14.0);
611 }
612
613 #[test]
614 fn test_find_body_font_size_empty() {
615 let entries: Vec<TextEntry> = vec![];
616 assert_eq!(find_body_font_size(&entries), 12.0);
617 }
618
619 #[test]
620 fn test_is_structural_wrapper() {
621 assert!(is_structural_wrapper(&ContentRole::Other("Document".to_string())));
622 assert!(is_structural_wrapper(&ContentRole::Other("Part".to_string())));
623 assert!(is_structural_wrapper(&ContentRole::Other("Div".to_string())));
624 assert!(is_structural_wrapper(&ContentRole::Other("Sect".to_string())));
625 assert!(is_structural_wrapper(&ContentRole::Other("Art".to_string())));
626 assert!(is_structural_wrapper(&ContentRole::Other("NonStruct".to_string())));
627 assert!(is_structural_wrapper(&ContentRole::Other(String::new())));
628
629 assert!(!is_structural_wrapper(&ContentRole::Paragraph));
630 assert!(!is_structural_wrapper(&ContentRole::Heading { level: 1 }));
631 assert!(!is_structural_wrapper(&ContentRole::Other("Table".to_string())));
632 }
633
634 #[test]
635 fn test_flatten_structural_wrappers_lifts_children() {
636 let blocks = vec![make_block(
637 ContentRole::Other("Document".to_string()),
638 "",
639 vec![
640 make_block(ContentRole::Heading { level: 1 }, "Title", vec![]),
641 make_block(ContentRole::Paragraph, "Body text", vec![]),
642 ],
643 )];
644
645 let flattened = flatten_structural_wrappers(blocks);
646 assert_eq!(flattened.len(), 2);
647 assert_eq!(flattened[0].text, "Title");
648 assert_eq!(flattened[1].text, "Body text");
649 }
650
651 #[test]
652 fn test_flatten_structural_wrappers_preserves_semantic_blocks() {
653 let blocks = vec![
654 make_block(ContentRole::Heading { level: 1 }, "Title", vec![]),
655 make_block(ContentRole::Paragraph, "Body", vec![]),
656 ];
657
658 let flattened = flatten_structural_wrappers(blocks);
659 assert_eq!(flattened.len(), 2);
660 assert_eq!(flattened[0].text, "Title");
661 assert_eq!(flattened[1].text, "Body");
662 }
663
664 #[test]
665 fn test_flatten_structural_wrappers_nested() {
666 let blocks = vec![make_block(
667 ContentRole::Other("Document".to_string()),
668 "",
669 vec![make_block(
670 ContentRole::Other("Sect".to_string()),
671 "",
672 vec![make_block(ContentRole::Paragraph, "Deep content", vec![])],
673 )],
674 )];
675
676 let flattened = flatten_structural_wrappers(blocks);
677 assert_eq!(flattened.len(), 1);
678 assert_eq!(flattened[0].text, "Deep content");
679 }
680
681 #[test]
682 fn test_flatten_keeps_wrapper_with_text() {
683 let blocks = vec![make_block(
684 ContentRole::Other("Div".to_string()),
685 "Div with text",
686 vec![],
687 )];
688
689 let flattened = flatten_structural_wrappers(blocks);
690 assert_eq!(flattened.len(), 1);
691 assert_eq!(flattened[0].text, "Div with text");
692 }
693
694 #[test]
695 fn test_finalize_block_paragraph() {
696 let group = vec![
697 make_entry("Hello", 12.0, 100.0, 90.0),
698 make_entry("world", 12.0, 100.0, 90.0),
699 ];
700 let block = finalize_block(group, 12.0);
701 assert_eq!(block.role, ContentRole::Paragraph);
702 assert_eq!(block.text, "Hello world");
703 }
704
705 #[test]
706 fn test_finalize_block_heading_detection() {
707 let group = vec![make_entry("Title", 24.0, 100.0, 80.0)];
708 let block = finalize_block(group, 12.0);
709 assert_eq!(block.role, ContentRole::Heading { level: 1 });
710 }
711
712 #[test]
713 fn test_finalize_block_h2_detection() {
714 let group = vec![make_entry("Subtitle", 20.0, 100.0, 80.0)];
715 let block = finalize_block(group, 12.0);
716 assert_eq!(block.role, ContentRole::Heading { level: 2 });
717 }
718
719 #[test]
720 fn test_finalize_block_h3_detection() {
721 let group = vec![make_entry("Section", 16.5, 100.0, 80.0)];
722 let block = finalize_block(group, 12.0);
723 assert_eq!(block.role, ContentRole::Heading { level: 3 });
724 }
725
726 #[test]
727 fn test_union_rect() {
728 let a = PdfRect::new(
729 PdfPoints::new(10.0),
730 PdfPoints::new(5.0),
731 PdfPoints::new(50.0),
732 PdfPoints::new(100.0),
733 );
734 let b = PdfRect::new(
735 PdfPoints::new(5.0),
736 PdfPoints::new(10.0),
737 PdfPoints::new(60.0),
738 PdfPoints::new(80.0),
739 );
740 let u = union_rect(&a, &b);
741 assert_eq!(u.bottom().value, 5.0);
742 assert_eq!(u.left().value, 5.0);
743 assert_eq!(u.top().value, 60.0);
744 assert_eq!(u.right().value, 100.0);
745 }
746
747 #[test]
748 fn test_compute_union_bounds_empty() {
749 let group: Vec<TextEntry> = vec![];
750 assert!(compute_union_bounds(&group).is_none());
751 }
752
753 #[test]
754 fn test_compute_union_bounds_no_bounds() {
755 let group = vec![TextEntry {
756 text: "test".to_string(),
757 font_size: 12.0,
758 is_bold: false,
759 is_italic: false,
760 is_monospace: false,
761 bounds: None,
762 }];
763 assert!(compute_union_bounds(&group).is_none());
764 }
765
766 #[test]
767 fn test_group_text_into_blocks_empty() {
768 let blocks = group_text_into_blocks(Vec::new(), 12.0, PdfPoints::new(800.0));
769 assert!(blocks.is_empty());
770 }
771
772 #[test]
773 fn test_group_text_into_blocks_single() {
774 let entries = vec![make_entry("Hello", 12.0, 100.0, 88.0)];
775 let blocks = group_text_into_blocks(entries, 12.0, PdfPoints::new(800.0));
776 assert_eq!(blocks.len(), 1);
777 assert_eq!(blocks[0].text, "Hello");
778 }
779
780 #[test]
781 fn test_vertical_gap_calculation() {
782 let a = make_entry("first", 12.0, 700.0, 688.0);
783 let b = make_entry("second", 12.0, 680.0, 668.0);
784 let gap = vertical_gap(&a, &b, PdfPoints::new(800.0));
785 assert!((gap - 8.0).abs() < 0.01);
786 }
787}