use crate::error::PdfiumError;
use crate::pdf::document::page::PdfPage;
use crate::pdf::document::page::object::PdfPageObjectCommon;
use crate::pdf::document::page::objects::common::PdfPageObjectsCommon;
use crate::pdf::document::page::struct_element::{PdfStructElement, PdfStructElementType};
use crate::pdf::font::PdfFontWeight;
use crate::pdf::points::PdfPoints;
use crate::pdf::rect::PdfRect;
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PageExtractionMethod {
StructureTree,
Heuristic,
}
#[derive(Debug, Clone, PartialEq)]
pub enum ContentRole {
Heading { level: u8 },
Paragraph,
ListItem { label: Option<String> },
TableCell { row: usize, col: usize, is_header: bool },
Figure { alt_text: Option<String> },
Caption,
Code,
BlockQuote,
Link { url: Option<String> },
Other(String),
}
#[derive(Debug, Clone)]
pub struct ExtractedBlock {
pub role: ContentRole,
pub text: String,
pub bounds: Option<PdfRect>,
pub font_size: Option<f32>,
pub is_bold: bool,
pub is_italic: bool,
pub is_monospace: bool,
pub children: Vec<ExtractedBlock>,
}
#[derive(Debug)]
pub struct PageExtraction {
pub method: PageExtractionMethod,
pub blocks: Vec<ExtractedBlock>,
}
pub fn extract_page_content(page: &PdfPage<'_>) -> Result<PageExtraction, PdfiumError> {
if let Some(extraction) = extract_via_structure_tree(page)? {
return Ok(extraction);
}
extract_via_heuristics(page)
}
fn extract_via_structure_tree(page: &PdfPage<'_>) -> Result<Option<PageExtraction>, PdfiumError> {
let tree = match page.struct_tree() {
Some(tree) => tree,
None => return Ok(None),
};
if tree.children_count() == 0 {
return Ok(None);
}
let (mcid_text_map, mcid_style_map) = build_mcid_maps(page)?;
if mcid_text_map.is_empty() {
return Ok(None);
}
let mut blocks = Vec::new();
let mut resolved = false;
for child in tree.children() {
if let Some(block) = extract_element_block(&child, &mcid_text_map, &mcid_style_map)
&& (!block.text.is_empty() || !block.children.is_empty())
{
resolved = true;
blocks.push(block);
}
}
if !resolved {
return Ok(None);
}
let blocks = flatten_structural_wrappers(blocks);
Ok(Some(PageExtraction {
method: PageExtractionMethod::StructureTree,
blocks,
}))
}
#[derive(Debug, Clone)]
struct TextStyle {
font_size: f32,
is_bold: bool,
is_italic: bool,
is_monospace: bool,
bounds: Option<PdfRect>,
}
type McidMaps = (HashMap<i32, String>, HashMap<i32, TextStyle>);
fn build_mcid_maps(page: &PdfPage<'_>) -> Result<McidMaps, PdfiumError> {
let objects = page.objects();
let text_page = page.text()?;
let mut text_map: HashMap<i32, String> = HashMap::new();
let mut style_map: HashMap<i32, TextStyle> = HashMap::new();
for i in 0..objects.len() {
let object = objects.get(i)?;
if let Some(text_obj) = object.as_text_object()
&& let Some(mcid) = object.marked_content_id()
{
let text = text_page.for_object(text_obj);
if !text.is_empty() {
text_map
.entry(mcid)
.and_modify(|existing| existing.push_str(&text))
.or_insert(text);
}
style_map.entry(mcid).or_insert_with(|| {
let font = text_obj.font();
let is_bold = font.weight().ok().is_some_and(|w| {
matches!(
w,
PdfFontWeight::Weight700Bold | PdfFontWeight::Weight800 | PdfFontWeight::Weight900
)
}) || font.is_bold_reenforced()
|| font.name().to_ascii_lowercase().contains("bold");
let is_italic = font.is_italic();
let is_monospace = font.is_fixed_pitch();
let font_size = text_obj.scaled_font_size().value;
let bounds = object.bounds().ok().map(|qp| qp.to_rect());
TextStyle {
font_size,
is_bold,
is_italic,
is_monospace,
bounds,
}
});
}
}
Ok((text_map, style_map))
}
fn extract_element_block(
element: &PdfStructElement<'_>,
mcid_text_map: &HashMap<i32, String>,
mcid_style_map: &HashMap<i32, TextStyle>,
) -> Option<ExtractedBlock> {
let element_type = element.element_type();
let role = element_type_to_role(&element_type, element);
let mcids = element.all_marked_content_ids();
let mut text_parts: Vec<&str> = Vec::new();
let mut style: Option<&TextStyle> = None;
for mcid in &mcids {
if let Some(t) = mcid_text_map.get(mcid) {
text_parts.push(t);
}
if style.is_none() {
style = mcid_style_map.get(mcid);
}
}
let actual_text = element.actual_text();
let alt_text = element.alt_text();
let text = if !text_parts.is_empty() {
text_parts.join("")
} else if let Some(ref at) = actual_text {
at.clone()
} else if let Some(ref alt) = alt_text {
alt.clone()
} else {
String::new()
};
let children = extract_children_blocks(element, mcid_text_map, mcid_style_map);
if text.is_empty() && children.is_empty() {
return None;
}
Some(ExtractedBlock {
role,
text,
bounds: style.and_then(|s| s.bounds),
font_size: style.map(|s| s.font_size),
is_bold: style.is_some_and(|s| s.is_bold),
is_italic: style.is_some_and(|s| s.is_italic),
is_monospace: style.is_some_and(|s| s.is_monospace),
children,
})
}
fn extract_children_blocks(
element: &PdfStructElement<'_>,
mcid_text_map: &HashMap<i32, String>,
mcid_style_map: &HashMap<i32, TextStyle>,
) -> Vec<ExtractedBlock> {
let mut children = Vec::new();
for child in element.children() {
if let Some(block) = extract_element_block(&child, mcid_text_map, mcid_style_map)
&& (!block.text.is_empty() || !block.children.is_empty())
{
children.push(block);
}
}
children
}
fn element_type_to_role(element_type: &PdfStructElementType, element: &PdfStructElement<'_>) -> ContentRole {
match element_type {
PdfStructElementType::H => ContentRole::Heading { level: 1 },
PdfStructElementType::H1 => ContentRole::Heading { level: 1 },
PdfStructElementType::H2 => ContentRole::Heading { level: 2 },
PdfStructElementType::H3 => ContentRole::Heading { level: 3 },
PdfStructElementType::H4 => ContentRole::Heading { level: 4 },
PdfStructElementType::H5 => ContentRole::Heading { level: 5 },
PdfStructElementType::H6 => ContentRole::Heading { level: 6 },
PdfStructElementType::P | PdfStructElementType::Span => ContentRole::Paragraph,
PdfStructElementType::LI => {
let label = find_child_text_by_type(element, &PdfStructElementType::Lbl);
ContentRole::ListItem { label }
}
PdfStructElementType::Figure => {
let alt = element.alt_text();
ContentRole::Figure { alt_text: alt }
}
PdfStructElementType::Caption => ContentRole::Caption,
PdfStructElementType::Code => ContentRole::Code,
PdfStructElementType::BlockQuote => ContentRole::BlockQuote,
PdfStructElementType::Link => {
let url = element.string_attribute("O");
ContentRole::Link { url }
}
PdfStructElementType::TD => ContentRole::TableCell {
row: 0,
col: 0,
is_header: false,
},
PdfStructElementType::TH => ContentRole::TableCell {
row: 0,
col: 0,
is_header: true,
},
_ => {
let type_str = element.element_type_raw().unwrap_or_default();
ContentRole::Other(type_str)
}
}
}
fn find_child_text_by_type(element: &PdfStructElement<'_>, target_type: &PdfStructElementType) -> Option<String> {
for child in element.children() {
if child.element_type() == *target_type {
if let Some(text) = child.actual_text() {
return Some(text);
}
if let Some(alt) = child.alt_text() {
return Some(alt);
}
}
}
None
}
fn flatten_structural_wrappers(blocks: Vec<ExtractedBlock>) -> Vec<ExtractedBlock> {
let mut result = Vec::new();
for block in blocks {
if is_structural_wrapper(&block.role) && block.text.is_empty() {
let children = flatten_structural_wrappers(block.children);
result.extend(children);
} else {
let children = flatten_structural_wrappers(block.children);
result.push(ExtractedBlock { children, ..block });
}
}
result
}
fn is_structural_wrapper(role: &ContentRole) -> bool {
matches!(
role,
ContentRole::Other(s) if matches!(s.as_str(), "Document" | "Part" | "Div" | "Sect" | "Art" | "NonStruct" | "")
)
}
fn extract_via_heuristics(page: &PdfPage<'_>) -> Result<PageExtraction, PdfiumError> {
let objects = page.objects();
let text_page = page.text()?;
let mut text_entries: Vec<TextEntry> = Vec::new();
for i in 0..objects.len() {
let object = objects.get(i)?;
if let Some(text_obj) = object.as_text_object() {
let text = text_page.for_object(text_obj);
if text.is_empty() {
continue;
}
let font = text_obj.font();
let font_size = text_obj.scaled_font_size().value;
let is_bold = font.weight().ok().is_some_and(|w| {
matches!(
w,
PdfFontWeight::Weight700Bold | PdfFontWeight::Weight800 | PdfFontWeight::Weight900
)
}) || font.is_bold_reenforced()
|| font.name().to_ascii_lowercase().contains("bold");
let is_italic = font.is_italic();
let is_monospace = font.is_fixed_pitch();
let bounds = object.bounds().ok().map(|qp| qp.to_rect());
text_entries.push(TextEntry {
text,
font_size,
is_bold,
is_italic,
is_monospace,
bounds,
});
}
}
if text_entries.is_empty() {
return Ok(PageExtraction {
method: PageExtractionMethod::Heuristic,
blocks: Vec::new(),
});
}
let body_font_size = find_body_font_size(&text_entries);
let blocks = group_text_into_blocks(text_entries, body_font_size, page.height());
Ok(PageExtraction {
method: PageExtractionMethod::Heuristic,
blocks,
})
}
struct TextEntry {
text: String,
font_size: f32,
is_bold: bool,
is_italic: bool,
is_monospace: bool,
bounds: Option<PdfRect>,
}
fn find_body_font_size(entries: &[TextEntry]) -> f32 {
let mut size_counts: HashMap<u32, usize> = HashMap::new();
for entry in entries {
let key = (entry.font_size * 2.0).round() as u32;
*size_counts.entry(key).or_insert(0) += 1;
}
size_counts
.into_iter()
.max_by_key(|(_, count)| *count)
.map(|(key, _)| key as f32 / 2.0)
.unwrap_or(12.0)
}
fn group_text_into_blocks(entries: Vec<TextEntry>, body_font_size: f32, page_height: PdfPoints) -> Vec<ExtractedBlock> {
if entries.is_empty() {
return Vec::new();
}
let mut sorted = entries;
sorted.sort_by(|a, b| {
let a_top = a
.bounds
.as_ref()
.map(|r| page_height.value - r.top().value)
.unwrap_or(0.0);
let b_top = b
.bounds
.as_ref()
.map(|r| page_height.value - r.top().value)
.unwrap_or(0.0);
a_top.total_cmp(&b_top).then_with(|| {
let a_left = a.bounds.as_ref().map(|r| r.left().value).unwrap_or(0.0);
let b_left = b.bounds.as_ref().map(|r| r.left().value).unwrap_or(0.0);
a_left.total_cmp(&b_left)
})
});
let mut blocks = Vec::new();
let mut current_group: Vec<TextEntry> = vec![sorted.remove(0)];
for entry in sorted {
let should_break = {
let last = current_group.last().unwrap();
let gap = vertical_gap(last, &entry, page_height);
gap > body_font_size * 1.2
};
if should_break {
blocks.push(finalize_block(current_group, body_font_size));
current_group = vec![entry];
} else {
current_group.push(entry);
}
}
if !current_group.is_empty() {
blocks.push(finalize_block(current_group, body_font_size));
}
blocks
}
fn vertical_gap(a: &TextEntry, b: &TextEntry, page_height: PdfPoints) -> f32 {
let a_bottom = a
.bounds
.as_ref()
.map(|r| page_height.value - r.bottom().value)
.unwrap_or(0.0);
let b_top = b
.bounds
.as_ref()
.map(|r| page_height.value - r.top().value)
.unwrap_or(0.0);
(b_top - a_bottom).abs()
}
fn finalize_block(group: Vec<TextEntry>, body_font_size: f32) -> ExtractedBlock {
let text: String = group.iter().map(|e| e.text.as_str()).collect::<Vec<_>>().join(" ");
let first = &group[0];
let font_size = first.font_size;
let is_bold = first.is_bold;
let is_italic = first.is_italic;
let is_monospace = first.is_monospace;
let role = if font_size > body_font_size * 1.3 {
let level = if font_size > body_font_size * 1.8 {
1
} else if font_size > body_font_size * 1.5 {
2
} else {
3
};
ContentRole::Heading { level }
} else {
ContentRole::Paragraph
};
let bounds = compute_union_bounds(&group);
ExtractedBlock {
role,
text,
bounds,
font_size: Some(font_size),
is_bold,
is_italic,
is_monospace,
children: Vec::new(),
}
}
fn compute_union_bounds(group: &[TextEntry]) -> Option<PdfRect> {
let mut result: Option<PdfRect> = None;
for entry in group {
if let Some(bounds) = &entry.bounds {
result = Some(match result {
None => *bounds,
Some(r) => union_rect(&r, bounds),
});
}
}
result
}
fn union_rect(a: &PdfRect, b: &PdfRect) -> PdfRect {
PdfRect::new(
PdfPoints::new(a.bottom().value.min(b.bottom().value)),
PdfPoints::new(a.left().value.min(b.left().value)),
PdfPoints::new(a.top().value.max(b.top().value)),
PdfPoints::new(a.right().value.max(b.right().value)),
)
}
#[cfg(test)]
mod tests {
use super::*;
fn make_entry(text: &str, font_size: f32, y_top: f32, y_bottom: f32) -> TextEntry {
TextEntry {
text: text.to_string(),
font_size,
is_bold: false,
is_italic: false,
is_monospace: false,
bounds: Some(PdfRect::new(
PdfPoints::new(y_bottom),
PdfPoints::new(0.0),
PdfPoints::new(y_top),
PdfPoints::new(100.0),
)),
}
}
fn make_block(role: ContentRole, text: &str, children: Vec<ExtractedBlock>) -> ExtractedBlock {
ExtractedBlock {
role,
text: text.to_string(),
bounds: None,
font_size: Some(12.0),
is_bold: false,
is_italic: false,
is_monospace: false,
children,
}
}
#[test]
fn test_find_body_font_size_most_common() {
let entries = vec![
make_entry("a", 12.0, 100.0, 90.0),
make_entry("b", 12.0, 90.0, 80.0),
make_entry("c", 12.0, 80.0, 70.0),
make_entry("d", 24.0, 60.0, 50.0),
];
assert_eq!(find_body_font_size(&entries), 12.0);
}
#[test]
fn test_find_body_font_size_single_entry() {
let entries = vec![make_entry("a", 14.0, 100.0, 90.0)];
assert_eq!(find_body_font_size(&entries), 14.0);
}
#[test]
fn test_find_body_font_size_empty() {
let entries: Vec<TextEntry> = vec![];
assert_eq!(find_body_font_size(&entries), 12.0);
}
#[test]
fn test_is_structural_wrapper() {
assert!(is_structural_wrapper(&ContentRole::Other("Document".to_string())));
assert!(is_structural_wrapper(&ContentRole::Other("Part".to_string())));
assert!(is_structural_wrapper(&ContentRole::Other("Div".to_string())));
assert!(is_structural_wrapper(&ContentRole::Other("Sect".to_string())));
assert!(is_structural_wrapper(&ContentRole::Other("Art".to_string())));
assert!(is_structural_wrapper(&ContentRole::Other("NonStruct".to_string())));
assert!(is_structural_wrapper(&ContentRole::Other(String::new())));
assert!(!is_structural_wrapper(&ContentRole::Paragraph));
assert!(!is_structural_wrapper(&ContentRole::Heading { level: 1 }));
assert!(!is_structural_wrapper(&ContentRole::Other("Table".to_string())));
}
#[test]
fn test_flatten_structural_wrappers_lifts_children() {
let blocks = vec![make_block(
ContentRole::Other("Document".to_string()),
"",
vec![
make_block(ContentRole::Heading { level: 1 }, "Title", vec![]),
make_block(ContentRole::Paragraph, "Body text", vec![]),
],
)];
let flattened = flatten_structural_wrappers(blocks);
assert_eq!(flattened.len(), 2);
assert_eq!(flattened[0].text, "Title");
assert_eq!(flattened[1].text, "Body text");
}
#[test]
fn test_flatten_structural_wrappers_preserves_semantic_blocks() {
let blocks = vec![
make_block(ContentRole::Heading { level: 1 }, "Title", vec![]),
make_block(ContentRole::Paragraph, "Body", vec![]),
];
let flattened = flatten_structural_wrappers(blocks);
assert_eq!(flattened.len(), 2);
assert_eq!(flattened[0].text, "Title");
assert_eq!(flattened[1].text, "Body");
}
#[test]
fn test_flatten_structural_wrappers_nested() {
let blocks = vec![make_block(
ContentRole::Other("Document".to_string()),
"",
vec![make_block(
ContentRole::Other("Sect".to_string()),
"",
vec![make_block(ContentRole::Paragraph, "Deep content", vec![])],
)],
)];
let flattened = flatten_structural_wrappers(blocks);
assert_eq!(flattened.len(), 1);
assert_eq!(flattened[0].text, "Deep content");
}
#[test]
fn test_flatten_keeps_wrapper_with_text() {
let blocks = vec![make_block(
ContentRole::Other("Div".to_string()),
"Div with text",
vec![],
)];
let flattened = flatten_structural_wrappers(blocks);
assert_eq!(flattened.len(), 1);
assert_eq!(flattened[0].text, "Div with text");
}
#[test]
fn test_finalize_block_paragraph() {
let group = vec![
make_entry("Hello", 12.0, 100.0, 90.0),
make_entry("world", 12.0, 100.0, 90.0),
];
let block = finalize_block(group, 12.0);
assert_eq!(block.role, ContentRole::Paragraph);
assert_eq!(block.text, "Hello world");
}
#[test]
fn test_finalize_block_heading_detection() {
let group = vec![make_entry("Title", 24.0, 100.0, 80.0)];
let block = finalize_block(group, 12.0);
assert_eq!(block.role, ContentRole::Heading { level: 1 });
}
#[test]
fn test_finalize_block_h2_detection() {
let group = vec![make_entry("Subtitle", 20.0, 100.0, 80.0)];
let block = finalize_block(group, 12.0);
assert_eq!(block.role, ContentRole::Heading { level: 2 });
}
#[test]
fn test_finalize_block_h3_detection() {
let group = vec![make_entry("Section", 16.5, 100.0, 80.0)];
let block = finalize_block(group, 12.0);
assert_eq!(block.role, ContentRole::Heading { level: 3 });
}
#[test]
fn test_union_rect() {
let a = PdfRect::new(
PdfPoints::new(10.0),
PdfPoints::new(5.0),
PdfPoints::new(50.0),
PdfPoints::new(100.0),
);
let b = PdfRect::new(
PdfPoints::new(5.0),
PdfPoints::new(10.0),
PdfPoints::new(60.0),
PdfPoints::new(80.0),
);
let u = union_rect(&a, &b);
assert_eq!(u.bottom().value, 5.0);
assert_eq!(u.left().value, 5.0);
assert_eq!(u.top().value, 60.0);
assert_eq!(u.right().value, 100.0);
}
#[test]
fn test_compute_union_bounds_empty() {
let group: Vec<TextEntry> = vec![];
assert!(compute_union_bounds(&group).is_none());
}
#[test]
fn test_compute_union_bounds_no_bounds() {
let group = vec![TextEntry {
text: "test".to_string(),
font_size: 12.0,
is_bold: false,
is_italic: false,
is_monospace: false,
bounds: None,
}];
assert!(compute_union_bounds(&group).is_none());
}
#[test]
fn test_group_text_into_blocks_empty() {
let blocks = group_text_into_blocks(Vec::new(), 12.0, PdfPoints::new(800.0));
assert!(blocks.is_empty());
}
#[test]
fn test_group_text_into_blocks_single() {
let entries = vec![make_entry("Hello", 12.0, 100.0, 88.0)];
let blocks = group_text_into_blocks(entries, 12.0, PdfPoints::new(800.0));
assert_eq!(blocks.len(), 1);
assert_eq!(blocks[0].text, "Hello");
}
#[test]
fn test_vertical_gap_calculation() {
let a = make_entry("first", 12.0, 700.0, 688.0);
let b = make_entry("second", 12.0, 680.0, 668.0);
let gap = vertical_gap(&a, &b, PdfPoints::new(800.0));
assert!((gap - 8.0).abs() < 0.01);
}
}