use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use super::extraction::BoundingBox;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "api", schema(value_type = u32))]
pub struct NodeIndex(pub u32);
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "api", schema(value_type = String))]
pub struct NodeId(pub String);
impl NodeId {
pub(crate) fn generate(node_type: &str, text: &str, page: Option<u32>, index: u32) -> Self {
let type_hash = node_type
.bytes()
.fold(0u64, |acc, b| acc.wrapping_mul(31).wrapping_add(b as u64));
let text_hash = text
.bytes()
.fold(0u64, |acc, b| acc.wrapping_mul(31).wrapping_add(b as u64));
let page_hash = page.map(|p| p as u64).unwrap_or(u64::MAX);
let combined = type_hash
.wrapping_mul(65599)
.wrapping_add(text_hash)
.wrapping_mul(65599)
.wrapping_add(page_hash)
.wrapping_mul(65599)
.wrapping_add(index as u64);
Self(format!("node-{:x}", combined))
}
}
impl AsRef<str> for NodeId {
fn as_ref(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for NodeId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "api", schema(no_recursion))]
pub struct DocumentStructure {
pub nodes: Vec<DocumentNode>,
#[serde(skip_serializing_if = "Option::is_none")]
pub source_format: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub relationships: Vec<DocumentRelationship>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub node_types: Vec<String>,
}
impl DocumentStructure {
pub fn finalize_node_types(&mut self) {
let mut types: Vec<&'static str> = self.nodes.iter().map(|n| n.content.node_type_name()).collect();
types.sort_unstable();
types.dedup();
self.node_types = types.into_iter().map(|s| s.to_string()).collect();
}
#[deprecated(since = "1.1.0", note = "Always returns None; scheduled for removal in 2.0")]
#[cfg_attr(alef, alef(skip))]
#[must_use]
pub fn node_rendered_offset(&self, _node_index: NodeIndex) -> Option<(usize, usize)> {
None
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
pub struct DocumentRelationship {
pub source: NodeIndex,
pub target: NodeIndex,
pub kind: RelationshipKind,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[serde(rename_all = "snake_case")]
pub enum RelationshipKind {
FootnoteReference,
CitationReference,
InternalLink,
Caption,
Label,
TocEntry,
CrossReference,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
pub struct DocumentNode {
#[serde(default)]
pub id: String,
pub content: NodeContent,
#[serde(skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "api", schema(nullable = false))]
pub parent: Option<NodeIndex>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub children: Vec<NodeIndex>,
#[serde(default)]
pub content_layer: ContentLayer,
#[serde(skip_serializing_if = "Option::is_none")]
pub page: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub page_end: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "api", schema(nullable = false))]
pub bbox: Option<BoundingBox>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub annotations: Vec<TextAnnotation>,
#[serde(skip_serializing_if = "Option::is_none")]
pub attributes: Option<HashMap<String, String>>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[serde(rename_all = "snake_case")]
pub enum ContentLayer {
#[default]
Body,
Header,
Footer,
Footnote,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[serde(tag = "node_type", rename_all = "snake_case")]
pub enum NodeContent {
Title {
text: String,
},
Heading {
level: u8,
text: String,
},
Paragraph {
text: String,
},
List {
ordered: bool,
},
ListItem {
text: String,
},
Table {
grid: TableGrid,
},
Image {
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
image_index: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
src: Option<String>,
},
Code {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
language: Option<String>,
},
Quote,
Formula {
text: String,
},
Footnote {
text: String,
},
#[cfg_attr(feature = "alef-meta", alef(since = "1.1.0"))]
Comment {
text: String,
},
Group {
#[serde(skip_serializing_if = "Option::is_none")]
label: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
heading_level: Option<u8>,
#[serde(skip_serializing_if = "Option::is_none")]
heading_text: Option<String>,
},
PageBreak,
Slide {
number: u32,
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<String>,
},
DefinitionList,
DefinitionItem {
term: String,
definition: String,
},
Citation {
key: String,
text: String,
},
Admonition {
kind: String,
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<String>,
},
RawBlock {
format: String,
content: String,
},
MetadataBlock {
entries: Vec<super::metadata::KeyValueAttribute>,
},
}
impl NodeContent {
pub fn node_type_name(&self) -> &'static str {
match self {
Self::Title { .. } => "title",
Self::Heading { .. } => "heading",
Self::Paragraph { .. } => "paragraph",
Self::List { .. } => "list",
Self::ListItem { .. } => "list_item",
Self::Table { .. } => "table",
Self::Image { .. } => "image",
Self::Code { .. } => "code",
Self::Quote => "quote",
Self::Formula { .. } => "formula",
Self::Footnote { .. } => "footnote",
Self::Comment { .. } => "comment",
Self::Group { .. } => "group",
Self::PageBreak => "page_break",
Self::Slide { .. } => "slide",
Self::DefinitionList => "definition_list",
Self::DefinitionItem { .. } => "definition_item",
Self::Citation { .. } => "citation",
Self::Admonition { .. } => "admonition",
Self::RawBlock { .. } => "raw_block",
Self::MetadataBlock { .. } => "metadata_block",
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
pub struct TableGrid {
pub rows: u32,
pub cols: u32,
pub cells: Vec<GridCell>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
pub struct GridCell {
pub content: String,
pub row: u32,
pub col: u32,
#[serde(default = "default_span")]
pub row_span: u32,
#[serde(default = "default_span")]
pub col_span: u32,
#[serde(default)]
pub is_header: bool,
#[serde(skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "api", schema(nullable = false))]
pub bbox: Option<BoundingBox>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub heading_level: Option<u8>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub style_name: Option<String>,
}
fn default_span() -> u32 {
1
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
pub struct TextAnnotation {
pub start: u32,
pub end: u32,
pub kind: AnnotationKind,
}
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[serde(tag = "annotation_type", rename_all = "snake_case")]
pub enum AnnotationKind {
#[default]
Bold,
Italic,
Underline,
Strikethrough,
Code,
Subscript,
Superscript,
Link {
url: String,
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<String>,
},
Highlight,
Color {
value: String,
},
FontSize {
value: String,
},
Custom {
name: String,
#[serde(skip_serializing_if = "Option::is_none")]
value: Option<String>,
},
}
impl From<(f32, f32, f32, f32)> for BoundingBox {
fn from((left, top, right, bottom): (f32, f32, f32, f32)) -> Self {
BoundingBox {
x0: left as f64,
y0: top as f64,
x1: right as f64,
y1: bottom as f64,
}
}
}
impl Default for NodeContent {
fn default() -> Self {
NodeContent::Paragraph { text: String::new() }
}
}
impl NodeContent {
pub fn text(&self) -> Option<&str> {
match self {
NodeContent::Title { text }
| NodeContent::Heading { text, .. }
| NodeContent::Paragraph { text }
| NodeContent::ListItem { text }
| NodeContent::Code { text, .. }
| NodeContent::Formula { text }
| NodeContent::Footnote { text }
| NodeContent::Comment { text }
| NodeContent::Citation { text, .. }
| NodeContent::RawBlock { content: text, .. } => Some(text),
NodeContent::DefinitionItem { term, .. } => Some(term),
NodeContent::Table { .. }
| NodeContent::Image { .. }
| NodeContent::List { .. }
| NodeContent::Quote
| NodeContent::Group { .. }
| NodeContent::PageBreak
| NodeContent::Slide { .. }
| NodeContent::DefinitionList
| NodeContent::Admonition { .. }
| NodeContent::MetadataBlock { .. } => None,
}
}
#[cfg(any(feature = "redaction", feature = "translation"))]
pub(crate) fn for_each_text_field_mut(&mut self, mut redact: impl FnMut(&mut String)) {
match self {
NodeContent::Title { text }
| NodeContent::Heading { text, .. }
| NodeContent::Paragraph { text }
| NodeContent::ListItem { text }
| NodeContent::Code { text, .. }
| NodeContent::Formula { text }
| NodeContent::Footnote { text }
| NodeContent::Comment { text }
| NodeContent::Citation { text, .. }
| NodeContent::RawBlock { content: text, .. } => redact(text),
NodeContent::DefinitionItem { term, definition } => {
redact(term);
redact(definition);
}
NodeContent::Group { heading_text, .. } => {
if let Some(text) = heading_text.as_mut() {
redact(text);
}
}
NodeContent::Slide { title, .. } => {
if let Some(text) = title.as_mut() {
redact(text);
}
}
NodeContent::Image { description, .. } => {
if let Some(text) = description.as_mut() {
redact(text);
}
}
NodeContent::Admonition { title, .. } => {
if let Some(text) = title.as_mut() {
redact(text);
}
}
NodeContent::Table { grid } => {
for cell in grid.cells.iter_mut() {
redact(&mut cell.content);
}
}
NodeContent::MetadataBlock { entries } => {
for attribute in entries.iter_mut() {
redact(&mut attribute.value);
}
}
NodeContent::List { .. } | NodeContent::Quote | NodeContent::PageBreak | NodeContent::DefinitionList => {}
}
}
pub fn node_type_str(&self) -> &'static str {
match self {
NodeContent::Title { .. } => "title",
NodeContent::Heading { .. } => "heading",
NodeContent::Paragraph { .. } => "paragraph",
NodeContent::List { .. } => "list",
NodeContent::ListItem { .. } => "list_item",
NodeContent::Table { .. } => "table",
NodeContent::Image { .. } => "image",
NodeContent::Code { .. } => "code",
NodeContent::Quote => "quote",
NodeContent::Formula { .. } => "formula",
NodeContent::Footnote { .. } => "footnote",
NodeContent::Comment { .. } => "comment",
NodeContent::Group { .. } => "group",
NodeContent::PageBreak => "page_break",
NodeContent::Slide { .. } => "slide",
NodeContent::DefinitionList => "definition_list",
NodeContent::DefinitionItem { .. } => "definition_item",
NodeContent::Citation { .. } => "citation",
NodeContent::Admonition { .. } => "admonition",
NodeContent::RawBlock { .. } => "raw_block",
NodeContent::MetadataBlock { .. } => "metadata_block",
}
}
}
impl DocumentStructure {
pub(crate) fn new() -> Self {
Self {
nodes: Vec::new(),
source_format: None,
relationships: Vec::new(),
node_types: Vec::new(),
}
}
pub(crate) fn with_capacity(capacity: usize) -> Self {
Self {
nodes: Vec::with_capacity(capacity),
source_format: None,
relationships: Vec::new(),
node_types: Vec::new(),
}
}
pub(crate) fn push_node(&mut self, node: DocumentNode) -> NodeIndex {
let idx = NodeIndex(self.nodes.len() as u32);
self.nodes.push(node);
idx
}
pub(crate) fn add_child(&mut self, parent: NodeIndex, child: NodeIndex) {
self.nodes[parent.0 as usize].children.push(child);
self.nodes[child.0 as usize].parent = Some(parent);
}
pub(crate) fn validate(&self) -> std::result::Result<(), String> {
let len = self.nodes.len() as u32;
for (i, node) in self.nodes.iter().enumerate() {
let idx = i as u32;
if let Some(parent) = node.parent {
if parent.0 >= len {
return Err(format!(
"Node {} has parent index {} which is out of bounds (len={})",
idx, parent.0, len
));
}
if !self.nodes[parent.0 as usize].children.contains(&NodeIndex(idx)) {
return Err(format!(
"Node {} claims parent {}, but parent's children list does not contain {}",
idx, parent.0, idx
));
}
}
for child in &node.children {
if child.0 >= len {
return Err(format!(
"Node {} has child index {} which is out of bounds (len={})",
idx, child.0, len
));
}
if self.nodes[child.0 as usize].parent != Some(NodeIndex(idx)) {
return Err(format!(
"Node {} lists child {}, but child's parent is {:?} instead of {}",
idx, child.0, self.nodes[child.0 as usize].parent, idx
));
}
}
}
Ok(())
}
#[cfg(test)]
#[cfg_attr(alef, alef(skip))]
pub(crate) fn body_roots(&self) -> impl Iterator<Item = (NodeIndex, &DocumentNode)> {
self.nodes.iter().enumerate().filter_map(|(i, node)| {
if node.parent.is_none() && node.content_layer == ContentLayer::Body {
Some((NodeIndex(i as u32), node))
} else {
None
}
})
}
#[cfg(test)]
#[cfg_attr(alef, alef(skip))]
pub(crate) fn furniture_roots(&self) -> impl Iterator<Item = (NodeIndex, &DocumentNode)> {
self.nodes.iter().enumerate().filter_map(|(i, node)| {
if node.parent.is_none() && node.content_layer != ContentLayer::Body {
Some((NodeIndex(i as u32), node))
} else {
None
}
})
}
pub(crate) fn len(&self) -> usize {
self.nodes.len()
}
pub fn is_empty(&self) -> bool {
self.nodes.is_empty()
}
}
impl Default for DocumentStructure {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests;