use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct Rect {
pub x0: f32,
pub y0: f32,
pub x1: f32,
pub y1: f32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum BlockKind {
Heading,
#[default]
Paragraph,
List,
Code,
Table,
Image,
Quote,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct TableData {
pub rows: Vec<Vec<String>>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Block {
#[serde(rename = "kind")]
pub kind: BlockKind,
#[serde(default)]
pub content: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bbox: Option<Rect>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub level: Option<u8>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ordered: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub lang: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub table: Option<TableData>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Image {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub data: Vec<u8>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bbox: Option<Rect>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dedup_key: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ocr_text: Option<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Page {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(default)]
pub blocks: Vec<Block>,
#[serde(default)]
pub text: String,
#[serde(default)]
pub images: Vec<Image>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub width: Option<f32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub height: Option<f32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rotation: Option<i32>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct DocMeta {
pub file_type: String,
pub page_count: usize,
pub text_length: usize,
pub image_count: usize,
pub image_text_length: usize,
#[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
pub extra: std::collections::BTreeMap<String, String>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Document {
pub meta: DocMeta,
#[serde(default)]
pub pages: Vec<Page>,
}
impl Document {
pub fn finalize(mut self) -> Self {
self.meta.page_count = self.pages.len();
self.meta.text_length = self.pages.iter().map(|p| p.text.chars().count()).sum();
self.meta.image_count = self.pages.iter().map(|p| p.images.len()).sum();
self.meta.image_text_length = self
.pages
.iter()
.flat_map(|p| &p.images)
.map(|im| {
im.ocr_text
.as_deref()
.map(|s| s.chars().count())
.unwrap_or(0)
})
.sum();
self
}
}
pub fn blocks_to_text(blocks: &[Block], sep: &str) -> String {
let parts: Vec<String> = blocks
.iter()
.filter_map(|b| match b.kind {
BlockKind::Table => b.table.as_ref().map(|t| {
t.rows
.iter()
.map(|r| r.join(","))
.collect::<Vec<_>>()
.join("\n")
}),
_ if !b.content.is_empty() => Some(b.content.clone()),
_ => None,
})
.collect();
parts.join(sep)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn document_finalize_counts() {
let doc = Document {
pages: vec![
Page {
text: "你好".into(),
images: vec![Image {
ocr_text: Some("x".into()),
..Default::default()
}],
..Default::default()
},
Page {
text: "world".into(),
..Default::default()
},
],
..Default::default()
}
.finalize();
assert_eq!(doc.meta.page_count, 2);
assert_eq!(doc.meta.text_length, 7); assert_eq!(doc.meta.image_count, 1);
assert_eq!(doc.meta.image_text_length, 1);
}
#[test]
fn blocks_to_text_expands_tables() {
let b = Block {
kind: BlockKind::Table,
table: Some(TableData {
rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
}),
..Default::default()
};
let txt = blocks_to_text(&[b], "\n\n");
assert_eq!(txt, "a,b\n1,2");
}
#[test]
fn roundtrip_json() {
let doc = Document {
pages: vec![Page {
text: "t".into(),
blocks: vec![Block {
kind: BlockKind::Paragraph,
content: "hi".into(),
..Default::default()
}],
..Default::default()
}],
..Default::default()
};
let s = serde_json::to_string(&doc).unwrap();
let back: Document = serde_json::from_str(&s).unwrap();
assert_eq!(back, doc);
}
}