use bytes::Bytes;
use serde::{Deserialize, Deserializer, Serialize};
use std::collections::HashMap;
use super::document_structure::DocumentStructure;
use super::extraction::ExtractedImage;
use super::metadata::PptxMetadata;
use super::page::{PageContent, PageStructure};
fn deserialize_languages<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
use serde::de::Error;
let value: serde_json::Value = serde_json::Value::deserialize(deserializer)?;
match value {
serde_json::Value::String(s) => {
if s.contains('+') {
Ok(s.split('+').map(|l| l.to_string()).collect())
} else {
Ok(vec![s])
}
}
serde_json::Value::Array(arr) => arr
.into_iter()
.map(|v| {
v.as_str()
.map(String::from)
.ok_or_else(|| Error::custom("each language must be a string"))
})
.collect(),
_ => Err(Error::custom(
"language must be a string (e.g., \"eng\") or an array of strings (e.g., [\"eng\", \"deu\"])",
)),
}
}
fn deserialize_thresholding_method<'de, D>(deserializer: D) -> Result<i32, D::Error>
where
D: Deserializer<'de>,
{
use serde::de::Error;
match serde_json::Value::deserialize(deserializer)? {
serde_json::Value::Bool(false) => Ok(0),
serde_json::Value::Bool(true) => Ok(1),
serde_json::Value::Number(n) => n
.as_i64()
.and_then(|n| i32::try_from(n).ok())
.ok_or_else(|| Error::custom("thresholding_method must be an integer (0-2)")),
_ => Err(Error::custom("thresholding_method must be an integer (0-2) or a bool")),
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExcelWorkbook {
pub sheets: Vec<ExcelSheet>,
pub metadata: HashMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub revisions: Option<Vec<super::revisions::DocumentRevision>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExcelSheet {
pub name: String,
pub markdown: String,
pub row_count: usize,
pub col_count: usize,
pub cell_count: usize,
#[serde(skip)]
pub table_cells: Option<Vec<Vec<String>>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct XmlExtractionResult {
pub content: String,
pub element_count: usize,
pub unique_elements: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TextExtractionResult {
pub content: String,
pub line_count: usize,
pub word_count: usize,
pub character_count: usize,
#[serde(skip_serializing_if = "Option::is_none")]
pub headers: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub links: Option<Vec<super::metadata::MarkdownLink>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub code_blocks: Option<Vec<super::metadata::MarkdownCodeBlock>>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct PresentationHyperlink {
pub url: String,
pub label: Option<String>,
}
#[derive(Deserialize)]
#[serde(untagged)]
enum PresentationHyperlinkWire {
Positional((String, Option<String>)),
Named {
url: String,
#[serde(default)]
label: Option<String>,
},
}
impl<'de> Deserialize<'de> for PresentationHyperlink {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(match PresentationHyperlinkWire::deserialize(deserializer)? {
PresentationHyperlinkWire::Positional(hyperlink) => hyperlink.into(),
PresentationHyperlinkWire::Named { url, label } => Self { url, label },
})
}
}
#[cfg(feature = "api")]
impl utoipa::PartialSchema for PresentationHyperlink {
fn schema() -> utoipa::openapi::RefOr<utoipa::openapi::schema::Schema> {
use utoipa::openapi::schema::{Object, ObjectBuilder, SchemaType, Type};
let nullable_string = ObjectBuilder::new()
.schema_type(SchemaType::from_iter([Type::String, Type::Null]))
.build();
ObjectBuilder::new()
.property("url", Object::with_type(Type::String))
.required("url")
.property("label", nullable_string)
.required("label")
.into()
}
}
#[cfg(feature = "api")]
impl utoipa::ToSchema for PresentationHyperlink {}
impl From<(String, Option<String>)> for PresentationHyperlink {
fn from((url, label): (String, Option<String>)) -> Self {
Self { url, label }
}
}
impl From<PresentationHyperlink> for (String, Option<String>) {
fn from(hyperlink: PresentationHyperlink) -> Self {
(hyperlink.url, hyperlink.label)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PptxExtractionResult {
pub content: String,
pub metadata: PptxMetadata,
pub slide_count: usize,
pub image_count: usize,
pub table_count: usize,
pub images: Vec<ExtractedImage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub page_structure: Option<PageStructure>,
#[serde(skip_serializing_if = "Option::is_none")]
pub page_contents: Option<Vec<PageContent>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub document: Option<DocumentStructure>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub hyperlinks: Vec<PresentationHyperlink>,
#[serde(skip_serializing_if = "HashMap::is_empty", default)]
pub office_metadata: HashMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub revisions: Option<Vec<super::revisions::DocumentRevision>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EmailExtractionResult {
pub subject: Option<String>,
pub from_email: Option<String>,
pub to_emails: Vec<String>,
pub cc_emails: Vec<String>,
pub bcc_emails: Vec<String>,
pub date: Option<String>,
pub message_id: Option<String>,
pub plain_text: Option<String>,
pub html_content: Option<String>,
#[serde(alias = "cleaned_text")]
pub content: String,
pub attachments: Vec<EmailAttachment>,
pub metadata: HashMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EmailAttachment {
pub name: Option<String>,
pub filename: Option<String>,
pub mime_type: Option<String>,
pub size: Option<usize>,
pub is_image: bool,
pub data: Option<Bytes>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct OcrExtractionResult {
pub content: String,
pub mime_type: String,
pub metadata: HashMap<String, serde_json::Value>,
pub tables: Vec<OcrTable>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ocr_elements: Option<Vec<super::OcrElement>>,
#[serde(skip)]
#[allow(dead_code)]
#[cfg_attr(alef, alef(skip))]
pub(crate) internal_document: Option<super::internal::InternalDocument>,
}
impl OcrExtractionResult {
#[must_use]
pub fn new(
content: String,
mime_type: String,
metadata: HashMap<String, serde_json::Value>,
tables: Vec<OcrTable>,
ocr_elements: Option<Vec<super::OcrElement>>,
) -> Self {
Self {
content,
mime_type,
metadata,
tables,
ocr_elements,
internal_document: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OcrTable {
pub cells: Vec<Vec<String>>,
pub markdown: String,
pub page_number: u32,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub bounding_box: Option<OcrTableBoundingBox>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct OcrTableBoundingBox {
pub left: u32,
pub top: u32,
pub right: u32,
pub bottom: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[serde(default, deny_unknown_fields)]
pub struct ImagePreprocessingConfig {
pub target_dpi: i32,
pub auto_rotate: bool,
pub deskew: bool,
pub denoise: bool,
pub contrast_enhance: bool,
pub binarization_method: String,
pub invert_colors: bool,
pub normalize_shaded_rows: bool,
}
impl Default for ImagePreprocessingConfig {
fn default() -> Self {
Self {
target_dpi: 300,
auto_rotate: false,
deskew: true,
denoise: false,
contrast_enhance: false,
binarization_method: "otsu".to_string(),
invert_colors: false,
normalize_shaded_rows: false,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
#[serde(default, deny_unknown_fields)]
pub struct TesseractConfig {
#[serde(deserialize_with = "deserialize_languages")]
pub language: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub psm: Option<i32>,
pub output_format: String,
pub oem: i32,
pub min_confidence: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub preprocessing: Option<ImagePreprocessingConfig>,
pub enable_table_detection: bool,
pub table_min_confidence: f64,
pub table_column_threshold: i32,
pub table_row_threshold_ratio: f64,
pub use_cache: bool,
pub classify_use_pre_adapted_templates: bool,
pub language_model_ngram_on: bool,
pub tessedit_dont_blkrej_good_wds: bool,
pub tessedit_dont_rowrej_good_wds: bool,
pub tessedit_enable_dict_correction: bool,
pub tessedit_char_whitelist: String,
pub tessedit_char_blacklist: String,
pub tessedit_use_primary_params_model: bool,
pub textord_space_size_is_variable: bool,
#[serde(deserialize_with = "deserialize_thresholding_method")]
pub thresholding_method: i32,
}
impl Default for TesseractConfig {
fn default() -> Self {
Self {
language: vec!["eng".to_string()],
psm: None,
output_format: "markdown".to_string(),
oem: 3,
min_confidence: 0.0,
preprocessing: None,
enable_table_detection: true,
table_min_confidence: 0.0,
table_column_threshold: 50,
table_row_threshold_ratio: 0.5,
use_cache: true,
classify_use_pre_adapted_templates: true,
language_model_ngram_on: true,
tessedit_dont_blkrej_good_wds: true,
tessedit_dont_rowrej_good_wds: true,
tessedit_enable_dict_correction: true,
tessedit_char_whitelist: String::new(),
tessedit_char_blacklist: String::new(),
tessedit_use_primary_params_model: true,
textord_space_size_is_variable: true,
thresholding_method: 0,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
pub struct PixelDimensions {
pub width: usize,
pub height: usize,
}
#[derive(Deserialize)]
#[serde(untagged)]
enum PixelDimensionsWire {
Positional((usize, usize)),
Named { width: usize, height: usize },
}
impl<'de> Deserialize<'de> for PixelDimensions {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(match PixelDimensionsWire::deserialize(deserializer)? {
PixelDimensionsWire::Positional(dimensions) => dimensions.into(),
PixelDimensionsWire::Named { width, height } => Self { width, height },
})
}
}
#[cfg(feature = "api")]
impl utoipa::PartialSchema for PixelDimensions {
fn schema() -> utoipa::openapi::RefOr<utoipa::openapi::schema::Schema> {
use utoipa::openapi::schema::{Object, ObjectBuilder, Type};
ObjectBuilder::new()
.property("width", Object::with_type(Type::Integer))
.required("width")
.property("height", Object::with_type(Type::Integer))
.required("height")
.into()
}
}
#[cfg(feature = "api")]
impl utoipa::ToSchema for PixelDimensions {}
impl From<(usize, usize)> for PixelDimensions {
fn from((width, height): (usize, usize)) -> Self {
Self { width, height }
}
}
impl From<PixelDimensions> for (usize, usize) {
fn from(dimensions: PixelDimensions) -> Self {
(dimensions.width, dimensions.height)
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize)]
pub struct ImageDpi {
pub horizontal: f64,
pub vertical: f64,
}
#[derive(Deserialize)]
#[serde(untagged)]
enum ImageDpiWire {
Positional((f64, f64)),
Named { horizontal: f64, vertical: f64 },
}
impl<'de> Deserialize<'de> for ImageDpi {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(match ImageDpiWire::deserialize(deserializer)? {
ImageDpiWire::Positional(dpi) => dpi.into(),
ImageDpiWire::Named { horizontal, vertical } => Self { horizontal, vertical },
})
}
}
#[cfg(feature = "api")]
impl utoipa::PartialSchema for ImageDpi {
fn schema() -> utoipa::openapi::RefOr<utoipa::openapi::schema::Schema> {
use utoipa::openapi::schema::{Object, ObjectBuilder, Type};
ObjectBuilder::new()
.property("horizontal", Object::with_type(Type::Number))
.required("horizontal")
.property("vertical", Object::with_type(Type::Number))
.required("vertical")
.into()
}
}
#[cfg(feature = "api")]
impl utoipa::ToSchema for ImageDpi {}
impl From<(f64, f64)> for ImageDpi {
fn from((horizontal, vertical): (f64, f64)) -> Self {
Self { horizontal, vertical }
}
}
impl From<ImageDpi> for (f64, f64) {
fn from(dpi: ImageDpi) -> Self {
(dpi.horizontal, dpi.vertical)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "api", derive(utoipa::ToSchema))]
pub struct ImagePreprocessingMetadata {
pub original_dimensions: PixelDimensions,
pub original_dpi: ImageDpi,
pub target_dpi: i32,
pub scale_factor: f64,
pub auto_adjusted: bool,
pub final_dpi: i32,
pub new_dimensions: Option<PixelDimensions>,
pub resample_method: String,
pub dimension_clamped: bool,
pub calculated_dpi: Option<i32>,
pub skipped_resize: bool,
pub resize_error: Option<String>,
}
#[cfg_attr(alef, alef(skip))]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImageDpiConfig {
pub target_dpi: i32,
pub max_image_dimension: i32,
pub auto_adjust_dpi: bool,
pub min_dpi: i32,
pub max_dpi: i32,
}
impl Default for ImageDpiConfig {
fn default() -> Self {
Self {
target_dpi: 300,
max_image_dimension: 4096,
auto_adjust_dpi: true,
min_dpi: 72,
max_dpi: 600,
}
}
}
#[cfg(test)]
mod tests;