use serde::Serialize;
use crate::config::MarkkitBuilder;
use crate::error::Error;
use crate::format::{DocumentHint, Format};
#[derive(Debug, Clone, PartialEq, Serialize)]
#[non_exhaustive]
pub struct Extraction {
pub markdown: String,
pub title: Option<String>,
pub format: Format,
pub warnings: Vec<String>,
pub stats: ExtractionStats,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
#[non_exhaustive]
pub struct ExtractionStats {
pub pages: Option<usize>,
pub sheets: Option<usize>,
pub rows: Option<usize>,
pub paragraphs: Option<usize>,
pub tables: Option<usize>,
}
impl Extraction {
fn new(markdown: String, format: Format) -> Self {
Self {
markdown,
title: None,
format,
warnings: Vec::new(),
stats: ExtractionStats::default(),
}
}
}
#[cfg(any(feature = "docx", feature = "spreadsheet"))]
fn guarded<T>(
what: &str,
make_err: fn(String) -> Error,
f: impl FnOnce() -> crate::Result<T>,
) -> crate::Result<T> {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).unwrap_or_else(|_| {
Err(make_err(format!(
"{what} parser panicked on malformed input"
)))
})
}
pub struct Markkit {
config: MarkkitBuilder,
}
impl Markkit {
pub fn new(config: MarkkitBuilder) -> Self {
Self { config }
}
pub fn builder() -> MarkkitBuilder {
MarkkitBuilder::default()
}
pub async fn extract_html(&self, html: &str) -> crate::Result<String> {
self.extract_html_sync(html)
}
pub fn extract_html_sync(&self, html: &str) -> crate::Result<String> {
if html.len() > self.config.max_input_size {
return Err(Error::ResourceLimit(format!(
"Input size {} exceeds limit {}",
html.len(),
self.config.max_input_size
)));
}
#[cfg(feature = "html")]
{
let parser = crate::html::HtmlParser::new(&self.config);
let doc = parser.parse(html)?;
let mut renderer = crate::render::MarkdownRenderer::new();
Ok(renderer.render(&doc))
}
#[cfg(not(feature = "html"))]
Err(Error::UnsupportedFormat(
"HTML support requires the 'html' feature".to_string(),
))
}
#[cfg(feature = "html")]
pub fn parse_html(&self, html: &str) -> crate::Result<crate::ir::Document> {
if html.len() > self.config.max_input_size {
return Err(Error::ResourceLimit(format!(
"Input size {} exceeds limit {}",
html.len(),
self.config.max_input_size
)));
}
let parser = crate::html::HtmlParser::new(&self.config);
parser.parse(html)
}
pub async fn extract_bytes(
&self,
bytes: &[u8],
content_type: Option<&str>,
) -> crate::Result<String> {
self.extract_document_sync(
bytes,
DocumentHint {
content_type,
file_name: None,
},
)
.map(|e| e.markdown)
}
pub async fn extract_document(
&self,
bytes: &[u8],
hint: DocumentHint<'_>,
) -> crate::Result<Extraction> {
self.extract_document_sync(bytes, hint)
}
pub fn extract_document_sync(
&self,
bytes: &[u8],
hint: DocumentHint<'_>,
) -> crate::Result<Extraction> {
if bytes.len() > self.config.max_input_size {
return Err(Error::ResourceLimit(format!(
"Input size {} exceeds limit {}",
bytes.len(),
self.config.max_input_size
)));
}
match Format::detect(bytes, hint) {
Format::Html => {
let text = String::from_utf8_lossy(bytes);
let md = self.extract_html_sync(&text)?;
Ok(Extraction::new(md, Format::Html))
}
Format::Pdf => self.extract_pdf_sync(bytes),
Format::Docx => self.extract_docx(bytes),
Format::Xlsx | Format::Xls | Format::Ods => self.extract_spreadsheet(bytes),
Format::Doc => Err(Error::UnsupportedFormat(
"legacy Word 97-2003 (.doc) documents are not supported".into(),
)),
Format::Pptx => Err(Error::UnsupportedFormat(
"presentations (.pptx) are not supported".into(),
)),
other => Err(Error::UnsupportedFormat(match hint.content_type {
Some(ct) => format!("{ct} ({other})"),
None => other.to_string(),
})),
}
}
#[cfg(feature = "docx")]
fn extract_docx(&self, bytes: &[u8]) -> crate::Result<Extraction> {
let out = guarded("DOCX", Error::DocxExtract, || {
crate::docx::parse(bytes, &self.config)
})?;
let mut renderer = crate::render::MarkdownRenderer::new();
let mut e = Extraction::new(renderer.render(&out.doc), Format::Docx);
e.title = out.title;
e.stats.paragraphs = Some(out.paragraphs);
e.stats.tables = Some(out.tables);
Ok(e)
}
#[cfg(not(feature = "docx"))]
fn extract_docx(&self, _bytes: &[u8]) -> crate::Result<Extraction> {
Err(Error::UnsupportedFormat(
"DOCX support requires the 'docx' feature".to_string(),
))
}
#[cfg(feature = "spreadsheet")]
fn extract_spreadsheet(&self, bytes: &[u8]) -> crate::Result<Extraction> {
let out = guarded("spreadsheet", Error::SpreadsheetExtract, || {
crate::spreadsheet::parse(bytes, &self.config)
})?;
let mut renderer = crate::render::MarkdownRenderer::new();
let mut e = Extraction::new(renderer.render(&out.doc), out.format);
e.warnings = out.warnings;
e.stats.sheets = Some(out.sheets);
e.stats.rows = Some(out.rows);
Ok(e)
}
#[cfg(not(feature = "spreadsheet"))]
fn extract_spreadsheet(&self, _bytes: &[u8]) -> crate::Result<Extraction> {
Err(Error::UnsupportedFormat(
"spreadsheet support requires the 'spreadsheet' feature".to_string(),
))
}
#[cfg(feature = "http_client")]
pub async fn extract_url(&self, url: &str) -> crate::Result<String> {
let response = reqwest::get(url)
.await
.map_err(|e| Error::Http(e.to_string()))?;
let content_type = response
.headers()
.get("content-type")
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
let bytes = response
.bytes()
.await
.map_err(|e| Error::Http(e.to_string()))?;
self.extract_bytes(&bytes, content_type.as_deref()).await
}
fn extract_pdf_sync(&self, _bytes: &[u8]) -> crate::Result<Extraction> {
#[cfg(feature = "pdf")]
let msg = "PDF extraction is not implemented yet";
#[cfg(not(feature = "pdf"))]
let msg = "PDF support requires the 'pdf' feature";
Err(Error::UnsupportedFormat(msg.to_string()))
}
pub fn config(&self) -> &MarkkitBuilder {
&self.config
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_builder_defaults() {
let m = Markkit::builder().build();
assert_eq!(m.config().max_input_size, 50 * 1024 * 1024);
assert!(m.config().preserve_links);
}
#[tokio::test]
async fn test_resource_limit() {
let m = Markkit::builder().max_input_size(10).build();
let result = m.extract_html("hello world!").await;
assert!(matches!(result, Err(Error::ResourceLimit(_))));
}
}