#![cfg(feature = "content")]
use crate::application::dto::scrape_request::ScrapeRequestDto;
use crate::domain::services::content_extractor::ContentExtractionFacade;
use crate::domain::services::markdown_service::{MarkdownConverter, MarkdownError};
use log::{debug, warn};
use std::sync::Arc;
#[derive(Debug, thiserror::Error)]
pub enum MarkdownPostProcessorError {
#[error("markdown conversion failed: {0}")]
ConversionFailed(#[from] MarkdownError),
#[error("markdown conversion returned empty result")]
EmptyResult,
}
#[derive(Clone)]
pub struct MarkdownPostProcessor {
markdown_converter: Arc<dyn MarkdownConverter>,
content_extractor: Option<Arc<ContentExtractionFacade>>,
}
enum MainContentSource {
StructuredHtml(String),
PlainText(String),
FullHtml,
}
impl MarkdownPostProcessor {
#[must_use]
pub fn new(
markdown_converter: Arc<dyn MarkdownConverter>,
content_extractor: Option<Arc<ContentExtractionFacade>>,
) -> Self {
Self {
markdown_converter,
content_extractor,
}
}
pub async fn generate(
&self,
task_id: crate::domain::id::Id,
req: &ScrapeRequestDto,
content: &str,
) -> Result<Option<String>, MarkdownPostProcessorError> {
let want_markdown = req
.formats
.as_ref()
.map(|fs| fs.iter().any(|f| f == "markdown"))
.unwrap_or(false);
if !want_markdown {
return Ok(None);
}
let content_source = self
.resolve_main_content_source(task_id, req, content)
.await;
let markdown_result = self.convert_main_content_to_markdown(content, req, content_source);
match markdown_result {
Ok(md) if !md.trim().is_empty() => {
debug!(
"task_id: {}, Markdown generated ({} bytes)",
task_id,
md.len()
);
Ok(Some(md))
}
Ok(_md) => {
debug!(
"task_id: {}, Markdown conversion returned empty result",
task_id
);
Err(MarkdownPostProcessorError::EmptyResult)
}
Err(e) => {
warn!("task_id: {}, Markdown conversion failed: {}", task_id, e);
Err(MarkdownPostProcessorError::ConversionFailed(e))
}
}
}
async fn resolve_main_content_source(
&self,
task_id: crate::domain::id::Id,
req: &ScrapeRequestDto,
content: &str,
) -> MainContentSource {
let only_main = req
.options
.as_ref()
.and_then(|o| o.only_main_content)
.unwrap_or(false);
if only_main {
if let Some(extractor) = &self.content_extractor {
match extractor.extract(content, &req.url).await {
Ok(extracted) if !extracted.text.trim().is_empty() => {
debug!(
"task_id: {}, main content extracted (confidence={:.2}, {} bytes)",
task_id,
extracted.confidence,
extracted.text.len()
);
match extracted.html.filter(|h| !h.trim().is_empty()) {
Some(html) => MainContentSource::StructuredHtml(html),
None => {
warn!(
"task_id: {}, extractor produced no main-content HTML; \
Markdown will be structureless (plain text)",
task_id
);
MainContentSource::PlainText(extracted.text)
}
}
}
Ok(_) => {
warn!(
"task_id: {}, content extraction returned empty, falling back to full HTML",
task_id
);
MainContentSource::FullHtml
}
Err(e) => {
warn!(
"task_id: {}, content extraction failed: {}, falling back to full HTML",
task_id, e
);
MainContentSource::FullHtml
}
}
} else {
MainContentSource::FullHtml
}
} else {
MainContentSource::FullHtml
}
}
fn convert_main_content_to_markdown(
&self,
content: &str,
req: &ScrapeRequestDto,
content_source: MainContentSource,
) -> Result<String, MarkdownError> {
match content_source {
MainContentSource::PlainText(text) => Ok(escape_leading_markdown_chars(&text)),
MainContentSource::StructuredHtml(html) => {
let absolute = absoluteize_links(&html, &req.url);
self.markdown_converter.convert(&absolute)
}
MainContentSource::FullHtml => self.markdown_converter.convert(content),
}
}
}
fn absoluteize_links(html: &str, base_url: &str) -> String {
use regex::Regex;
let base = match url::Url::parse(base_url) {
Ok(b) => b,
Err(_) => return html.to_string(),
};
let attr_re = match Regex::new(r#"(?i)(\s(?:href|src)=")([^"]*)(")"#) {
Ok(r) => r,
Err(_) => return html.to_string(),
};
attr_re
.replace_all(html, |caps: ®ex::Captures| {
let value = &caps[2];
let is_special = value.is_empty()
|| value.starts_with('#')
|| value.starts_with("//")
|| value.starts_with("mailto:")
|| value.starts_with("tel:")
|| value.starts_with("javascript:")
|| value.starts_with("data:");
if is_special {
return caps[0].to_string();
}
match base.join(value) {
Ok(absolute) => format!("{}{}{}", &caps[1], absolute, &caps[3]),
Err(_) => caps[0].to_string(),
}
})
.into_owned()
}
fn escape_leading_markdown_chars(text: &str) -> String {
fn needs_escape(trimmed: &str) -> bool {
if trimmed.starts_with('#') || trimmed.starts_with('>') {
return true;
}
for marker in ["- ", "+ ", "* "] {
if trimmed.starts_with(marker) {
return true;
}
}
match trimmed.split_once('.') {
Some((num, rest)) if !num.is_empty() && num.chars().all(|c| c.is_ascii_digit()) => {
rest.starts_with(' ') || rest.is_empty()
}
_ => false,
}
}
let mut out = String::with_capacity(text.len());
for line in text.split_inclusive('\n') {
let leading = line.len() - line.trim_start().len();
if needs_escape(line.trim_start()) {
out.push_str(&line[..leading]);
out.push('\\');
out.push_str(&line[leading..]);
} else {
out.push_str(line);
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::services::markdown_service::HtmdMarkdownService;
use serde_json::json;
fn make_req(payload: serde_json::Value) -> ScrapeRequestDto {
serde_json::from_value(payload).expect("valid ScrapeRequestDto")
}
fn make_processor() -> MarkdownPostProcessor {
MarkdownPostProcessor::new(Arc::new(HtmdMarkdownService::new()), None)
}
#[tokio::test]
async fn generates_markdown_when_formats_contains_markdown() {
let p = make_processor();
let req = make_req(json!({
"url": "https://example.com",
"formats": ["markdown"]
}));
let result = p
.generate(
crate::domain::id::Id::generate(),
&req,
"<html><body><h1>Title</h1></body></html>",
)
.await;
let md = result.expect("expected Ok, got Err");
assert!(md.is_some(), "expected Some(markdown)");
let md = md.expect("some markdown");
assert!(
md.contains("Title"),
"expected Title in markdown, got: {md}"
);
}
#[tokio::test]
async fn returns_ok_none_when_formats_does_not_contain_markdown() {
let p = make_processor();
let req = make_req(json!({
"url": "https://example.com",
"formats": ["html"]
}));
let result = p
.generate(
crate::domain::id::Id::generate(),
&req,
"<html><body>Hi</body></html>",
)
.await;
assert!(result.is_ok(), "expected Ok, got Err: {:?}", result.err());
assert!(
result.unwrap().is_none(),
"expected Ok(None) when markdown not requested"
);
}
#[tokio::test]
async fn returns_ok_none_when_formats_is_none() {
let p = make_processor();
let req = make_req(json!({"url": "https://example.com"}));
let result = p
.generate(
crate::domain::id::Id::generate(),
&req,
"<html><body>Hi</body></html>",
)
.await;
assert!(result.is_ok(), "expected Ok, got Err: {:?}", result.err());
assert!(
result.unwrap().is_none(),
"expected Ok(None) when formats absent"
);
}
#[tokio::test]
async fn returns_err_empty_result_for_empty_html_when_markdown_requested() {
let p = make_processor();
let req = make_req(json!({
"url": "https://example.com",
"formats": ["markdown"]
}));
let result = p
.generate(crate::domain::id::Id::generate(), &req, "")
.await;
match result {
Err(MarkdownPostProcessorError::EmptyResult) => {}
other => panic!("expected Err(EmptyResult), got {other:?}"),
}
}
#[tokio::test]
async fn returns_err_empty_result_for_whitespace_only_html() {
let p = make_processor();
let req = make_req(json!({
"url": "https://example.com",
"formats": ["markdown"]
}));
let result = p
.generate(crate::domain::id::Id::generate(), &req, " \n\t \n ")
.await;
match result {
Err(MarkdownPostProcessorError::EmptyResult) => {}
other => panic!("expected Err(EmptyResult), got {other:?}"),
}
}
#[tokio::test]
async fn caller_strategy_continues_on_markdown_error() {
let p = make_processor();
let req = make_req(json!({"url": "https://example.com"}));
let result = p
.generate(crate::domain::id::Id::generate(), &req, "<html></html>")
.await;
let generated: Option<String> = match result {
Ok(md) => md,
Err(e) => {
warn!("markdown post-processing failed: {}", e);
None
}
};
assert!(
generated.is_none(),
"expected no markdown when not requested"
);
let req = make_req(json!({
"url": "https://example.com",
"formats": ["markdown"]
}));
let result = p
.generate(crate::domain::id::Id::generate(), &req, "")
.await;
let generated: Option<String> = match result {
Ok(md) => md,
Err(e) => {
warn!("markdown post-processing failed: {}", e);
None
}
};
assert!(
generated.is_none(),
"expected no markdown on empty html error"
);
}
#[cfg(feature = "trafilatura")]
#[tokio::test]
async fn only_main_content_preserves_markdown_structure_from_extracted_html() {
use crate::domain::services::content_extractor::ContentExtractionFacade;
let p = MarkdownPostProcessor::new(
Arc::new(HtmdMarkdownService::new()),
Some(Arc::new(ContentExtractionFacade::new(None))),
);
let req = make_req(json!({
"url": "https://example.com/article",
"formats": ["markdown"],
"options": { "only_main_content": true }
}));
let html = "<html><head><title>Engine Benchmarks Report</title></head><body>\
<nav>home | about | contact</nav>\
<article>\
<h1>Engine Benchmarks Report</h1>\
<p>This report covers the latest round of performance testing across \
all supported crawling engines. The methodology focuses on sustained \
request throughput under realistic network latency conditions.\
Read the <a href=\"https://example.com/docs\">full documentation</a> for setup details.</p>\
<h2>Methodology</h2>\
<p>Each engine ran for ten minutes against a controlled test cluster. \
We measured requests per second, error rates, and memory overhead. \
The test harness pinned CPU affinity to eliminate scheduler noise.</p>\
<h2>Results</h2>\
<ul><li>wreq reached 9000 rps</li><li>reqwest reached 6500 rps</li></ul>\
<p>For deeper analysis see the <a href=\"https://example.com/analysis\">analysis</a> page.</p>\
</article>\
</body></html>";
let result = p
.generate(crate::domain::id::Id::generate(), &req, html)
.await;
let md = result
.expect("expected Ok")
.expect("expected Some(markdown)");
assert!(
md.contains("documentation") && md.contains("]("),
"links must survive as Markdown links, got: {md}"
);
assert!(
md.contains('#'),
"heading structure must survive as Markdown heading, got: {md}"
);
}
#[tokio::test]
async fn only_main_content_without_extractor_falls_back_to_full_html() {
let p = make_processor();
let req = make_req(json!({
"url": "https://example.com/article",
"formats": ["markdown"],
"options": { "only_main_content": true }
}));
let result = p
.generate(
crate::domain::id::Id::generate(),
&req,
"<html><body><h1>Title</h1><p>Body text.</p></body></html>",
)
.await;
let md = result
.expect("expected Ok")
.expect("expected Some(markdown)");
assert!(
md.contains("Title"),
"full HTML conversion should still work, got: {md}"
);
}
}