use dom_smoothie::{Article, Readability};
use super::traits::{ContentExtractor, ExtractError, ExtractedContent, PageType};
#[derive(Debug, Clone, Default)]
pub struct DomSmoothieExtractor;
impl DomSmoothieExtractor {
pub fn new() -> Self {
Self
}
fn calculate_confidence(text_len: usize) -> f32 {
if text_len < 100 {
0.6
} else if text_len < 1000 {
0.6 + 0.15 * ((text_len - 100) as f32 / 900.0)
} else {
0.85
}
}
}
impl ContentExtractor for DomSmoothieExtractor {
fn extract(&self, html: &str, url: &str) -> Result<ExtractedContent, ExtractError> {
let trimmed_html = html.trim();
if trimmed_html.is_empty() {
return Err(ExtractError::NoContent);
}
let mut readability = Readability::new(trimmed_html, Some(url), None)
.map_err(|e| ExtractError::ExtractorFailed(format!("dom-smoothie: {e}")))?;
let article: Article = readability
.parse()
.map_err(|e| ExtractError::ExtractorFailed(format!("dom-smoothie parse: {e}")))?;
let text = article.text_content.to_string();
let trimmed_text = text.trim().to_string();
if trimmed_text.is_empty() {
return Err(ExtractError::NoContent);
}
let confidence = Self::calculate_confidence(trimmed_text.len());
Ok(ExtractedContent {
text: trimmed_text,
html: None,
title: Some(article.title),
author: article.byline,
confidence,
page_type: PageType::Unknown,
})
}
fn name(&self) -> &'static str {
"dom-smoothie"
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn extracts_main_content_from_simple_article() {
let html = r#"<html><head><title>Test Article</title>
<meta name="author" content="Kirky.X"></head>
<body>
<nav>Home About</nav>
<article><p>This is the main content of the article.</p>
<p>Second paragraph with more text.</p></article>
<footer>Copyright 2025</footer>
</body></html>"#;
let result = DomSmoothieExtractor::new()
.extract(html, "https://example.com/article")
.expect("extract ok");
assert!(!result.text.is_empty(), "text should not be empty");
}
#[test]
fn returns_error_for_empty_html() {
let result = DomSmoothieExtractor::new().extract("", "https://example.com/");
assert!(matches!(result, Err(ExtractError::NoContent)));
}
#[test]
fn returns_error_for_whitespace_only_html() {
let result = DomSmoothieExtractor::new().extract(" \n\t ", "https://example.com/");
assert!(matches!(result, Err(ExtractError::NoContent)));
}
#[test]
fn confidence_is_within_valid_range() {
let html = r#"<html><head><title>Article</title></head>
<body><article><p>Body content with enough text to be considered as main content.
More text here to ensure it passes length thresholds.</p></article></body></html>"#;
let result = DomSmoothieExtractor::new()
.extract(html, "https://example.com/a")
.expect("extract ok");
assert!(
result.confidence >= 0.6 && result.confidence <= 0.85,
"confidence should be in [0.6, 0.85], got: {}",
result.confidence
);
}
#[test]
fn name_returns_dom_smoothie() {
assert_eq!(DomSmoothieExtractor::new().name(), "dom-smoothie");
}
#[test]
fn default_equals_new() {
let a = DomSmoothieExtractor::new();
let b = DomSmoothieExtractor;
let html = r#"<html><body><article><p>some content</p></article></body></html>"#;
let ra = a
.extract(html, "https://example.com/")
.expect("a extract ok");
let rb = b
.extract(html, "https://example.com/")
.expect("b extract ok");
assert_eq!(ra.text, rb.text);
assert_eq!(ra.confidence, rb.confidence);
}
#[test]
fn trait_object_dispatch_works() {
let extractor: Box<dyn ContentExtractor> = Box::new(DomSmoothieExtractor::new());
let html = r#"<html><body><article><p>via trait</p></article></body></html>"#;
let result = extractor
.extract(html, "https://example.com/")
.expect("trait dispatch ok");
assert!(!result.text.is_empty());
assert_eq!(extractor.name(), "dom-smoothie");
}
#[test]
fn calculate_confidence_uses_length_thresholds() {
assert!((DomSmoothieExtractor::calculate_confidence(0) - 0.6).abs() < f32::EPSILON);
assert!((DomSmoothieExtractor::calculate_confidence(99) - 0.6).abs() < f32::EPSILON);
let c100 = DomSmoothieExtractor::calculate_confidence(100);
let c500 = DomSmoothieExtractor::calculate_confidence(500);
let c999 = DomSmoothieExtractor::calculate_confidence(999);
assert!(
(c100 - 0.6).abs() < 0.01,
"100 chars should be ~0.6, got: {c100}"
);
assert!(
c500 > c100 && c999 > c500,
"confidence should increase with text length"
);
let c1000 = DomSmoothieExtractor::calculate_confidence(1000);
let c10000 = DomSmoothieExtractor::calculate_confidence(10000);
assert!((c1000 - 0.85).abs() < f32::EPSILON, "got: {c1000}");
assert!((c10000 - 0.85).abs() < f32::EPSILON);
}
#[test]
fn calculate_confidence_at_500_is_between_0_6_and_0_75() {
let c = DomSmoothieExtractor::calculate_confidence(500);
assert!(
c > 0.6 && c < 0.75,
"500 chars should be between 0.6 and 0.75, got: {c}"
);
}
#[test]
fn handles_very_long_html() {
let mut body = String::from("<html><body><article>");
for _ in 0..1_000 {
body.push_str("<p>paragraph content here. </p>");
}
body.push_str("</article></body></html>");
let result = DomSmoothieExtractor::new()
.extract(&body, "https://example.com/long")
.expect("extract ok");
assert!(!result.text.is_empty());
}
}