use rs_trafilatura::{self, page_type, Options};
use super::traits::{ContentExtractor, ExtractError, ExtractedContent, PageType};
#[derive(Debug, Clone, Default)]
pub struct TrafilaturaExtractor;
impl TrafilaturaExtractor {
pub fn new() -> Self {
Self
}
fn map_page_type(pt: page_type::PageType) -> PageType {
match pt {
page_type::PageType::Article | page_type::PageType::Documentation => PageType::Article,
page_type::PageType::Category | page_type::PageType::Listing => PageType::Listing,
page_type::PageType::Product
| page_type::PageType::Forum
| page_type::PageType::Service => PageType::Unknown,
}
}
fn build_options(url: &str) -> Options {
Options {
url: Some(url.to_string()),
include_links: true,
..Default::default()
}
}
fn calculate_confidence(extracted_len: usize, html_len: usize) -> f32 {
if html_len == 0 || extracted_len == 0 {
return 0.7;
}
let ratio = (extracted_len as f32 / html_len as f32).clamp(0.0, 1.0);
0.7 + ratio * 0.25
}
}
impl ContentExtractor for TrafilaturaExtractor {
fn extract(&self, html: &str, url: &str) -> Result<ExtractedContent, ExtractError> {
let trimmed_html = html.trim();
if trimmed_html.is_empty() {
return Err(ExtractError::NoContent);
}
let opts = Self::build_options(url);
let result = rs_trafilatura::extract_with_options(trimmed_html, &opts)
.map_err(|e| ExtractError::ExtractorFailed(format!("trafilatura: {e}")))?;
let text = result.content_text.trim().to_string();
if text.is_empty() {
return Err(ExtractError::NoContent);
}
let raw_page_type = page_type::classify_url(url);
let page_type = Self::map_page_type(raw_page_type);
let confidence = Self::calculate_confidence(text.len(), trimmed_html.len());
Ok(ExtractedContent {
text,
html: result.content_html.filter(|h| !h.trim().is_empty()),
title: result.metadata.title,
author: result.metadata.author,
confidence,
page_type,
})
}
fn name(&self) -> &'static str {
"trafilatura"
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn extracts_main_content_from_simple_article() {
let html = r#"<html><head><title>Test Article</title>
<meta name="author" content="Kirky.X"></head>
<body>
<nav>Home About</nav>
<article><p>This is the main content of the article.</p>
<p>Second paragraph with more text.</p></article>
<footer>Copyright 2025</footer>
</body></html>"#;
let result = TrafilaturaExtractor::new()
.extract(html, "https://example.com/article")
.expect("extract ok");
assert!(!result.text.is_empty(), "text should not be empty");
assert!(
result.text.contains("main content"),
"text should contain main content, got: {}",
result.text
);
}
#[test]
fn returns_error_for_empty_html() {
let result = TrafilaturaExtractor::new().extract("", "https://example.com/");
assert!(matches!(result, Err(ExtractError::NoContent)));
}
#[test]
fn returns_error_for_whitespace_only_html() {
let result = TrafilaturaExtractor::new().extract(" \n\t ", "https://example.com/");
assert!(matches!(result, Err(ExtractError::NoContent)));
}
#[test]
fn confidence_is_within_valid_range() {
let html = r#"<html><head><title>Article</title></head>
<body><article><p>Body content with enough text to be considered as main content.
More text here to ensure it passes length thresholds.</p></article></body></html>"#;
let result = TrafilaturaExtractor::new()
.extract(html, "https://example.com/a")
.expect("extract ok");
assert!(
result.confidence >= 0.7 && result.confidence <= 0.95,
"confidence should be in [0.7, 0.95], got: {}",
result.confidence
);
}
#[test]
fn name_returns_trafilatura() {
assert_eq!(TrafilaturaExtractor::new().name(), "trafilatura");
}
#[test]
fn default_equals_new() {
let a = TrafilaturaExtractor::new();
let b = TrafilaturaExtractor;
let html = r#"<html><body><article><p>some content</p></article></body></html>"#;
let ra = a
.extract(html, "https://example.com/")
.expect("a extract ok");
let rb = b
.extract(html, "https://example.com/")
.expect("b extract ok");
assert_eq!(ra.text, rb.text);
assert_eq!(ra.confidence, rb.confidence);
}
#[test]
fn trait_object_dispatch_works() {
let extractor: Box<dyn ContentExtractor> = Box::new(TrafilaturaExtractor::new());
let html = r#"<html><body><article><p>via trait</p></article></body></html>"#;
let result = extractor
.extract(html, "https://example.com/")
.expect("trait dispatch ok");
assert!(!result.text.is_empty());
assert_eq!(extractor.name(), "trafilatura");
}
#[test]
fn map_page_type_correctly_translates_all_variants() {
assert_eq!(
TrafilaturaExtractor::map_page_type(page_type::PageType::Article),
PageType::Article
);
assert_eq!(
TrafilaturaExtractor::map_page_type(page_type::PageType::Documentation),
PageType::Article
);
assert_eq!(
TrafilaturaExtractor::map_page_type(page_type::PageType::Category),
PageType::Listing
);
assert_eq!(
TrafilaturaExtractor::map_page_type(page_type::PageType::Listing),
PageType::Listing
);
assert_eq!(
TrafilaturaExtractor::map_page_type(page_type::PageType::Product),
PageType::Unknown
);
assert_eq!(
TrafilaturaExtractor::map_page_type(page_type::PageType::Forum),
PageType::Unknown
);
assert_eq!(
TrafilaturaExtractor::map_page_type(page_type::PageType::Service),
PageType::Unknown
);
}
#[test]
fn build_options_injects_url() {
let opts = TrafilaturaExtractor::build_options("https://example.com/x");
assert_eq!(opts.url.as_deref(), Some("https://example.com/x"));
}
#[test]
fn calculate_confidence_in_valid_range() {
assert!((TrafilaturaExtractor::calculate_confidence(0, 0) - 0.7).abs() < f32::EPSILON);
assert!((TrafilaturaExtractor::calculate_confidence(10, 0) - 0.7).abs() < f32::EPSILON);
let c = TrafilaturaExtractor::calculate_confidence(100, 100);
assert!(
(c - 0.95).abs() < f32::EPSILON,
"ratio=1.0 → 0.95, got: {c}"
);
let c = TrafilaturaExtractor::calculate_confidence(50, 100);
assert!(
c > 0.7 && c < 0.95,
"ratio=0.5 → between 0.7 and 0.95, got: {c}"
);
let c = TrafilaturaExtractor::calculate_confidence(1, 1000);
assert!((0.7..0.75).contains(&c), "ratio=0.001 → near 0.7, got: {c}");
}
#[test]
fn handles_very_long_html() {
let mut body = String::from("<html><body><article>");
for _ in 0..1_000 {
body.push_str("<p>paragraph content here. </p>");
}
body.push_str("</article></body></html>");
let result = TrafilaturaExtractor::new()
.extract(&body, "https://example.com/long")
.expect("extract ok");
assert!(!result.text.is_empty());
}
}