use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PageType {
Article,
Listing,
Unknown,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExtractedContent {
pub text: String,
#[serde(default)]
pub html: Option<String>,
pub title: Option<String>,
pub author: Option<String>,
pub confidence: f32,
pub page_type: PageType,
}
impl ExtractedContent {
pub fn new(text: String) -> Self {
Self {
text,
html: None,
title: None,
author: None,
confidence: 0.0,
page_type: PageType::Unknown,
}
}
pub fn should_fallback_to_llm(&self) -> bool {
self.confidence < 0.7
}
}
pub trait ContentExtractor: Send + Sync {
fn extract(&self, html: &str, url: &str) -> Result<ExtractedContent>;
fn name(&self) -> &'static str;
}
#[derive(Debug, thiserror::Error)]
pub enum ExtractError {
#[error("HTML parse error: {0}")]
HtmlParseError(String),
#[error("no content extracted")]
NoContent,
#[error("extractor failed: {0}")]
ExtractorFailed(String),
}
pub type Result<T> = std::result::Result<T, ExtractError>;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn new_initializes_with_low_confidence_and_unknown_type() {
let c = ExtractedContent::new("hello".to_string());
assert_eq!(c.text, "hello");
assert_eq!(c.confidence, 0.0);
assert_eq!(c.page_type, PageType::Unknown);
assert!(c.title.is_none() && c.author.is_none());
}
#[test]
fn should_fallback_to_llm_when_confidence_below_threshold() {
let mut c = ExtractedContent::new("text".to_string());
c.confidence = 0.69;
assert!(c.should_fallback_to_llm(), "0.69 should trigger fallback");
c.confidence = 0.7;
assert!(
!c.should_fallback_to_llm(),
"0.7 should NOT trigger fallback (boundary)"
);
c.confidence = 1.0;
assert!(!c.should_fallback_to_llm());
}
#[test]
fn page_type_serializes_as_snake_case() {
let json = serde_json::to_string(&PageType::Article).unwrap();
assert_eq!(json, "\"article\"");
let parsed: PageType = serde_json::from_str("\"listing\"").unwrap();
assert_eq!(parsed, PageType::Listing);
}
#[test]
fn page_type_all_variants_roundtrip() {
for variant in [PageType::Article, PageType::Listing, PageType::Unknown] {
let json = serde_json::to_string(&variant).unwrap();
let back: PageType = serde_json::from_str(&json).unwrap();
assert_eq!(back, variant);
}
}
#[test]
fn extracted_content_roundtrips_through_serde() {
let c = ExtractedContent {
text: "body".to_string(),
html: Some("<p>body</p>".to_string()),
title: Some("Title".to_string()),
author: Some("Author".to_string()),
confidence: 0.85,
page_type: PageType::Article,
};
let json = serde_json::to_string(&c).unwrap();
let back: ExtractedContent = serde_json::from_str(&json).unwrap();
assert_eq!(back.text, c.text);
assert_eq!(back.title, c.title);
assert_eq!(back.author, c.author);
assert!((back.confidence - c.confidence).abs() < f32::EPSILON);
assert_eq!(back.page_type, c.page_type);
}
#[test]
fn extract_error_display_includes_message() {
let e = ExtractError::HtmlParseError("bad html".to_string());
let s = format!("{e}");
assert!(s.contains("HTML parse error"));
assert!(s.contains("bad html"));
let e = ExtractError::NoContent;
assert_eq!(format!("{e}"), "no content extracted");
let e = ExtractError::ExtractorFailed("boom".to_string());
let s = format!("{e}");
assert!(s.contains("extractor failed"));
assert!(s.contains("boom"));
}
#[test]
fn extract_error_implements_std_error() {
fn assert_error<E: std::error::Error>(_: &E) {}
let e = ExtractError::NoContent;
assert_error(&e);
}
#[test]
fn extract_error_can_be_boxed() {
let e: Box<dyn std::error::Error> =
Box::new(ExtractError::ExtractorFailed("oops".to_string()));
assert!(e.to_string().contains("oops"));
}
#[test]
fn result_alias_is_correct() {
let r: Result<()> = Err(ExtractError::NoContent);
assert!(r.is_err());
let r: Result<i32> = Ok(42);
assert!(r.is_ok_and(|v| v == 42));
}
}