Skip to main content

fetchium_core/validate/
content.rs

1//! V2 Content validation — relevance, paywall detection, minimum length (PRD §19).
2
3use crate::validate::types::*;
4
5/// Input for content validation.
6#[derive(Debug, Clone)]
7pub struct ContentInput {
8    pub url: String,
9    pub text: String,
10}
11
12/// V2 Content Validator.
13pub struct ContentValidator {
14    pub min_relevance_score: f64,
15    pub min_content_length: usize,
16}
17
18impl Default for ContentValidator {
19    fn default() -> Self {
20        Self {
21            min_relevance_score: 0.2,
22            min_content_length: 100,
23        }
24    }
25}
26
27impl ContentValidator {
28    /// V2: Validate content quality of extracted text against the query.
29    pub fn validate(&self, sources: &[ContentInput], query: &str) -> LayerResult {
30        let start = std::time::Instant::now();
31        let mut issues = Vec::new();
32        let mut scores = Vec::new();
33
34        for src in sources {
35            let mut score = 1.0;
36
37            if src.text.len() < self.min_content_length {
38                score *= 0.3;
39                issues.push(ValidationIssue {
40                    severity: IssueSeverity::Warning,
41                    code: "V2_TOO_SHORT".into(),
42                    message: format!("Content too short: {} chars", src.text.len()),
43                    source_url: Some(src.url.clone()),
44                });
45            }
46
47            if Self::is_error_page(&src.text) {
48                score *= 0.1;
49                issues.push(ValidationIssue {
50                    severity: IssueSeverity::Error,
51                    code: "V2_ERROR_PAGE".into(),
52                    message: "Detected error or paywall page".into(),
53                    source_url: Some(src.url.clone()),
54                });
55            }
56
57            let query_terms: Vec<&str> = query.split_whitespace().filter(|w| w.len() > 2).collect();
58            let lower = src.text.to_lowercase();
59            let term_hits = query_terms
60                .iter()
61                .filter(|t| lower.contains(&t.to_lowercase()))
62                .count();
63            let relevance = if query_terms.is_empty() {
64                0.5
65            } else {
66                term_hits as f64 / query_terms.len() as f64
67            };
68
69            if relevance < self.min_relevance_score {
70                score *= 0.4;
71                issues.push(ValidationIssue {
72                    severity: IssueSeverity::Warning,
73                    code: "V2_LOW_RELEVANCE".into(),
74                    message: format!("Low relevance score: {relevance:.2}"),
75                    source_url: Some(src.url.clone()),
76                });
77            }
78
79            scores.push(score * relevance.max(0.1));
80        }
81
82        let avg = if scores.is_empty() {
83            0.5
84        } else {
85            scores.iter().sum::<f64>() / scores.len() as f64
86        };
87
88        LayerResult {
89            layer: ValidationLayerId::V2Content,
90            passed: avg >= 0.2,
91            score: avg,
92            issues,
93            duration_ms: start.elapsed().as_millis() as u64,
94        }
95    }
96
97    fn is_error_page(text: &str) -> bool {
98        let lower = text.to_lowercase();
99        let patterns = [
100            "404 not found",
101            "403 forbidden",
102            "access denied",
103            "subscribe to continue",
104            "sign in to view",
105            "paywall",
106            "enable javascript",
107            "captcha",
108            "too many requests",
109        ];
110        patterns.iter().any(|p| lower.contains(p))
111    }
112}
113
114#[cfg(test)]
115mod tests {
116    use super::*;
117
118    #[test]
119    fn relevant_content_passes() {
120        let v = ContentValidator::default();
121        let src = ContentInput {
122            url: "https://rust-lang.org".into(),
123            text: "Rust is a systems programming language focused on safety, speed, and concurrency. It provides memory safety without a garbage collector.".into(),
124        };
125        let r = v.validate(&[src], "Rust programming language");
126        assert!(r.passed);
127    }
128
129    #[test]
130    fn paywall_detected() {
131        let v = ContentValidator::default();
132        let src = ContentInput {
133            url: "https://example.com".into(),
134            text: "Subscribe to continue reading this article about Rust programming.".into(),
135        };
136        let r = v.validate(&[src], "Rust");
137        assert!(r.issues.iter().any(|i| i.code == "V2_ERROR_PAGE"));
138    }
139
140    #[test]
141    fn short_content_penalized() {
142        let v = ContentValidator::default();
143        let src = ContentInput {
144            url: "https://example.com".into(),
145            text: "Short text about Rust.".into(),
146        };
147        let r = v.validate(&[src], "Rust");
148        assert!(r.issues.iter().any(|i| i.code == "V2_TOO_SHORT"));
149    }
150}