fetchium_core/validate/
content.rs1use crate::validate::types::*;
4
5#[derive(Debug, Clone)]
7pub struct ContentInput {
8 pub url: String,
9 pub text: String,
10}
11
12pub struct ContentValidator {
14 pub min_relevance_score: f64,
15 pub min_content_length: usize,
16}
17
18impl Default for ContentValidator {
19 fn default() -> Self {
20 Self {
21 min_relevance_score: 0.2,
22 min_content_length: 100,
23 }
24 }
25}
26
27impl ContentValidator {
28 pub fn validate(&self, sources: &[ContentInput], query: &str) -> LayerResult {
30 let start = std::time::Instant::now();
31 let mut issues = Vec::new();
32 let mut scores = Vec::new();
33
34 for src in sources {
35 let mut score = 1.0;
36
37 if src.text.len() < self.min_content_length {
38 score *= 0.3;
39 issues.push(ValidationIssue {
40 severity: IssueSeverity::Warning,
41 code: "V2_TOO_SHORT".into(),
42 message: format!("Content too short: {} chars", src.text.len()),
43 source_url: Some(src.url.clone()),
44 });
45 }
46
47 if Self::is_error_page(&src.text) {
48 score *= 0.1;
49 issues.push(ValidationIssue {
50 severity: IssueSeverity::Error,
51 code: "V2_ERROR_PAGE".into(),
52 message: "Detected error or paywall page".into(),
53 source_url: Some(src.url.clone()),
54 });
55 }
56
57 let query_terms: Vec<&str> = query.split_whitespace().filter(|w| w.len() > 2).collect();
58 let lower = src.text.to_lowercase();
59 let term_hits = query_terms
60 .iter()
61 .filter(|t| lower.contains(&t.to_lowercase()))
62 .count();
63 let relevance = if query_terms.is_empty() {
64 0.5
65 } else {
66 term_hits as f64 / query_terms.len() as f64
67 };
68
69 if relevance < self.min_relevance_score {
70 score *= 0.4;
71 issues.push(ValidationIssue {
72 severity: IssueSeverity::Warning,
73 code: "V2_LOW_RELEVANCE".into(),
74 message: format!("Low relevance score: {relevance:.2}"),
75 source_url: Some(src.url.clone()),
76 });
77 }
78
79 scores.push(score * relevance.max(0.1));
80 }
81
82 let avg = if scores.is_empty() {
83 0.5
84 } else {
85 scores.iter().sum::<f64>() / scores.len() as f64
86 };
87
88 LayerResult {
89 layer: ValidationLayerId::V2Content,
90 passed: avg >= 0.2,
91 score: avg,
92 issues,
93 duration_ms: start.elapsed().as_millis() as u64,
94 }
95 }
96
97 fn is_error_page(text: &str) -> bool {
98 let lower = text.to_lowercase();
99 let patterns = [
100 "404 not found",
101 "403 forbidden",
102 "access denied",
103 "subscribe to continue",
104 "sign in to view",
105 "paywall",
106 "enable javascript",
107 "captcha",
108 "too many requests",
109 ];
110 patterns.iter().any(|p| lower.contains(p))
111 }
112}
113
114#[cfg(test)]
115mod tests {
116 use super::*;
117
118 #[test]
119 fn relevant_content_passes() {
120 let v = ContentValidator::default();
121 let src = ContentInput {
122 url: "https://rust-lang.org".into(),
123 text: "Rust is a systems programming language focused on safety, speed, and concurrency. It provides memory safety without a garbage collector.".into(),
124 };
125 let r = v.validate(&[src], "Rust programming language");
126 assert!(r.passed);
127 }
128
129 #[test]
130 fn paywall_detected() {
131 let v = ContentValidator::default();
132 let src = ContentInput {
133 url: "https://example.com".into(),
134 text: "Subscribe to continue reading this article about Rust programming.".into(),
135 };
136 let r = v.validate(&[src], "Rust");
137 assert!(r.issues.iter().any(|i| i.code == "V2_ERROR_PAGE"));
138 }
139
140 #[test]
141 fn short_content_penalized() {
142 let v = ContentValidator::default();
143 let src = ContentInput {
144 url: "https://example.com".into(),
145 text: "Short text about Rust.".into(),
146 };
147 let r = v.validate(&[src], "Rust");
148 assert!(r.issues.iter().any(|i| i.code == "V2_TOO_SHORT"));
149 }
150}