fetchium_core/validate/
extraction.rs1use crate::validate::types::*;
4
5#[derive(Debug, Clone)]
7pub struct ExtractionInput {
8 pub url: String,
9 pub truncated: bool,
10 pub segment_count: usize,
11 pub encoding_errors: usize,
12}
13
14pub struct ExtractionValidator;
16
17impl ExtractionValidator {
18 pub fn validate(sources: &[ExtractionInput]) -> LayerResult {
20 let start = std::time::Instant::now();
21 let mut issues = Vec::new();
22 let mut scores = Vec::new();
23
24 for src in sources {
25 let mut score = 1.0;
26
27 if src.truncated {
28 score *= 0.7;
29 issues.push(ValidationIssue {
30 severity: IssueSeverity::Warning,
31 code: "V5_TRUNCATED".into(),
32 message: "Content was truncated during extraction".into(),
33 source_url: Some(src.url.clone()),
34 });
35 }
36
37 if src.segment_count == 0 {
38 score *= 0.2;
39 issues.push(ValidationIssue {
40 severity: IssueSeverity::Error,
41 code: "V5_NO_SEGMENTS".into(),
42 message: "Extraction produced zero segments".into(),
43 source_url: Some(src.url.clone()),
44 });
45 }
46
47 if src.encoding_errors > 0 {
48 score *= 0.8;
49 }
50
51 scores.push(score);
52 }
53
54 let avg = if scores.is_empty() {
55 0.5
56 } else {
57 scores.iter().sum::<f64>() / scores.len() as f64
58 };
59
60 LayerResult {
61 layer: ValidationLayerId::V5ExtractionQuality,
62 passed: avg >= 0.3,
63 score: avg,
64 issues,
65 duration_ms: start.elapsed().as_millis() as u64,
66 }
67 }
68}
69
70#[cfg(test)]
71mod tests {
72 use super::*;
73
74 #[test]
75 fn good_extraction_passes() {
76 let r = ExtractionValidator::validate(&[ExtractionInput {
77 url: "https://example.com".into(),
78 truncated: false,
79 segment_count: 10,
80 encoding_errors: 0,
81 }]);
82 assert!(r.passed);
83 assert_eq!(r.score, 1.0);
84 }
85
86 #[test]
87 fn truncated_penalized() {
88 let r = ExtractionValidator::validate(&[ExtractionInput {
89 url: "https://example.com".into(),
90 truncated: true,
91 segment_count: 5,
92 encoding_errors: 0,
93 }]);
94 assert!(r.issues.iter().any(|i| i.code == "V5_TRUNCATED"));
95 assert!(r.score < 1.0);
96 }
97
98 #[test]
99 fn no_segments_fails() {
100 let r = ExtractionValidator::validate(&[ExtractionInput {
101 url: "https://example.com".into(),
102 truncated: false,
103 segment_count: 0,
104 encoding_errors: 0,
105 }]);
106 assert!(r.issues.iter().any(|i| i.code == "V5_NO_SEGMENTS"));
107 assert!(!r.passed);
108 }
109}