Skip to main content

fetchium_core/validate/
extraction.rs

1//! V5 Extraction quality validation — completeness, truncation, segments (PRD §19).
2
3use crate::validate::types::*;
4
5/// Input for extraction quality validation.
6#[derive(Debug, Clone)]
7pub struct ExtractionInput {
8    pub url: String,
9    pub truncated: bool,
10    pub segment_count: usize,
11    pub encoding_errors: usize,
12}
13
14/// V5 Extraction Quality Validator.
15pub struct ExtractionValidator;
16
17impl ExtractionValidator {
18    /// V5: Validate extraction quality metrics.
19    pub fn validate(sources: &[ExtractionInput]) -> LayerResult {
20        let start = std::time::Instant::now();
21        let mut issues = Vec::new();
22        let mut scores = Vec::new();
23
24        for src in sources {
25            let mut score = 1.0;
26
27            if src.truncated {
28                score *= 0.7;
29                issues.push(ValidationIssue {
30                    severity: IssueSeverity::Warning,
31                    code: "V5_TRUNCATED".into(),
32                    message: "Content was truncated during extraction".into(),
33                    source_url: Some(src.url.clone()),
34                });
35            }
36
37            if src.segment_count == 0 {
38                score *= 0.2;
39                issues.push(ValidationIssue {
40                    severity: IssueSeverity::Error,
41                    code: "V5_NO_SEGMENTS".into(),
42                    message: "Extraction produced zero segments".into(),
43                    source_url: Some(src.url.clone()),
44                });
45            }
46
47            if src.encoding_errors > 0 {
48                score *= 0.8;
49            }
50
51            scores.push(score);
52        }
53
54        let avg = if scores.is_empty() {
55            0.5
56        } else {
57            scores.iter().sum::<f64>() / scores.len() as f64
58        };
59
60        LayerResult {
61            layer: ValidationLayerId::V5ExtractionQuality,
62            passed: avg >= 0.3,
63            score: avg,
64            issues,
65            duration_ms: start.elapsed().as_millis() as u64,
66        }
67    }
68}
69
70#[cfg(test)]
71mod tests {
72    use super::*;
73
74    #[test]
75    fn good_extraction_passes() {
76        let r = ExtractionValidator::validate(&[ExtractionInput {
77            url: "https://example.com".into(),
78            truncated: false,
79            segment_count: 10,
80            encoding_errors: 0,
81        }]);
82        assert!(r.passed);
83        assert_eq!(r.score, 1.0);
84    }
85
86    #[test]
87    fn truncated_penalized() {
88        let r = ExtractionValidator::validate(&[ExtractionInput {
89            url: "https://example.com".into(),
90            truncated: true,
91            segment_count: 5,
92            encoding_errors: 0,
93        }]);
94        assert!(r.issues.iter().any(|i| i.code == "V5_TRUNCATED"));
95        assert!(r.score < 1.0);
96    }
97
98    #[test]
99    fn no_segments_fails() {
100        let r = ExtractionValidator::validate(&[ExtractionInput {
101            url: "https://example.com".into(),
102            truncated: false,
103            segment_count: 0,
104            encoding_errors: 0,
105        }]);
106        assert!(r.issues.iter().any(|i| i.code == "V5_NO_SEGMENTS"));
107        assert!(!r.passed);
108    }
109}