datasynth_eval/
lib.rs

1#![deny(clippy::unwrap_used)]
2// Allow some clippy lints that are common in test/evaluation code
3#![allow(clippy::field_reassign_with_default)]
4#![allow(clippy::too_many_arguments)]
5#![allow(clippy::upper_case_acronyms)] // MCAR, MAR, MNAR, ISO are standard abbreviations
6
7//! Synthetic Data Evaluation Framework
8//!
9//! This crate provides comprehensive evaluation capabilities for validating
10//! the quality and correctness of generated synthetic financial data.
11//!
12//! # Features
13//!
14//! - **Statistical Quality**: Benford's Law, amount distributions, line item patterns
15//! - **Semantic Coherence**: Balance sheet validation, subledger reconciliation
16//! - **Data Quality**: Uniqueness, completeness, format consistency
17//! - **ML-Readiness**: Feature distributions, label quality, graph structure
18//! - **Reporting**: HTML and JSON reports with pass/fail thresholds
19//!
20//! # Example
21//!
22//! ```ignore
23//! use datasynth_eval::{Evaluator, EvaluationConfig};
24//!
25//! let config = EvaluationConfig::default();
26//! let evaluator = Evaluator::new(config);
27//!
28//! // Evaluate generated data
29//! let result = evaluator.evaluate(&generation_result)?;
30//!
31//! // Generate report
32//! result.generate_html_report("evaluation_report.html")?;
33//! ```
34
35pub mod benchmarks;
36pub mod config;
37pub mod enhancement;
38pub mod error;
39pub mod gates;
40pub mod privacy;
41
42pub mod coherence;
43pub mod ml;
44pub mod quality;
45pub mod report;
46pub mod statistical;
47pub mod tuning;
48
49// Re-exports
50pub use config::{EvaluationConfig, EvaluationThresholds, PrivacyEvaluationConfig};
51pub use error::{EvalError, EvalResult};
52
53pub use statistical::{
54    AmountDistributionAnalysis, AmountDistributionAnalyzer, BenfordAnalysis, BenfordAnalyzer,
55    BenfordConformity, DetectionDifficulty, DriftDetectionAnalysis, DriftDetectionAnalyzer,
56    DriftDetectionEntry, DriftDetectionMetrics, DriftEventCategory, LabeledDriftEvent,
57    LabeledEventAnalysis, LineItemAnalysis, LineItemAnalyzer, LineItemEntry, StatisticalEvaluation,
58    TemporalAnalysis, TemporalAnalyzer, TemporalEntry,
59};
60
61pub use coherence::{
62    AuditTrailEvaluation, AuditTrailGap, BalanceSheetEvaluation, BalanceSheetEvaluator,
63    CoherenceEvaluation, ConcentrationMetrics, DocumentChainEvaluation, DocumentChainEvaluator,
64    FairValueEvaluation, FrameworkViolation, ICMatchingEvaluation, ICMatchingEvaluator,
65    ImpairmentEvaluation, IsaComplianceEvaluation, LeaseAccountingEvaluation,
66    LeaseAccountingEvaluator, LeaseEvaluation, NetworkEdge, NetworkEvaluation, NetworkEvaluator,
67    NetworkNode, NetworkThresholds, PcaobComplianceEvaluation, PerformanceObligation,
68    ReferentialIntegrityEvaluation, ReferentialIntegrityEvaluator, RevenueContract,
69    RevenueRecognitionEvaluation, RevenueRecognitionEvaluator, SoxComplianceEvaluation,
70    StandardsComplianceEvaluation, StandardsThresholds, StrengthStats, SubledgerEvaluator,
71    SubledgerReconciliationEvaluation, VariableConsideration, ViolationSeverity,
72};
73
74pub use quality::{
75    CompletenessAnalysis, CompletenessAnalyzer, ConsistencyAnalysis, ConsistencyAnalyzer,
76    ConsistencyRule, DuplicateInfo, FieldCompleteness, FormatAnalysis, FormatAnalyzer,
77    FormatVariation, QualityEvaluation, UniquenessAnalysis, UniquenessAnalyzer,
78};
79
80pub use ml::{
81    FeatureAnalysis, FeatureAnalyzer, FeatureStats, GraphAnalysis, GraphAnalyzer, GraphMetrics,
82    LabelAnalysis, LabelAnalyzer, LabelDistribution, MLReadinessEvaluation, SplitAnalysis,
83    SplitAnalyzer, SplitMetrics,
84};
85
86pub use report::{
87    BaselineComparison, ComparisonResult, EvaluationReport, HtmlReportGenerator,
88    JsonReportGenerator, MetricChange, ReportMetadata, ThresholdChecker, ThresholdResult,
89};
90
91pub use tuning::{
92    ConfigSuggestion, ConfigSuggestionGenerator, TuningAnalyzer, TuningCategory, TuningOpportunity,
93};
94
95pub use enhancement::{
96    AutoTuneResult, AutoTuner, ConfigPatch, EnhancementReport, Recommendation,
97    RecommendationCategory, RecommendationEngine, RecommendationPriority, RootCause,
98    SuggestedAction,
99};
100
101pub use privacy::{
102    LinkageAttack, LinkageConfig, LinkageResults, MembershipInferenceAttack, MiaConfig, MiaResults,
103    NistAlignmentReport, NistCriterion, PrivacyEvaluation, SynQPMatrix, SynQPQuadrant,
104};
105
106pub use benchmarks::{
107    // ACFE-calibrated benchmarks
108    acfe_calibrated_1k,
109    acfe_collusion_5k,
110    acfe_management_override_2k,
111    all_acfe_benchmarks,
112    all_benchmarks,
113    // Industry-specific benchmarks
114    all_industry_benchmarks,
115    anomaly_bench_1k,
116    data_quality_100k,
117    entity_match_5k,
118    financial_services_fraud_5k,
119    fraud_detect_10k,
120    get_benchmark,
121    get_industry_benchmark,
122    graph_fraud_10k,
123    healthcare_fraud_5k,
124    manufacturing_fraud_5k,
125    retail_fraud_10k,
126    technology_fraud_3k,
127    AcfeAlignment,
128    AcfeCalibration,
129    AcfeCategoryDistribution,
130    BaselineModelType,
131    BaselineResult,
132    BenchmarkBuilder,
133    BenchmarkSuite,
134    BenchmarkTaskType,
135    CostMatrix,
136    DatasetSpec,
137    EvaluationSpec,
138    FeatureSet,
139    IndustryBenchmarkAnalysis,
140    LeaderboardEntry,
141    MetricType,
142    SplitRatios,
143};
144
145use serde::{Deserialize, Serialize};
146
147/// Comprehensive evaluation result combining all evaluation modules.
148#[derive(Debug, Clone, Serialize, Deserialize)]
149pub struct ComprehensiveEvaluation {
150    /// Statistical quality evaluation.
151    pub statistical: StatisticalEvaluation,
152    /// Semantic coherence evaluation.
153    pub coherence: CoherenceEvaluation,
154    /// Data quality evaluation.
155    pub quality: QualityEvaluation,
156    /// ML-readiness evaluation.
157    pub ml_readiness: MLReadinessEvaluation,
158    /// Privacy evaluation (optional — only populated when privacy testing is enabled).
159    #[serde(default, skip_serializing_if = "Option::is_none")]
160    pub privacy: Option<PrivacyEvaluation>,
161    /// Overall pass/fail status.
162    pub passes: bool,
163    /// Summary of all failures.
164    pub failures: Vec<String>,
165    /// Tuning opportunities identified.
166    pub tuning_opportunities: Vec<TuningOpportunity>,
167    /// Configuration suggestions.
168    pub config_suggestions: Vec<ConfigSuggestion>,
169}
170
171impl ComprehensiveEvaluation {
172    /// Create a new empty evaluation.
173    pub fn new() -> Self {
174        Self {
175            statistical: StatisticalEvaluation::default(),
176            coherence: CoherenceEvaluation::default(),
177            quality: QualityEvaluation::default(),
178            ml_readiness: MLReadinessEvaluation::default(),
179            privacy: None,
180            passes: true,
181            failures: Vec::new(),
182            tuning_opportunities: Vec::new(),
183            config_suggestions: Vec::new(),
184        }
185    }
186
187    /// Check all evaluations against thresholds and update overall status.
188    pub fn check_all_thresholds(&mut self, thresholds: &EvaluationThresholds) {
189        self.failures.clear();
190
191        // Check statistical thresholds
192        self.statistical.check_thresholds(thresholds);
193        self.failures.extend(self.statistical.failures.clone());
194
195        // Check coherence thresholds
196        self.coherence.check_thresholds(thresholds);
197        self.failures.extend(self.coherence.failures.clone());
198
199        // Check quality thresholds
200        self.quality.check_thresholds(thresholds);
201        self.failures.extend(self.quality.failures.clone());
202
203        // Check ML thresholds
204        self.ml_readiness.check_thresholds(thresholds);
205        self.failures.extend(self.ml_readiness.failures.clone());
206
207        // Check privacy evaluation (if present)
208        if let Some(ref mut privacy) = self.privacy {
209            privacy.update_status();
210            self.failures.extend(privacy.failures.clone());
211        }
212
213        self.passes = self.failures.is_empty();
214    }
215}
216
217impl Default for ComprehensiveEvaluation {
218    fn default() -> Self {
219        Self::new()
220    }
221}
222
223/// Main evaluator that coordinates all evaluation modules.
224pub struct Evaluator {
225    /// Evaluation configuration.
226    config: EvaluationConfig,
227}
228
229impl Evaluator {
230    /// Create a new evaluator with the given configuration.
231    pub fn new(config: EvaluationConfig) -> Self {
232        Self { config }
233    }
234
235    /// Create an evaluator with default configuration.
236    pub fn with_defaults() -> Self {
237        Self::new(EvaluationConfig::default())
238    }
239
240    /// Get the configuration.
241    pub fn config(&self) -> &EvaluationConfig {
242        &self.config
243    }
244
245    /// Run a comprehensive evaluation and return results.
246    ///
247    /// This is a placeholder - actual implementation would take
248    /// generation results as input.
249    pub fn run_evaluation(&self) -> ComprehensiveEvaluation {
250        let mut evaluation = ComprehensiveEvaluation::new();
251        evaluation.check_all_thresholds(&self.config.thresholds);
252        evaluation
253    }
254}
255
256#[cfg(test)]
257#[allow(clippy::unwrap_used)]
258mod tests {
259    use super::*;
260
261    #[test]
262    fn test_comprehensive_evaluation_new() {
263        let eval = ComprehensiveEvaluation::new();
264        assert!(eval.passes);
265        assert!(eval.failures.is_empty());
266    }
267
268    #[test]
269    fn test_evaluator_creation() {
270        let evaluator = Evaluator::with_defaults();
271        assert_eq!(evaluator.config().thresholds.benford_p_value_min, 0.05);
272    }
273}
datasynth_eval/lib.rs

datasynth_eval/
lib.rs
