datasynth_eval/
lib.rs

1#![cfg_attr(not(test), deny(clippy::unwrap_used))]
2// Allow some clippy lints that are common in test/evaluation code
3#![allow(clippy::field_reassign_with_default)]
4#![allow(clippy::too_many_arguments)]
5#![allow(clippy::upper_case_acronyms)] // MCAR, MAR, MNAR, ISO are standard abbreviations
6
7//! Synthetic Data Evaluation Framework
8//!
9//! This crate provides comprehensive evaluation capabilities for validating
10//! the quality and correctness of generated synthetic financial data.
11//!
12//! # Features
13//!
14//! - **Statistical Quality**: Benford's Law, amount distributions, line item patterns
15//! - **Semantic Coherence**: Balance sheet validation, subledger reconciliation
16//! - **Data Quality**: Uniqueness, completeness, format consistency
17//! - **ML-Readiness**: Feature distributions, label quality, graph structure
18//! - **Reporting**: HTML and JSON reports with pass/fail thresholds
19//!
20//! # Example
21//!
22//! ```ignore
23//! use datasynth_eval::{Evaluator, EvaluationConfig};
24//!
25//! let config = EvaluationConfig::default();
26//! let evaluator = Evaluator::new(config);
27//!
28//! // Evaluate generated data
29//! let result = evaluator.evaluate(&generation_result)?;
30//!
31//! // Generate report
32//! result.generate_html_report("evaluation_report.html")?;
33//! ```
34
35pub mod benchmarks;
36pub mod config;
37pub mod enhancement;
38pub mod error;
39pub mod gates;
40pub mod privacy;
41
42pub mod behavioral_fidelity;
43pub mod coherence;
44pub mod ml;
45pub mod quality;
46pub mod report;
47pub mod statistical;
48pub mod tuning;
49
50#[cfg(feature = "adversarial")]
51pub mod adversarial;
52pub mod banking;
53pub mod causal;
54pub mod diff_engine;
55pub mod enrichment;
56pub mod process_mining;
57pub mod scenario_diff;
58
59// Re-exports
60pub use config::{EvaluationConfig, EvaluationThresholds, PrivacyEvaluationConfig};
61pub use error::{EvalError, EvalResult};
62
63pub use statistical::{
64    AmountDistributionAnalysis, AmountDistributionAnalyzer, AnomalyRealismEvaluation,
65    AnomalyRealismEvaluator, BenfordAnalysis, BenfordAnalyzer, BenfordConformity,
66    CorrelationAnalysis, CorrelationAnalyzer, CorrelationCheckResult, DetectionDifficulty,
67    DriftDetectionAnalysis, DriftDetectionAnalyzer, DriftDetectionEntry, DriftDetectionMetrics,
68    DriftEventCategory, ExpectedCorrelation, LabeledDriftEvent, LabeledEventAnalysis,
69    LineItemAnalysis, LineItemAnalyzer, LineItemEntry, SecondDigitAnalysis, StatisticalEvaluation,
70    TemporalAnalysis, TemporalAnalyzer, TemporalEntry,
71};
72
73pub use coherence::{
74    AccountType,
75    ApprovalLevelData,
76    AuditEvaluation,
77    AuditEvaluator,
78    AuditFindingData,
79    AuditRiskData,
80    AuditTrailEvaluation,
81    AuditTrailGap,
82    BalanceSheetEvaluation,
83    BalanceSheetEvaluator,
84    BalanceSnapshot,
85    BankReconciliationEvaluation,
86    BankReconciliationEvaluator,
87    BidEvaluationData,
88    BudgetVarianceData,
89    CashPositionData,
90    CoherenceEvaluation,
91    ConcentrationMetrics,
92    CountryPackData,
93    CountryPackEvaluation,
94    CountryPackEvaluator,
95    CountryPackThresholds,
96    CovenantData,
97    CrossProcessEvaluation,
98    CrossProcessEvaluator,
99    CycleCountData,
100    DocumentChainEvaluation,
101    DocumentChainEvaluator,
102    DocumentReferenceData,
103    EarnedValueData,
104    EntityReferenceData,
105    EsgEvaluation,
106    EsgEvaluator,
107    EsgThresholds,
108    ExpenseReportData,
109    FairValueEvaluation,
110    // Task 4.1: Financial Ratio Evaluator
111    FinancialRatios,
112    FinancialReportingEvaluation,
113    FinancialReportingEvaluator,
114    FinancialStatementData,
115    FrameworkViolation,
116    GovernanceData,
117    HedgeEffectivenessData,
118    HolidayData,
119    HrPayrollEvaluation,
120    HrPayrollEvaluator,
121    ICMatchingData,
122    ICMatchingEvaluation,
123    ICMatchingEvaluator,
124    ImpairmentEvaluation,
125    IsaComplianceEvaluation,
126    // Task 4.2: JE Risk Scoring Evaluator
127    JeRiskScoringResult,
128    KpiData,
129    LeaseAccountingEvaluation,
130    LeaseAccountingEvaluator,
131    LeaseEvaluation,
132    ManufacturingEvaluation,
133    ManufacturingEvaluator,
134    MaterialityData,
135    NettingData,
136    NetworkEdge,
137    NetworkEvaluation,
138    NetworkEvaluator,
139    NetworkNode,
140    NetworkThresholds,
141    O2CChainData,
142    P2PChainData,
143    PayrollHoursData,
144    PayrollLineItemData,
145    PayrollRunData,
146    PcaobComplianceEvaluation,
147    PerformanceObligation,
148    ProductionOrderData,
149    ProjectAccountingEvaluation,
150    ProjectAccountingEvaluator,
151    ProjectAccountingThresholds,
152    ProjectRevenueData,
153    QualityInspectionData,
154    QuoteLineData,
155    RatioAnalysisResult,
156    RatioCheck,
157    ReconciliationData,
158    ReferentialData,
159    ReferentialIntegrityEvaluation,
160    ReferentialIntegrityEvaluator,
161    RetainageData,
162    RevenueContract,
163    RevenueRecognitionEvaluation,
164    RevenueRecognitionEvaluator,
165    RiskAttributeStats,
166    RiskDistribution,
167    RoutingOperationData,
168    SafetyMetricData,
169    SalesQuoteData,
170    SalesQuoteEvaluation,
171    SalesQuoteEvaluator,
172    SalesQuoteThresholds,
173    ScorecardCoverageData,
174    SourcingEvaluation,
175    SourcingEvaluator,
176    SourcingProjectData,
177    SoxComplianceEvaluation,
178    SpendAnalysisData,
179    StandardsComplianceEvaluation,
180    StandardsThresholds,
181    StrengthStats,
182    SubledgerEvaluator,
183    SubledgerReconciliationEvaluation,
184    SupplierEsgData,
185    TaxEvaluation,
186    TaxEvaluator,
187    TaxLineData,
188    TaxRateData,
189    TaxReturnData,
190    TaxThresholds,
191    TimeEntryData,
192    TreasuryEvaluation,
193    TreasuryEvaluator,
194    TreasuryThresholds,
195    UnmatchedICItem,
196    VariableConsideration,
197    ViolationSeverity,
198    WaterUsageData,
199    WithholdingData,
200    WorkpaperData,
201};
202
203pub use quality::{
204    CompletenessAnalysis, CompletenessAnalyzer, ConsistencyAnalysis, ConsistencyAnalyzer,
205    ConsistencyRule, DuplicateInfo, FieldCompleteness, FieldDefinition, FieldValue, FormatAnalysis,
206    FormatAnalyzer, FormatVariation, QualityEvaluation, UniqueRecord, UniquenessAnalysis,
207    UniquenessAnalyzer,
208};
209
210pub use ml::{
211    AnomalyScoringAnalysis, AnomalyScoringAnalyzer, CrossModalAnalysis, CrossModalAnalyzer,
212    DomainGapAnalysis, DomainGapAnalyzer, EmbeddingReadinessAnalysis, EmbeddingReadinessAnalyzer,
213    FeatureAnalysis, FeatureAnalyzer, FeatureQualityAnalysis, FeatureQualityAnalyzer, FeatureStats,
214    GnnReadinessAnalysis, GnnReadinessAnalyzer, GraphAnalysis, GraphAnalyzer, GraphMetrics,
215    LabelAnalysis, LabelAnalyzer, LabelDistribution, MLReadinessEvaluation,
216    SchemeDetectabilityAnalysis, SchemeDetectabilityAnalyzer, SplitAnalysis, SplitAnalyzer,
217    SplitMetrics, TemporalFidelityAnalysis, TemporalFidelityAnalyzer,
218};
219
220pub use report::{
221    BaselineComparison, ComparisonResult, EvaluationReport, HtmlReportGenerator,
222    JsonReportGenerator, MetricChange, ReportMetadata, ThresholdChecker, ThresholdResult,
223};
224
225pub use tuning::{
226    ConfigSuggestion, ConfigSuggestionGenerator, TuningAnalyzer, TuningCategory, TuningOpportunity,
227};
228
229pub use enhancement::{
230    AiTuneResult, AiTuner, AiTunerConfig, AutoTuneResult, AutoTuner, ConfigPatch,
231    EnhancementReport, Recommendation, RecommendationCategory, RecommendationEngine,
232    RecommendationPriority, RootCause, SuggestedAction, TuningIteration,
233};
234
235pub use privacy::{
236    LinkageAttack, LinkageConfig, LinkageResults, MembershipInferenceAttack, MiaConfig, MiaResults,
237    NistAlignmentReport, NistCriterion, PrivacyEvaluation, SynQPMatrix, SynQPQuadrant,
238};
239
240pub use benchmarks::{
241    // ACFE-calibrated benchmarks
242    acfe_calibrated_1k,
243    acfe_collusion_5k,
244    acfe_management_override_2k,
245    all_acfe_benchmarks,
246    all_benchmarks,
247    // Industry-specific benchmarks
248    all_industry_benchmarks,
249    anomaly_bench_1k,
250    data_quality_100k,
251    entity_match_5k,
252    financial_services_fraud_5k,
253    fraud_detect_10k,
254    get_benchmark,
255    get_industry_benchmark,
256    graph_fraud_10k,
257    healthcare_fraud_5k,
258    manufacturing_fraud_5k,
259    retail_fraud_10k,
260    technology_fraud_3k,
261    AcfeAlignment,
262    AcfeCalibration,
263    AcfeCategoryDistribution,
264    BaselineModelType,
265    BaselineResult,
266    BenchmarkBuilder,
267    BenchmarkSuite,
268    BenchmarkTaskType,
269    CostMatrix,
270    DatasetSpec,
271    EvaluationSpec,
272    FeatureSet,
273    IndustryBenchmarkAnalysis,
274    LeaderboardEntry,
275    MetricType,
276    SplitRatios,
277};
278
279pub use banking::{
280    AmlDetectabilityAnalysis, AmlDetectabilityAnalyzer, AmlTransactionData, BankingEvaluation,
281    KycCompletenessAnalysis, KycCompletenessAnalyzer, KycProfileData, TypologyData,
282};
283
284pub use process_mining::{
285    EventSequenceAnalysis, EventSequenceAnalyzer, ProcessEventData, ProcessMiningEvaluation,
286    VariantAnalysis, VariantAnalyzer, VariantData,
287};
288
289pub use causal::{CausalModelEvaluation, CausalModelEvaluator};
290
291pub use enrichment::{EnrichmentQualityEvaluation, EnrichmentQualityEvaluator};
292
293use serde::{Deserialize, Serialize};
294
295/// Comprehensive evaluation result combining all evaluation modules.
296#[derive(Debug, Clone, Serialize, Deserialize)]
297pub struct ComprehensiveEvaluation {
298    /// Statistical quality evaluation.
299    pub statistical: StatisticalEvaluation,
300    /// Semantic coherence evaluation.
301    pub coherence: CoherenceEvaluation,
302    /// Data quality evaluation.
303    pub quality: QualityEvaluation,
304    /// ML-readiness evaluation.
305    pub ml_readiness: MLReadinessEvaluation,
306    /// Privacy evaluation (optional — only populated when privacy testing is enabled).
307    #[serde(default, skip_serializing_if = "Option::is_none")]
308    pub privacy: Option<PrivacyEvaluation>,
309    /// Banking/KYC/AML evaluation (optional).
310    #[serde(default, skip_serializing_if = "Option::is_none")]
311    pub banking: Option<BankingEvaluation>,
312    /// OCEL 2.0 process mining evaluation (optional).
313    #[serde(default, skip_serializing_if = "Option::is_none")]
314    pub process_mining: Option<ProcessMiningEvaluation>,
315    /// Causal model evaluation (optional).
316    #[serde(default, skip_serializing_if = "Option::is_none")]
317    pub causal: Option<CausalModelEvaluation>,
318    /// LLM enrichment quality evaluation (optional).
319    #[serde(default, skip_serializing_if = "Option::is_none")]
320    pub enrichment_quality: Option<EnrichmentQualityEvaluation>,
321    /// Overall pass/fail status.
322    pub passes: bool,
323    /// Summary of all failures.
324    pub failures: Vec<String>,
325    /// Tuning opportunities identified.
326    pub tuning_opportunities: Vec<TuningOpportunity>,
327    /// Configuration suggestions.
328    pub config_suggestions: Vec<ConfigSuggestion>,
329}
330
331impl ComprehensiveEvaluation {
332    /// Create a new empty evaluation.
333    pub fn new() -> Self {
334        Self {
335            statistical: StatisticalEvaluation::default(),
336            coherence: CoherenceEvaluation::default(),
337            quality: QualityEvaluation::default(),
338            ml_readiness: MLReadinessEvaluation::default(),
339            privacy: None,
340            banking: None,
341            process_mining: None,
342            causal: None,
343            enrichment_quality: None,
344            passes: true,
345            failures: Vec::new(),
346            tuning_opportunities: Vec::new(),
347            config_suggestions: Vec::new(),
348        }
349    }
350
351    /// Check all evaluations against thresholds and update overall status.
352    pub fn check_all_thresholds(&mut self, thresholds: &EvaluationThresholds) {
353        self.failures.clear();
354
355        // Check statistical thresholds
356        self.statistical.check_thresholds(thresholds);
357        self.failures.extend(self.statistical.failures.clone());
358
359        // Check coherence thresholds
360        self.coherence.check_thresholds(thresholds);
361        self.failures.extend(self.coherence.failures.clone());
362
363        // Check quality thresholds
364        self.quality.check_thresholds(thresholds);
365        self.failures.extend(self.quality.failures.clone());
366
367        // Check ML thresholds
368        self.ml_readiness.check_thresholds(thresholds);
369        self.failures.extend(self.ml_readiness.failures.clone());
370
371        // Check privacy evaluation (if present)
372        if let Some(ref mut privacy) = self.privacy {
373            privacy.update_status();
374            self.failures.extend(privacy.failures.clone());
375        }
376
377        // Check banking evaluation
378        if let Some(ref mut banking) = self.banking {
379            banking.check_thresholds();
380            self.failures.extend(banking.issues.clone());
381        }
382
383        // Check process mining evaluation
384        if let Some(ref mut pm) = self.process_mining {
385            pm.check_thresholds();
386            self.failures.extend(pm.issues.clone());
387        }
388
389        // Check causal model evaluation
390        if let Some(ref causal) = self.causal {
391            if !causal.passes {
392                self.failures.extend(causal.issues.clone());
393            }
394        }
395
396        // Check enrichment quality evaluation
397        if let Some(ref enrichment) = self.enrichment_quality {
398            if !enrichment.passes {
399                self.failures.extend(enrichment.issues.clone());
400            }
401        }
402
403        self.passes = self.failures.is_empty();
404    }
405}
406
407impl Default for ComprehensiveEvaluation {
408    fn default() -> Self {
409        Self::new()
410    }
411}
412
413/// Main evaluator that coordinates all evaluation modules.
414pub struct Evaluator {
415    /// Evaluation configuration.
416    config: EvaluationConfig,
417}
418
419impl Evaluator {
420    /// Create a new evaluator with the given configuration.
421    pub fn new(config: EvaluationConfig) -> Self {
422        Self { config }
423    }
424
425    /// Create an evaluator with default configuration.
426    pub fn with_defaults() -> Self {
427        Self::new(EvaluationConfig::default())
428    }
429
430    /// Get the configuration.
431    pub fn config(&self) -> &EvaluationConfig {
432        &self.config
433    }
434
435    /// Run a comprehensive evaluation and return results.
436    ///
437    /// # Architectural note
438    ///
439    /// This zero-argument variant returns a default (passing) evaluation because the
440    /// `Evaluator` struct holds only configuration — it has no access to the generated
441    /// journal entry or balance data that the sub-module evaluators require.
442    ///
443    /// To evaluate actual generation output, use `run_evaluation_with_amounts` which
444    /// accepts raw JE amounts and runs the Benford analysis.  Full wiring of all
445    /// sub-modules (BalanceSheetEvaluator, DocumentChainEvaluator, etc.) requires
446    /// passing the complete `EnhancedGenerationResult` from the runtime crate, which
447    /// would create a circular dependency.  The recommended integration point is the
448    /// orchestrator layer (datasynth-runtime) which already calls the gate engine with
449    /// a populated `ComprehensiveEvaluation`.
450    pub fn run_evaluation(&self) -> ComprehensiveEvaluation {
451        let mut evaluation = ComprehensiveEvaluation::new();
452        evaluation.check_all_thresholds(&self.config.thresholds);
453        evaluation
454    }
455
456    /// Run a Benford-augmented evaluation given raw JE amounts.
457    ///
458    /// This method calls the [`BenfordAnalyzer`] sub-module and populates the
459    /// `statistical.benford` field of the returned [`ComprehensiveEvaluation`].
460    /// All other sub-module fields remain at their default (passing) values.
461    pub fn run_evaluation_with_amounts(
462        &self,
463        je_amounts: &[rust_decimal::Decimal],
464    ) -> ComprehensiveEvaluation {
465        let mut evaluation = ComprehensiveEvaluation::new();
466
467        if !je_amounts.is_empty() {
468            let analyzer = BenfordAnalyzer::new(self.config.thresholds.benford_p_value_min);
469            match analyzer.analyze(je_amounts) {
470                Ok(benford) => {
471                    evaluation.statistical.benford = Some(benford);
472                }
473                Err(e) => {
474                    evaluation
475                        .failures
476                        .push(format!("Benford analysis failed: {e}"));
477                    evaluation.passes = false;
478                }
479            }
480        }
481
482        evaluation.check_all_thresholds(&self.config.thresholds);
483        evaluation
484    }
485}
486
487#[cfg(test)]
488mod tests {
489    use super::*;
490
491    #[test]
492    fn test_comprehensive_evaluation_new() {
493        let eval = ComprehensiveEvaluation::new();
494        assert!(eval.passes);
495        assert!(eval.failures.is_empty());
496    }
497
498    #[test]
499    fn test_evaluator_creation() {
500        let evaluator = Evaluator::with_defaults();
501        assert_eq!(evaluator.config().thresholds.benford_p_value_min, 0.05);
502    }
503}
datasynth_eval/lib.rs

datasynth_eval/
lib.rs
