datasynth_eval/
lib.rs

1#![deny(clippy::unwrap_used)]
2// Allow some clippy lints that are common in test/evaluation code
3#![allow(clippy::field_reassign_with_default)]
4#![allow(clippy::too_many_arguments)]
5#![allow(clippy::upper_case_acronyms)] // MCAR, MAR, MNAR, ISO are standard abbreviations
6
7//! Synthetic Data Evaluation Framework
8//!
9//! This crate provides comprehensive evaluation capabilities for validating
10//! the quality and correctness of generated synthetic financial data.
11//!
12//! # Features
13//!
14//! - **Statistical Quality**: Benford's Law, amount distributions, line item patterns
15//! - **Semantic Coherence**: Balance sheet validation, subledger reconciliation
16//! - **Data Quality**: Uniqueness, completeness, format consistency
17//! - **ML-Readiness**: Feature distributions, label quality, graph structure
18//! - **Reporting**: HTML and JSON reports with pass/fail thresholds
19//!
20//! # Example
21//!
22//! ```ignore
23//! use datasynth_eval::{Evaluator, EvaluationConfig};
24//!
25//! let config = EvaluationConfig::default();
26//! let evaluator = Evaluator::new(config);
27//!
28//! // Evaluate generated data
29//! let result = evaluator.evaluate(&generation_result)?;
30//!
31//! // Generate report
32//! result.generate_html_report("evaluation_report.html")?;
33//! ```
34
35pub mod benchmarks;
36pub mod config;
37pub mod enhancement;
38pub mod error;
39pub mod gates;
40pub mod privacy;
41
42pub mod coherence;
43pub mod ml;
44pub mod quality;
45pub mod report;
46pub mod statistical;
47pub mod tuning;
48
49pub mod banking;
50pub mod causal;
51pub mod diff_engine;
52pub mod enrichment;
53pub mod process_mining;
54pub mod scenario_diff;
55
56// Re-exports
57pub use config::{EvaluationConfig, EvaluationThresholds, PrivacyEvaluationConfig};
58pub use error::{EvalError, EvalResult};
59
60pub use statistical::{
61    AmountDistributionAnalysis, AmountDistributionAnalyzer, AnomalyRealismEvaluation,
62    AnomalyRealismEvaluator, BenfordAnalysis, BenfordAnalyzer, BenfordConformity,
63    DetectionDifficulty, DriftDetectionAnalysis, DriftDetectionAnalyzer, DriftDetectionEntry,
64    DriftDetectionMetrics, DriftEventCategory, LabeledDriftEvent, LabeledEventAnalysis,
65    LineItemAnalysis, LineItemAnalyzer, LineItemEntry, StatisticalEvaluation, TemporalAnalysis,
66    TemporalAnalyzer, TemporalEntry,
67};
68
69pub use coherence::{
70    AccountType,
71    ApprovalLevelData,
72    AuditEvaluation,
73    AuditEvaluator,
74    AuditFindingData,
75    AuditRiskData,
76    AuditTrailEvaluation,
77    AuditTrailGap,
78    BalanceSheetEvaluation,
79    BalanceSheetEvaluator,
80    BalanceSnapshot,
81    BankReconciliationEvaluation,
82    BankReconciliationEvaluator,
83    BidEvaluationData,
84    BudgetVarianceData,
85    CashPositionData,
86    CoherenceEvaluation,
87    ConcentrationMetrics,
88    CountryPackData,
89    CountryPackEvaluation,
90    CountryPackEvaluator,
91    CountryPackThresholds,
92    CovenantData,
93    CrossProcessEvaluation,
94    CrossProcessEvaluator,
95    CycleCountData,
96    DocumentChainEvaluation,
97    DocumentChainEvaluator,
98    DocumentReferenceData,
99    EarnedValueData,
100    EntityReferenceData,
101    EsgEvaluation,
102    EsgEvaluator,
103    EsgThresholds,
104    ExpenseReportData,
105    FairValueEvaluation,
106    // Task 4.1: Financial Ratio Evaluator
107    FinancialRatios,
108    FinancialReportingEvaluation,
109    FinancialReportingEvaluator,
110    FinancialStatementData,
111    FrameworkViolation,
112    GovernanceData,
113    HedgeEffectivenessData,
114    HolidayData,
115    HrPayrollEvaluation,
116    HrPayrollEvaluator,
117    ICMatchingData,
118    ICMatchingEvaluation,
119    ICMatchingEvaluator,
120    ImpairmentEvaluation,
121    IsaComplianceEvaluation,
122    // Task 4.2: JE Risk Scoring Evaluator
123    JeRiskScoringResult,
124    KpiData,
125    LeaseAccountingEvaluation,
126    LeaseAccountingEvaluator,
127    LeaseEvaluation,
128    ManufacturingEvaluation,
129    ManufacturingEvaluator,
130    MaterialityData,
131    NettingData,
132    NetworkEdge,
133    NetworkEvaluation,
134    NetworkEvaluator,
135    NetworkNode,
136    NetworkThresholds,
137    O2CChainData,
138    P2PChainData,
139    PayrollHoursData,
140    PayrollLineItemData,
141    PayrollRunData,
142    PcaobComplianceEvaluation,
143    PerformanceObligation,
144    ProductionOrderData,
145    ProjectAccountingEvaluation,
146    ProjectAccountingEvaluator,
147    ProjectAccountingThresholds,
148    ProjectRevenueData,
149    QualityInspectionData,
150    QuoteLineData,
151    RatioAnalysisResult,
152    RatioCheck,
153    ReconciliationData,
154    ReferentialData,
155    ReferentialIntegrityEvaluation,
156    ReferentialIntegrityEvaluator,
157    RetainageData,
158    RevenueContract,
159    RevenueRecognitionEvaluation,
160    RevenueRecognitionEvaluator,
161    RiskAttributeStats,
162    RiskDistribution,
163    RoutingOperationData,
164    SafetyMetricData,
165    SalesQuoteData,
166    SalesQuoteEvaluation,
167    SalesQuoteEvaluator,
168    SalesQuoteThresholds,
169    ScorecardCoverageData,
170    SourcingEvaluation,
171    SourcingEvaluator,
172    SourcingProjectData,
173    SoxComplianceEvaluation,
174    SpendAnalysisData,
175    StandardsComplianceEvaluation,
176    StandardsThresholds,
177    StrengthStats,
178    SubledgerEvaluator,
179    SubledgerReconciliationEvaluation,
180    SupplierEsgData,
181    TaxEvaluation,
182    TaxEvaluator,
183    TaxLineData,
184    TaxRateData,
185    TaxReturnData,
186    TaxThresholds,
187    TimeEntryData,
188    TreasuryEvaluation,
189    TreasuryEvaluator,
190    TreasuryThresholds,
191    UnmatchedICItem,
192    VariableConsideration,
193    ViolationSeverity,
194    WaterUsageData,
195    WithholdingData,
196    WorkpaperData,
197};
198
199pub use quality::{
200    CompletenessAnalysis, CompletenessAnalyzer, ConsistencyAnalysis, ConsistencyAnalyzer,
201    ConsistencyRule, DuplicateInfo, FieldCompleteness, FieldDefinition, FieldValue, FormatAnalysis,
202    FormatAnalyzer, FormatVariation, QualityEvaluation, UniqueRecord, UniquenessAnalysis,
203    UniquenessAnalyzer,
204};
205
206pub use ml::{
207    AnomalyScoringAnalysis, AnomalyScoringAnalyzer, CrossModalAnalysis, CrossModalAnalyzer,
208    DomainGapAnalysis, DomainGapAnalyzer, EmbeddingReadinessAnalysis, EmbeddingReadinessAnalyzer,
209    FeatureAnalysis, FeatureAnalyzer, FeatureQualityAnalysis, FeatureQualityAnalyzer, FeatureStats,
210    GnnReadinessAnalysis, GnnReadinessAnalyzer, GraphAnalysis, GraphAnalyzer, GraphMetrics,
211    LabelAnalysis, LabelAnalyzer, LabelDistribution, MLReadinessEvaluation,
212    SchemeDetectabilityAnalysis, SchemeDetectabilityAnalyzer, SplitAnalysis, SplitAnalyzer,
213    SplitMetrics, TemporalFidelityAnalysis, TemporalFidelityAnalyzer,
214};
215
216pub use report::{
217    BaselineComparison, ComparisonResult, EvaluationReport, HtmlReportGenerator,
218    JsonReportGenerator, MetricChange, ReportMetadata, ThresholdChecker, ThresholdResult,
219};
220
221pub use tuning::{
222    ConfigSuggestion, ConfigSuggestionGenerator, TuningAnalyzer, TuningCategory, TuningOpportunity,
223};
224
225pub use enhancement::{
226    AutoTuneResult, AutoTuner, ConfigPatch, EnhancementReport, Recommendation,
227    RecommendationCategory, RecommendationEngine, RecommendationPriority, RootCause,
228    SuggestedAction,
229};
230
231pub use privacy::{
232    LinkageAttack, LinkageConfig, LinkageResults, MembershipInferenceAttack, MiaConfig, MiaResults,
233    NistAlignmentReport, NistCriterion, PrivacyEvaluation, SynQPMatrix, SynQPQuadrant,
234};
235
236pub use benchmarks::{
237    // ACFE-calibrated benchmarks
238    acfe_calibrated_1k,
239    acfe_collusion_5k,
240    acfe_management_override_2k,
241    all_acfe_benchmarks,
242    all_benchmarks,
243    // Industry-specific benchmarks
244    all_industry_benchmarks,
245    anomaly_bench_1k,
246    data_quality_100k,
247    entity_match_5k,
248    financial_services_fraud_5k,
249    fraud_detect_10k,
250    get_benchmark,
251    get_industry_benchmark,
252    graph_fraud_10k,
253    healthcare_fraud_5k,
254    manufacturing_fraud_5k,
255    retail_fraud_10k,
256    technology_fraud_3k,
257    AcfeAlignment,
258    AcfeCalibration,
259    AcfeCategoryDistribution,
260    BaselineModelType,
261    BaselineResult,
262    BenchmarkBuilder,
263    BenchmarkSuite,
264    BenchmarkTaskType,
265    CostMatrix,
266    DatasetSpec,
267    EvaluationSpec,
268    FeatureSet,
269    IndustryBenchmarkAnalysis,
270    LeaderboardEntry,
271    MetricType,
272    SplitRatios,
273};
274
275pub use banking::{
276    AmlDetectabilityAnalysis, AmlDetectabilityAnalyzer, AmlTransactionData, BankingEvaluation,
277    KycCompletenessAnalysis, KycCompletenessAnalyzer, KycProfileData, TypologyData,
278};
279
280pub use process_mining::{
281    EventSequenceAnalysis, EventSequenceAnalyzer, ProcessEventData, ProcessMiningEvaluation,
282    VariantAnalysis, VariantAnalyzer, VariantData,
283};
284
285pub use causal::{CausalModelEvaluation, CausalModelEvaluator};
286
287pub use enrichment::{EnrichmentQualityEvaluation, EnrichmentQualityEvaluator};
288
289use serde::{Deserialize, Serialize};
290
291/// Comprehensive evaluation result combining all evaluation modules.
292#[derive(Debug, Clone, Serialize, Deserialize)]
293pub struct ComprehensiveEvaluation {
294    /// Statistical quality evaluation.
295    pub statistical: StatisticalEvaluation,
296    /// Semantic coherence evaluation.
297    pub coherence: CoherenceEvaluation,
298    /// Data quality evaluation.
299    pub quality: QualityEvaluation,
300    /// ML-readiness evaluation.
301    pub ml_readiness: MLReadinessEvaluation,
302    /// Privacy evaluation (optional — only populated when privacy testing is enabled).
303    #[serde(default, skip_serializing_if = "Option::is_none")]
304    pub privacy: Option<PrivacyEvaluation>,
305    /// Banking/KYC/AML evaluation (optional).
306    #[serde(default, skip_serializing_if = "Option::is_none")]
307    pub banking: Option<BankingEvaluation>,
308    /// OCEL 2.0 process mining evaluation (optional).
309    #[serde(default, skip_serializing_if = "Option::is_none")]
310    pub process_mining: Option<ProcessMiningEvaluation>,
311    /// Causal model evaluation (optional).
312    #[serde(default, skip_serializing_if = "Option::is_none")]
313    pub causal: Option<CausalModelEvaluation>,
314    /// LLM enrichment quality evaluation (optional).
315    #[serde(default, skip_serializing_if = "Option::is_none")]
316    pub enrichment_quality: Option<EnrichmentQualityEvaluation>,
317    /// Overall pass/fail status.
318    pub passes: bool,
319    /// Summary of all failures.
320    pub failures: Vec<String>,
321    /// Tuning opportunities identified.
322    pub tuning_opportunities: Vec<TuningOpportunity>,
323    /// Configuration suggestions.
324    pub config_suggestions: Vec<ConfigSuggestion>,
325}
326
327impl ComprehensiveEvaluation {
328    /// Create a new empty evaluation.
329    pub fn new() -> Self {
330        Self {
331            statistical: StatisticalEvaluation::default(),
332            coherence: CoherenceEvaluation::default(),
333            quality: QualityEvaluation::default(),
334            ml_readiness: MLReadinessEvaluation::default(),
335            privacy: None,
336            banking: None,
337            process_mining: None,
338            causal: None,
339            enrichment_quality: None,
340            passes: true,
341            failures: Vec::new(),
342            tuning_opportunities: Vec::new(),
343            config_suggestions: Vec::new(),
344        }
345    }
346
347    /// Check all evaluations against thresholds and update overall status.
348    pub fn check_all_thresholds(&mut self, thresholds: &EvaluationThresholds) {
349        self.failures.clear();
350
351        // Check statistical thresholds
352        self.statistical.check_thresholds(thresholds);
353        self.failures.extend(self.statistical.failures.clone());
354
355        // Check coherence thresholds
356        self.coherence.check_thresholds(thresholds);
357        self.failures.extend(self.coherence.failures.clone());
358
359        // Check quality thresholds
360        self.quality.check_thresholds(thresholds);
361        self.failures.extend(self.quality.failures.clone());
362
363        // Check ML thresholds
364        self.ml_readiness.check_thresholds(thresholds);
365        self.failures.extend(self.ml_readiness.failures.clone());
366
367        // Check privacy evaluation (if present)
368        if let Some(ref mut privacy) = self.privacy {
369            privacy.update_status();
370            self.failures.extend(privacy.failures.clone());
371        }
372
373        // Check banking evaluation
374        if let Some(ref mut banking) = self.banking {
375            banking.check_thresholds();
376            self.failures.extend(banking.issues.clone());
377        }
378
379        // Check process mining evaluation
380        if let Some(ref mut pm) = self.process_mining {
381            pm.check_thresholds();
382            self.failures.extend(pm.issues.clone());
383        }
384
385        // Check causal model evaluation
386        if let Some(ref causal) = self.causal {
387            if !causal.passes {
388                self.failures.extend(causal.issues.clone());
389            }
390        }
391
392        // Check enrichment quality evaluation
393        if let Some(ref enrichment) = self.enrichment_quality {
394            if !enrichment.passes {
395                self.failures.extend(enrichment.issues.clone());
396            }
397        }
398
399        self.passes = self.failures.is_empty();
400    }
401}
402
403impl Default for ComprehensiveEvaluation {
404    fn default() -> Self {
405        Self::new()
406    }
407}
408
409/// Main evaluator that coordinates all evaluation modules.
410pub struct Evaluator {
411    /// Evaluation configuration.
412    config: EvaluationConfig,
413}
414
415impl Evaluator {
416    /// Create a new evaluator with the given configuration.
417    pub fn new(config: EvaluationConfig) -> Self {
418        Self { config }
419    }
420
421    /// Create an evaluator with default configuration.
422    pub fn with_defaults() -> Self {
423        Self::new(EvaluationConfig::default())
424    }
425
426    /// Get the configuration.
427    pub fn config(&self) -> &EvaluationConfig {
428        &self.config
429    }
430
431    /// Run a comprehensive evaluation and return results.
432    ///
433    /// # Architectural note
434    ///
435    /// This zero-argument variant returns a default (passing) evaluation because the
436    /// `Evaluator` struct holds only configuration — it has no access to the generated
437    /// journal entry or balance data that the sub-module evaluators require.
438    ///
439    /// To evaluate actual generation output, use [`run_evaluation_with_amounts`] which
440    /// accepts raw JE amounts and runs the Benford analysis.  Full wiring of all
441    /// sub-modules (BalanceSheetEvaluator, DocumentChainEvaluator, etc.) requires
442    /// passing the complete `EnhancedGenerationResult` from the runtime crate, which
443    /// would create a circular dependency.  The recommended integration point is the
444    /// orchestrator layer (datasynth-runtime) which already calls the gate engine with
445    /// a populated `ComprehensiveEvaluation`.
446    pub fn run_evaluation(&self) -> ComprehensiveEvaluation {
447        let mut evaluation = ComprehensiveEvaluation::new();
448        evaluation.check_all_thresholds(&self.config.thresholds);
449        evaluation
450    }
451
452    /// Run a Benford-augmented evaluation given raw JE amounts.
453    ///
454    /// This method calls the [`BenfordAnalyzer`] sub-module and populates the
455    /// `statistical.benford` field of the returned [`ComprehensiveEvaluation`].
456    /// All other sub-module fields remain at their default (passing) values.
457    pub fn run_evaluation_with_amounts(
458        &self,
459        je_amounts: &[rust_decimal::Decimal],
460    ) -> ComprehensiveEvaluation {
461        let mut evaluation = ComprehensiveEvaluation::new();
462
463        if !je_amounts.is_empty() {
464            let analyzer = BenfordAnalyzer::new(self.config.thresholds.benford_p_value_min);
465            match analyzer.analyze(je_amounts) {
466                Ok(benford) => {
467                    evaluation.statistical.benford = Some(benford);
468                }
469                Err(e) => {
470                    evaluation
471                        .failures
472                        .push(format!("Benford analysis failed: {e}"));
473                    evaluation.passes = false;
474                }
475            }
476        }
477
478        evaluation.check_all_thresholds(&self.config.thresholds);
479        evaluation
480    }
481}
482
483#[cfg(test)]
484#[allow(clippy::unwrap_used)]
485mod tests {
486    use super::*;
487
488    #[test]
489    fn test_comprehensive_evaluation_new() {
490        let eval = ComprehensiveEvaluation::new();
491        assert!(eval.passes);
492        assert!(eval.failures.is_empty());
493    }
494
495    #[test]
496    fn test_evaluator_creation() {
497        let evaluator = Evaluator::with_defaults();
498        assert_eq!(evaluator.config().thresholds.benford_p_value_min, 0.05);
499    }
500}
datasynth_eval/lib.rs

datasynth_eval/
lib.rs
