datasynth_eval/
lib.rs

1#![deny(clippy::unwrap_used)]
2// Allow some clippy lints that are common in test/evaluation code
3#![allow(clippy::field_reassign_with_default)]
4#![allow(clippy::too_many_arguments)]
5#![allow(clippy::upper_case_acronyms)] // MCAR, MAR, MNAR, ISO are standard abbreviations
6
7//! Synthetic Data Evaluation Framework
8//!
9//! This crate provides comprehensive evaluation capabilities for validating
10//! the quality and correctness of generated synthetic financial data.
11//!
12//! # Features
13//!
14//! - **Statistical Quality**: Benford's Law, amount distributions, line item patterns
15//! - **Semantic Coherence**: Balance sheet validation, subledger reconciliation
16//! - **Data Quality**: Uniqueness, completeness, format consistency
17//! - **ML-Readiness**: Feature distributions, label quality, graph structure
18//! - **Reporting**: HTML and JSON reports with pass/fail thresholds
19//!
20//! # Example
21//!
22//! ```ignore
23//! use datasynth_eval::{Evaluator, EvaluationConfig};
24//!
25//! let config = EvaluationConfig::default();
26//! let evaluator = Evaluator::new(config);
27//!
28//! // Evaluate generated data
29//! let result = evaluator.evaluate(&generation_result)?;
30//!
31//! // Generate report
32//! result.generate_html_report("evaluation_report.html")?;
33//! ```
34
35pub mod benchmarks;
36pub mod config;
37pub mod enhancement;
38pub mod error;
39pub mod gates;
40pub mod privacy;
41
42pub mod coherence;
43pub mod ml;
44pub mod quality;
45pub mod report;
46pub mod statistical;
47pub mod tuning;
48
49#[cfg(feature = "adversarial")]
50pub mod adversarial;
51pub mod banking;
52pub mod causal;
53pub mod diff_engine;
54pub mod enrichment;
55pub mod process_mining;
56pub mod scenario_diff;
57
58// Re-exports
59pub use config::{EvaluationConfig, EvaluationThresholds, PrivacyEvaluationConfig};
60pub use error::{EvalError, EvalResult};
61
62pub use statistical::{
63    AmountDistributionAnalysis, AmountDistributionAnalyzer, AnomalyRealismEvaluation,
64    AnomalyRealismEvaluator, BenfordAnalysis, BenfordAnalyzer, BenfordConformity,
65    CorrelationAnalysis, CorrelationAnalyzer, CorrelationCheckResult, DetectionDifficulty,
66    DriftDetectionAnalysis, DriftDetectionAnalyzer, DriftDetectionEntry, DriftDetectionMetrics,
67    DriftEventCategory, ExpectedCorrelation, LabeledDriftEvent, LabeledEventAnalysis,
68    LineItemAnalysis, LineItemAnalyzer, LineItemEntry, SecondDigitAnalysis, StatisticalEvaluation,
69    TemporalAnalysis, TemporalAnalyzer, TemporalEntry,
70};
71
72pub use coherence::{
73    AccountType,
74    ApprovalLevelData,
75    AuditEvaluation,
76    AuditEvaluator,
77    AuditFindingData,
78    AuditRiskData,
79    AuditTrailEvaluation,
80    AuditTrailGap,
81    BalanceSheetEvaluation,
82    BalanceSheetEvaluator,
83    BalanceSnapshot,
84    BankReconciliationEvaluation,
85    BankReconciliationEvaluator,
86    BidEvaluationData,
87    BudgetVarianceData,
88    CashPositionData,
89    CoherenceEvaluation,
90    ConcentrationMetrics,
91    CountryPackData,
92    CountryPackEvaluation,
93    CountryPackEvaluator,
94    CountryPackThresholds,
95    CovenantData,
96    CrossProcessEvaluation,
97    CrossProcessEvaluator,
98    CycleCountData,
99    DocumentChainEvaluation,
100    DocumentChainEvaluator,
101    DocumentReferenceData,
102    EarnedValueData,
103    EntityReferenceData,
104    EsgEvaluation,
105    EsgEvaluator,
106    EsgThresholds,
107    ExpenseReportData,
108    FairValueEvaluation,
109    // Task 4.1: Financial Ratio Evaluator
110    FinancialRatios,
111    FinancialReportingEvaluation,
112    FinancialReportingEvaluator,
113    FinancialStatementData,
114    FrameworkViolation,
115    GovernanceData,
116    HedgeEffectivenessData,
117    HolidayData,
118    HrPayrollEvaluation,
119    HrPayrollEvaluator,
120    ICMatchingData,
121    ICMatchingEvaluation,
122    ICMatchingEvaluator,
123    ImpairmentEvaluation,
124    IsaComplianceEvaluation,
125    // Task 4.2: JE Risk Scoring Evaluator
126    JeRiskScoringResult,
127    KpiData,
128    LeaseAccountingEvaluation,
129    LeaseAccountingEvaluator,
130    LeaseEvaluation,
131    ManufacturingEvaluation,
132    ManufacturingEvaluator,
133    MaterialityData,
134    NettingData,
135    NetworkEdge,
136    NetworkEvaluation,
137    NetworkEvaluator,
138    NetworkNode,
139    NetworkThresholds,
140    O2CChainData,
141    P2PChainData,
142    PayrollHoursData,
143    PayrollLineItemData,
144    PayrollRunData,
145    PcaobComplianceEvaluation,
146    PerformanceObligation,
147    ProductionOrderData,
148    ProjectAccountingEvaluation,
149    ProjectAccountingEvaluator,
150    ProjectAccountingThresholds,
151    ProjectRevenueData,
152    QualityInspectionData,
153    QuoteLineData,
154    RatioAnalysisResult,
155    RatioCheck,
156    ReconciliationData,
157    ReferentialData,
158    ReferentialIntegrityEvaluation,
159    ReferentialIntegrityEvaluator,
160    RetainageData,
161    RevenueContract,
162    RevenueRecognitionEvaluation,
163    RevenueRecognitionEvaluator,
164    RiskAttributeStats,
165    RiskDistribution,
166    RoutingOperationData,
167    SafetyMetricData,
168    SalesQuoteData,
169    SalesQuoteEvaluation,
170    SalesQuoteEvaluator,
171    SalesQuoteThresholds,
172    ScorecardCoverageData,
173    SourcingEvaluation,
174    SourcingEvaluator,
175    SourcingProjectData,
176    SoxComplianceEvaluation,
177    SpendAnalysisData,
178    StandardsComplianceEvaluation,
179    StandardsThresholds,
180    StrengthStats,
181    SubledgerEvaluator,
182    SubledgerReconciliationEvaluation,
183    SupplierEsgData,
184    TaxEvaluation,
185    TaxEvaluator,
186    TaxLineData,
187    TaxRateData,
188    TaxReturnData,
189    TaxThresholds,
190    TimeEntryData,
191    TreasuryEvaluation,
192    TreasuryEvaluator,
193    TreasuryThresholds,
194    UnmatchedICItem,
195    VariableConsideration,
196    ViolationSeverity,
197    WaterUsageData,
198    WithholdingData,
199    WorkpaperData,
200};
201
202pub use quality::{
203    CompletenessAnalysis, CompletenessAnalyzer, ConsistencyAnalysis, ConsistencyAnalyzer,
204    ConsistencyRule, DuplicateInfo, FieldCompleteness, FieldDefinition, FieldValue, FormatAnalysis,
205    FormatAnalyzer, FormatVariation, QualityEvaluation, UniqueRecord, UniquenessAnalysis,
206    UniquenessAnalyzer,
207};
208
209pub use ml::{
210    AnomalyScoringAnalysis, AnomalyScoringAnalyzer, CrossModalAnalysis, CrossModalAnalyzer,
211    DomainGapAnalysis, DomainGapAnalyzer, EmbeddingReadinessAnalysis, EmbeddingReadinessAnalyzer,
212    FeatureAnalysis, FeatureAnalyzer, FeatureQualityAnalysis, FeatureQualityAnalyzer, FeatureStats,
213    GnnReadinessAnalysis, GnnReadinessAnalyzer, GraphAnalysis, GraphAnalyzer, GraphMetrics,
214    LabelAnalysis, LabelAnalyzer, LabelDistribution, MLReadinessEvaluation,
215    SchemeDetectabilityAnalysis, SchemeDetectabilityAnalyzer, SplitAnalysis, SplitAnalyzer,
216    SplitMetrics, TemporalFidelityAnalysis, TemporalFidelityAnalyzer,
217};
218
219pub use report::{
220    BaselineComparison, ComparisonResult, EvaluationReport, HtmlReportGenerator,
221    JsonReportGenerator, MetricChange, ReportMetadata, ThresholdChecker, ThresholdResult,
222};
223
224pub use tuning::{
225    ConfigSuggestion, ConfigSuggestionGenerator, TuningAnalyzer, TuningCategory, TuningOpportunity,
226};
227
228pub use enhancement::{
229    AiTuneResult, AiTuner, AiTunerConfig, AutoTuneResult, AutoTuner, ConfigPatch,
230    EnhancementReport, Recommendation, RecommendationCategory, RecommendationEngine,
231    RecommendationPriority, RootCause, SuggestedAction, TuningIteration,
232};
233
234pub use privacy::{
235    LinkageAttack, LinkageConfig, LinkageResults, MembershipInferenceAttack, MiaConfig, MiaResults,
236    NistAlignmentReport, NistCriterion, PrivacyEvaluation, SynQPMatrix, SynQPQuadrant,
237};
238
239pub use benchmarks::{
240    // ACFE-calibrated benchmarks
241    acfe_calibrated_1k,
242    acfe_collusion_5k,
243    acfe_management_override_2k,
244    all_acfe_benchmarks,
245    all_benchmarks,
246    // Industry-specific benchmarks
247    all_industry_benchmarks,
248    anomaly_bench_1k,
249    data_quality_100k,
250    entity_match_5k,
251    financial_services_fraud_5k,
252    fraud_detect_10k,
253    get_benchmark,
254    get_industry_benchmark,
255    graph_fraud_10k,
256    healthcare_fraud_5k,
257    manufacturing_fraud_5k,
258    retail_fraud_10k,
259    technology_fraud_3k,
260    AcfeAlignment,
261    AcfeCalibration,
262    AcfeCategoryDistribution,
263    BaselineModelType,
264    BaselineResult,
265    BenchmarkBuilder,
266    BenchmarkSuite,
267    BenchmarkTaskType,
268    CostMatrix,
269    DatasetSpec,
270    EvaluationSpec,
271    FeatureSet,
272    IndustryBenchmarkAnalysis,
273    LeaderboardEntry,
274    MetricType,
275    SplitRatios,
276};
277
278pub use banking::{
279    AmlDetectabilityAnalysis, AmlDetectabilityAnalyzer, AmlTransactionData, BankingEvaluation,
280    KycCompletenessAnalysis, KycCompletenessAnalyzer, KycProfileData, TypologyData,
281};
282
283pub use process_mining::{
284    EventSequenceAnalysis, EventSequenceAnalyzer, ProcessEventData, ProcessMiningEvaluation,
285    VariantAnalysis, VariantAnalyzer, VariantData,
286};
287
288pub use causal::{CausalModelEvaluation, CausalModelEvaluator};
289
290pub use enrichment::{EnrichmentQualityEvaluation, EnrichmentQualityEvaluator};
291
292use serde::{Deserialize, Serialize};
293
294/// Comprehensive evaluation result combining all evaluation modules.
295#[derive(Debug, Clone, Serialize, Deserialize)]
296pub struct ComprehensiveEvaluation {
297    /// Statistical quality evaluation.
298    pub statistical: StatisticalEvaluation,
299    /// Semantic coherence evaluation.
300    pub coherence: CoherenceEvaluation,
301    /// Data quality evaluation.
302    pub quality: QualityEvaluation,
303    /// ML-readiness evaluation.
304    pub ml_readiness: MLReadinessEvaluation,
305    /// Privacy evaluation (optional — only populated when privacy testing is enabled).
306    #[serde(default, skip_serializing_if = "Option::is_none")]
307    pub privacy: Option<PrivacyEvaluation>,
308    /// Banking/KYC/AML evaluation (optional).
309    #[serde(default, skip_serializing_if = "Option::is_none")]
310    pub banking: Option<BankingEvaluation>,
311    /// OCEL 2.0 process mining evaluation (optional).
312    #[serde(default, skip_serializing_if = "Option::is_none")]
313    pub process_mining: Option<ProcessMiningEvaluation>,
314    /// Causal model evaluation (optional).
315    #[serde(default, skip_serializing_if = "Option::is_none")]
316    pub causal: Option<CausalModelEvaluation>,
317    /// LLM enrichment quality evaluation (optional).
318    #[serde(default, skip_serializing_if = "Option::is_none")]
319    pub enrichment_quality: Option<EnrichmentQualityEvaluation>,
320    /// Overall pass/fail status.
321    pub passes: bool,
322    /// Summary of all failures.
323    pub failures: Vec<String>,
324    /// Tuning opportunities identified.
325    pub tuning_opportunities: Vec<TuningOpportunity>,
326    /// Configuration suggestions.
327    pub config_suggestions: Vec<ConfigSuggestion>,
328}
329
330impl ComprehensiveEvaluation {
331    /// Create a new empty evaluation.
332    pub fn new() -> Self {
333        Self {
334            statistical: StatisticalEvaluation::default(),
335            coherence: CoherenceEvaluation::default(),
336            quality: QualityEvaluation::default(),
337            ml_readiness: MLReadinessEvaluation::default(),
338            privacy: None,
339            banking: None,
340            process_mining: None,
341            causal: None,
342            enrichment_quality: None,
343            passes: true,
344            failures: Vec::new(),
345            tuning_opportunities: Vec::new(),
346            config_suggestions: Vec::new(),
347        }
348    }
349
350    /// Check all evaluations against thresholds and update overall status.
351    pub fn check_all_thresholds(&mut self, thresholds: &EvaluationThresholds) {
352        self.failures.clear();
353
354        // Check statistical thresholds
355        self.statistical.check_thresholds(thresholds);
356        self.failures.extend(self.statistical.failures.clone());
357
358        // Check coherence thresholds
359        self.coherence.check_thresholds(thresholds);
360        self.failures.extend(self.coherence.failures.clone());
361
362        // Check quality thresholds
363        self.quality.check_thresholds(thresholds);
364        self.failures.extend(self.quality.failures.clone());
365
366        // Check ML thresholds
367        self.ml_readiness.check_thresholds(thresholds);
368        self.failures.extend(self.ml_readiness.failures.clone());
369
370        // Check privacy evaluation (if present)
371        if let Some(ref mut privacy) = self.privacy {
372            privacy.update_status();
373            self.failures.extend(privacy.failures.clone());
374        }
375
376        // Check banking evaluation
377        if let Some(ref mut banking) = self.banking {
378            banking.check_thresholds();
379            self.failures.extend(banking.issues.clone());
380        }
381
382        // Check process mining evaluation
383        if let Some(ref mut pm) = self.process_mining {
384            pm.check_thresholds();
385            self.failures.extend(pm.issues.clone());
386        }
387
388        // Check causal model evaluation
389        if let Some(ref causal) = self.causal {
390            if !causal.passes {
391                self.failures.extend(causal.issues.clone());
392            }
393        }
394
395        // Check enrichment quality evaluation
396        if let Some(ref enrichment) = self.enrichment_quality {
397            if !enrichment.passes {
398                self.failures.extend(enrichment.issues.clone());
399            }
400        }
401
402        self.passes = self.failures.is_empty();
403    }
404}
405
406impl Default for ComprehensiveEvaluation {
407    fn default() -> Self {
408        Self::new()
409    }
410}
411
412/// Main evaluator that coordinates all evaluation modules.
413pub struct Evaluator {
414    /// Evaluation configuration.
415    config: EvaluationConfig,
416}
417
418impl Evaluator {
419    /// Create a new evaluator with the given configuration.
420    pub fn new(config: EvaluationConfig) -> Self {
421        Self { config }
422    }
423
424    /// Create an evaluator with default configuration.
425    pub fn with_defaults() -> Self {
426        Self::new(EvaluationConfig::default())
427    }
428
429    /// Get the configuration.
430    pub fn config(&self) -> &EvaluationConfig {
431        &self.config
432    }
433
434    /// Run a comprehensive evaluation and return results.
435    ///
436    /// # Architectural note
437    ///
438    /// This zero-argument variant returns a default (passing) evaluation because the
439    /// `Evaluator` struct holds only configuration — it has no access to the generated
440    /// journal entry or balance data that the sub-module evaluators require.
441    ///
442    /// To evaluate actual generation output, use [`run_evaluation_with_amounts`] which
443    /// accepts raw JE amounts and runs the Benford analysis.  Full wiring of all
444    /// sub-modules (BalanceSheetEvaluator, DocumentChainEvaluator, etc.) requires
445    /// passing the complete `EnhancedGenerationResult` from the runtime crate, which
446    /// would create a circular dependency.  The recommended integration point is the
447    /// orchestrator layer (datasynth-runtime) which already calls the gate engine with
448    /// a populated `ComprehensiveEvaluation`.
449    pub fn run_evaluation(&self) -> ComprehensiveEvaluation {
450        let mut evaluation = ComprehensiveEvaluation::new();
451        evaluation.check_all_thresholds(&self.config.thresholds);
452        evaluation
453    }
454
455    /// Run a Benford-augmented evaluation given raw JE amounts.
456    ///
457    /// This method calls the [`BenfordAnalyzer`] sub-module and populates the
458    /// `statistical.benford` field of the returned [`ComprehensiveEvaluation`].
459    /// All other sub-module fields remain at their default (passing) values.
460    pub fn run_evaluation_with_amounts(
461        &self,
462        je_amounts: &[rust_decimal::Decimal],
463    ) -> ComprehensiveEvaluation {
464        let mut evaluation = ComprehensiveEvaluation::new();
465
466        if !je_amounts.is_empty() {
467            let analyzer = BenfordAnalyzer::new(self.config.thresholds.benford_p_value_min);
468            match analyzer.analyze(je_amounts) {
469                Ok(benford) => {
470                    evaluation.statistical.benford = Some(benford);
471                }
472                Err(e) => {
473                    evaluation
474                        .failures
475                        .push(format!("Benford analysis failed: {e}"));
476                    evaluation.passes = false;
477                }
478            }
479        }
480
481        evaluation.check_all_thresholds(&self.config.thresholds);
482        evaluation
483    }
484}
485
486#[cfg(test)]
487#[allow(clippy::unwrap_used)]
488mod tests {
489    use super::*;
490
491    #[test]
492    fn test_comprehensive_evaluation_new() {
493        let eval = ComprehensiveEvaluation::new();
494        assert!(eval.passes);
495        assert!(eval.failures.is_empty());
496    }
497
498    #[test]
499    fn test_evaluator_creation() {
500        let evaluator = Evaluator::with_defaults();
501        assert_eq!(evaluator.config().thresholds.benford_p_value_min, 0.05);
502    }
503}
datasynth_eval/lib.rs

datasynth_eval/
lib.rs
