datasynth_eval/
lib.rs

1#![cfg_attr(not(test), deny(clippy::unwrap_used))]
2// Allow some clippy lints that are common in test/evaluation code
3#![allow(clippy::field_reassign_with_default)]
4#![allow(clippy::too_many_arguments)]
5#![allow(clippy::upper_case_acronyms)] // MCAR, MAR, MNAR, ISO are standard abbreviations
6
7//! Synthetic Data Evaluation Framework
8//!
9//! This crate provides comprehensive evaluation capabilities for validating
10//! the quality and correctness of generated synthetic financial data.
11//!
12//! # Features
13//!
14//! - **Statistical Quality**: Benford's Law, amount distributions, line item patterns
15//! - **Semantic Coherence**: Balance sheet validation, subledger reconciliation
16//! - **Data Quality**: Uniqueness, completeness, format consistency
17//! - **ML-Readiness**: Feature distributions, label quality, graph structure
18//! - **Reporting**: HTML and JSON reports with pass/fail thresholds
19//!
20//! # Example
21//!
22//! ```ignore
23//! use datasynth_eval::{Evaluator, EvaluationConfig};
24//!
25//! let config = EvaluationConfig::default();
26//! let evaluator = Evaluator::new(config);
27//!
28//! // Evaluate generated data
29//! let result = evaluator.evaluate(&generation_result)?;
30//!
31//! // Generate report
32//! result.generate_html_report("evaluation_report.html")?;
33//! ```
34
35pub mod benchmarks;
36pub mod calibration;
37pub mod config;
38pub mod enhancement;
39pub mod error;
40pub mod gates;
41pub mod privacy;
42
43pub mod behavioral_fidelity;
44pub mod coherence;
45pub mod ml;
46pub mod quality;
47pub mod report;
48pub mod statistical;
49pub mod tuning;
50
51#[cfg(feature = "adversarial")]
52pub mod adversarial;
53pub mod banking;
54pub mod causal;
55pub mod diff_engine;
56pub mod enrichment;
57pub mod process_mining;
58pub mod scenario_diff;
59
60// Re-exports
61pub use config::{EvaluationConfig, EvaluationThresholds, PrivacyEvaluationConfig};
62pub use error::{EvalError, EvalResult};
63
64pub use statistical::{
65    AmountDistributionAnalysis, AmountDistributionAnalyzer, AnomalyRealismEvaluation,
66    AnomalyRealismEvaluator, BenfordAnalysis, BenfordAnalyzer, BenfordConformity,
67    CorrelationAnalysis, CorrelationAnalyzer, CorrelationCheckResult, DetectionDifficulty,
68    DriftDetectionAnalysis, DriftDetectionAnalyzer, DriftDetectionEntry, DriftDetectionMetrics,
69    DriftEventCategory, ExpectedCorrelation, LabeledDriftEvent, LabeledEventAnalysis,
70    LineItemAnalysis, LineItemAnalyzer, LineItemEntry, SecondDigitAnalysis, StatisticalEvaluation,
71    TemporalAnalysis, TemporalAnalyzer, TemporalEntry,
72};
73
74pub use coherence::{
75    AccountType,
76    ApprovalLevelData,
77    AuditEvaluation,
78    AuditEvaluator,
79    AuditFindingData,
80    AuditRiskData,
81    AuditTrailEvaluation,
82    AuditTrailGap,
83    BalanceSheetEvaluation,
84    BalanceSheetEvaluator,
85    BalanceSnapshot,
86    BankReconciliationEvaluation,
87    BankReconciliationEvaluator,
88    BidEvaluationData,
89    BudgetVarianceData,
90    CashPositionData,
91    CoherenceEvaluation,
92    ConcentrationMetrics,
93    CountryPackData,
94    CountryPackEvaluation,
95    CountryPackEvaluator,
96    CountryPackThresholds,
97    CovenantData,
98    CrossProcessEvaluation,
99    CrossProcessEvaluator,
100    CycleCountData,
101    DocumentChainEvaluation,
102    DocumentChainEvaluator,
103    DocumentReferenceData,
104    EarnedValueData,
105    EntityReferenceData,
106    EsgEvaluation,
107    EsgEvaluator,
108    EsgThresholds,
109    ExpenseReportData,
110    FairValueEvaluation,
111    // Task 4.1: Financial Ratio Evaluator
112    FinancialRatios,
113    FinancialReportingEvaluation,
114    FinancialReportingEvaluator,
115    FinancialStatementData,
116    FrameworkViolation,
117    GovernanceData,
118    HedgeEffectivenessData,
119    HolidayData,
120    HrPayrollEvaluation,
121    HrPayrollEvaluator,
122    ICMatchingData,
123    ICMatchingEvaluation,
124    ICMatchingEvaluator,
125    ImpairmentEvaluation,
126    IsaComplianceEvaluation,
127    // Task 4.2: JE Risk Scoring Evaluator
128    JeRiskScoringResult,
129    KpiData,
130    LeaseAccountingEvaluation,
131    LeaseAccountingEvaluator,
132    LeaseEvaluation,
133    ManufacturingEvaluation,
134    ManufacturingEvaluator,
135    MaterialityData,
136    NettingData,
137    NetworkEdge,
138    NetworkEvaluation,
139    NetworkEvaluator,
140    NetworkNode,
141    NetworkThresholds,
142    O2CChainData,
143    P2PChainData,
144    PayrollHoursData,
145    PayrollLineItemData,
146    PayrollRunData,
147    PcaobComplianceEvaluation,
148    PerformanceObligation,
149    ProductionOrderData,
150    ProjectAccountingEvaluation,
151    ProjectAccountingEvaluator,
152    ProjectAccountingThresholds,
153    ProjectRevenueData,
154    QualityInspectionData,
155    QuoteLineData,
156    RatioAnalysisResult,
157    RatioCheck,
158    ReconciliationData,
159    ReferentialData,
160    ReferentialIntegrityEvaluation,
161    ReferentialIntegrityEvaluator,
162    RetainageData,
163    RevenueContract,
164    RevenueRecognitionEvaluation,
165    RevenueRecognitionEvaluator,
166    RiskAttributeStats,
167    RiskDistribution,
168    RoutingOperationData,
169    SafetyMetricData,
170    SalesQuoteData,
171    SalesQuoteEvaluation,
172    SalesQuoteEvaluator,
173    SalesQuoteThresholds,
174    ScorecardCoverageData,
175    SourcingEvaluation,
176    SourcingEvaluator,
177    SourcingProjectData,
178    SoxComplianceEvaluation,
179    SpendAnalysisData,
180    StandardsComplianceEvaluation,
181    StandardsThresholds,
182    StrengthStats,
183    SubledgerEvaluator,
184    SubledgerReconciliationEvaluation,
185    SupplierEsgData,
186    TaxEvaluation,
187    TaxEvaluator,
188    TaxLineData,
189    TaxRateData,
190    TaxReturnData,
191    TaxThresholds,
192    TimeEntryData,
193    TreasuryEvaluation,
194    TreasuryEvaluator,
195    TreasuryThresholds,
196    UnmatchedICItem,
197    VariableConsideration,
198    ViolationSeverity,
199    WaterUsageData,
200    WithholdingData,
201    WorkpaperData,
202};
203
204pub use quality::{
205    CompletenessAnalysis, CompletenessAnalyzer, ConsistencyAnalysis, ConsistencyAnalyzer,
206    ConsistencyRule, DuplicateInfo, FieldCompleteness, FieldDefinition, FieldValue, FormatAnalysis,
207    FormatAnalyzer, FormatVariation, QualityEvaluation, UniqueRecord, UniquenessAnalysis,
208    UniquenessAnalyzer,
209};
210
211pub use ml::{
212    AnomalyScoringAnalysis, AnomalyScoringAnalyzer, CrossModalAnalysis, CrossModalAnalyzer,
213    DomainGapAnalysis, DomainGapAnalyzer, EmbeddingReadinessAnalysis, EmbeddingReadinessAnalyzer,
214    FeatureAnalysis, FeatureAnalyzer, FeatureQualityAnalysis, FeatureQualityAnalyzer, FeatureStats,
215    GnnReadinessAnalysis, GnnReadinessAnalyzer, GraphAnalysis, GraphAnalyzer, GraphMetrics,
216    LabelAnalysis, LabelAnalyzer, LabelDistribution, MLReadinessEvaluation,
217    SchemeDetectabilityAnalysis, SchemeDetectabilityAnalyzer, SplitAnalysis, SplitAnalyzer,
218    SplitMetrics, TemporalFidelityAnalysis, TemporalFidelityAnalyzer,
219};
220
221pub use report::{
222    BaselineComparison, ComparisonResult, EvaluationReport, HtmlReportGenerator,
223    JsonReportGenerator, MetricChange, ReportMetadata, ThresholdChecker, ThresholdResult,
224};
225
226pub use tuning::{
227    ConfigSuggestion, ConfigSuggestionGenerator, TuningAnalyzer, TuningCategory, TuningOpportunity,
228};
229
230pub use enhancement::{
231    AiTuneResult, AiTuner, AiTunerConfig, AutoTuneResult, AutoTuner, ConfigPatch,
232    EnhancementReport, Recommendation, RecommendationCategory, RecommendationEngine,
233    RecommendationPriority, RootCause, SuggestedAction, TuningIteration,
234};
235
236pub use privacy::{
237    LinkageAttack, LinkageConfig, LinkageResults, MembershipInferenceAttack, MiaConfig, MiaResults,
238    NistAlignmentReport, NistCriterion, PrivacyEvaluation, SynQPMatrix, SynQPQuadrant,
239};
240
241pub use benchmarks::{
242    // ACFE-calibrated benchmarks
243    acfe_calibrated_1k,
244    acfe_collusion_5k,
245    acfe_management_override_2k,
246    all_acfe_benchmarks,
247    all_benchmarks,
248    // Industry-specific benchmarks
249    all_industry_benchmarks,
250    anomaly_bench_1k,
251    data_quality_100k,
252    entity_match_5k,
253    financial_services_fraud_5k,
254    fraud_detect_10k,
255    get_benchmark,
256    get_industry_benchmark,
257    graph_fraud_10k,
258    healthcare_fraud_5k,
259    manufacturing_fraud_5k,
260    retail_fraud_10k,
261    technology_fraud_3k,
262    AcfeAlignment,
263    AcfeCalibration,
264    AcfeCategoryDistribution,
265    BaselineModelType,
266    BaselineResult,
267    BenchmarkBuilder,
268    BenchmarkSuite,
269    BenchmarkTaskType,
270    CostMatrix,
271    DatasetSpec,
272    EvaluationSpec,
273    FeatureSet,
274    IndustryBenchmarkAnalysis,
275    LeaderboardEntry,
276    MetricType,
277    SplitRatios,
278};
279
280pub use banking::{
281    AmlDetectabilityAnalysis, AmlDetectabilityAnalyzer, AmlTransactionData, BankingEvaluation,
282    KycCompletenessAnalysis, KycCompletenessAnalyzer, KycProfileData, TypologyData,
283};
284
285pub use process_mining::{
286    EventSequenceAnalysis, EventSequenceAnalyzer, ProcessEventData, ProcessMiningEvaluation,
287    VariantAnalysis, VariantAnalyzer, VariantData,
288};
289
290pub use causal::{CausalModelEvaluation, CausalModelEvaluator};
291
292pub use enrichment::{EnrichmentQualityEvaluation, EnrichmentQualityEvaluator};
293
294use serde::{Deserialize, Serialize};
295
296/// Comprehensive evaluation result combining all evaluation modules.
297#[derive(Debug, Clone, Serialize, Deserialize)]
298pub struct ComprehensiveEvaluation {
299    /// Statistical quality evaluation.
300    pub statistical: StatisticalEvaluation,
301    /// Semantic coherence evaluation.
302    pub coherence: CoherenceEvaluation,
303    /// Data quality evaluation.
304    pub quality: QualityEvaluation,
305    /// ML-readiness evaluation.
306    pub ml_readiness: MLReadinessEvaluation,
307    /// Privacy evaluation (optional — only populated when privacy testing is enabled).
308    #[serde(default, skip_serializing_if = "Option::is_none")]
309    pub privacy: Option<PrivacyEvaluation>,
310    /// Banking/KYC/AML evaluation (optional).
311    #[serde(default, skip_serializing_if = "Option::is_none")]
312    pub banking: Option<BankingEvaluation>,
313    /// OCEL 2.0 process mining evaluation (optional).
314    #[serde(default, skip_serializing_if = "Option::is_none")]
315    pub process_mining: Option<ProcessMiningEvaluation>,
316    /// Causal model evaluation (optional).
317    #[serde(default, skip_serializing_if = "Option::is_none")]
318    pub causal: Option<CausalModelEvaluation>,
319    /// LLM enrichment quality evaluation (optional).
320    #[serde(default, skip_serializing_if = "Option::is_none")]
321    pub enrichment_quality: Option<EnrichmentQualityEvaluation>,
322    /// Overall pass/fail status.
323    pub passes: bool,
324    /// Summary of all failures.
325    pub failures: Vec<String>,
326    /// Tuning opportunities identified.
327    pub tuning_opportunities: Vec<TuningOpportunity>,
328    /// Configuration suggestions.
329    pub config_suggestions: Vec<ConfigSuggestion>,
330}
331
332impl ComprehensiveEvaluation {
333    /// Create a new empty evaluation.
334    pub fn new() -> Self {
335        Self {
336            statistical: StatisticalEvaluation::default(),
337            coherence: CoherenceEvaluation::default(),
338            quality: QualityEvaluation::default(),
339            ml_readiness: MLReadinessEvaluation::default(),
340            privacy: None,
341            banking: None,
342            process_mining: None,
343            causal: None,
344            enrichment_quality: None,
345            passes: true,
346            failures: Vec::new(),
347            tuning_opportunities: Vec::new(),
348            config_suggestions: Vec::new(),
349        }
350    }
351
352    /// Check all evaluations against thresholds and update overall status.
353    pub fn check_all_thresholds(&mut self, thresholds: &EvaluationThresholds) {
354        self.failures.clear();
355
356        // Check statistical thresholds
357        self.statistical.check_thresholds(thresholds);
358        self.failures.extend(self.statistical.failures.clone());
359
360        // Check coherence thresholds
361        self.coherence.check_thresholds(thresholds);
362        self.failures.extend(self.coherence.failures.clone());
363
364        // Check quality thresholds
365        self.quality.check_thresholds(thresholds);
366        self.failures.extend(self.quality.failures.clone());
367
368        // Check ML thresholds
369        self.ml_readiness.check_thresholds(thresholds);
370        self.failures.extend(self.ml_readiness.failures.clone());
371
372        // Check privacy evaluation (if present)
373        if let Some(ref mut privacy) = self.privacy {
374            privacy.update_status();
375            self.failures.extend(privacy.failures.clone());
376        }
377
378        // Check banking evaluation
379        if let Some(ref mut banking) = self.banking {
380            banking.check_thresholds();
381            self.failures.extend(banking.issues.clone());
382        }
383
384        // Check process mining evaluation
385        if let Some(ref mut pm) = self.process_mining {
386            pm.check_thresholds();
387            self.failures.extend(pm.issues.clone());
388        }
389
390        // Check causal model evaluation
391        if let Some(ref causal) = self.causal {
392            if !causal.passes {
393                self.failures.extend(causal.issues.clone());
394            }
395        }
396
397        // Check enrichment quality evaluation
398        if let Some(ref enrichment) = self.enrichment_quality {
399            if !enrichment.passes {
400                self.failures.extend(enrichment.issues.clone());
401            }
402        }
403
404        self.passes = self.failures.is_empty();
405    }
406}
407
408impl Default for ComprehensiveEvaluation {
409    fn default() -> Self {
410        Self::new()
411    }
412}
413
414/// Main evaluator that coordinates all evaluation modules.
415pub struct Evaluator {
416    /// Evaluation configuration.
417    config: EvaluationConfig,
418}
419
420impl Evaluator {
421    /// Create a new evaluator with the given configuration.
422    pub fn new(config: EvaluationConfig) -> Self {
423        Self { config }
424    }
425
426    /// Create an evaluator with default configuration.
427    pub fn with_defaults() -> Self {
428        Self::new(EvaluationConfig::default())
429    }
430
431    /// Get the configuration.
432    pub fn config(&self) -> &EvaluationConfig {
433        &self.config
434    }
435
436    /// Run a comprehensive evaluation and return results.
437    ///
438    /// # Architectural note
439    ///
440    /// This zero-argument variant returns a default (passing) evaluation because the
441    /// `Evaluator` struct holds only configuration — it has no access to the generated
442    /// journal entry or balance data that the sub-module evaluators require.
443    ///
444    /// To evaluate actual generation output, use `run_evaluation_with_amounts` which
445    /// accepts raw JE amounts and runs the Benford analysis.  Full wiring of all
446    /// sub-modules (BalanceSheetEvaluator, DocumentChainEvaluator, etc.) requires
447    /// passing the complete `EnhancedGenerationResult` from the runtime crate, which
448    /// would create a circular dependency.  The recommended integration point is the
449    /// orchestrator layer (datasynth-runtime) which already calls the gate engine with
450    /// a populated `ComprehensiveEvaluation`.
451    pub fn run_evaluation(&self) -> ComprehensiveEvaluation {
452        let mut evaluation = ComprehensiveEvaluation::new();
453        evaluation.check_all_thresholds(&self.config.thresholds);
454        evaluation
455    }
456
457    /// Run a Benford-augmented evaluation given raw JE amounts.
458    ///
459    /// This method calls the [`BenfordAnalyzer`] sub-module and populates the
460    /// `statistical.benford` field of the returned [`ComprehensiveEvaluation`].
461    /// All other sub-module fields remain at their default (passing) values.
462    pub fn run_evaluation_with_amounts(
463        &self,
464        je_amounts: &[rust_decimal::Decimal],
465    ) -> ComprehensiveEvaluation {
466        let mut evaluation = ComprehensiveEvaluation::new();
467
468        if !je_amounts.is_empty() {
469            let analyzer = BenfordAnalyzer::new(self.config.thresholds.benford_p_value_min);
470            match analyzer.analyze(je_amounts) {
471                Ok(benford) => {
472                    evaluation.statistical.benford = Some(benford);
473                }
474                Err(e) => {
475                    evaluation
476                        .failures
477                        .push(format!("Benford analysis failed: {e}"));
478                    evaluation.passes = false;
479                }
480            }
481        }
482
483        evaluation.check_all_thresholds(&self.config.thresholds);
484        evaluation
485    }
486}
487
488#[cfg(test)]
489mod tests {
490    use super::*;
491
492    #[test]
493    fn test_comprehensive_evaluation_new() {
494        let eval = ComprehensiveEvaluation::new();
495        assert!(eval.passes);
496        assert!(eval.failures.is_empty());
497    }
498
499    #[test]
500    fn test_evaluator_creation() {
501        let evaluator = Evaluator::with_defaults();
502        assert_eq!(evaluator.config().thresholds.benford_p_value_min, 0.05);
503    }
504}
datasynth_eval/lib.rs

datasynth_eval/
lib.rs
