anno_eval/eval/
unified_evaluator.rs

1//! Unified Evaluation System
2//!
3//! Provides a single, consistent API for all evaluation types, replacing
4//! the multiple entry points (TaskEvaluator, EvalHarness, direct functions, etc.)
5//!
6//! # Design Goals
7//!
8//! - **Single Entry Point**: One API for all evaluation needs
9//! - **Unified Results**: Consistent result types across all evaluations
10//! - **Composable**: Easy to combine standard + bias + calibration
11//! - **Type-Safe**: Compile-time checking for configurations
12//!
13//! # Example
14//!
15//! ```rust,ignore
16//! #[cfg(feature = "eval")]
17//! {
18//! use anno_eval::eval::unified_evaluator::EvalSystem;
19//! use anno_eval::eval::task_mapping::Task;
20//!
21//! let results = EvalSystem::new()
22//!     .with_tasks(vec![Task::NER])
23//!     .with_datasets(vec![])  // All suitable datasets
24//!     .with_backends(vec!["gliner_multitask".to_string()])
25//!     .run()?;
26//!
27//! println!("Standard F1: {:.1}%", results.standard.as_ref().map(|s| s.f1 * 100.0).unwrap_or(0.0));
28//! }
29//! ```
30
31use anno::{Model, Result};
32use serde::{Deserialize, Serialize};
33#[cfg(feature = "eval")]
34use std::collections::HashMap;
35
36#[cfg(feature = "eval")]
37use crate::eval::loader::DatasetId;
38#[cfg(feature = "eval")]
39use crate::eval::task_evaluator::{TaskEvalConfig, TaskEvaluator};
40#[cfg(feature = "eval")]
41use crate::eval::task_mapping::Task;
42
43#[cfg(feature = "eval-bias")]
44use crate::eval::bias_config::BiasDatasetConfig;
45#[cfg(feature = "eval-bias")]
46use crate::eval::coref_resolver::SimpleCorefResolver;
47#[cfg(feature = "eval-bias")]
48use crate::eval::demographic_bias::{create_diverse_name_dataset, DemographicBiasEvaluator};
49#[cfg(feature = "eval-bias")]
50use crate::eval::gender_bias::{create_winobias_templates, GenderBiasEvaluator};
51#[cfg(feature = "eval-bias")]
52use crate::eval::length_bias::{create_length_varied_dataset, EntityLengthEvaluator};
53#[cfg(feature = "eval-bias")]
54use crate::eval::temporal_bias::{create_temporal_name_dataset, TemporalBiasEvaluator};
55
56#[cfg(feature = "eval")]
57use crate::eval::backend_name::BackendName;
58
59// =============================================================================
60// Unified Results
61// =============================================================================
62
63/// Unified evaluation results combining all evaluation types.
64#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct UnifiedEvalResults {
66    /// Standard task evaluation results (NER, Coref, etc.)
67    #[cfg(feature = "eval")]
68    pub standard: Option<StandardEvalResults>,
69
70    /// Bias evaluation results
71    #[cfg(feature = "eval-bias")]
72    pub bias: Option<BiasEvalResults>,
73
74    /// Calibration results (if enabled)
75    #[cfg(feature = "eval")]
76    pub calibration: Option<CalibrationEvalResults>,
77
78    /// Data quality results (if enabled)
79    #[cfg(feature = "eval")]
80    pub data_quality: Option<DataQualityEvalResults>,
81
82    /// Warnings and notes
83    pub warnings: Vec<String>,
84
85    /// Evaluation metadata
86    pub metadata: EvalMetadata,
87}
88
89/// Standard task evaluation results.
90#[derive(Debug, Clone, Serialize, Deserialize)]
91#[cfg(feature = "eval")]
92pub struct StandardEvalResults {
93    /// Overall F1 score
94    pub f1: f64,
95    /// Precision
96    pub precision: f64,
97    /// Recall
98    pub recall: f64,
99    /// Per-task results
100    pub per_task: HashMap<String, TaskResults>,
101    /// Per-dataset results
102    pub per_dataset: HashMap<String, DatasetResults>,
103    /// Per-backend results
104    pub per_backend: HashMap<String, BackendResults>,
105}
106
107/// Task-specific results.
108#[derive(Debug, Clone, Serialize, Deserialize)]
109#[cfg(feature = "eval")]
110pub struct TaskResults {
111    /// Task identifier (e.g., "NER", "Coref").
112    pub task: String,
113    /// F1 score for this task.
114    pub f1: f64,
115    /// Precision score for this task.
116    pub precision: f64,
117    /// Recall score for this task.
118    pub recall: f64,
119    /// Number of examples evaluated.
120    pub num_examples: usize,
121}
122
123/// Dataset-specific results.
124#[derive(Debug, Clone, Serialize, Deserialize)]
125#[cfg(feature = "eval")]
126pub struct DatasetResults {
127    /// Dataset identifier/name.
128    pub dataset: String,
129    /// F1 score on this dataset.
130    pub f1: f64,
131    /// Precision on this dataset.
132    pub precision: f64,
133    /// Recall on this dataset.
134    pub recall: f64,
135    /// Number of evaluated examples for this dataset.
136    pub num_examples: usize,
137}
138
139/// Backend-specific results.
140#[derive(Debug, Clone, Serialize, Deserialize)]
141#[cfg(feature = "eval")]
142pub struct BackendResults {
143    /// Backend identifier/name.
144    pub backend: String,
145    /// F1 score for this backend.
146    pub f1: f64,
147    /// Precision for this backend.
148    pub precision: f64,
149    /// Recall for this backend.
150    pub recall: f64,
151    /// Number of evaluated examples for this backend.
152    pub num_examples: usize,
153}
154
155/// Bias evaluation results.
156#[derive(Debug, Clone, Serialize, Deserialize)]
157#[cfg(feature = "eval-bias")]
158pub struct BiasEvalResults {
159    /// Gender bias results
160    pub gender: Option<GenderBiasSummary>,
161    /// Demographic bias results
162    pub demographic: Option<DemographicBiasSummary>,
163    /// Temporal bias results
164    pub temporal: Option<TemporalBiasSummary>,
165    /// Length bias results
166    pub length: Option<LengthBiasSummary>,
167}
168
169/// Gender bias summary.
170#[derive(Debug, Clone, Serialize, Deserialize)]
171#[cfg(feature = "eval-bias")]
172pub struct GenderBiasSummary {
173    /// Difference between pro- and anti-stereotype accuracy.
174    pub bias_gap: f64,
175    /// Accuracy on pro-stereotype examples.
176    pub pro_stereotype_accuracy: f64,
177    /// Accuracy on anti-stereotype examples.
178    pub anti_stereotype_accuracy: f64,
179}
180
181/// Demographic bias summary.
182#[derive(Debug, Clone, Serialize, Deserialize)]
183#[cfg(feature = "eval-bias")]
184pub struct DemographicBiasSummary {
185    /// Parity gap across ethnicity groups.
186    pub ethnicity_parity_gap: f64,
187    /// Bias gap across different scripts (Latin vs non-Latin).
188    pub script_bias_gap: f64,
189    /// Overall recognition rate across all demographic groups.
190    pub overall_recognition_rate: f64,
191}
192
193/// Temporal bias summary.
194#[derive(Debug, Clone, Serialize, Deserialize)]
195#[cfg(feature = "eval-bias")]
196pub struct TemporalBiasSummary {
197    /// Gap between historical and modern entity recognition.
198    pub historical_modern_gap: f64,
199    /// Recognition rate for historical entities.
200    pub historical_rate: f64,
201    /// Recognition rate for modern entities.
202    pub modern_rate: f64,
203}
204
205/// Length bias summary.
206#[derive(Debug, Clone, Serialize, Deserialize)]
207#[cfg(feature = "eval-bias")]
208pub struct LengthBiasSummary {
209    /// Gap between short and long entity recognition.
210    pub short_vs_long_gap: f64,
211    /// F1 score for single-word entities.
212    pub short_entity_f1: f64,
213    /// F1 score for four-or-more-word entities.
214    pub long_entity_f1: f64,
215}
216
217/// Calibration results.
218#[derive(Debug, Clone, Serialize, Deserialize)]
219#[cfg(feature = "eval")]
220pub struct CalibrationEvalResults {
221    /// Expected calibration error.
222    pub ece: f64,
223    /// Maximum calibration error.
224    pub mce: f64,
225    /// Brier score (lower is better).
226    pub brier_score: f64,
227}
228
229/// Data quality results.
230#[derive(Debug, Clone, Serialize, Deserialize)]
231#[cfg(feature = "eval")]
232pub struct DataQualityEvalResults {
233    /// Whether train/test leakage was detected.
234    pub leakage_detected: bool,
235    /// Proportion of redundant examples (0.0 to 1.0).
236    pub redundancy_rate: f64,
237    /// Number of ambiguous annotations found.
238    pub ambiguous_count: usize,
239}
240
241/// Evaluation metadata captured during an evaluation run.
242///
243/// Contains timing, model identification, and basic statistics.
244#[derive(Debug, Clone, Serialize, Deserialize)]
245pub struct EvalMetadata {
246    /// ISO 8601 timestamp when evaluation started.
247    pub timestamp: String,
248    /// Name of the model being evaluated, if known.
249    pub model_name: Option<String>,
250    /// Total wall-clock duration in milliseconds.
251    pub total_duration_ms: Option<f64>,
252    /// Number of examples processed.
253    pub num_examples: usize,
254}
255
256// =============================================================================
257// Unified Evaluator
258// =============================================================================
259
260/// Unified evaluation system - single entry point for all evaluations.
261pub struct EvalSystem {
262    #[cfg(feature = "eval")]
263    tasks: Vec<Task>,
264    #[cfg(feature = "eval")]
265    datasets: Vec<DatasetId>,
266    #[cfg(feature = "eval")]
267    backends: Vec<String>,
268    #[cfg(feature = "eval")]
269    max_examples: Option<usize>,
270    #[cfg(feature = "eval")]
271    seed: Option<u64>,
272
273    #[cfg(feature = "eval-bias")]
274    include_bias: bool,
275    #[cfg(feature = "eval-bias")]
276    bias_config: Option<BiasDatasetConfig>,
277
278    #[cfg(feature = "eval")]
279    include_calibration: bool,
280    #[cfg(feature = "eval")]
281    include_data_quality: bool,
282
283    model: Option<Box<dyn Model>>,
284    model_name: Option<String>,
285
286    /// Coreference resolver for coreference evaluation tasks
287    /// Uses Arc to allow sharing across multiple evaluation calls
288    coref_resolver: Option<std::sync::Arc<dyn crate::eval::coref_resolver::CoreferenceResolver>>,
289}
290
291impl EvalSystem {
292    /// Create a new unified evaluation system.
293    pub fn new() -> Self {
294        Self {
295            #[cfg(feature = "eval")]
296            tasks: vec![],
297            #[cfg(feature = "eval")]
298            datasets: vec![],
299            #[cfg(feature = "eval")]
300            backends: vec![],
301            #[cfg(feature = "eval")]
302            max_examples: None,
303            #[cfg(feature = "eval")]
304            seed: Some(42),
305
306            #[cfg(feature = "eval-bias")]
307            include_bias: false,
308            #[cfg(feature = "eval-bias")]
309            bias_config: None,
310
311            #[cfg(feature = "eval")]
312            include_calibration: false,
313            #[cfg(feature = "eval")]
314            include_data_quality: false,
315
316            model: None,
317            model_name: None,
318            coref_resolver: None,
319        }
320    }
321
322    /// Set tasks to evaluate.
323    #[cfg(feature = "eval")]
324    pub fn with_tasks(mut self, tasks: Vec<Task>) -> Self {
325        self.tasks = tasks;
326        self
327    }
328
329    /// Set datasets to use.
330    #[cfg(feature = "eval")]
331    pub fn with_datasets(mut self, datasets: Vec<DatasetId>) -> Self {
332        self.datasets = datasets;
333        self
334    }
335
336    /// Set backends to test.
337    #[cfg(feature = "eval")]
338    pub fn with_backends(mut self, backends: Vec<String>) -> Self {
339        self.backends = backends;
340        self
341    }
342
343    /// Set backends using type-safe BackendName enum.
344    #[cfg(feature = "eval")]
345    pub fn with_backend_names(mut self, backends: Vec<BackendName>) -> Self {
346        self.backends = backends
347            .into_iter()
348            .map(|b| b.as_str().to_string())
349            .collect();
350        self
351    }
352
353    /// Set maximum examples per dataset.
354    ///
355    /// Pass `None` to remove limit (evaluate all examples).
356    #[cfg(feature = "eval")]
357    pub fn with_max_examples(mut self, max: Option<usize>) -> Self {
358        self.max_examples = max;
359        self
360    }
361
362    /// Add a task to evaluate.
363    #[cfg(feature = "eval")]
364    pub fn add_task(mut self, task: Task) -> Self {
365        if !self.tasks.contains(&task) {
366            self.tasks.push(task);
367        }
368        self
369    }
370
371    /// Add a dataset to use.
372    #[cfg(feature = "eval")]
373    pub fn add_dataset(mut self, dataset: DatasetId) -> Self {
374        if !self.datasets.contains(&dataset) {
375            self.datasets.push(dataset);
376        }
377        self
378    }
379
380    /// Add a backend to test.
381    #[cfg(feature = "eval")]
382    pub fn add_backend(mut self, backend: String) -> Self {
383        if !self.backends.contains(&backend) {
384            self.backends.push(backend);
385        }
386        self
387    }
388
389    /// Add a backend using type-safe BackendName enum.
390    #[cfg(feature = "eval")]
391    pub fn add_backend_name(mut self, backend: BackendName) -> Self {
392        let backend_str = backend.as_str().to_string();
393        if !self.backends.contains(&backend_str) {
394            self.backends.push(backend_str);
395        }
396        self
397    }
398
399    /// Set random seed.
400    #[cfg(feature = "eval")]
401    pub fn with_seed(mut self, seed: u64) -> Self {
402        self.seed = Some(seed);
403        self
404    }
405
406    /// Enable bias analysis.
407    #[cfg(feature = "eval-bias")]
408    pub fn with_bias_analysis(mut self, enable: bool) -> Self {
409        self.include_bias = enable;
410        if enable && self.bias_config.is_none() {
411            self.bias_config = Some(
412                BiasDatasetConfig::default()
413                    .with_frequency_weighting()
414                    .with_validation(),
415            );
416        }
417        self
418    }
419
420    /// Set bias evaluation configuration.
421    #[cfg(feature = "eval-bias")]
422    pub fn with_bias_config(mut self, config: BiasDatasetConfig) -> Self {
423        self.bias_config = Some(config);
424        self.include_bias = true;
425        self
426    }
427
428    /// Enable calibration analysis.
429    #[cfg(feature = "eval")]
430    pub fn with_calibration(mut self, enable: bool) -> Self {
431        self.include_calibration = enable;
432        self
433    }
434
435    /// Enable data quality checks.
436    #[cfg(feature = "eval")]
437    pub fn with_data_quality(mut self, enable: bool) -> Self {
438        self.include_data_quality = enable;
439        self
440    }
441
442    /// Set model to evaluate (for bias/calibration that need model instance).
443    pub fn with_model(mut self, model: Box<dyn Model>, name: Option<String>) -> Self {
444        self.model = Some(model);
445        self.model_name = name;
446        self
447    }
448
449    /// Set coreference resolver to evaluate.
450    ///
451    /// This allows evaluating coreference resolvers (e.g., `TrainedBoxCorefResolver` from matryoshka-box)
452    /// using anno's evaluation infrastructure.
453    ///
454    /// # Example
455    ///
456    /// ```rust,ignore
457    /// use matryoshka_box_inference::trained_resolver::TrainedBoxCorefResolver;
458    /// use anno_eval::eval::unified_evaluator::EvalSystem;
459    /// use anno_eval::eval::task_mapping::Task;
460    ///
461    /// let resolver = TrainedBoxCorefResolver::new(trained_boxes, config);
462    /// let results = EvalSystem::new()
463    ///     .with_coref_resolver(Box::new(resolver))
464    ///     .with_tasks(vec![Task::Coreference { metrics: vec![] }])
465    ///     .run()?;
466    /// ```
467    pub fn with_coref_resolver(
468        mut self,
469        resolver: Box<dyn crate::eval::coref_resolver::CoreferenceResolver>,
470    ) -> Self {
471        self.coref_resolver = Some(std::sync::Arc::from(resolver));
472        self
473    }
474
475    /// Run all enabled evaluations.
476    pub fn run(self) -> Result<UnifiedEvalResults> {
477        use std::time::Instant;
478
479        let start = Instant::now();
480        #[allow(unused_mut)]
481        let mut warnings = Vec::new();
482
483        // Run standard evaluation
484        #[cfg(feature = "eval")]
485        let standard_result = self.run_standard_evaluation(&mut warnings)?;
486
487        // Run bias evaluation
488        #[cfg(feature = "eval-bias")]
489        let bias = if self.include_bias {
490            match self.run_bias_evaluation(&mut warnings) {
491                Ok(results) => Some(results),
492                Err(e) => {
493                    warnings.push(format!("Bias evaluation failed: {}", e));
494                    None
495                }
496            }
497        } else {
498            None
499        };
500
501        // Run calibration (if model provided)
502        #[cfg(feature = "eval")]
503        let calibration = if self.include_calibration && self.model.is_some() {
504            match self.run_calibration(&mut warnings) {
505                Ok(results) => Some(results),
506                Err(e) => {
507                    warnings.push(format!("Calibration evaluation failed: {}", e));
508                    None
509                }
510            }
511        } else {
512            None
513        };
514
515        // Run data quality checks
516        #[cfg(feature = "eval")]
517        let data_quality = if self.include_data_quality {
518            match self.run_data_quality(&mut warnings) {
519                Ok(results) => Some(results),
520                Err(e) => {
521                    warnings.push(format!("Data quality checks failed: {}", e));
522                    None
523                }
524            }
525        } else {
526            None
527        };
528
529        let duration = start.elapsed();
530
531        #[cfg(feature = "eval")]
532        let num_examples = standard_result
533            .as_ref()
534            .map(|s| s.per_task.values().map(|t| t.num_examples).sum::<usize>())
535            .unwrap_or(0);
536        #[cfg(not(feature = "eval"))]
537        let num_examples = 0;
538
539        Ok(UnifiedEvalResults {
540            #[cfg(feature = "eval")]
541            standard: standard_result,
542            #[cfg(feature = "eval-bias")]
543            bias,
544            #[cfg(feature = "eval")]
545            calibration,
546            #[cfg(feature = "eval")]
547            data_quality,
548            warnings,
549            metadata: EvalMetadata {
550                timestamp: chrono::Utc::now().to_rfc3339(),
551                model_name: self.model_name.clone(),
552                total_duration_ms: Some(duration.as_secs_f64() * 1000.0),
553                num_examples,
554            },
555        })
556    }
557
558    /// Run standard task evaluation.
559    ///
560    /// **Empty vector semantics**:
561    /// - Empty `tasks` → uses all available tasks
562    /// - Empty `datasets` → uses all suitable datasets for each task
563    /// - Empty `backends` → uses all compatible backends for each task
564    #[cfg(feature = "eval")]
565    fn run_standard_evaluation(
566        &self,
567        _warnings: &mut Vec<String>,
568    ) -> Result<Option<StandardEvalResults>> {
569        // Note: get_task_datasets and get_task_backends are available but not used here
570        // as we rely on TaskEvaluator's internal logic for dataset/backend selection
571
572        // Empty tasks = use all tasks
573        let tasks = if self.tasks.is_empty() {
574            Task::all().to_vec()
575        } else {
576            self.tasks.clone()
577        };
578
579        if tasks.is_empty() {
580            return Ok(None);
581        }
582
583        let evaluator = TaskEvaluator::new().map_err(|e| {
584            crate::Error::InvalidInput(format!("Failed to create TaskEvaluator: {}", e))
585        })?;
586
587        let config = TaskEvalConfig {
588            tasks: tasks.clone(),
589            datasets: self.datasets.clone(),
590            backends: self.backends.clone(),
591            max_examples: self.max_examples,
592            seed: self.seed,
593            require_cached: false,
594            relation_threshold: 0.5,
595            robustness: false,
596            compute_familiarity: true,
597            temporal_stratification: false,
598            confidence_intervals: true,
599            custom_coref_resolver: self.coref_resolver.clone(),
600            coref_use_gold_mentions: false,
601        };
602
603        let comprehensive_results = evaluator.evaluate_all(config)?;
604
605        // Aggregate results
606        let mut per_task: HashMap<String, TaskResults> = HashMap::new();
607        let mut per_dataset: HashMap<String, DatasetResults> = HashMap::new();
608        let mut per_backend: HashMap<String, BackendResults> = HashMap::new();
609
610        let mut total_f1_weighted = 0.0;
611        let mut total_precision_weighted = 0.0;
612        let mut total_recall_weighted = 0.0;
613        let mut total_examples = 0;
614
615        for result in &comprehensive_results.results {
616            if !result.success {
617                continue;
618            }
619
620            let f1 = result.metrics.get("f1").copied().unwrap_or(0.0);
621            let precision = result.metrics.get("precision").copied().unwrap_or(0.0);
622            let recall = result.metrics.get("recall").copied().unwrap_or(0.0);
623            let examples = result.num_examples;
624
625            // Weight by number of examples for overall average
626            total_f1_weighted += f1 * examples as f64;
627            total_precision_weighted += precision * examples as f64;
628            total_recall_weighted += recall * examples as f64;
629            total_examples += examples;
630
631            // Per-task aggregation (weighted by number of examples)
632            let task_key = format!("{:?}", result.task);
633            per_task
634                .entry(task_key.clone())
635                .and_modify(|t| {
636                    // Weighted average: (old_f1 * old_count + new_f1 * new_count) / total_count
637                    let old_count = t.num_examples as f64;
638                    let new_count = result.num_examples as f64;
639                    let total_count = old_count + new_count;
640
641                    if total_count > 0.0 {
642                        t.f1 = (t.f1 * old_count + f1 * new_count) / total_count;
643                        t.precision =
644                            (t.precision * old_count + precision * new_count) / total_count;
645                        t.recall = (t.recall * old_count + recall * new_count) / total_count;
646                    }
647                    t.num_examples += result.num_examples;
648                })
649                .or_insert_with(|| TaskResults {
650                    task: task_key,
651                    f1,
652                    precision,
653                    recall,
654                    num_examples: result.num_examples,
655                });
656
657            // Per-dataset aggregation (weighted by number of examples)
658            let dataset_key = format!("{:?}", result.dataset);
659            per_dataset
660                .entry(dataset_key.clone())
661                .and_modify(|d| {
662                    let old_count = d.num_examples as f64;
663                    let new_count = result.num_examples as f64;
664                    let total_count = old_count + new_count;
665
666                    if total_count > 0.0 {
667                        d.f1 = (d.f1 * old_count + f1 * new_count) / total_count;
668                        d.precision =
669                            (d.precision * old_count + precision * new_count) / total_count;
670                        d.recall = (d.recall * old_count + recall * new_count) / total_count;
671                    }
672                    d.num_examples += result.num_examples;
673                })
674                .or_insert_with(|| DatasetResults {
675                    dataset: dataset_key,
676                    f1,
677                    precision,
678                    recall,
679                    num_examples: result.num_examples,
680                });
681
682            // Per-backend aggregation (weighted by number of examples)
683            per_backend
684                .entry(result.backend.clone())
685                .and_modify(|b| {
686                    let old_count = b.num_examples as f64;
687                    let new_count = result.num_examples as f64;
688                    let total_count = old_count + new_count;
689
690                    if total_count > 0.0 {
691                        b.f1 = (b.f1 * old_count + f1 * new_count) / total_count;
692                        b.precision =
693                            (b.precision * old_count + precision * new_count) / total_count;
694                        b.recall = (b.recall * old_count + recall * new_count) / total_count;
695                    }
696                    b.num_examples += result.num_examples;
697                })
698                .or_insert_with(|| BackendResults {
699                    backend: result.backend.clone(),
700                    f1,
701                    precision,
702                    recall,
703                    num_examples: result.num_examples,
704                });
705        }
706
707        // Weighted average across all results
708        let avg_f1 = if total_examples > 0 {
709            total_f1_weighted / total_examples as f64
710        } else {
711            0.0
712        };
713        let avg_precision = if total_examples > 0 {
714            total_precision_weighted / total_examples as f64
715        } else {
716            0.0
717        };
718        let avg_recall = if total_examples > 0 {
719            total_recall_weighted / total_examples as f64
720        } else {
721            0.0
722        };
723
724        Ok(Some(StandardEvalResults {
725            f1: avg_f1,
726            precision: avg_precision,
727            recall: avg_recall,
728            per_task,
729            per_dataset,
730            per_backend,
731        }))
732    }
733
734    /// Run bias evaluation.
735    #[cfg(feature = "eval-bias")]
736    fn run_bias_evaluation(&self, warnings: &mut Vec<String>) -> Result<BiasEvalResults> {
737        let model = self.model.as_deref().ok_or_else(|| {
738            crate::Error::InvalidInput(
739                "Bias evaluation requires a model instance. Use with_model()".to_string(),
740            )
741        })?;
742
743        let config = self.bias_config.clone().unwrap_or_else(|| {
744            BiasDatasetConfig::default()
745                .with_frequency_weighting()
746                .with_validation()
747        });
748
749        // Gender bias (coreference)
750        // Note: Gender bias requires CoreferenceResolver, not Model.
751        // If the provided model implements CoreferenceResolver, we could use it,
752        // but for now we use a default resolver. This is a known limitation.
753        warnings.push(
754            "Gender bias evaluation uses default SimpleCorefResolver, not the provided model."
755                .to_string(),
756        );
757        let resolver = SimpleCorefResolver::default();
758        let templates = create_winobias_templates();
759        let evaluator = GenderBiasEvaluator::new(true);
760        let gender_results = evaluator.evaluate_resolver(&resolver, &templates);
761        let gender = Some(GenderBiasSummary {
762            bias_gap: gender_results.bias_gap,
763            pro_stereotype_accuracy: gender_results.pro_stereotype_accuracy,
764            anti_stereotype_accuracy: gender_results.anti_stereotype_accuracy,
765        });
766
767        // Demographic bias
768        let names = create_diverse_name_dataset();
769        let demo_evaluator = DemographicBiasEvaluator::with_config(true, config.clone());
770        let demo_results = demo_evaluator.evaluate_ner(model, &names);
771        let demographic = Some(DemographicBiasSummary {
772            ethnicity_parity_gap: demo_results.ethnicity_parity_gap,
773            script_bias_gap: demo_results.script_bias_gap,
774            overall_recognition_rate: demo_results.overall_recognition_rate,
775        });
776
777        // Temporal bias
778        let temporal_names = create_temporal_name_dataset();
779        let temporal_evaluator = TemporalBiasEvaluator::new(true);
780        let temporal_results = temporal_evaluator.evaluate(model, &temporal_names);
781        let temporal = Some(TemporalBiasSummary {
782            historical_modern_gap: temporal_results.historical_modern_gap,
783            historical_rate: temporal_results.historical_rate,
784            modern_rate: temporal_results.modern_rate,
785        });
786
787        // Length bias
788        let length_examples = create_length_varied_dataset();
789        let length_evaluator = EntityLengthEvaluator::new(true);
790        let length_results = length_evaluator.evaluate(model, &length_examples);
791        let length = Some(LengthBiasSummary {
792            short_vs_long_gap: length_results.short_vs_long_gap,
793            short_entity_f1: length_results
794                .by_word_bucket
795                .get("SingleWord")
796                .copied()
797                .unwrap_or(0.0),
798            long_entity_f1: length_results
799                .by_word_bucket
800                .get("FourPlusWords")
801                .copied()
802                .unwrap_or(0.0),
803        });
804
805        Ok(BiasEvalResults {
806            gender,
807            demographic,
808            temporal,
809            length,
810        })
811    }
812
813    /// Run calibration analysis.
814    #[cfg(feature = "eval")]
815    fn run_calibration(&self, warnings: &mut Vec<String>) -> Result<CalibrationEvalResults> {
816        use crate::eval::calibration::CalibrationEvaluator;
817
818        let model = self.model.as_deref().ok_or_else(|| {
819            crate::Error::InvalidInput(
820                "Calibration analysis requires a model instance. Use with_model()".to_string(),
821            )
822        })?;
823
824        // Try to load a sample dataset for calibration
825        // For now, use a simple synthetic dataset if no datasets are configured
826        let test_texts = if self.datasets.is_empty() {
827            warnings.push(
828                "No datasets configured for calibration. Using synthetic test data.".to_string(),
829            );
830            vec![
831                "John Smith works at Google in New York.".to_string(),
832                "Jane Doe is a professor at MIT.".to_string(),
833                "Microsoft was founded by Bill Gates.".to_string(),
834            ]
835        } else {
836            // Load first dataset for calibration
837            // Note: This is a simplified implementation
838            // A full implementation would load actual test data from the dataset
839            warnings.push(
840                "Calibration using configured datasets requires dataset loading (not yet fully implemented). Using synthetic data.".to_string(),
841            );
842            vec![
843                "John Smith works at Google in New York.".to_string(),
844                "Jane Doe is a professor at MIT.".to_string(),
845                "Microsoft was founded by Bill Gates.".to_string(),
846            ]
847        };
848
849        // Collect predictions with confidence scores
850        let mut predictions = Vec::new();
851        let mut has_calibrated_entities = false;
852
853        for text in &test_texts {
854            let entities = model
855                .extract_entities(text, None)
856                .unwrap_or_else(|_| Vec::new());
857
858            for entity in &entities {
859                // Check if this entity's extraction method is calibrated
860                let is_calibrated = entity
861                    .provenance
862                    .as_ref()
863                    .map(|p| p.method.is_calibrated())
864                    .unwrap_or(false);
865
866                if !is_calibrated {
867                    continue; // Skip uncalibrated entities
868                }
869
870                has_calibrated_entities = true;
871
872                // For calibration, we need gold labels to determine correctness
873                // Since we're using synthetic data, we'll use a simple heuristic:
874                // Assume entities are correct if they have reasonable confidence
875                // Without gold labels, approximate correctness from confidence threshold
876                let is_correct = entity.confidence > 0.5;
877
878                predictions.push((entity.confidence.into(), is_correct));
879            }
880        }
881
882        // If no calibrated entities found, return default (zero) metrics
883        if !has_calibrated_entities || predictions.is_empty() {
884            warnings.push(
885                "No calibrated entities found for calibration analysis. Model may not provide calibrated confidence scores.".to_string(),
886            );
887            return Ok(CalibrationEvalResults {
888                ece: 0.0,
889                mce: 0.0,
890                brier_score: 0.0,
891            });
892        }
893
894        // Compute calibration metrics
895        let results = CalibrationEvaluator::compute(&predictions);
896
897        Ok(CalibrationEvalResults {
898            ece: results.ece,
899            mce: results.mce,
900            brier_score: results.brier_score,
901        })
902    }
903
904    /// Run data quality checks.
905    #[cfg(feature = "eval")]
906    fn run_data_quality(&self, warnings: &mut Vec<String>) -> Result<DataQualityEvalResults> {
907        // Try to load datasets for data quality analysis
908        // For now, use a simple check on configured datasets
909        if self.datasets.is_empty() {
910            warnings.push(
911                "No datasets configured for data quality checks. Cannot check for leakage without train/test split.".to_string(),
912            );
913            return Ok(DataQualityEvalResults {
914                leakage_detected: false,
915                redundancy_rate: 0.0,
916                ambiguous_count: 0,
917            });
918        }
919
920        // Note: Full implementation would:
921        // 1. Load train and test splits from datasets
922        // 2. Use DatasetQualityAnalyzer to check for leakage, redundancy, ambiguity
923        // 3. Return comprehensive quality metrics
924        //
925        warnings.push(
926            "Data quality checks require dataset loading (not yet fully implemented). Returning default results.".to_string(),
927        );
928
929        Ok(DataQualityEvalResults {
930            leakage_detected: false, // Cannot determine without actual data
931            redundancy_rate: 0.0,    // Cannot determine without actual data
932            ambiguous_count: 0,      // Cannot determine without actual data
933        })
934    }
935}
936
937impl Default for EvalSystem {
938    fn default() -> Self {
939        Self::new()
940    }
941}
anno_eval/eval/unified_evaluator.rs

anno_eval/eval/
unified_evaluator.rs
