anno_eval/eval/
low_resource.rs

1//! Low-resource and morphologically complex language evaluation metrics.
2//!
3//! This module provides specialized evaluation tools for:
4//! - Indigenous/Native American languages (Quechua, Cherokee, Navajo, etc.)
5//! - Polysynthetic languages with complex morphology
6//! - Languages with orthographic variation
7//! - Low-resource scenarios with limited training data
8//!
9//! # Key Metrics
10//!
11//! - **Morpheme-level F1**: Evaluation at morpheme boundaries (important for polysynthetic languages)
12//! - **Character-level F1**: Robust to tokenization differences
13//! - **Normalized Entity Ratio**: Compares entity density across languages
14//! - **Transfer Efficiency**: Measures how well high-resource models transfer
15//! - **Orthographic Robustness**: Handles spelling variations
16//!
17//! # Example
18//!
19//! ```rust,ignore
20//! use anno_eval::eval::low_resource::{LowResourceEvaluator, MorphemeConfig};
21//!
22//! let evaluator = LowResourceEvaluator::new()
23//!     .with_morpheme_boundaries(true)
24//!     .with_orthographic_normalization(true);
25//!
26//! let results = evaluator.evaluate(&model, &quechua_dataset)?;
27//! println!("Morpheme F1: {:.3}", results.morpheme_f1);
28//! println!("Transfer efficiency: {:.3}", results.transfer_efficiency);
29//! ```
30//!
31//! # References
32//!
33//! - qxoRef: Galarreta et al., AmericasNLP 2021 (Quechua coreference)
34//! - AmericasNLI: Ebrahimi et al., EMNLP 2022 (Indigenous NLI)
35//! - CorefUD 1.3: Nedoluzhko et al., 2022 (Multilingual coreference)
36
37use anno::{Error, Model, Result};
38use serde::{Deserialize, Serialize};
39use std::collections::HashMap;
40
41/// Configuration for morpheme-level evaluation.
42#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct MorphemeConfig {
44    /// Character used as morpheme boundary marker
45    pub boundary_char: char,
46    /// Whether to use character-level fallback when morpheme boundaries unavailable
47    pub char_level_fallback: bool,
48    /// Minimum morpheme length to count
49    pub min_morpheme_len: usize,
50}
51
52impl Default for MorphemeConfig {
53    fn default() -> Self {
54        Self {
55            boundary_char: '-',
56            char_level_fallback: true,
57            min_morpheme_len: 1,
58        }
59    }
60}
61
62/// Configuration for orthographic normalization.
63#[derive(Debug, Clone, Serialize, Deserialize)]
64pub struct OrthographicConfig {
65    /// Enable Unicode normalization (NFC)
66    pub unicode_normalize: bool,
67    /// Case-insensitive matching
68    pub case_insensitive: bool,
69    /// Diacritic-insensitive matching
70    pub ignore_diacritics: bool,
71    /// Custom character mappings (e.g., for non-standard orthographies)
72    pub char_mappings: HashMap<char, char>,
73}
74
75impl Default for OrthographicConfig {
76    fn default() -> Self {
77        Self {
78            unicode_normalize: true,
79            case_insensitive: false,
80            ignore_diacritics: false,
81            char_mappings: HashMap::new(),
82        }
83    }
84}
85
86/// Results from low-resource language evaluation.
87#[derive(Debug, Clone, Serialize, Deserialize)]
88pub struct LowResourceResults {
89    /// Standard token-level F1
90    pub token_f1: f64,
91    /// Morpheme-level F1 (for polysynthetic languages)
92    pub morpheme_f1: Option<f64>,
93    /// Character-level F1 (robust to tokenization)
94    pub char_f1: f64,
95    /// Entity density ratio compared to English baseline
96    pub entity_density_ratio: f64,
97    /// Transfer efficiency (F1 / English F1 baseline)
98    pub transfer_efficiency: Option<f64>,
99    /// Per-entity-type breakdown
100    pub per_type: HashMap<String, TypeMetrics>,
101    /// Orthographic normalization impact
102    pub normalization_impact: Option<NormalizationImpact>,
103    /// Language-specific metadata
104    pub metadata: LowResourceMetadata,
105}
106
107/// Per-type metrics for low-resource evaluation.
108#[derive(Debug, Clone, Serialize, Deserialize)]
109pub struct TypeMetrics {
110    /// Precision score (0-1)
111    pub precision: f64,
112    /// Recall score (0-1)
113    pub recall: f64,
114    /// F1 score (0-1)
115    pub f1: f64,
116    /// Number of examples for this type
117    pub support: usize,
118}
119
120/// Impact of orthographic normalization on results.
121#[derive(Debug, Clone, Serialize, Deserialize)]
122pub struct NormalizationImpact {
123    /// F1 without normalization
124    pub raw_f1: f64,
125    /// F1 with normalization
126    pub normalized_f1: f64,
127    /// Improvement from normalization
128    pub improvement: f64,
129    /// Number of entities affected
130    pub entities_affected: usize,
131}
132
133/// Metadata about the low-resource evaluation.
134#[derive(Debug, Clone, Serialize, Deserialize)]
135pub struct LowResourceMetadata {
136    /// ISO 639-3 language code
137    pub language_code: String,
138    /// Language family
139    pub language_family: Option<String>,
140    /// Whether language is polysynthetic
141    pub is_polysynthetic: bool,
142    /// Whether language has standardized orthography
143    pub has_standard_orthography: bool,
144    /// Estimated speaker population
145    pub speaker_population: Option<u64>,
146    /// UNESCO endangerment level
147    pub endangerment_level: Option<EndangermentLevel>,
148    /// Number of training examples available
149    pub training_examples: Option<usize>,
150}
151
152/// UNESCO language endangerment levels.
153#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
154pub enum EndangermentLevel {
155    /// Language is used by all ages
156    Safe,
157    /// Most children speak the language
158    Vulnerable,
159    /// Children speak at home but not in school
160    DefinitelyEndangered,
161    /// Only spoken by grandparents
162    SeverelyEndangered,
163    /// Only spoken by a few elderly
164    CriticallyEndangered,
165    /// No living speakers
166    Extinct,
167}
168
169/// Evaluator for low-resource and morphologically complex languages.
170pub struct LowResourceEvaluator {
171    morpheme_config: Option<MorphemeConfig>,
172    orthographic_config: Option<OrthographicConfig>,
173    english_baseline_f1: Option<f64>,
174}
175
176impl LowResourceEvaluator {
177    /// Create a new low-resource evaluator with default settings.
178    pub fn new() -> Self {
179        Self {
180            morpheme_config: None,
181            orthographic_config: None,
182            english_baseline_f1: None,
183        }
184    }
185
186    /// Enable morpheme-level evaluation.
187    pub fn with_morpheme_boundaries(mut self, config: MorphemeConfig) -> Self {
188        self.morpheme_config = Some(config);
189        self
190    }
191
192    /// Enable orthographic normalization.
193    pub fn with_orthographic_normalization(mut self, config: OrthographicConfig) -> Self {
194        self.orthographic_config = Some(config);
195        self
196    }
197
198    /// Set English baseline F1 for transfer efficiency calculation.
199    pub fn with_english_baseline(mut self, f1: f64) -> Self {
200        self.english_baseline_f1 = Some(f1);
201        self
202    }
203
204    /// Evaluate model on low-resource dataset.
205    pub fn evaluate(
206        &self,
207        model: &dyn Model,
208        test_cases: &[(String, Vec<super::GoldEntity>)],
209        metadata: LowResourceMetadata,
210    ) -> Result<LowResourceResults> {
211        if test_cases.is_empty() {
212            return Err(Error::InvalidInput("Empty test cases".to_string()));
213        }
214
215        // Calculate standard token-level metrics
216        let standard_results = super::evaluate_ner_model(model, test_cases)?;
217
218        // Calculate character-level F1
219        let char_f1 = self.calculate_char_f1(model, test_cases)?;
220
221        // Calculate morpheme-level F1 if configured
222        let morpheme_f1 = if self.morpheme_config.is_some() && metadata.is_polysynthetic {
223            Some(self.calculate_morpheme_f1(model, test_cases)?)
224        } else {
225            None
226        };
227
228        // Calculate entity density ratio
229        let total_chars: usize = test_cases.iter().map(|(text, _)| text.len()).sum();
230        let total_entities: usize = test_cases.iter().map(|(_, entities)| entities.len()).sum();
231        let entity_density = if total_chars > 0 {
232            total_entities as f64 / total_chars as f64
233        } else {
234            0.0
235        };
236        // English baseline entity density (approximate from CoNLL-2003)
237        let english_baseline_density = 0.05;
238        let entity_density_ratio = entity_density / english_baseline_density;
239
240        // Calculate transfer efficiency
241        let transfer_efficiency = self
242            .english_baseline_f1
243            .map(|baseline| standard_results.f1 / baseline);
244
245        // Calculate normalization impact if configured
246        let normalization_impact = if self.orthographic_config.is_some() {
247            Some(self.calculate_normalization_impact(model, test_cases)?)
248        } else {
249            None
250        };
251
252        // Convert per-type metrics
253        let per_type: HashMap<String, TypeMetrics> = standard_results
254            .per_type
255            .into_iter()
256            .map(|(k, v)| {
257                (
258                    k,
259                    TypeMetrics {
260                        precision: v.precision,
261                        recall: v.recall,
262                        f1: v.f1,
263                        support: v.expected,
264                    },
265                )
266            })
267            .collect();
268
269        Ok(LowResourceResults {
270            token_f1: standard_results.f1,
271            morpheme_f1,
272            char_f1,
273            entity_density_ratio,
274            transfer_efficiency,
275            per_type,
276            normalization_impact,
277            metadata,
278        })
279    }
280
281    /// Calculate character-level F1.
282    ///
283    /// This is more robust to tokenization differences across languages.
284    fn calculate_char_f1(
285        &self,
286        model: &dyn Model,
287        test_cases: &[(String, Vec<super::GoldEntity>)],
288    ) -> Result<f64> {
289        let mut total_gold_chars = 0;
290        let mut total_pred_chars = 0;
291        let mut total_correct_chars = 0;
292
293        for (text, gold_entities) in test_cases {
294            // Get predictions
295            let predictions = model.extract_entities(text, None)?;
296
297            // Create character-level gold mask
298            let text_char_len = text.chars().count();
299            let mut gold_mask = vec![false; text_char_len];
300            for entity in gold_entities {
301                let start = entity.start.min(text_char_len);
302                let end = entity.end.min(text_char_len);
303                for slot in gold_mask.iter_mut().take(end).skip(start) {
304                    *slot = true;
305                }
306            }
307
308            // Create character-level prediction mask
309            let mut pred_mask = vec![false; text_char_len];
310            for entity in &predictions {
311                let start = entity.start().min(text_char_len);
312                let end = entity.end().min(text_char_len);
313                for slot in pred_mask.iter_mut().take(end).skip(start) {
314                    *slot = true;
315                }
316            }
317
318            // Count matches
319            for i in 0..text_char_len {
320                if gold_mask[i] {
321                    total_gold_chars += 1;
322                }
323                if pred_mask[i] {
324                    total_pred_chars += 1;
325                }
326                if gold_mask[i] && pred_mask[i] {
327                    total_correct_chars += 1;
328                }
329            }
330        }
331
332        let precision = if total_pred_chars > 0 {
333            total_correct_chars as f64 / total_pred_chars as f64
334        } else {
335            0.0
336        };
337        let recall = if total_gold_chars > 0 {
338            total_correct_chars as f64 / total_gold_chars as f64
339        } else {
340            0.0
341        };
342        let f1 = if precision + recall > 0.0 {
343            2.0 * precision * recall / (precision + recall)
344        } else {
345            0.0
346        };
347
348        Ok(f1)
349    }
350
351    /// Calculate morpheme-level F1 for polysynthetic languages.
352    fn calculate_morpheme_f1(
353        &self,
354        model: &dyn Model,
355        test_cases: &[(String, Vec<super::GoldEntity>)],
356    ) -> Result<f64> {
357        let config = self.morpheme_config.as_ref().ok_or_else(|| {
358            Error::evaluation(
359                "morpheme-level evaluation requested without MorphemeConfig (call with_morpheme_boundaries(true))",
360            )
361        })?;
362
363        let mut total_gold_morphemes = 0;
364        let mut total_pred_morphemes = 0;
365        let mut total_correct_morphemes = 0;
366
367        for (text, gold_entities) in test_cases {
368            let predictions = model.extract_entities(text, None)?;
369
370            // Count morphemes in gold entities
371            for entity in gold_entities {
372                let morpheme_count = entity
373                    .text
374                    .split(config.boundary_char)
375                    .filter(|m| m.len() >= config.min_morpheme_len)
376                    .count()
377                    .max(1);
378                total_gold_morphemes += morpheme_count;
379            }
380
381            // Count morphemes in predicted entities
382            for entity in &predictions {
383                // Note: entity.start/end are CHARACTER offsets, not byte offsets
384                let char_count = text.chars().count();
385                let entity_text: String = text
386                    .chars()
387                    .skip(entity.start())
388                    .take(entity.end().min(char_count).saturating_sub(entity.start()))
389                    .collect();
390                let morpheme_count = entity_text
391                    .split(config.boundary_char)
392                    .filter(|m| m.len() >= config.min_morpheme_len)
393                    .count()
394                    .max(1);
395                total_pred_morphemes += morpheme_count;
396
397                // Check if this prediction matches a gold entity
398                for gold in gold_entities {
399                    if entity.start() == gold.start && entity.end() == gold.end {
400                        total_correct_morphemes += morpheme_count;
401                        break;
402                    }
403                }
404            }
405        }
406
407        let precision = if total_pred_morphemes > 0 {
408            total_correct_morphemes as f64 / total_pred_morphemes as f64
409        } else {
410            0.0
411        };
412        let recall = if total_gold_morphemes > 0 {
413            total_correct_morphemes as f64 / total_gold_morphemes as f64
414        } else {
415            0.0
416        };
417        let f1 = if precision + recall > 0.0 {
418            2.0 * precision * recall / (precision + recall)
419        } else {
420            0.0
421        };
422
423        Ok(f1)
424    }
425
426    /// Calculate impact of orthographic normalization.
427    fn calculate_normalization_impact(
428        &self,
429        model: &dyn Model,
430        test_cases: &[(String, Vec<super::GoldEntity>)],
431    ) -> Result<NormalizationImpact> {
432        let config = self.orthographic_config.as_ref().ok_or_else(|| {
433            Error::evaluation(
434                "normalization impact requested without OrthographicConfig (call with_orthographic_normalization(true))",
435            )
436        })?;
437
438        // Evaluate without normalization
439        let raw_results = super::evaluate_ner_model(model, test_cases)?;
440
441        // Apply normalization to test cases
442        let normalized_cases: Vec<(String, Vec<super::GoldEntity>)> = test_cases
443            .iter()
444            .map(|(text, entities)| {
445                let normalized_text = self.normalize_text(text, config);
446                let normalized_entities: Vec<super::GoldEntity> = entities
447                    .iter()
448                    .map(|e| super::GoldEntity {
449                        text: self.normalize_text(&e.text, config),
450                        entity_type: e.entity_type.clone(),
451                        original_label: e.original_label.clone(),
452                        start: e.start,
453                        end: e.end,
454                    })
455                    .collect();
456                (normalized_text, normalized_entities)
457            })
458            .collect();
459
460        // Evaluate with normalization
461        let normalized_results = super::evaluate_ner_model(model, &normalized_cases)?;
462
463        // Count affected entities
464        let mut entities_affected = 0;
465        for ((orig_text, _), (norm_text, _)) in test_cases.iter().zip(normalized_cases.iter()) {
466            if orig_text != norm_text {
467                entities_affected += 1;
468            }
469        }
470
471        Ok(NormalizationImpact {
472            raw_f1: raw_results.f1,
473            normalized_f1: normalized_results.f1,
474            improvement: normalized_results.f1 - raw_results.f1,
475            entities_affected,
476        })
477    }
478
479    /// Apply orthographic normalization to text.
480    fn normalize_text(&self, text: &str, config: &OrthographicConfig) -> String {
481        let mut result = text.to_string();
482
483        // Unicode normalization (NFC)
484        if config.unicode_normalize {
485            use unicode_normalization::UnicodeNormalization;
486            result = result.nfc().collect();
487        }
488
489        // Case normalization
490        if config.case_insensitive {
491            result = result.to_lowercase();
492        }
493
494        // Diacritic removal
495        if config.ignore_diacritics {
496            result = remove_diacritics(&result);
497        }
498
499        // Custom character mappings
500        for (from, to) in &config.char_mappings {
501            result = result.replace(*from, &to.to_string());
502        }
503
504        result
505    }
506}
507
508impl Default for LowResourceEvaluator {
509    fn default() -> Self {
510        Self::new()
511    }
512}
513
514/// Remove diacritics from text.
515fn remove_diacritics(text: &str) -> String {
516    use unicode_normalization::UnicodeNormalization;
517    text.nfd()
518        .filter(|c| !unicode_normalization::char::is_combining_mark(*c))
519        .collect()
520}
521
522/// Create metadata for common Indigenous American languages.
523pub fn language_metadata(language_code: &str) -> Option<LowResourceMetadata> {
524    match language_code {
525        // Quechua (Conchucos dialect - qxoRef)
526        "qxo" => Some(LowResourceMetadata {
527            language_code: "qxo".to_string(),
528            language_family: Some("Quechuan".to_string()),
529            is_polysynthetic: false, // Quechua is agglutinative, not polysynthetic
530            has_standard_orthography: false,
531            speaker_population: Some(200_000),
532            endangerment_level: Some(EndangermentLevel::Vulnerable),
533            training_examples: Some(12), // qxoRef has 12 documents
534        }),
535        // Cherokee
536        "chr" => Some(LowResourceMetadata {
537            language_code: "chr".to_string(),
538            language_family: Some("Iroquoian".to_string()),
539            is_polysynthetic: true,
540            has_standard_orthography: true, // Cherokee syllabary is standardized
541            speaker_population: Some(2_000),
542            endangerment_level: Some(EndangermentLevel::SeverelyEndangered),
543            training_examples: None,
544        }),
545        // Navajo
546        "nav" => Some(LowResourceMetadata {
547            language_code: "nav".to_string(),
548            language_family: Some("Na-Dené".to_string()),
549            is_polysynthetic: true,
550            has_standard_orthography: true,
551            speaker_population: Some(170_000),
552            endangerment_level: Some(EndangermentLevel::Vulnerable),
553            training_examples: None,
554        }),
555        // Guarani
556        "gn" | "grn" => Some(LowResourceMetadata {
557            language_code: "grn".to_string(),
558            language_family: Some("Tupian".to_string()),
559            is_polysynthetic: false,
560            has_standard_orthography: true,
561            speaker_population: Some(6_000_000),
562            endangerment_level: Some(EndangermentLevel::Safe),
563            training_examples: None,
564        }),
565        // Nahuatl
566        "nah" => Some(LowResourceMetadata {
567            language_code: "nah".to_string(),
568            language_family: Some("Uto-Aztecan".to_string()),
569            is_polysynthetic: true,
570            has_standard_orthography: false, // Multiple competing orthographies
571            speaker_population: Some(1_700_000),
572            endangerment_level: Some(EndangermentLevel::Vulnerable),
573            training_examples: None,
574        }),
575        // Shipibo-Konibo
576        "shp" => Some(LowResourceMetadata {
577            language_code: "shp".to_string(),
578            language_family: Some("Panoan".to_string()),
579            is_polysynthetic: false,
580            has_standard_orthography: true,
581            speaker_population: Some(35_000),
582            endangerment_level: Some(EndangermentLevel::Vulnerable),
583            training_examples: None,
584        }),
585
586        // =========================================================================
587        // African Languages (MasakhaNER 2.0 languages)
588        // =========================================================================
589
590        // Swahili (Kiswahili) - East Africa's lingua franca
591        "sw" | "swa" => Some(LowResourceMetadata {
592            language_code: "swa".to_string(),
593            language_family: Some("Atlantic-Congo (Bantu)".to_string()),
594            is_polysynthetic: false, // Agglutinative, not polysynthetic
595            has_standard_orthography: true,
596            speaker_population: Some(100_000_000), // Including L2 speakers
597            endangerment_level: Some(EndangermentLevel::Safe),
598            training_examples: Some(9_418), // MasakhaNER 2.0 train+dev+test
599        }),
600
601        // Yoruba - Tonal language with diacritics
602        "yo" | "yor" => Some(LowResourceMetadata {
603            language_code: "yor".to_string(),
604            language_family: Some("Atlantic-Congo (Volta-Niger)".to_string()),
605            is_polysynthetic: false,
606            has_standard_orthography: true, // Standard with tone marks
607            speaker_population: Some(45_000_000),
608            endangerment_level: Some(EndangermentLevel::Safe),
609            training_examples: Some(9_824), // MasakhaNER 2.0
610        }),
611
612        // Hausa - Major West African trade language
613        "ha" | "hau" => Some(LowResourceMetadata {
614            language_code: "hau".to_string(),
615            language_family: Some("Afro-Asiatic (Chadic)".to_string()),
616            is_polysynthetic: false,
617            has_standard_orthography: true, // Latin (Boko) standard
618            speaker_population: Some(80_000_000),
619            endangerment_level: Some(EndangermentLevel::Safe),
620            training_examples: Some(8_165), // MasakhaNER 2.0
621        }),
622
623        // Amharic - Ethiopian Semitic with Ge'ez script
624        "am" | "amh" => Some(LowResourceMetadata {
625            language_code: "amh".to_string(),
626            language_family: Some("Afro-Asiatic (Semitic)".to_string()),
627            is_polysynthetic: false,
628            has_standard_orthography: true, // Ge'ez (Ethiopic) script
629            speaker_population: Some(57_000_000),
630            endangerment_level: Some(EndangermentLevel::Safe),
631            training_examples: Some(1_750), // MasakhaNER 1.0
632        }),
633
634        // Igbo - Tonal language of Nigeria
635        "ig" | "ibo" => Some(LowResourceMetadata {
636            language_code: "ibo".to_string(),
637            language_family: Some("Atlantic-Congo (Volta-Niger)".to_string()),
638            is_polysynthetic: false,
639            has_standard_orthography: true, // Önwu alphabet
640            speaker_population: Some(45_000_000),
641            endangerment_level: Some(EndangermentLevel::Safe),
642            training_examples: Some(10_905), // MasakhaNER 2.0
643        }),
644
645        // Kinyarwanda - Rwanda/Burundi
646        "rw" | "kin" => Some(LowResourceMetadata {
647            language_code: "kin".to_string(),
648            language_family: Some("Atlantic-Congo (Bantu)".to_string()),
649            is_polysynthetic: false, // Agglutinative
650            has_standard_orthography: true,
651            speaker_population: Some(12_000_000),
652            endangerment_level: Some(EndangermentLevel::Safe),
653            training_examples: Some(11_178), // MasakhaNER 2.0
654        }),
655
656        // Nigerian Pidgin - English-based creole
657        "pcm" => Some(LowResourceMetadata {
658            language_code: "pcm".to_string(),
659            language_family: Some("English Creole".to_string()),
660            is_polysynthetic: false,
661            has_standard_orthography: false, // No standardized spelling
662            speaker_population: Some(100_000_000), // L2 speakers
663            endangerment_level: Some(EndangermentLevel::Safe),
664            training_examples: Some(7_746), // MasakhaNER 2.0
665        }),
666
667        // Wolof - Senegal/Gambia
668        "wo" | "wol" => Some(LowResourceMetadata {
669            language_code: "wol".to_string(),
670            language_family: Some("Atlantic-Congo (Atlantic)".to_string()),
671            is_polysynthetic: false,
672            has_standard_orthography: true,
673            speaker_population: Some(12_000_000),
674            endangerment_level: Some(EndangermentLevel::Safe),
675            training_examples: Some(6_561), // MasakhaNER 2.0
676        }),
677
678        // Zulu (isiZulu) - South Africa
679        "zu" | "zul" => Some(LowResourceMetadata {
680            language_code: "zul".to_string(),
681            language_family: Some("Atlantic-Congo (Bantu/Nguni)".to_string()),
682            is_polysynthetic: false, // Agglutinative
683            has_standard_orthography: true,
684            speaker_population: Some(27_000_000),
685            endangerment_level: Some(EndangermentLevel::Safe),
686            training_examples: Some(8_354), // MasakhaNER 2.0
687        }),
688
689        // Xhosa (isiXhosa) - South Africa, with clicks
690        "xh" | "xho" => Some(LowResourceMetadata {
691            language_code: "xho".to_string(),
692            language_family: Some("Atlantic-Congo (Bantu/Nguni)".to_string()),
693            is_polysynthetic: false, // Agglutinative
694            has_standard_orthography: true,
695            speaker_population: Some(19_000_000),
696            endangerment_level: Some(EndangermentLevel::Safe),
697            training_examples: Some(8_168), // MasakhaNER 2.0
698        }),
699
700        // Luganda - Uganda
701        "lg" | "lug" => Some(LowResourceMetadata {
702            language_code: "lug".to_string(),
703            language_family: Some("Atlantic-Congo (Bantu)".to_string()),
704            is_polysynthetic: false, // Agglutinative
705            has_standard_orthography: true,
706            speaker_population: Some(10_000_000),
707            endangerment_level: Some(EndangermentLevel::Safe),
708            training_examples: Some(7_060), // MasakhaNER 2.0
709        }),
710
711        // Luo (Dholuo) - Kenya/Tanzania
712        "luo" => Some(LowResourceMetadata {
713            language_code: "luo".to_string(),
714            language_family: Some("Nilo-Saharan (Nilotic)".to_string()),
715            is_polysynthetic: false,
716            has_standard_orthography: true,
717            speaker_population: Some(6_000_000),
718            endangerment_level: Some(EndangermentLevel::Safe),
719            training_examples: Some(7_372), // MasakhaNER 2.0
720        }),
721
722        // Twi (Akan) - Ghana
723        "tw" | "twi" | "aka" => Some(LowResourceMetadata {
724            language_code: "twi".to_string(),
725            language_family: Some("Atlantic-Congo (Kwa)".to_string()),
726            is_polysynthetic: false,
727            has_standard_orthography: true, // Tonal diacritics
728            speaker_population: Some(11_000_000),
729            endangerment_level: Some(EndangermentLevel::Safe),
730            training_examples: Some(6_056), // MasakhaNER 2.0
731        }),
732
733        // Shona (chiShona) - Zimbabwe
734        "sn" | "sna" => Some(LowResourceMetadata {
735            language_code: "sna".to_string(),
736            language_family: Some("Atlantic-Congo (Bantu)".to_string()),
737            is_polysynthetic: false, // Agglutinative
738            has_standard_orthography: true,
739            speaker_population: Some(15_000_000),
740            endangerment_level: Some(EndangermentLevel::Safe),
741            training_examples: Some(8_867), // MasakhaNER 2.0
742        }),
743
744        // Tigrinya - Eritrea/Ethiopia, Ge'ez script
745        "ti" | "tir" => Some(LowResourceMetadata {
746            language_code: "tir".to_string(),
747            language_family: Some("Afro-Asiatic (Semitic)".to_string()),
748            is_polysynthetic: false,
749            has_standard_orthography: true, // Ge'ez script
750            speaker_population: Some(9_000_000),
751            endangerment_level: Some(EndangermentLevel::Safe),
752            training_examples: None, // Not in MasakhaNER, but in AfriSenti
753        }),
754
755        // Bambara - Mali
756        "bm" | "bam" => Some(LowResourceMetadata {
757            language_code: "bam".to_string(),
758            language_family: Some("Atlantic-Congo (Mande)".to_string()),
759            is_polysynthetic: false,
760            has_standard_orthography: true, // N'Ko or Latin
761            speaker_population: Some(14_000_000),
762            endangerment_level: Some(EndangermentLevel::Safe),
763            training_examples: Some(6_375), // MasakhaNER 2.0
764        }),
765
766        // Ewe - Ghana/Togo
767        "ee" | "ewe" => Some(LowResourceMetadata {
768            language_code: "ewe".to_string(),
769            language_family: Some("Atlantic-Congo (Kwa)".to_string()),
770            is_polysynthetic: false,
771            has_standard_orthography: true, // Tonal diacritics
772            speaker_population: Some(7_000_000),
773            endangerment_level: Some(EndangermentLevel::Safe),
774            training_examples: Some(5_007), // MasakhaNER 2.0
775        }),
776
777        // Fon - Benin
778        "fon" => Some(LowResourceMetadata {
779            language_code: "fon".to_string(),
780            language_family: Some("Atlantic-Congo (Kwa)".to_string()),
781            is_polysynthetic: false,
782            has_standard_orthography: true, // Tonal diacritics
783            speaker_population: Some(2_200_000),
784            endangerment_level: Some(EndangermentLevel::Safe),
785            training_examples: Some(6_204), // MasakhaNER 2.0
786        }),
787
788        // Mossi (Mooré) - Burkina Faso
789        "mos" => Some(LowResourceMetadata {
790            language_code: "mos".to_string(),
791            language_family: Some("Atlantic-Congo (Gur)".to_string()),
792            is_polysynthetic: false,
793            has_standard_orthography: true,
794            speaker_population: Some(8_000_000),
795            endangerment_level: Some(EndangermentLevel::Safe),
796            training_examples: Some(6_793), // MasakhaNER 2.0
797        }),
798
799        // Setswana - Botswana/South Africa
800        "tn" | "tsn" => Some(LowResourceMetadata {
801            language_code: "tsn".to_string(),
802            language_family: Some("Atlantic-Congo (Bantu)".to_string()),
803            is_polysynthetic: false, // Agglutinative
804            has_standard_orthography: true,
805            speaker_population: Some(8_000_000),
806            endangerment_level: Some(EndangermentLevel::Safe),
807            training_examples: Some(4_784), // MasakhaNER 2.0
808        }),
809
810        // Chichewa (Nyanja) - Malawi
811        "ny" | "nya" => Some(LowResourceMetadata {
812            language_code: "nya".to_string(),
813            language_family: Some("Atlantic-Congo (Bantu)".to_string()),
814            is_polysynthetic: false, // Agglutinative
815            has_standard_orthography: true,
816            speaker_population: Some(15_000_000),
817            endangerment_level: Some(EndangermentLevel::Safe),
818            training_examples: Some(8_928), // MasakhaNER 2.0
819        }),
820
821        // Ghomala - Cameroon (lower resource)
822        "bbj" => Some(LowResourceMetadata {
823            language_code: "bbj".to_string(),
824            language_family: Some("Atlantic-Congo (Grassfields Bantu)".to_string()),
825            is_polysynthetic: false,
826            has_standard_orthography: false, // Developing
827            speaker_population: Some(1_000_000),
828            endangerment_level: Some(EndangermentLevel::Vulnerable),
829            training_examples: Some(4_833), // MasakhaNER 2.0
830        }),
831
832        _ => None,
833    }
834}
835
836/// MasakhaNER 2.0 language codes.
837///
838/// These codes can be used to load specific language splits from MasakhaNER 2.0.
839/// Example: `load_dataset("masakhane/masakhaner2", "yor")` for Yoruba.
840pub const MASAKHANER2_LANGUAGES: &[(&str, &str)] = &[
841    ("bam", "Bambara"),
842    ("bbj", "Ghomala"),
843    ("ewe", "Ewe"),
844    ("fon", "Fon"),
845    ("hau", "Hausa"),
846    ("ibo", "Igbo"),
847    ("kin", "Kinyarwanda"),
848    ("lug", "Luganda"),
849    ("luo", "Dholuo"),
850    ("mos", "Mossi"),
851    ("nya", "Chichewa"),
852    ("pcm", "Nigerian Pidgin"),
853    ("sna", "Shona"),
854    ("swa", "Swahili"),
855    ("tsn", "Setswana"),
856    ("twi", "Twi"),
857    ("wol", "Wolof"),
858    ("xho", "Xhosa"),
859    ("yor", "Yoruba"),
860    ("zul", "Zulu"),
861];
862
863/// Get language name from MasakhaNER 2.0 code.
864pub fn masakhaner2_language_name(code: &str) -> Option<&'static str> {
865    MASAKHANER2_LANGUAGES
866        .iter()
867        .find(|(c, _)| *c == code)
868        .map(|(_, name)| *name)
869}
870
871#[cfg(test)]
872mod tests {
873    use super::*;
874
875    #[test]
876    fn test_language_metadata() {
877        let quechua = language_metadata("qxo").unwrap();
878        assert_eq!(quechua.language_family, Some("Quechuan".to_string()));
879        assert!(!quechua.is_polysynthetic);
880
881        let cherokee = language_metadata("chr").unwrap();
882        assert!(cherokee.is_polysynthetic);
883        assert_eq!(
884            cherokee.endangerment_level,
885            Some(EndangermentLevel::SeverelyEndangered)
886        );
887    }
888
889    #[test]
890    fn test_orthographic_normalization() {
891        let config = OrthographicConfig {
892            unicode_normalize: true,
893            case_insensitive: true,
894            ignore_diacritics: true,
895            char_mappings: HashMap::new(),
896        };
897
898        let evaluator = LowResourceEvaluator::new();
899        let normalized = evaluator.normalize_text("Café", &config);
900        assert_eq!(normalized, "cafe");
901    }
902
903    #[test]
904    fn test_evaluator_creation() {
905        let evaluator = LowResourceEvaluator::new()
906            .with_morpheme_boundaries(MorphemeConfig::default())
907            .with_orthographic_normalization(OrthographicConfig::default())
908            .with_english_baseline(0.92);
909
910        assert!(evaluator.morpheme_config.is_some());
911        assert!(evaluator.orthographic_config.is_some());
912        assert_eq!(evaluator.english_baseline_f1, Some(0.92));
913    }
914}
anno_eval/eval/low_resource.rs

anno_eval/eval/
low_resource.rs
