lling-llang 0.1.0

WFST framework for text normalization and grammar correction
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

//! Error model transducers for correction pipelines.
//!
//! This module provides WFSTs that model various types of errors in text,
//! enabling systematic correction through composition with dictionaries
//! and language models.
//!
//! # Available Error Models
//!
//! - **Edit Distance**: Levenshtein/Damerau-Levenshtein transducers for spelling correction
//! - **Confusion**: Character-level confusion matrices (OCR, keyboard typos)
//! - **Homophone**: Sound-alike word mappings for spoken language errors
//!
//! # Example
//!
//! ```rust,ignore
//! use lling_llang::error_models::EditDistanceTransducer;
//! use lling_llang::composition::compose;
//!
//! // Build an edit distance transducer with max distance 2
//! let edit_fst = EditDistanceTransducer::new(2)
//!     .with_alphabet("abcdefghijklmnopqrstuvwxyz")
//!     .build();
//!
//! // Compose with a dictionary for fuzzy lookup
//! let fuzzy = compose(&edit_fst, &dictionary_fst);
//! ```

pub mod confusion;
pub mod edit_distance;
pub mod homophone;
pub mod normalize;

pub use edit_distance::{
    DamerauLevenshteinTransducer, EditCosts, EditDistanceConfig, EditDistanceTransducer,
};

pub use confusion::{
    combined_confusion_matrix, dvorak_confusion_matrix, ocr_confusion_matrix,
    qwerty_confusion_matrix, train_confusion_matrix, ConfusionConfig, ConfusionMatrix,
    ConfusionTransducer,
};

pub use homophone::{
    common_english_homophones, english_homophone_transducer, HomophoneConfig, HomophoneEntry,
    HomophoneTransducer, PhoneticAlgorithm, PhoneticCode, PhoneticEncoder,
};

pub use normalize::{
    ascii_normalizer, search_normalizer, unicode_normalizer, CharacterMapping, NormalizationConfig,
    NormalizationResult, NormalizationTransducer,
};