libgrammstein 0.1.0

Hybrid language model (N-gram + Embeddings) for WFST text correction
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

//! Integration with lling-llang WFST framework.
//!
//! This module provides:
//! - `LanguageModel` trait implementation for lattice rescoring
//! - WFST export for n-gram models (eager and lazy)
//! - Vocabulary mapping between words and WFST labels
//!
//! # WFST Export
//!
//! The WFST export creates a transducer representing the n-gram language model:
//! - States represent n-gram histories (previous n-1 words)
//! - Transitions represent word emissions with log probability weights
//! - Backoff ε-transitions implement smoothing fallback
//!
//! ```ignore
//! use libgrammstein::ngram::NgramModel;
//! use libgrammstein::integration::{NgramWfstExport, FromLogProb};
//! use lling_llang::semiring::LogWeight;
//!
//! let model: NgramModel<D> = /* ... */;
//! let (wfst, vocab) = model.to_wfst::<LogWeight>();
//!
//! // Use in ASR cascade
//! let cascade = CascadeBuilder::new()
//!     .grammar(wfst)
//!     .build();
//! ```
//!
//! # Language Model Scoring
//!
//! ```ignore
//! use libgrammstein::integration::GrammsteinLanguageModel;
//! use libgrammstein::ngram::NgramModel;
//! use lling_llang::layers::LanguageModel;
//!
//! let ngram_model = NgramModel::load("model.bin")?;
//! let lm = GrammsteinLanguageModel::from_ngram(ngram_model);
//!
//! // Use with lling-llang's LanguageModelLayer
//! let score = lm.score_sequence(&["the", "quick", "brown", "fox"]);
//! ```

pub mod lazy_ngram;
mod lling_llang;
pub mod vocabulary;
pub mod wfst_export;

pub use self::lazy_ngram::{NgramHistoryKey, NgramLazyWfst, NgramStateRegistry, NgramStateSource};
pub use self::lling_llang::GrammsteinLanguageModel;
pub use self::vocabulary::{WordId, WordVocabulary, EOS_WORD_ID, UNK_WORD_ID};
pub use self::wfst_export::{
    FromLogProb, NgramTransducerBuilder, NgramWfstBuilder, NgramWfstExport,
};