libgrammstein 0.1.0

Hybrid language model (N-gram + Embeddings) for WFST text correction
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

//! Perplexity and sentence scoring.
//!
//! This module provides:
//! - Sentence log-probability computation
//! - Corpus perplexity evaluation
//! - OOV rate calculation
//!
//! # Example
//!
//! ```ignore
//! use libgrammstein::scoring::Perplexity;
//! use libgrammstein::ngram::NgramModel;
//!
//! let model = NgramModel::load("model.bin")?;
//! let perplexity = Perplexity::new(&model);
//!
//! // Score a sentence
//! let log_prob = perplexity.sentence_log_prob(&["the", "quick", "brown", "fox"]);
//!
//! // Evaluate corpus perplexity
//! let corpus_ppl = perplexity.corpus_perplexity(&corpus_reader)?;
//! ```

mod perplexity;
mod sentence;

pub use perplexity::Perplexity;
pub use sentence::SentenceScorer;