Struct Model 

pub struct Model { /* private fields */ }

A KenLM model loaded from an ARPA or KenLM binary file.

Implementations

impl Model

pub fn new(path: impl AsRef<Path>) -> Result<Self>

Load a language model with default configuration.

Examples found in repository

examples/full_scores.rs (line 14)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let sentence = args.collect::<Vec<_>>().join(" ");
    let sentence = if sentence.is_empty() {
        "looking on a little"
    } else {
        sentence.as_str()
    };

    let model = Model::new(model_path)?;
    let scores = model.full_scores(sentence, true, true)?;

    for (word, score) in sentence
        .split_whitespace()
        .chain(std::iter::once("</s>"))
        .zip(scores)
    {
        println!(
            "{word}\tlog10={:.6}\tngram_length={}\toov={}",
            score.log_prob, score.ngram_length, score.oov
        );
    }

    Ok(())
}

examples/score_sentence.rs (line 14)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let sentence = args.collect::<Vec<_>>().join(" ");
    let sentence = if sentence.is_empty() {
        "looking on a little"
    } else {
        sentence.as_str()
    };

    let model = Model::new(model_path)?;
    let score = model.score(sentence, true, true)?;
    let fragment_score = model.score(sentence, false, false)?;
    let perplexity = model.perplexity(sentence)?;

    println!("sentence: {sentence}");
    println!("order: {}", model.order());
    println!("score with <s> and </s>: {score}");
    println!("fragment score: {fragment_score}");
    println!("perplexity: {perplexity}");

    Ok(())
}

examples/stateful_scoring.rs (line 14)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let words = args.collect::<Vec<_>>();
    let words = if words.is_empty() {
        vec!["looking".to_string(), "on".to_string(), "a".to_string()]
    } else {
        words
    };

    let model = Model::new(model_path)?;
    let mut state = model.begin_sentence_state();
    let mut next = model.null_context_state();
    let mut total = 0.0;

    for word in &words {
        let word_index = model.index(word)?;
        let full = model.base_full_score(&state, word_index, &mut next)?;
        total += full.log_prob;
        println!(
            "{word}\tindex={word_index}\tlog10={:.6}\tngram_length={}",
            full.log_prob, full.ngram_length
        );
        std::mem::swap(&mut state, &mut next);
    }

    let eos = model.base_full_score(&state, model.end_sentence_index(), &mut next)?;
    total += eos.log_prob;
    println!(
        "</s>\tindex={}\tlog10={:.6}\tngram_length={}",
        model.end_sentence_index(),
        eos.log_prob,
        eos.ngram_length
    );
    println!("total: {total}");

    Ok(())
}

pub fn with_config(path: impl AsRef<Path>, config: Config) -> Result<Self>

Load a language model with explicit configuration.

Examples found in repository

examples/custom_config.rs (line 16)

fn main() -> Result<(), kenlm::KenlmError> {
    let model_path = env::args()
        .nth(1)
        .unwrap_or_else(|| "lm/test.arpa".to_string());

    let config = Config {
        show_progress: false,
        arpa_complain: ArpaLoadComplain::None,
        load_method: LoadMethod::Lazy,
        ..Config::default()
    };

    let model = Model::with_config(model_path, config)?;

    for word in ["looking", "definitely-not-in-this-model", "<s>", "</s>"] {
        let index = model.index(word)?;
        println!("{word}\tindex={index}\tin_vocab={}", model.contains(word)?);
    }

    Ok(())
}

pub fn order(&self) -> u8

Return the n-gram order of the model.

Examples found in repository

examples/score_sentence.rs (line 20)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let sentence = args.collect::<Vec<_>>().join(" ");
    let sentence = if sentence.is_empty() {
        "looking on a little"
    } else {
        sentence.as_str()
    };

    let model = Model::new(model_path)?;
    let score = model.score(sentence, true, true)?;
    let fragment_score = model.score(sentence, false, false)?;
    let perplexity = model.perplexity(sentence)?;

    println!("sentence: {sentence}");
    println!("order: {}", model.order());
    println!("score with <s> and </s>: {score}");
    println!("fragment score: {fragment_score}");
    println!("perplexity: {perplexity}");

    Ok(())
}

pub fn contains(&self, word: &str) -> Result<bool>

Return true when word exists in the model vocabulary.

Examples found in repository

examples/custom_config.rs (line 20)

fn main() -> Result<(), kenlm::KenlmError> {
    let model_path = env::args()
        .nth(1)
        .unwrap_or_else(|| "lm/test.arpa".to_string());

    let config = Config {
        show_progress: false,
        arpa_complain: ArpaLoadComplain::None,
        load_method: LoadMethod::Lazy,
        ..Config::default()
    };

    let model = Model::with_config(model_path, config)?;

    for word in ["looking", "definitely-not-in-this-model", "<s>", "</s>"] {
        let index = model.index(word)?;
        println!("{word}\tindex={index}\tin_vocab={}", model.contains(word)?);
    }

    Ok(())
}

pub fn index(&self, word: &str) -> Result<WordIndex>

Return KenLM’s vocabulary index for word, or the not-found index for OOV words.

Examples found in repository

examples/custom_config.rs (line 19)

fn main() -> Result<(), kenlm::KenlmError> {
    let model_path = env::args()
        .nth(1)
        .unwrap_or_else(|| "lm/test.arpa".to_string());

    let config = Config {
        show_progress: false,
        arpa_complain: ArpaLoadComplain::None,
        load_method: LoadMethod::Lazy,
        ..Config::default()
    };

    let model = Model::with_config(model_path, config)?;

    for word in ["looking", "definitely-not-in-this-model", "<s>", "</s>"] {
        let index = model.index(word)?;
        println!("{word}\tindex={index}\tin_vocab={}", model.contains(word)?);
    }

    Ok(())
}

examples/stateful_scoring.rs (line 20)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let words = args.collect::<Vec<_>>();
    let words = if words.is_empty() {
        vec!["looking".to_string(), "on".to_string(), "a".to_string()]
    } else {
        words
    };

    let model = Model::new(model_path)?;
    let mut state = model.begin_sentence_state();
    let mut next = model.null_context_state();
    let mut total = 0.0;

    for word in &words {
        let word_index = model.index(word)?;
        let full = model.base_full_score(&state, word_index, &mut next)?;
        total += full.log_prob;
        println!(
            "{word}\tindex={word_index}\tlog10={:.6}\tngram_length={}",
            full.log_prob, full.ngram_length
        );
        std::mem::swap(&mut state, &mut next);
    }

    let eos = model.base_full_score(&state, model.end_sentence_index(), &mut next)?;
    total += eos.log_prob;
    println!(
        "</s>\tindex={}\tlog10={:.6}\tngram_length={}",
        model.end_sentence_index(),
        eos.log_prob,
        eos.ngram_length
    );
    println!("total: {total}");

    Ok(())
}

pub fn begin_sentence_index(&self) -> WordIndex

Return the index for <s>.

pub fn end_sentence_index(&self) -> WordIndex

Return the index for </s>.

Examples found in repository

examples/stateful_scoring.rs (line 30)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let words = args.collect::<Vec<_>>();
    let words = if words.is_empty() {
        vec!["looking".to_string(), "on".to_string(), "a".to_string()]
    } else {
        words
    };

    let model = Model::new(model_path)?;
    let mut state = model.begin_sentence_state();
    let mut next = model.null_context_state();
    let mut total = 0.0;

    for word in &words {
        let word_index = model.index(word)?;
        let full = model.base_full_score(&state, word_index, &mut next)?;
        total += full.log_prob;
        println!(
            "{word}\tindex={word_index}\tlog10={:.6}\tngram_length={}",
            full.log_prob, full.ngram_length
        );
        std::mem::swap(&mut state, &mut next);
    }

    let eos = model.base_full_score(&state, model.end_sentence_index(), &mut next)?;
    total += eos.log_prob;
    println!(
        "</s>\tindex={}\tlog10={:.6}\tngram_length={}",
        model.end_sentence_index(),
        eos.log_prob,
        eos.ngram_length
    );
    println!("total: {total}");

    Ok(())
}

pub fn not_found_index(&self) -> WordIndex

Return the vocabulary index used for out-of-vocabulary words.

pub fn score(&self, sentence: &str, bos: bool, eos: bool) -> Result<f32>

Score a whitespace-tokenized sentence, returning log10 probability.

With bos = true and eos = true, this returns log10 p(sentence </s> | <s>).

Examples found in repository

examples/score_sentence.rs (line 15)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let sentence = args.collect::<Vec<_>>().join(" ");
    let sentence = if sentence.is_empty() {
        "looking on a little"
    } else {
        sentence.as_str()
    };

    let model = Model::new(model_path)?;
    let score = model.score(sentence, true, true)?;
    let fragment_score = model.score(sentence, false, false)?;
    let perplexity = model.perplexity(sentence)?;

    println!("sentence: {sentence}");
    println!("order: {}", model.order());
    println!("score with <s> and </s>: {score}");
    println!("fragment score: {fragment_score}");
    println!("perplexity: {perplexity}");

    Ok(())
}

pub fn score_words<'a>( &self, words: impl IntoIterator<Item = &'a str>, bos: bool, eos: bool, ) -> Result<f32>

Score pre-tokenized words, returning log10 probability.

pub fn perplexity(&self, sentence: &str) -> Result<f32>

Return perplexity for a complete whitespace-tokenized sentence.

Examples found in repository

examples/score_sentence.rs (line 17)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let sentence = args.collect::<Vec<_>>().join(" ");
    let sentence = if sentence.is_empty() {
        "looking on a little"
    } else {
        sentence.as_str()
    };

    let model = Model::new(model_path)?;
    let score = model.score(sentence, true, true)?;
    let fragment_score = model.score(sentence, false, false)?;
    let perplexity = model.perplexity(sentence)?;

    println!("sentence: {sentence}");
    println!("order: {}", model.order());
    println!("score with <s> and </s>: {score}");
    println!("fragment score: {fragment_score}");
    println!("perplexity: {perplexity}");

    Ok(())
}

pub fn full_scores( &self, sentence: &str, bos: bool, eos: bool, ) -> Result<Vec<TokenScore>>

Return per-token full scores for a whitespace-tokenized sentence.

Examples found in repository

examples/full_scores.rs (line 15)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let sentence = args.collect::<Vec<_>>().join(" ");
    let sentence = if sentence.is_empty() {
        "looking on a little"
    } else {
        sentence.as_str()
    };

    let model = Model::new(model_path)?;
    let scores = model.full_scores(sentence, true, true)?;

    for (word, score) in sentence
        .split_whitespace()
        .chain(std::iter::once("</s>"))
        .zip(scores)
    {
        println!(
            "{word}\tlog10={:.6}\tngram_length={}\toov={}",
            score.log_prob, score.ngram_length, score.oov
        );
    }

    Ok(())
}

pub fn full_scores_words<'a>( &self, words: impl IntoIterator<Item = &'a str>, bos: bool, eos: bool, ) -> Result<Vec<TokenScore>>

Return per-token full scores for pre-tokenized words.

pub fn begin_sentence_state(&self) -> State

Create a state initialized to beginning-of-sentence context.

Examples found in repository

examples/stateful_scoring.rs (line 15)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let words = args.collect::<Vec<_>>();
    let words = if words.is_empty() {
        vec!["looking".to_string(), "on".to_string(), "a".to_string()]
    } else {
        words
    };

    let model = Model::new(model_path)?;
    let mut state = model.begin_sentence_state();
    let mut next = model.null_context_state();
    let mut total = 0.0;

    for word in &words {
        let word_index = model.index(word)?;
        let full = model.base_full_score(&state, word_index, &mut next)?;
        total += full.log_prob;
        println!(
            "{word}\tindex={word_index}\tlog10={:.6}\tngram_length={}",
            full.log_prob, full.ngram_length
        );
        std::mem::swap(&mut state, &mut next);
    }

    let eos = model.base_full_score(&state, model.end_sentence_index(), &mut next)?;
    total += eos.log_prob;
    println!(
        "</s>\tindex={}\tlog10={:.6}\tngram_length={}",
        model.end_sentence_index(),
        eos.log_prob,
        eos.ngram_length
    );
    println!("total: {total}");

    Ok(())
}

pub fn null_context_state(&self) -> State

Create a state initialized to null context.

Examples found in repository

examples/stateful_scoring.rs (line 16)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let words = args.collect::<Vec<_>>();
    let words = if words.is_empty() {
        vec!["looking".to_string(), "on".to_string(), "a".to_string()]
    } else {
        words
    };

    let model = Model::new(model_path)?;
    let mut state = model.begin_sentence_state();
    let mut next = model.null_context_state();
    let mut total = 0.0;

    for word in &words {
        let word_index = model.index(word)?;
        let full = model.base_full_score(&state, word_index, &mut next)?;
        total += full.log_prob;
        println!(
            "{word}\tindex={word_index}\tlog10={:.6}\tngram_length={}",
            full.log_prob, full.ngram_length
        );
        std::mem::swap(&mut state, &mut next);
    }

    let eos = model.base_full_score(&state, model.end_sentence_index(), &mut next)?;
    total += eos.log_prob;
    println!(
        "</s>\tindex={}\tlog10={:.6}\tngram_length={}",
        model.end_sentence_index(),
        eos.log_prob,
        eos.ngram_length
    );
    println!("total: {total}");

    Ok(())
}

pub fn base_score( &self, in_state: &State, word_index: WordIndex, out_state: &mut State, ) -> Result<f32>

Score word_index from in_state, writing the next state into out_state.

pub fn base_full_score( &self, in_state: &State, word_index: WordIndex, out_state: &mut State, ) -> Result<FullScore>

Return KenLM’s full score metadata for a state transition.

Examples found in repository

examples/stateful_scoring.rs (line 21)

fn main() -> Result<(), kenlm::KenlmError> {
    let mut args = env::args().skip(1);
    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
    let words = args.collect::<Vec<_>>();
    let words = if words.is_empty() {
        vec!["looking".to_string(), "on".to_string(), "a".to_string()]
    } else {
        words
    };

    let model = Model::new(model_path)?;
    let mut state = model.begin_sentence_state();
    let mut next = model.null_context_state();
    let mut total = 0.0;

    for word in &words {
        let word_index = model.index(word)?;
        let full = model.base_full_score(&state, word_index, &mut next)?;
        total += full.log_prob;
        println!(
            "{word}\tindex={word_index}\tlog10={:.6}\tngram_length={}",
            full.log_prob, full.ngram_length
        );
        std::mem::swap(&mut state, &mut next);
    }

    let eos = model.base_full_score(&state, model.end_sentence_index(), &mut next)?;
    total += eos.log_prob;
    println!(
        "</s>\tindex={}\tlog10={:.6}\tngram_length={}",
        model.end_sentence_index(),
        eos.log_prob,
        eos.ngram_length
    );
    println!("total: {total}");

    Ok(())
}

Trait Implementations

impl Drop for Model

fn drop(&mut self)

Executes the destructor for this type.

impl Send for Model

impl Sync for Model