Skip to main content

score_sentence/
score_sentence.rs

1use kenlm::Model;
2use std::env;
3
4fn main() -> Result<(), kenlm::KenlmError> {
5    let mut args = env::args().skip(1);
6    let model_path = args.next().unwrap_or_else(|| "lm/test.arpa".to_string());
7    let sentence = args.collect::<Vec<_>>().join(" ");
8    let sentence = if sentence.is_empty() {
9        "looking on a little"
10    } else {
11        sentence.as_str()
12    };
13
14    let model = Model::new(model_path)?;
15    let score = model.score(sentence, true, true)?;
16    let fragment_score = model.score(sentence, false, false)?;
17    let perplexity = model.perplexity(sentence)?;
18
19    println!("sentence: {sentence}");
20    println!("order: {}", model.order());
21    println!("score with <s> and </s>: {score}");
22    println!("fragment score: {fragment_score}");
23    println!("perplexity: {perplexity}");
24
25    Ok(())
26}