punkt 1.0.5

An implementation of a Punkt sentence tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

extern crate punkt;

use punkt::{SentenceTokenizer, Trainer, TrainingData};
use punkt::params::Standard;

fn main() {
  let docs = [
    "This is sentence one. This is sentence two.",
    "The U.S. is a great country.",
    "I bought $5.50 worth of apples from the store. I gave them to my dog when I came home.",
  ];

  // The punkt algorithm can derive all the knowledge it needs to know from
  // the input document.
  println!("\n-- Trained only using document --");
  for d in docs.iter() {
    let trainer: Trainer<Standard> = Trainer::new();
    let mut data = TrainingData::new();

    trainer.train(d, &mut data);

    for s in SentenceTokenizer::<Standard>::new(d, &data) {
      println!("{:?}", s);
    }
  }

  // Alternatively, you can use some pretrained data.
  let english = TrainingData::english();

  println!("\n-- Using pretrained data --");
  for d in docs.iter() {
    for s in SentenceTokenizer::<Standard>::new(d, &english) {
      println!("{:?}", s);
    }
  }

  // You can incrementally build up training data too.
  let trainer: Trainer<Standard> = Trainer::new();
  let mut data = TrainingData::new();

  println!("\n-- Trained incrementally --");
  for d in docs.iter() {
    trainer.train(d, &mut data);

    for s in SentenceTokenizer::<Standard>::new(d, &data) {
      println!("{:?}", s);
    }
  }

  println!(
    "\nIs 'u.s' an abbreviation? : {:?}",
    data.contains_abbrev("u.s")
  );

  assert!(data.contains_abbrev("u.s"));
}