tinysegmenter 0.1.1

Compact Japanese tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14

extern crate tinysegmenter;

#[test]
fn tokenize() {
  assert_eq!(
    tinysegmenter::tokenize("私の名前は中野です"),
    ["私", "の", "名前", "は", "中野", "です"]);

  assert_eq!(
    tinysegmenter::tokenize("TinySegmenterは25kBで書かれています。"),
    ["TinySegmenter", "は", "2", "5", "kB", "で", "書か", "れ", "て", "い", "ます", "。"]);

  assert_eq!(tinysegmenter::tokenize(""), [] as [&str; 0]);
}