tinysegmenter 0.1.1

Compact Japanese tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

#![feature(test)]

extern crate test;
extern crate tinysegmenter;

use std::io::prelude::*;
use std::fs::File;
use test::Bencher;

#[bench]
fn run(b: &mut Bencher) {
  // http://www.genpaku.org/timemachine/timemachineu8j.txt
  let mut f =
    File::open("benchmark/timemachineu8j.txt")
    .expect("Failed to read a benchmark text.");
  let mut s = String::new();
  let _ = f.read_to_string(&mut s);

  b.iter(|| tinysegmenter::tokenize(&s));
}