instant-segment 0.11.1

Fast English word segmentation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#![cfg(feature = "__test_data")]

use bencher::{benchmark_group, benchmark_main, Bencher};

use instant_segment::test_data::{crate_data_dir, segmenter};
use instant_segment::Search;

benchmark_group!(benches, short, long, extra_long);
benchmark_main!(benches);

fn short(bench: &mut Bencher) {
    let segmenter = segmenter(crate_data_dir());
    let mut search = Search::default();
    bench.iter(|| {
        let _ = segmenter.segment("thisisatest", &mut search);
    });
}

fn long(bench: &mut Bencher) {
    let segmenter = segmenter(crate_data_dir());
    let mut search = Search::default();
    bench.iter(|| {
        let _ = segmenter.segment(
            "itwasabrightcolddayinaprilandtheclockswerestrikingthirteen",
            &mut search,
        );
    });
}

fn extra_long(bench: &mut Bencher) {
    let segmenter = segmenter(crate_data_dir());
    let mut search = Search::default();
    bench.iter(|| {
        let _ = segmenter.segment(
"itwasthebestoftimesitwastheworstoftimesitwastheageofwisdomitwastheageoffoolishnessitwastheepochofbeliefitwastheepochofincredulityitwastheseasonoflightitwastheseasonofdarknessitwasthespringofhopeitwasthewinterofdespair",
            &mut search,
        );
    });
}