saku 0.1.6

A simple yet efficient rule-based Japanese Sentence Tokenizer.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

use crate::{CharTable, ControlFlow, SentenceTokenizer};

const DEFAULT_EOS: char = '。';
const DEFAULT_LEFT_PATTERNS: [char; 3] = ['（', '「', '『'];
const DEFAULT_RIGHT_PATTERNS: [char; 3] = ['）', '」', '』'];

#[derive(Clone, Debug)]
pub struct SentenceTokenizerBuilder {
    pub(crate) eos: char,
    pub(crate) left_patterns: Vec<char>,
    pub(crate) right_patterns: Vec<char>,
}

impl SentenceTokenizerBuilder {
    #[inline(always)]
    pub fn new() -> SentenceTokenizerBuilder {
        SentenceTokenizerBuilder {
            eos: DEFAULT_EOS,
            left_patterns: DEFAULT_LEFT_PATTERNS.to_vec(),
            right_patterns: DEFAULT_RIGHT_PATTERNS.to_vec(),
        }
    }
}

impl SentenceTokenizerBuilder {
    #[inline(always)]
    pub fn eos(self, eos: char) -> SentenceTokenizerBuilder {
        SentenceTokenizerBuilder {
            eos,
            left_patterns: self.left_patterns,
            right_patterns: self.right_patterns,
        }
    }

    #[inline(always)]
    pub fn patterns(self, patterns: &[[char; 2]]) -> SentenceTokenizerBuilder {
        let left_patterns: Vec<char> = patterns.iter().map(|p| p[0]).collect();
        let right_patterns: Vec<char> = patterns.iter().map(|p| p[1]).collect();
        SentenceTokenizerBuilder {
            eos: self.eos,
            left_patterns,
            right_patterns,
        }
    }

    #[inline(always)]
    pub fn build(&self) -> SentenceTokenizer {
        let eos = self.eos;
        let num_parens: u8 = self.left_patterns.len() as u8;

        let mut char_table: CharTable = CharTable::default();
        char_table.insert(eos, ControlFlow::Eos);
        for (flag_id, &l) in self.left_patterns.iter().enumerate() {
            char_table.insert(l, ControlFlow::LeftParens(flag_id as u8));
        }
        for (flag_id, &r) in self.right_patterns.iter().enumerate() {
            char_table.insert(r, ControlFlow::RightParens(flag_id as u8));
        }
        char_table.insert('\n', ControlFlow::LineBreaks);
        char_table.insert('\r', ControlFlow::LineBreaks);
        let eos_size = eos.len_utf8();

        SentenceTokenizer {
            eos,
            eos_size,
            num_parens,
            char_table,
        }
    }
}
#[cfg(test)]
mod test {
    use crate::*;

    #[test]
    fn it_works() {
        let tokenizer = SentenceTokenizerBuilder::new()
            .eos('。')
            .patterns(&[['（', '）'], ['「', '」']])
            .build();
        assert_eq!('。', tokenizer.eos);
    }
}