saku 0.1.6

A simple yet efficient rule-based Japanese Sentence Tokenizer.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

mod builder;
mod char_table;
mod tokenizer;

#[derive(Clone, Debug)]
pub(crate) enum ControlFlow {
    Eos,
    LineBreaks,
    RightParens(u8),
    LeftParens(u8),
}
pub(crate) use char_table::CharTable;

pub use builder::SentenceTokenizerBuilder;
pub use tokenizer::SentenceTokenizer;