jpreprocess_core/
token.rs

1use crate::{word_entry::WordEntry, JPreprocessResult};
2
3pub trait Tokenizer {
4    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>>;
5}
6
7pub trait Token {
8    fn fetch(&mut self) -> JPreprocessResult<(&str, WordEntry)>;
9}
10
11impl Token for (String, WordEntry) {
12    fn fetch(&mut self) -> JPreprocessResult<(&str, WordEntry)> {
13        let (string, entry) = self;
14        Ok((string.as_str(), entry.to_owned()))
15    }
16}
17
18#[cfg(feature = "lindera")]
19impl Tokenizer for lindera::tokenizer::Tokenizer {
20    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>> {
21        Ok(self.tokenize(text)?)
22    }
23}
24
25#[cfg(feature = "lindera")]
26impl Token for lindera::token::Token<'_> {
27    fn fetch(&mut self) -> JPreprocessResult<(&str, WordEntry)> {
28        use lindera_dictionary::dictionary::UNK;
29
30        let mut details = self.details();
31        let entry = if details == *UNK {
32            WordEntry::default()
33        } else {
34            details.resize(12, "");
35            WordEntry::load(&details)?
36        };
37
38        Ok((&self.surface, entry))
39    }
40}
41
42/// Vibrato support is experimental and may be removed or changed in the future. Use with caution.
43#[cfg(feature = "vibrato")]
44impl Tokenizer for vibrato::tokenizer::Tokenizer {
45    fn tokenize<'a>(&'a self, text: &'a str) -> JPreprocessResult<Vec<impl 'a + Token>> {
46        let mut worker = self.new_worker();
47        worker.reset_sentence(text);
48        worker.tokenize();
49
50        worker
51            .token_iter()
52            .map(|token| {
53                let features = token.feature();
54                let mut details = features.split(',').collect::<Vec<_>>();
55                details.resize(12, "*");
56                let entry = WordEntry::load(&details)?;
57
58                Ok((token.surface().to_string(), entry))
59            })
60            .collect()
61    }
62}
jpreprocess_core/token.rs

jpreprocess_core/
token.rs
