lindera 3.0.5

A morphological analysis library.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

use lindera::LinderaResult;

fn main() -> LinderaResult<()> {
    #[cfg(feature = "embed-ipadic")]
    {
        use lindera::dictionary::load_dictionary;
        use lindera::mode::Mode;
        use lindera::segmenter::Segmenter;
        use lindera::tokenizer::Tokenizer;

        let dictionary = load_dictionary("embedded://ipadic")?;
        let segmenter = Segmenter::new(Mode::Normal, dictionary, None);
        let tokenizer = Tokenizer::new(segmenter);

        let text = "関西国際空港限定トートバッグ";
        let mut tokens = tokenizer.tokenize(text)?;
        println!("text:\t{text}");
        for token in tokens.iter_mut() {
            let details = token.details().join(",");
            println!("token:\t{}\t{}", token.surface.as_ref(), details);
        }
    }

    #[cfg(not(feature = "embed-ipadic"))]
    {
        eprintln!("This example requires the 'embed-ipadic' feature to be enabled.");
        eprintln!("Run with: cargo run --features embed-ipadic --example tokenize");
    }

    Ok(())
}