lindera 3.0.0

A morphological analysis library.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

use lindera::LinderaResult;

fn main() -> LinderaResult<()> {
    #[cfg(feature = "embed-ipadic")]
    {
        use std::path::PathBuf;

        use lindera::tokenizer::TokenizerBuilder;

        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
            .join("../resources")
            .join("config")
            .join("lindera.yml");

        let builder = TokenizerBuilder::from_file(&path)?;

        let tokenizer = builder.build()?;

        let text = "Ｌｉｎｄｅｒａは形態素解析ｴﾝｼﾞﾝです。ユーザー辞書も利用可能です。".to_string();
        println!("text: {text}");

        let tokens = tokenizer.tokenize(&text)?;

        for token in tokens {
            println!(
                "token: {:?}, start: {:?}, end: {:?}, details: {:?}",
                token.surface, token.byte_start, token.byte_end, token.details
            );
        }
    }

    #[cfg(not(feature = "embed-ipadic"))]
    {
        eprintln!("This example requires the 'embed-ipadic' feature to be enabled.");
        eprintln!("Run with: cargo run --features embed-ipadic --example tokenize_with_config");
    }

    Ok(())
}