lindera/
lib.rs

1//! # Lindera Python Bindings
2//!
3//! Python bindings for [Lindera](https://github.com/lindera/lindera), a morphological analysis library for CJK text.
4//!
5//! Lindera provides high-performance tokenization and morphological analysis for:
6//! - Japanese (IPADIC, IPADIC NEologd, UniDic)
7//! - Korean (ko-dic)
8//! - Chinese (CC-CEDICT)
9//!
10//! ## Features
11//!
12//! - **Dictionary management**: Build, load, and use custom dictionaries
13//! - **Tokenization**: Multiple tokenization modes (normal, decompose)
14//! - **Filters**: Character and token filtering pipeline
15//! - **Training**: Train custom morphological models (with `train` feature)
16//! - **User dictionaries**: Support for custom user dictionaries
17//!
18//! ## Examples
19//!
20//! ```python
21//! import lindera
22//!
23//! # Create a tokenizer
24//! tokenizer = lindera.TokenizerBuilder().build()
25//!
26//! # Tokenize text
27//! tokens = tokenizer.tokenize("関西国際空港")
28//! for token in tokens:
29//!     print(token["text"], token["detail"])
30//! ```
31
32pub mod character_filter;
33pub mod dictionary;
34pub mod error;
35pub mod metadata;
36pub mod mode;
37pub mod schema;
38pub mod token;
39pub mod token_filter;
40pub mod tokenizer;
41pub mod util;
42
43#[cfg(feature = "train")]
44pub mod trainer;
45
46use pyo3::prelude::*;
47
48/// Returns the version of the lindera-python package.
49#[pyfunction]
50pub fn version() -> String {
51    env!("CARGO_PKG_VERSION").to_string()
52}
53
54/// Python module definition for lindera.
55#[pymodule]
56fn lindera(m: &Bound<'_, PyModule>) -> PyResult<()> {
57    // Register submodules
58    tokenizer::register(m)?;
59    dictionary::register(m)?;
60    token::register(m)?;
61    mode::register(m)?;
62    metadata::register(m)?;
63    schema::register(m)?;
64    character_filter::register(m)?;
65    token_filter::register(m)?;
66    error::register(m)?;
67
68    #[cfg(feature = "train")]
69    {
70        // For trainer, we can implement register similarly or just keeping it flat for now if complex
71        // Let's assume we want lindera.trainer.train() etc.
72        let py = m.py();
73        let trainer_mod = PyModule::new(py, "trainer")?;
74        trainer_mod.add_function(wrap_pyfunction!(crate::trainer::train, &trainer_mod)?)?;
75        trainer_mod.add_function(wrap_pyfunction!(crate::trainer::export, &trainer_mod)?)?;
76        m.add_submodule(&trainer_mod)?;
77    }
78
79    m.add_function(wrap_pyfunction!(version, m)?)?;
80
81    // --- Backward compatibility aliases (top-level) ---
82    // Classes
83    m.add("Tokenizer", m.getattr("tokenizer")?.getattr("Tokenizer")?)?;
84    m.add(
85        "TokenizerBuilder",
86        m.getattr("tokenizer")?.getattr("TokenizerBuilder")?,
87    )?;
88    m.add(
89        "Dictionary",
90        m.getattr("dictionary")?.getattr("Dictionary")?,
91    )?;
92    m.add(
93        "UserDictionary",
94        m.getattr("dictionary")?.getattr("UserDictionary")?,
95    )?;
96    m.add("Token", m.getattr("token")?.getattr("Token")?)?;
97    m.add("Mode", m.getattr("mode")?.getattr("Mode")?)?;
98    m.add("Penalty", m.getattr("mode")?.getattr("Penalty")?)?;
99    m.add("Metadata", m.getattr("metadata")?.getattr("Metadata")?)?;
100    m.add("Schema", m.getattr("schema")?.getattr("Schema")?)?;
101    m.add(
102        "FieldDefinition",
103        m.getattr("schema")?.getattr("FieldDefinition")?,
104    )?;
105    m.add("FieldType", m.getattr("schema")?.getattr("FieldType")?)?;
106    m.add("LinderaError", m.getattr("error")?.getattr("LinderaError")?)?;
107
108    // Functions
109    m.add(
110        "load_dictionary",
111        m.getattr("dictionary")?.getattr("load_dictionary")?,
112    )?;
113    m.add(
114        "load_user_dictionary",
115        m.getattr("dictionary")?.getattr("load_user_dictionary")?,
116    )?;
117    m.add(
118        "build_dictionary",
119        m.getattr("dictionary")?.getattr("build_dictionary")?,
120    )?;
121    m.add(
122        "build_user_dictionary",
123        m.getattr("dictionary")?.getattr("build_user_dictionary")?,
124    )?;
125
126    #[cfg(feature = "train")]
127    {
128        m.add("train", m.getattr("trainer")?.getattr("train")?)?;
129        m.add("export", m.getattr("trainer")?.getattr("export")?)?;
130    }
131
132    Ok(())
133}
lindera/lib.rs

lindera/
lib.rs
