lindera_binding_core/
metadata.rs

1//! Shared dictionary-metadata defaults and schema wiring for the bindings.
2//!
3//! The Python/PHP/Ruby/Node.js `Metadata` wrappers each hard-coded the same
4//! default values (name `"default"`, encoding `"UTF-8"`, word cost `-10000`,
5//! context ids `1288`, field value `"*"`, three `false` flags) and the same
6//! default user-dictionary schema (`surface`/`reading`/`pronunciation`). This
7//! module collects that into a single [`CoreMetadata`] the bindings can wrap.
8
9use lindera::dictionary::Metadata;
10use lindera::space_penalty::SpacePenaltyConfig;
11
12use crate::schema::CoreSchema;
13
14/// Default dictionary name.
15const DEFAULT_NAME: &str = "default";
16/// Default character encoding.
17const DEFAULT_ENCODING: &str = "UTF-8";
18/// Default cost assigned to simple user-dictionary entries.
19const DEFAULT_WORD_COST: i16 = -10000;
20/// Default left context id for simple user-dictionary entries.
21const DEFAULT_LEFT_CONTEXT_ID: u16 = 1288;
22/// Default right context id for simple user-dictionary entries.
23const DEFAULT_RIGHT_CONTEXT_ID: u16 = 1288;
24/// Default value substituted for missing fields.
25const DEFAULT_FIELD_VALUE: &str = "*";
26
27/// Returns the default user-dictionary schema (`surface`/`reading`/`pronunciation`).
28fn default_user_dictionary_schema() -> CoreSchema {
29    CoreSchema::new(vec![
30        "surface".to_string(),
31        "reading".to_string(),
32        "pronunciation".to_string(),
33    ])
34}
35
36/// Dictionary metadata shared by the bindings.
37///
38/// Mirrors the binding `Metadata` wrappers as a pure-Rust type (using
39/// [`CoreSchema`] for the two schema fields) so each binding can wrap a
40/// `CoreMetadata` instead of re-declaring the same defaults and conversions.
41/// Converts to and from [`lindera::dictionary::Metadata`]; the optional
42/// `model_info` carried by the lindera type is not retained (the bindings do
43/// not expose it), while `space_penalty` is, so a dictionary built from a
44/// `metadata.json` that ships left-space penalty rules (ko-dic) keeps them.
45#[derive(Debug, Clone)]
46pub struct CoreMetadata {
47    /// Dictionary name.
48    pub name: String,
49    /// Character encoding.
50    pub encoding: String,
51    /// Default word cost for simple user-dictionary entries.
52    pub default_word_cost: i16,
53    /// Default left context id for simple user-dictionary entries.
54    pub default_left_context_id: u16,
55    /// Default right context id for simple user-dictionary entries.
56    pub default_right_context_id: u16,
57    /// Default value substituted for missing fields.
58    pub default_field_value: String,
59    /// Whether CSV columns are handled flexibly.
60    pub flexible_csv: bool,
61    /// Whether entries with invalid cost or id are skipped.
62    pub skip_invalid_cost_or_id: bool,
63    /// Whether morphological details are normalized.
64    pub normalize_details: bool,
65    /// Schema for the main dictionary.
66    pub dictionary_schema: CoreSchema,
67    /// Schema for the user dictionary.
68    pub user_dictionary_schema: CoreSchema,
69    /// Left-space penalty rules the dictionary ships (ko-dic carries
70    /// mecab-ko-dic's `left-space-penalty-factor`); `None` when it ships none.
71    pub space_penalty: Option<SpacePenaltyConfig>,
72}
73
74impl CoreMetadata {
75    /// Creates metadata, falling back to the binding defaults for any `None`.
76    #[allow(clippy::too_many_arguments)]
77    pub fn new(
78        name: Option<String>,
79        encoding: Option<String>,
80        default_word_cost: Option<i16>,
81        default_left_context_id: Option<u16>,
82        default_right_context_id: Option<u16>,
83        default_field_value: Option<String>,
84        flexible_csv: Option<bool>,
85        skip_invalid_cost_or_id: Option<bool>,
86        normalize_details: Option<bool>,
87        dictionary_schema: Option<CoreSchema>,
88        user_dictionary_schema: Option<CoreSchema>,
89    ) -> Self {
90        Self {
91            name: name.unwrap_or_else(|| DEFAULT_NAME.to_string()),
92            encoding: encoding.unwrap_or_else(|| DEFAULT_ENCODING.to_string()),
93            default_word_cost: default_word_cost.unwrap_or(DEFAULT_WORD_COST),
94            default_left_context_id: default_left_context_id.unwrap_or(DEFAULT_LEFT_CONTEXT_ID),
95            default_right_context_id: default_right_context_id.unwrap_or(DEFAULT_RIGHT_CONTEXT_ID),
96            default_field_value: default_field_value
97                .unwrap_or_else(|| DEFAULT_FIELD_VALUE.to_string()),
98            flexible_csv: flexible_csv.unwrap_or(false),
99            skip_invalid_cost_or_id: skip_invalid_cost_or_id.unwrap_or(false),
100            normalize_details: normalize_details.unwrap_or(false),
101            dictionary_schema: dictionary_schema.unwrap_or_else(CoreSchema::create_default),
102            user_dictionary_schema: user_dictionary_schema
103                .unwrap_or_else(default_user_dictionary_schema),
104            space_penalty: None,
105        }
106    }
107
108    /// Creates metadata with all binding defaults.
109    pub fn create_default() -> Self {
110        Self::new(
111            None, None, None, None, None, None, None, None, None, None, None,
112        )
113    }
114}
115
116impl CoreMetadata {
117    /// Returns the left-space penalty rules as a JSON string, for the
118    /// bindings' string-valued `to_dict` / `to_hash` / `to_array` /
119    /// `to_object` maps.
120    ///
121    /// # Returns
122    ///
123    /// The rules serialized as JSON (e.g. `{"rules":[...]}`), or `None` when
124    /// the dictionary ships none.
125    pub fn space_penalty_json(&self) -> Option<String> {
126        self.space_penalty
127            .as_ref()
128            .and_then(|config| serde_json::to_string(config).ok())
129    }
130}
131
132impl Default for CoreMetadata {
133    /// Returns [`CoreMetadata::create_default`].
134    fn default() -> Self {
135        Self::create_default()
136    }
137}
138
139impl From<Metadata> for CoreMetadata {
140    /// Converts a lindera [`Metadata`] into a [`CoreMetadata`] (dropping
141    /// `model_info`, keeping `space_penalty`).
142    fn from(metadata: Metadata) -> Self {
143        Self {
144            name: metadata.name,
145            encoding: metadata.encoding,
146            default_word_cost: metadata.default_word_cost,
147            default_left_context_id: metadata.default_left_context_id,
148            default_right_context_id: metadata.default_right_context_id,
149            default_field_value: metadata.default_field_value,
150            flexible_csv: metadata.flexible_csv,
151            skip_invalid_cost_or_id: metadata.skip_invalid_cost_or_id,
152            normalize_details: metadata.normalize_details,
153            dictionary_schema: metadata.dictionary_schema.into(),
154            user_dictionary_schema: metadata.user_dictionary_schema.into(),
155            space_penalty: metadata.space_penalty,
156        }
157    }
158}
159
160impl From<CoreMetadata> for Metadata {
161    /// Converts a [`CoreMetadata`] into a lindera [`Metadata`] (`model_info` is
162    /// `None`; `space_penalty` is carried over).
163    fn from(metadata: CoreMetadata) -> Self {
164        let space_penalty = metadata.space_penalty;
165        let mut converted = Metadata::new(
166            metadata.name,
167            metadata.encoding,
168            metadata.default_word_cost,
169            metadata.default_left_context_id,
170            metadata.default_right_context_id,
171            metadata.default_field_value,
172            metadata.flexible_csv,
173            metadata.skip_invalid_cost_or_id,
174            metadata.normalize_details,
175            metadata.dictionary_schema.into(),
176            metadata.user_dictionary_schema.into(),
177        );
178        converted.space_penalty = space_penalty;
179        converted
180    }
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186
187    #[test]
188    fn create_default_uses_binding_defaults() {
189        let m = CoreMetadata::create_default();
190        assert_eq!(m.name, "default");
191        assert_eq!(m.encoding, "UTF-8");
192        assert_eq!(m.default_word_cost, -10000);
193        assert_eq!(m.default_left_context_id, 1288);
194        assert_eq!(m.default_right_context_id, 1288);
195        assert_eq!(m.default_field_value, "*");
196        assert!(!m.flexible_csv);
197        assert!(!m.skip_invalid_cost_or_id);
198        assert!(!m.normalize_details);
199        // The default dictionary schema uses the unified `pos_detail_*` names.
200        assert_eq!(m.dictionary_schema.field_count(), 13);
201        assert_eq!(m.dictionary_schema.fields()[5], "pos_detail_1");
202        assert_eq!(m.user_dictionary_schema.fields().len(), 3);
203        assert_eq!(m.user_dictionary_schema.fields()[1], "reading");
204    }
205
206    #[test]
207    fn new_applies_overrides() {
208        let m = CoreMetadata::new(
209            Some("custom".to_string()),
210            Some("EUC-JP".to_string()),
211            Some(-5000),
212            Some(100),
213            Some(200),
214            Some("N/A".to_string()),
215            Some(true),
216            Some(true),
217            Some(true),
218            None,
219            None,
220        );
221        assert_eq!(m.name, "custom");
222        assert_eq!(m.encoding, "EUC-JP");
223        assert_eq!(m.default_word_cost, -5000);
224        assert_eq!(m.default_left_context_id, 100);
225        assert_eq!(m.default_right_context_id, 200);
226        assert_eq!(m.default_field_value, "N/A");
227        assert!(m.flexible_csv && m.skip_invalid_cost_or_id && m.normalize_details);
228    }
229
230    #[test]
231    fn converts_to_and_from_lindera() {
232        let m = CoreMetadata::create_default();
233        let lindera: Metadata = m.into();
234        assert_eq!(lindera.name, "default");
235        assert_eq!(lindera.default_left_context_id, 1288);
236        assert_eq!(lindera.user_dictionary_schema.field_count(), 3);
237
238        let back: CoreMetadata = lindera.into();
239        assert_eq!(back.name, "default");
240        assert_eq!(back.dictionary_schema.fields()[5], "pos_detail_1");
241        assert!(back.space_penalty.is_none());
242        assert!(back.space_penalty_json().is_none());
243    }
244
245    /// ko-dic's `metadata.json` ships left-space penalty rules; loading it the
246    /// way the bindings' `Metadata.from_json_file` does and converting back
247    /// for a dictionary build must keep them (#1052).
248    #[test]
249    fn keeps_space_penalty_from_ko_dic_metadata() {
250        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
251            .join("../lindera-ko-dic/metadata.json");
252        let json = std::fs::read_to_string(&path).unwrap();
253        let metadata: Metadata = serde_json::from_str(&json).unwrap();
254        let shipped = metadata.space_penalty.clone().unwrap();
255        assert!(!shipped.rules.is_empty());
256
257        let core = CoreMetadata::from(metadata);
258        assert_eq!(core.space_penalty.as_ref(), Some(&shipped));
259        let as_json = core.space_penalty_json().unwrap();
260        assert!(as_json.contains("\"rules\""));
261        assert!(as_json.contains("JKS"));
262
263        let rebuilt: Metadata = core.into();
264        assert_eq!(rebuilt.space_penalty, Some(shipped));
265    }
266}
lindera_binding_core/metadata.rs

lindera_binding_core/
metadata.rs
