lindera_dictionary/dictionary/
metadata.rs

1use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
2use serde::{Deserialize, Serialize};
3
4use crate::dictionary::context_id_map::ContextIdMap;
5use crate::dictionary::schema::Schema;
6
7const DEFAULT_WORD_COST: i16 = -10000;
8const DEFAULT_LEFT_CONTEXT_ID: u16 = 1288;
9const DEFAULT_RIGHT_CONTEXT_ID: u16 = 1288;
10const DEFAULT_FIELD_VALUE: &str = "*";
11
12#[derive(Clone, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)]
13
14pub struct ModelInfo {
15    pub feature_count: usize,
16    pub label_count: usize,
17    pub max_left_context_id: usize,
18    pub max_right_context_id: usize,
19    pub connection_matrix_size: String,
20    pub version: String,
21    pub training_iterations: u64,
22    pub regularization: f64,
23    pub updated_at: u64,
24}
25
26#[derive(Clone, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)]
27
28pub struct Metadata {
29    pub name: String,                  // Name of the dictionary
30    pub encoding: String,              // Character encoding
31    pub default_word_cost: i16,        // Word cost for simple user dictionary
32    pub default_left_context_id: u16,  // Context ID for simple user dictionary
33    pub default_right_context_id: u16, // Context ID for simple user dictionary
34    pub default_field_value: String,   // Default value for fields in simple user dictionary
35    pub flexible_csv: bool,            // Handle CSV columns flexibly
36    pub skip_invalid_cost_or_id: bool, // Skip invalid cost or ID
37    pub normalize_details: bool,       // Normalize characters
38    /// Reorder connection-cost context IDs by frequency at build time so that
39    /// frequently-used connection-matrix cells cluster in cache. Optional and
40    /// defaults to `false`; when `false` the field is omitted from `metadata.json`
41    /// so existing files stay byte-identical, and the build output is unchanged.
42    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
43    pub connection_id_mapping: bool,
44    /// The context-ID permutation that was applied when this dictionary was built.
45    ///
46    /// Written into the *built* `metadata.json` when `connection_id_mapping` is on, so
47    /// that anything compiled later against this dictionary — most importantly a
48    /// detailed user dictionary — can be relabeled into the same ID space. Absent (and
49    /// omitted from the file) for an un-remapped dictionary, which keeps those builds
50    /// byte-identical. Source `metadata.json` files carry only the boolean flag.
51    #[serde(default, skip_serializing_if = "Option::is_none")]
52    pub context_id_map: Option<ContextIdMap>,
53    pub dictionary_schema: Schema,      // Schema for the dictionary
54    pub user_dictionary_schema: Schema, // Schema for user dictionary
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub model_info: Option<ModelInfo>, // Training model information (optional)
57}
58
59impl Default for Metadata {
60    fn default() -> Self {
61        // Default metadata values can be adjusted as needed
62        Metadata::new(
63            "default".to_string(),
64            "UTF-8".to_string(),
65            DEFAULT_WORD_COST,
66            DEFAULT_LEFT_CONTEXT_ID,
67            DEFAULT_RIGHT_CONTEXT_ID,
68            DEFAULT_FIELD_VALUE.to_string(),
69            false,
70            false,
71            false,
72            Schema::default(),
73            Schema::new(vec![
74                "surface".to_string(),
75                "reading".to_string(),
76                "pronunciation".to_string(),
77            ]),
78        )
79    }
80}
81
82impl Metadata {
83    #[allow(clippy::too_many_arguments)]
84    pub fn new(
85        name: String,
86        encoding: String,
87        simple_word_cost: i16,
88        default_left_context_id: u16,
89        default_right_context_id: u16,
90        default_field_value: String,
91        flexible_csv: bool,
92        skip_invalid_cost_or_id: bool,
93        normalize_details: bool,
94        schema: Schema,
95        userdic_schema: Schema,
96    ) -> Self {
97        Self {
98            encoding,
99            default_word_cost: simple_word_cost,
100            default_left_context_id,
101            default_right_context_id,
102            default_field_value,
103            dictionary_schema: schema,
104            name,
105            flexible_csv,
106            skip_invalid_cost_or_id,
107            normalize_details,
108            connection_id_mapping: false,
109            context_id_map: None,
110            user_dictionary_schema: userdic_schema,
111            model_info: None,
112        }
113    }
114
115    /// Load metadata from binary data (JSON format).
116    /// This provides a consistent interface with other dictionary components.
117    pub fn load(data: &[u8]) -> crate::LinderaResult<Self> {
118        // If data is empty, return an error since metadata is required
119        if data.is_empty() {
120            return Err(crate::error::LinderaErrorKind::Io
121                .with_error(anyhow::anyhow!("Empty metadata data")));
122        }
123
124        // Deserialize as JSON
125        serde_json::from_slice(data).map_err(|err| {
126            crate::error::LinderaErrorKind::Deserialize
127                .with_error(anyhow::anyhow!(err))
128                .add_context("Failed to deserialize metadata from JSON")
129        })
130    }
131
132    /// Load metadata with fallback to default values.
133    /// This is used when feature flags are disabled and data might be empty.
134    pub fn load_or_default(data: &[u8], default_fn: fn() -> Self) -> Self {
135        if data.is_empty() {
136            default_fn()
137        } else {
138            match Self::load(data) {
139                Ok(metadata) => metadata,
140                Err(_) => default_fn(),
141            }
142        }
143    }
144}
145
146#[cfg(test)]
147mod tests {
148    use super::*;
149
150    #[test]
151    fn test_metadata_default() {
152        let metadata = Metadata::default();
153        assert_eq!(metadata.name, "default");
154        // Schema no longer has name field
155    }
156
157    #[test]
158    fn test_metadata_new() {
159        let schema = Schema::default();
160        let metadata = Metadata::new(
161            "TestDict".to_string(),
162            "UTF-8".to_string(),
163            -10000,
164            0,
165            0,
166            "*".to_string(),
167            false,
168            false,
169            false,
170            schema.clone(),
171            Schema::new(vec!["surface".to_string(), "reading".to_string()]),
172        );
173        assert_eq!(metadata.name, "TestDict");
174        // Schema no longer has name field
175    }
176
177    #[test]
178    fn test_metadata_serialization() {
179        let metadata = Metadata::default();
180
181        // Test serialization
182        let serialized = serde_json::to_string(&metadata).unwrap();
183        assert!(serialized.contains("default"));
184        assert!(serialized.contains("schema"));
185        assert!(serialized.contains("name"));
186
187        // Test deserialization
188        let deserialized: Metadata = serde_json::from_str(&serialized).unwrap();
189        assert_eq!(deserialized.name, "default");
190        // Schema no longer has name field
191    }
192}
lindera_dictionary/dictionary/metadata.rs

lindera_dictionary/dictionary/
metadata.rs
