lindera_trainer/

model.rs

use std::collections::BTreeMap;
use std::io::{Read, Write};
use std::num::NonZeroU32;

use anyhow::Result;
use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
use serde::{Deserialize, Serialize};

use crate::corpus::Word;
use lindera_dictionary::viterbi::{LexType, WordEntry, WordId};

/// Convert CRF weight to MeCab-compatible cost using cost-factor.
///
/// MeCab's `tocost(d, n)` formula: `clamp(-n * d, -32767, 32767)`
/// where `d` is the cost-factor and `n` is the CRF weight.
fn tocost(weight: f64, cost_factor: i32) -> i16 {
    let raw = -(cost_factor as f64) * weight;
    raw.round().clamp(i16::MIN as f64, i16::MAX as f64) as i16
}

/// Returns whether `bytes` begins a JSON object, ignoring leading whitespace.
///
/// Used only to decide whether a payload that rkyv already rejected is worth
/// handing to the legacy JSON reader. rkyv is always tried first, so a false
/// positive here cannot misroute a valid model.
///
/// # 引数
///
/// * `bytes` - Raw model file contents.
///
/// # 戻り値
///
/// `true` when the first non-whitespace byte is `{`.
fn looks_like_json_object(bytes: &[u8]) -> bool {
    bytes
        .iter()
        .find(|byte| !byte.is_ascii_whitespace())
        .is_some_and(|byte| *byte == b'{')
}

/// Calculate the optimal cost factor from actual model weights.
///
/// This ensures the full i16 range is utilized, preserving relative differences
/// between weights. The cost factor is computed as `i16::MAX / max_abs_weight`
/// so that the largest weight maps to the boundary of the i16 range.
fn calculate_cost_factor(merged_model: &lindera_crf::MergedModel) -> i32 {
    let mut weight_abs_max = 0f64;

    // Find maximum absolute weight from unigram feature sets
    for feature_set in &merged_model.feature_sets {
        weight_abs_max = weight_abs_max.max(feature_set.weight.abs());
    }

    // Find maximum absolute weight from connection cost matrix
    for hm in &merged_model.matrix {
        for &w in hm.values() {
            weight_abs_max = weight_abs_max.max(w.abs());
        }
    }

    if weight_abs_max > f64::EPSILON {
        (f64::from(i16::MAX) / weight_abs_max) as i32
    } else {
        700 // MeCab default fallback
    }
}

/// Feature set information extracted from CRF training
#[derive(Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize, Clone, Debug)]

pub struct FeatureSetInfo {
    /// Left connection ID learned from CRF training
    pub left_id: u32,
    /// Right connection ID learned from CRF training
    pub right_id: u32,
    /// Feature weight learned from CRF training
    pub weight: f64,
}

/// Trained model with weights and configuration.
#[derive(Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)]

pub struct SerializableModel {
    /// Feature weights from CRF training
    pub feature_weights: Vec<f64>,
    /// Label information
    pub labels: Vec<String>,
    /// Part-of-speech information for each label
    pub pos_info: Vec<String>,
    /// Model metadata
    pub metadata: ModelMetadata,
    /// Connection cost matrix: (right_id, left_id) -> cost.
    ///
    /// Ordered maps rather than hashed ones, at both levels. rkyv archives a
    /// `HashMap` as a SwissTable whose slot assignment resolves collisions in
    /// source iteration order, and `HashMap` iteration order is seeded per
    /// instance -- so serializing the same trained model twice produced files
    /// of identical length with scattered byte differences (#974). A
    /// `BTreeMap` serializes from its key-ordered iterator into a tree whose
    /// shape depends only on the entry count, which makes `model.dat`
    /// byte-reproducible. No stored value changes: both levels are only ever
    /// filled key-by-key from `Vec`-ordered sources.
    pub connection_matrix: BTreeMap<usize, BTreeMap<usize, f64>>,
    /// Maximum left connection ID
    pub max_left_id: usize,
    /// Maximum right connection ID
    pub max_right_id: usize,
    /// Feature set information (left_id, right_id, weight) for each label
    pub feature_sets: Vec<FeatureSetInfo>,
    /// Unknown word category names (from char.def)
    pub unk_category_names: Vec<String>,
    /// Unknown word category features (from unk.def).
    ///
    /// Ordered for the same reason as
    /// [`SerializableModel::connection_matrix`]: rkyv's archived `HashMap`
    /// layout follows source iteration order (#974).
    pub unk_categories: BTreeMap<String, String>,
    /// Raw content of the character definition file (char.def)
    pub char_def_content: String,
    /// Raw content of the feature definition file (feature.def)
    pub feature_def_content: String,
    /// Raw content of the rewrite rule definition file (rewrite.def)
    pub rewrite_def_content: String,
    /// Cost factor for weight-to-cost conversion (MeCab default: 700)
    pub cost_factor: i32,
    /// Left context ID to feature string mapping (for left-id.def)
    pub left_id_map: Vec<(u32, String)>,
    /// Right context ID to feature string mapping (for right-id.def)
    pub right_id_map: Vec<(u32, String)>,
}

#[derive(Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize)]

pub struct ModelMetadata {
    pub version: String,
    pub regularization: f64,
    pub iterations: u64,
    pub feature_count: usize,
    pub label_count: usize,
}

/// Trained model.
pub struct Model {
    pub(crate) raw_model: lindera_crf::RawModel,
    pub(crate) config: super::config::TrainerConfig,
    pub(crate) feature_weights: Vec<f64>,
    pub(crate) labels: Vec<String>,
    pub(crate) user_entries: Vec<(Word, WordEntry, NonZeroU32)>,
    pub(crate) merged_model: Option<lindera_crf::MergedModel>,
    pub(crate) regularization_cost: f64,
    pub(crate) max_iterations: u64,
}

impl Model {
    /// Creates a new model with metadata.
    pub(crate) fn new_with_metadata(
        raw_model: lindera_crf::RawModel,
        config: super::config::TrainerConfig,
        feature_weights: Vec<f64>,
        labels: Vec<String>,
        regularization_cost: f64,
        max_iterations: u64,
    ) -> Self {
        Self {
            raw_model,
            config,
            feature_weights,
            labels,
            user_entries: Vec::new(),
            merged_model: None,
            regularization_cost,
            max_iterations,
        }
    }

    /// Reads the user-defined lexicon file.
    ///
    /// If you want to assign parameters to the user-defined lexicon file, you need to call this
    /// function before exporting the dictionary. The model overwrites the parameter only when it
    /// is `0,0,0`. Otherwise, the parameter is used as is.
    ///
    /// # Arguments
    ///
    /// * `rdr` - Read sink of the user-defined lexicon file.
    pub fn read_user_lexicon<R: Read>(&mut self, rdr: R) -> Result<()> {
        use std::io::BufRead;
        use std::io::BufReader;

        self.merged_model = None;
        let reader = BufReader::new(rdr);

        for line in reader.lines() {
            let line = line?;
            if line.trim().is_empty() || line.starts_with('#') {
                continue;
            }

            let parts: Vec<&str> = line.split(',').collect();
            if parts.len() >= 5 {
                let surface = parts[0];
                let left_id = parts[1].parse::<i32>().unwrap_or(0);
                let right_id = parts[2].parse::<i32>().unwrap_or(0);
                let cost = parts[3].parse::<i16>().unwrap_or(0);
                let features = parts[4..].join(",");

                let word = Word::new(surface, &features);

                // Create word ID for user dictionary entry
                let word_id = WordId::new(LexType::User, self.user_entries.len() as u32);

                let entry = WordEntry::new(word_id, cost, left_id as u16, right_id as u16);

                // Extract features and create feature set for this user entry
                let first_char = surface.chars().next().unwrap_or('\0');
                let cate_id = self.get_category_id(first_char);

                // Apply dictionary rewriter to get ufeature, lfeature, rfeature
                let (ufeature, lfeature, rfeature) =
                    self.config.dictionary_rewriter.rewrite(&features);
                let u_vec: Vec<&str> = ufeature.split(',').collect();
                let l_vec: Vec<&str> = lfeature.split(',').collect();
                let r_vec: Vec<&str> = rfeature.split(',').collect();

                let unigram_features = self
                    .config
                    .feature_extractor
                    .extract_unigram_feature_ids(&u_vec, cate_id);
                let left_features = self
                    .config
                    .feature_extractor
                    .extract_left_feature_ids(&l_vec);
                let right_features = self
                    .config
                    .feature_extractor
                    .extract_right_feature_ids(&r_vec);

                let _feature_set = lindera_crf::FeatureSet::new(
                    &unigram_features,
                    &right_features,
                    &left_features,
                );
                // TODO: Integrate feature_set into provider for proper user lexicon feature handling
                // Currently, we cannot access the provider from this context, which limits
                // the integration of user lexicon features into the trained model.
                // This should be refactored to allow proper feature integration.

                // Create a label ID without modifying the provider
                // Since we can't clone the provider, we'll use a fixed ID based on entry count
                let label_id = NonZeroU32::new(1000000 + self.user_entries.len() as u32 + 1)
                    .ok_or_else(|| anyhow::anyhow!("Failed to create label ID"))?;

                self.user_entries.push((word, entry, label_id));
            }
        }

        Ok(())
    }

    fn get_category_id(&self, ch: char) -> u32 {
        // Use CharacterDefinition to map character to category ID
        // This works for any dictionary (IPADIC, UniDic, ko-dic, CC-CEDICT, etc.)
        let char_def = &self.config.dict.character_definition;
        let categories = char_def.lookup_categories(ch);

        // Return the first category ID, or 0 (DEFAULT) if no categories match
        if !categories.is_empty() {
            categories[0].0 as u32
        } else {
            0 // DEFAULT category
        }
    }

    /// Writes the model to a writer.
    pub fn write_model<W: Write>(&self, writer: &mut W) -> Result<()> {
        let feature_weights = self.feature_weights.clone();

        // Extract connection cost matrix from the trained model
        let merged_model = self.raw_model.merge()?;

        // Compute optimal cost factor from actual weight range
        let cost_factor = calculate_cost_factor(&merged_model);

        let mut connection_matrix: BTreeMap<usize, BTreeMap<usize, f64>> = BTreeMap::new();
        let mut max_left_id = 0;
        let mut max_right_id = 0;

        for (right_id, left_map) in merged_model.matrix.iter().enumerate() {
            max_right_id = max_right_id.max(right_id);
            let mut inner_map = BTreeMap::new();

            for (&left_id, &weight) in left_map.iter() {
                max_left_id = max_left_id.max(left_id as usize);
                inner_map.insert(left_id as usize, weight);
            }

            if !inner_map.is_empty() {
                connection_matrix.insert(right_id, inner_map);
            }
        }

        // Extract feature_sets information from merged_model
        let feature_sets: Vec<FeatureSetInfo> = merged_model
            .feature_sets
            .iter()
            .map(|fs| FeatureSetInfo {
                left_id: fs.left_id.get(),
                right_id: fs.right_id.get(),
                weight: fs.weight,
            })
            .collect();

        // Extract unknown word category information
        let char_def = &self.config.dict.character_definition;
        let unk_category_names: Vec<String> = char_def
            .categories()
            .iter()
            .map(|s| s.to_string())
            .collect();

        // Build left/right ID maps from merged model's connection IDs
        // before feature_sets is moved into SerializableModel
        // Build left/right ID maps from merged model's connection IDs.
        // Use feature strings (POS info) from config.features, not surface forms.
        let features = &self.config.features;
        let unk_categories = &self.config.unk_categories;
        let unk_start_idx = self
            .labels
            .len()
            .saturating_sub(self.config.dict.character_definition.categories().len());

        let get_feature_string = |i: usize| -> String {
            if i < unk_start_idx {
                // Dictionary entry: use feature string from config.features
                if i < features.len() {
                    features[i].clone()
                } else {
                    "*".to_string()
                }
            } else {
                // Unknown word category: use unk feature string
                let category_idx = i - unk_start_idx;
                let char_def = &self.config.dict.character_definition;
                let categories = char_def.categories();
                if category_idx < categories.len() {
                    let cat_name = &categories[category_idx];
                    unk_categories
                        .get(cat_name)
                        .cloned()
                        .unwrap_or_else(|| format!("UNK_{cat_name}"))
                } else {
                    format!("UNK_{}", category_idx)
                }
            }
        };

        let left_id_map = {
            let mut id_to_feat: std::collections::HashMap<u32, String> =
                std::collections::HashMap::new();
            for (i, fs) in feature_sets.iter().enumerate() {
                let lid = fs.left_id;
                id_to_feat.entry(lid).or_insert_with(|| {
                    let feat = get_feature_string(i);
                    let (_, lfeature, _) = self.config.dictionary_rewriter.rewrite(&feat);
                    lfeature
                });
            }
            let mut entries: Vec<(u32, String)> = id_to_feat.into_iter().collect();
            entries.sort_by_key(|&(id, _)| id);
            entries
        };
        let right_id_map = {
            let mut id_to_feat: std::collections::HashMap<u32, String> =
                std::collections::HashMap::new();
            for (i, fs) in feature_sets.iter().enumerate() {
                let rid = fs.right_id;
                id_to_feat.entry(rid).or_insert_with(|| {
                    let feat = get_feature_string(i);
                    let (_, _, rfeature) = self.config.dictionary_rewriter.rewrite(&feat);
                    rfeature
                });
            }
            let mut entries: Vec<(u32, String)> = id_to_feat.into_iter().collect();
            entries.sort_by_key(|&(id, _)| id);
            entries
        };

        let serializable_model = SerializableModel {
            feature_weights,
            labels: self.labels.clone(),
            pos_info: self.extract_pos_info(),
            metadata: ModelMetadata {
                version: "1.0.0".to_string(),
                regularization: self.regularization_cost,
                iterations: self.max_iterations,
                feature_count: self.feature_weights.len(),
                label_count: self.labels.len(),
            },
            connection_matrix,
            max_left_id,
            max_right_id,
            feature_sets,
            unk_category_names,
            unk_categories: self.config.unk_categories.clone(),
            char_def_content: self.config.char_def_content.clone(),
            feature_def_content: self.config.feature_def_content.clone(),
            rewrite_def_content: self.config.rewrite_def_content.clone(),
            cost_factor,
            left_id_map,
            right_id_map,
        };

        // Use rkyv for efficient binary serialization
        let encoded = rkyv::to_bytes::<rkyv::rancor::Error>(&serializable_model)
            .map_err(|e| anyhow::anyhow!("Failed to serialize model: {}", e))?;
        writer.write_all(&encoded)?;

        Ok(())
    }

    /// Reads a trained model from a reader.
    ///
    /// Accepts the current rkyv binary format and, for backward
    /// compatibility, the legacy JSON format.
    ///
    /// # Arguments
    ///
    /// * `reader` - Reader containing the serialized model data
    ///
    /// # Returns
    ///
    /// The deserialized model.
    ///
    /// # Errors
    ///
    /// Returns an error naming the format that failed. A model written by a
    /// build with a different `SerializableModel` layout is reported as such,
    /// with the instruction to re-train -- it used to fall through to the
    /// legacy JSON reader and surface as an unrelated UTF-8 error (#974).
    pub fn read_model<R: Read>(mut reader: R) -> Result<SerializableModel> {
        let mut buffer = Vec::new();
        reader.read_to_end(&mut buffer)?;

        if buffer.is_empty() {
            return Err(anyhow::anyhow!(
                "model file is empty; re-run `lindera train` to produce one"
            ));
        }

        // Current format: rkyv. Tried first because it is self-validating, so
        // a successful parse proves the on-disk layout matched this build.
        let rkyv_error = match rkyv::from_bytes::<SerializableModel, rkyv::rancor::Error>(&buffer) {
            Ok(mut model) => {
                // Backward compatibility: if feature_sets is empty, generate from feature_weights
                if model.feature_sets.is_empty() {
                    model.feature_sets = model
                        .feature_weights
                        .iter()
                        .map(|&weight| FeatureSetInfo {
                            left_id: 0,
                            right_id: 0,
                            weight,
                        })
                        .collect();
                }

                return Ok(model);
            }
            Err(err) => err,
        };

        // Legacy format: JSON, attempted only when the payload actually looks
        // like JSON. Routing on the leading byte as the *primary* check would
        // be wrong -- a valid rkyv payload starts with the low byte of
        // `feature_weights[0]` as a little-endian f64, which is `{` about one
        // time in 256.
        if looks_like_json_object(&buffer) {
            let json_str = String::from_utf8(buffer).map_err(|err| {
                anyhow::anyhow!("model file starts like legacy JSON but is not valid UTF-8: {err}")
            })?;
            return serde_json::from_str(&json_str)
                .map_err(|err| anyhow::anyhow!("failed to parse legacy JSON model: {err}"));
        }

        Err(anyhow::anyhow!(
            "failed to deserialize model: {rkyv_error}. The file is not a legacy JSON \
             model either, so it was most likely written by a different version of \
             lindera-trainer; re-run `lindera train` to regenerate it."
        ))
    }

    /// Gets the merged model, creating it if necessary
    fn get_merged_model(&self) -> Result<lindera_crf::MergedModel> {
        Ok(self.raw_model.merge()?)
    }

    /// Generate a default feature string with appropriate number of fields
    /// based on existing entries in the dictionary
    fn generate_default_features(&self) -> String {
        // Try to infer field count from existing unk_categories
        if let Some(first_unk) = self.config.unk_categories.values().next() {
            let field_count = first_unk.split(',').count();
            return vec!["*"; field_count].join(",");
        }

        // Fallback: try from config.features
        if let Some(first_feature) = self.config.features.first() {
            let field_count = first_feature.split(',').count();
            return vec!["*"; field_count].join(",");
        }

        // Ultimate fallback (should rarely happen)
        "*".to_string()
    }

    /// Extracts part-of-speech information for each label
    fn extract_pos_info(&self) -> Vec<String> {
        // Get POS info from config.features (parallel to surfaces/labels)
        let mut pos_info = Vec::new();

        for (i, label) in self.labels.iter().enumerate() {
            // First check if this is within the vocabulary (config.features range)
            if i < self.config.features.len() {
                pos_info.push(self.config.features[i].clone());
            } else {
                // For unknown word categories (DEFAULT, HIRAGANA, etc.),
                // look up POS info from unk_categories
                let unk_features = self
                    .config
                    .unk_categories
                    .get(label)
                    .cloned()
                    .unwrap_or_else(|| {
                        // Fallback: use DEFAULT category if label not found in unk_categories
                        self.config
                            .unk_categories
                            .get("DEFAULT")
                            .cloned()
                            .unwrap_or_else(|| self.generate_default_features())
                    });
                pos_info.push(unk_features);
            }
        }

        pos_info
    }

    /// Writes the dictionary files in Lindera format.
    pub fn write_dictionary<W1, W2, W3, W4>(
        &self,
        lexicon_wtr: &mut W1,
        connector_wtr: &mut W2,
        unk_handler_wtr: &mut W3,
        user_lexicon_wtr: &mut W4,
    ) -> Result<()>
    where
        W1: Write,
        W2: Write,
        W3: Write,
        W4: Write,
    {
        // Write lexicon with trained weights
        self.write_lexicon(lexicon_wtr)?;

        // Write connection cost matrix with trained costs
        self.write_connection_costs(connector_wtr)?;

        // Write unknown word handler with trained parameters
        self.write_unknown_dictionary(unk_handler_wtr)?;

        // Write user lexicon with trained weights
        self.write_user_lexicon(user_lexicon_wtr)?;

        Ok(())
    }

    pub fn write_lexicon<W: Write>(&self, writer: &mut W) -> Result<()> {
        let merged_model = self.get_merged_model()?;
        let cost_factor = calculate_cost_factor(&merged_model);

        for (i, surface) in self.config.surfaces.iter().enumerate() {
            if i < merged_model.feature_sets.len() {
                let feature_set = merged_model.feature_sets[i];
                let cost = tocost(feature_set.weight, cost_factor);
                let features = &self.config.features[i];
                writeln!(
                    writer,
                    "{},{},{},{},{}",
                    surface,
                    feature_set.left_id.get(),
                    feature_set.right_id.get(),
                    cost,
                    features
                )?;
            } else {
                let features = &self.config.features[i];
                writeln!(writer, "{surface},0,0,0,{features}")?;
            }
        }

        Ok(())
    }

    pub fn write_connection_costs<W: Write>(&self, writer: &mut W) -> Result<()> {
        let merged_model = self.get_merged_model()?;
        let cost_factor = calculate_cost_factor(&merged_model);

        // Dense matrix dimensions: (right_size, left_size)
        // +1 for BOS/EOS (id=0)
        let right_size = merged_model.right_conn_to_left_feats.len() + 1;
        let left_size = merged_model.left_conn_to_right_feats.len() + 1;
        writeln!(writer, "{right_size} {left_size}")?;

        // Dense matrix: output all (right_id, left_id) pairs
        // Unseen pairs get maximum penalty cost to block unlearned transitions,
        // forcing Viterbi to prefer paths through learned POS bigram connections.
        for right_id in 0..right_size {
            for left_id in 0..left_size {
                let cost = if let Some(weight) = merged_model
                    .matrix
                    .get(right_id)
                    .and_then(|hm| hm.get(&(left_id as u32)))
                    .copied()
                {
                    tocost(weight, cost_factor)
                } else {
                    i16::MAX
                };
                writeln!(writer, "{right_id} {left_id} {cost}")?;
            }
        }

        Ok(())
    }

    pub fn write_unknown_dictionary<W: Write>(&self, writer: &mut W) -> Result<()> {
        let merged_model = self.get_merged_model()?;
        let cost_factor = calculate_cost_factor(&merged_model);

        let char_def = &self.config.dict.character_definition;
        let category_names = char_def.categories();
        let unk_start_idx = self.config.surfaces.len();

        for (i, category_name) in category_names.iter().enumerate() {
            let feature_set_idx = unk_start_idx + i;
            if feature_set_idx < merged_model.feature_sets.len() {
                let feature_set = merged_model.feature_sets[feature_set_idx];
                let cost = tocost(feature_set.weight, cost_factor);

                let default_features = self.generate_default_features();
                let features = self
                    .config
                    .unk_categories
                    .get(category_name.as_str())
                    .map(|s| s.as_str())
                    .unwrap_or(default_features.as_str());

                writeln!(
                    writer,
                    "{},{},{},{},{}",
                    category_name,
                    feature_set.left_id.get(),
                    feature_set.right_id.get(),
                    cost,
                    features
                )?;
            }
        }

        Ok(())
    }

    /// Writes the user lexicon with trained parameters.
    ///
    /// Entries are written in the order `read_user_lexicon` read them, so the
    /// output is byte-reproducible and diffable against the input file (#981).
    /// An entry whose left id, right id and cost are all zero receives
    /// inferred context ids and a trained cost; any other entry is re-emitted
    /// with its input parameters unchanged, per the contract documented on
    /// `read_user_lexicon`.
    ///
    /// # 引数
    ///
    /// * `writer` - Write sink for the user lexicon CSV.
    ///
    /// # 戻り値
    ///
    /// `Ok(())` on success; writing nothing when no user lexicon was read.
    fn write_user_lexicon<W: Write>(&self, writer: &mut W) -> Result<()> {
        if self.user_entries.is_empty() {
            return Ok(());
        }

        let merged_model = self.get_merged_model()?;
        let cost_factor = calculate_cost_factor(&merged_model);

        for (word, entry, _label_id) in &self.user_entries {
            let surface = word.surface();
            let features = word.feature();
            if entry.left_id() == 0 && entry.right_id() == 0 && entry.word_cost() == 0 {
                let (left_id, right_id) = self.infer_context_ids(surface, features);
                let raw_cost = self.get_user_word_cost(surface) as f64 / 1000.0;
                let cost = tocost(raw_cost, cost_factor);
                writeln!(writer, "{surface},{left_id},{right_id},{cost},{features}")?;
            } else {
                writeln!(
                    writer,
                    "{},{},{},{},{}",
                    surface,
                    entry.left_id(),
                    entry.right_id(),
                    entry.word_cost(),
                    features
                )?;
            }
        }

        Ok(())
    }

    fn get_user_word_cost(&self, _surface: &str) -> i32 {
        // Return trained cost for user lexicon words
        // Could be based on trained model weights
        800 // Slightly lower cost than default for user words
    }

    /// Calculate unknown word cost based on trained feature weights using dynamic calculation
    pub fn get_unknown_word_cost(&self, category: usize) -> i32 {
        // Get category name from character definition
        let char_def = &self.config.dict.character_definition;
        let category_names = char_def.categories();

        if category < category_names.len() {
            let category_name = &category_names[category];
            // Look up cost from unk_costs, with fallback to 2000
            self.config
                .unk_costs
                .get(category_name)
                .copied()
                .unwrap_or(2000)
        } else {
            2000 // Default fallback cost
        }
    }

    /// Infer context IDs (left_id, right_id) from surface form and features.
    /// Finds the most similar POS pattern from trained vocabulary and uses its ID.
    fn infer_context_ids(&self, surface: &str, features: &str) -> (u32, u32) {
        // Parse feature string to get POS information
        let feature_parts: Vec<&str> = features.split(',').collect();

        // Find best matching entry from trained vocabulary by comparing features
        // Try to match increasingly general patterns:
        // 1. Exact feature match (all fields)
        // 2. First 2 fields match (main POS + sub POS)
        // 3. First field match (main POS only)
        // 4. Same character category (via CharacterDefinition)

        let char_def = &self.config.dict.character_definition;

        // Strategy 1 & 2 & 3: Match by feature similarity
        let mut best_match_idx: Option<usize> = None;
        let mut best_match_score = 0;

        for (i, _label) in self.labels.iter().enumerate() {
            // Skip unknown word categories
            if i >= self.config.features.len() {
                break;
            }

            let vocab_features = &self.config.features[i];
            let vocab_parts: Vec<&str> = vocab_features.split(',').collect();

            // Calculate similarity score
            let mut score = 0;
            let max_fields = feature_parts.len().min(vocab_parts.len());

            for j in 0..max_fields {
                if feature_parts[j] == vocab_parts[j] {
                    // Weight earlier fields more heavily (POS > sub-POS > details)
                    score += (max_fields - j) * 10;
                }
            }

            if score > best_match_score {
                best_match_score = score;
                best_match_idx = Some(i);
            }
        }

        // If found a match from vocabulary, look up from user_entries
        if let Some(idx) = best_match_idx {
            // Try to find in user_entries first
            if idx < self.user_entries.len() {
                let (_, entry, _) = &self.user_entries[idx];
                return (entry.left_id(), entry.right_id());
            }
        }

        // Strategy 4: If no good match found, use character category
        if best_match_score == 0 && !surface.is_empty() {
            let first_char = surface.chars().next().unwrap();
            let categories = char_def.lookup_categories(first_char);

            if !categories.is_empty() {
                let category_id = categories[0].0 as u32;
                // Use category ID as both left and right ID
                return (category_id, category_id);
            }
        }

        // Ultimate fallback: use first user_entry's IDs or default to 0
        if let Some((_, entry, _)) = self.user_entries.first() {
            (entry.left_id(), entry.right_id())
        } else {
            (0, 0)
        }
    }

    /// Calculate maximum context ID from training data
    #[allow(dead_code)]
    fn calculate_max_context_id(&self) -> u32 {
        let mut max_id = 0u32;

        // Get maximum ID from all user_entries
        for (_, entry, _) in &self.user_entries {
            max_id = max_id.max(entry.left_id()).max(entry.right_id());
        }

        max_id
    }

    /// Calculate connection cost based on trained model
    #[allow(dead_code)]
    fn get_trained_connection_cost(&self, from_id: usize, to_id: usize) -> i32 {
        // Use CRF feature weights to calculate connection cost
        let weights = self.raw_model.weights();

        if weights.is_empty() {
            return 0; // Fallback
        }

        // Calculate cost based on context ID combination
        let cost_index = (from_id * 1000 + to_id) % weights.len();
        let raw_cost = weights[cost_index];

        // Negative weight = low cost (good connection), positive weight = high cost (bad connection)
        let scaled_cost = (-raw_cost * 1000.0) as i32;

        // Limit cost range (-10000 to 10000)
        scaled_cost.clamp(-10000, 10000)
    }

    /// Gets the number of features in the model.
    pub fn num_features(&self) -> usize {
        // Return the actual feature count from the raw model
        // The raw model contains the feature weights vector
        self.feature_weights.len()
    }

    /// Gets the number of labels in the model.
    pub fn num_labels(&self) -> usize {
        // Return the actual label count
        self.labels.len()
    }

    /// Gets the raw CRF model for advanced operations.
    pub fn raw_model(&self) -> &lindera_crf::RawModel {
        &self.raw_model
    }

    /// Writes the bigram details in three separate files.
    ///
    /// This method outputs:
    /// - Left features: per left connection id, the quoted right-side (`%R`)
    ///   feature strings it groups, `*` for absent slots
    /// - Right features: per right connection id, the quoted left-side (`%L`)
    ///   feature strings it groups
    /// - Costs: one `{left}/{right}\t{cost}` line per raw bigram feature
    ///   pair, where id 0 renders as `BOS` / `EOS` and names come from the
    ///   feature extractor
    ///
    /// All three files are byte-reproducible for the same model: the loops
    /// run over vectors or key-sorted map entries, never over raw map
    /// iteration order (#981).
    ///
    /// # 引数
    ///
    /// * `left_wtr` - Write sink for the left-feature file.
    /// * `right_wtr` - Write sink for the right-feature file.
    /// * `cost_wtr` - Write sink for the feature-pair cost file.
    ///
    /// # 戻り値
    ///
    /// `Ok(())` on success.
    pub fn write_bigram_details<L, R, C>(
        &self,
        left_wtr: L,
        right_wtr: R,
        cost_wtr: C,
    ) -> Result<()>
    where
        L: Write,
        R: Write,
        C: Write,
    {
        use std::collections::HashMap;
        use std::io::BufWriter;

        // Get merged model for detailed analysis
        let merged_model = self.get_merged_model()?;
        let cost_factor = self.config.cost_factor;

        // Real feature names, inverted from the extractor's string-to-id maps
        // (the same inversion write_left_id_def uses): id -> feature string.
        let right_features: HashMap<u32, &str> = self
            .config
            .feature_extractor
            .right_feature_ids
            .iter()
            .map(|(s, &id)| (id.get(), s.as_str()))
            .collect();
        let left_features: HashMap<u32, &str> = self
            .config
            .feature_extractor
            .left_feature_ids
            .iter()
            .map(|(s, &id)| (id.get(), s.as_str()))
            .collect();

        // Write left features
        let mut left_wtr = BufWriter::new(left_wtr);
        for (conn_id, left_feat_ids) in merged_model.left_conn_to_right_feats.iter().enumerate() {
            write!(&mut left_wtr, "{}\t", conn_id + 1)?;
            for (i, feat_id) in left_feat_ids.iter().enumerate() {
                if i != 0 {
                    write!(&mut left_wtr, ",")?;
                }
                if let Some(feat_id) = feat_id {
                    let feat_str = right_features.get(&feat_id.get()).copied().unwrap_or("*");
                    write!(&mut left_wtr, "\"{feat_str}\"")?;
                } else {
                    write!(&mut left_wtr, "*")?;
                }
            }
            writeln!(&mut left_wtr)?;
        }

        // Write right features
        let mut right_wtr = BufWriter::new(right_wtr);
        for (conn_id, right_feat_ids) in merged_model.right_conn_to_left_feats.iter().enumerate() {
            write!(&mut right_wtr, "{}\t", conn_id + 1)?;
            for (i, feat_id) in right_feat_ids.iter().enumerate() {
                if i != 0 {
                    write!(&mut right_wtr, ",")?;
                }
                if let Some(feat_id) = feat_id {
                    let feat_str = left_features.get(&feat_id.get()).copied().unwrap_or("*");
                    write!(&mut right_wtr, "\"{feat_str}\"")?;
                } else {
                    write!(&mut right_wtr, "*")?;
                }
            }
            writeln!(&mut right_wtr)?;
        }

        // Write bigram costs with feature pair names. The source is the raw
        // model's bigram weight table -- left bigram feature id (0 = BOS) to
        // right bigram feature id (0 = EOS) to weight index -- which is the
        // id space these name maps describe. The inner maps are key-sorted so
        // the output is byte-reproducible (#981).
        let mut cost_wtr = BufWriter::new(cost_wtr);
        for (left_feat_id, hm) in self.raw_model.bigram_weight_indices().iter().enumerate() {
            let left_feat_str = if left_feat_id == 0 {
                "BOS"
            } else {
                left_features
                    .get(&(left_feat_id as u32))
                    .copied()
                    .unwrap_or("*")
            };
            let mut entries: Vec<(u32, u32)> = hm.iter().map(|(&k, &v)| (k, v)).collect();
            entries.sort_unstable_by_key(|&(k, _)| k);
            for (right_feat_id, weight_index) in entries {
                let right_feat_str = if right_feat_id == 0 {
                    "EOS"
                } else {
                    right_features.get(&right_feat_id).copied().unwrap_or("*")
                };
                let weight = self
                    .raw_model
                    .weights()
                    .get(weight_index as usize)
                    .copied()
                    .ok_or_else(|| {
                        anyhow::anyhow!("bigram weight index {weight_index} out of range")
                    })?;
                let cost = tocost(weight, cost_factor);
                writeln!(&mut cost_wtr, "{left_feat_str}/{right_feat_str}\t{cost}")?;
            }
        }

        Ok(())
    }

    /// Evaluates the model on test data.
    /// Returns a simple evaluation score based on feature weights.
    pub fn evaluate(&self, _test_lattices: &[lindera_crf::Lattice]) -> f64 {
        // For now, return a simple evaluation based on the model's feature weights
        // A more sophisticated implementation would compute actual likelihood scores
        let weights = self.raw_model.weights();

        // Compute average absolute weight as a simple evaluation metric
        if weights.is_empty() {
            0.0
        } else {
            let sum: f64 = weights.iter().map(|w| w.abs()).sum();
            sum / weights.len() as f64
        }
    }

    /// Write dictionary components to separate buffers using optimized serialization
    pub fn write_dictionary_buffers(
        &self,
        lexicon: &mut Vec<u8>,
        connector: &mut Vec<u8>,
        unk_handler: &mut Vec<u8>,
        user_lexicon: &mut Vec<u8>,
    ) -> Result<()> {
        // Serialize lexicon data
        let lexicon_data = rkyv::to_bytes::<rkyv::rancor::Error>(&self.labels)
            .map_err(|e| anyhow::anyhow!("Failed to serialize lexicon: {}", e))?;
        lexicon.extend_from_slice(&lexicon_data);

        // Serialize connection costs (feature weights as connection matrix)
        let connection_data = rkyv::to_bytes::<rkyv::rancor::Error>(&self.feature_weights)
            .map_err(|e| anyhow::anyhow!("Failed to serialize connector: {}", e))?;
        connector.extend_from_slice(&connection_data);

        // Serialize unknown word handler (simplified data)
        let unk_data = rkyv::to_bytes::<rkyv::rancor::Error>(&self.user_entries.len())
            .map_err(|e| anyhow::anyhow!("Failed to serialize unknown handler: {}", e))?;
        unk_handler.extend_from_slice(&unk_data);

        // Serialize user lexicon (config info as user lexicon)
        let user_data = rkyv::to_bytes::<rkyv::rancor::Error>(&self.config.surfaces)
            .map_err(|e| anyhow::anyhow!("Failed to serialize user lexicon: {}", e))?;
        user_lexicon.extend_from_slice(&user_data);

        Ok(())
    }
}

impl SerializableModel {
    /// Generate a default feature string with appropriate number of fields
    /// based on existing entries in the dictionary
    fn generate_default_features(&self) -> String {
        // Try to infer field count from existing unk_categories
        if let Some(first_unk) = self.unk_categories.values().next() {
            let field_count = first_unk.split(',').count();
            return vec!["*"; field_count].join(",");
        }

        // Fallback: try from pos_info
        if let Some(first_pos) = self.pos_info.first() {
            let field_count = first_pos.split(',').count();
            return vec!["*"; field_count].join(",");
        }

        // Ultimate fallback (should rarely happen)
        "*".to_string()
    }

    /// Write lexicon file with proper cost calculation
    pub fn write_lexicon<W: std::io::Write>(&self, writer: &mut W) -> anyhow::Result<()> {
        // Unknown word categories are at the end of labels, skip them (they go to unk.def)
        let unk_start_idx = self
            .labels
            .len()
            .saturating_sub(self.unk_category_names.len());

        // Write lexicon entries using learned connection IDs and costs
        for (i, label) in self.labels.iter().enumerate() {
            if i >= unk_start_idx {
                continue;
            }

            if i < self.feature_sets.len() {
                let fs = &self.feature_sets[i];
                let pos_info_str;
                let pos_info = if i < self.pos_info.len() {
                    &self.pos_info[i]
                } else {
                    pos_info_str = self.generate_default_features();
                    &pos_info_str
                };

                let cost = tocost(fs.weight, self.cost_factor);

                writeln!(
                    writer,
                    "{},{},{},{},{}",
                    label, fs.left_id, fs.right_id, cost, pos_info
                )?;
            }
        }

        Ok(())
    }

    /// Write dense connection cost matrix (MeCab-compatible)
    pub fn write_connection_costs<W: std::io::Write>(&self, writer: &mut W) -> anyhow::Result<()> {
        if !self.connection_matrix.is_empty() {
            let right_size = self.max_right_id + 1;
            let left_size = self.max_left_id + 1;

            writeln!(writer, "{right_size} {left_size}")?;

            // Dense matrix: all (right_id, left_id) pairs
            // Unseen pairs get maximum penalty cost to block unlearned transitions,
            // forcing Viterbi to prefer paths through learned POS bigram connections.
            for right_id in 0..right_size {
                for left_id in 0..left_size {
                    let cost = if let Some(&weight) = self
                        .connection_matrix
                        .get(&right_id)
                        .and_then(|inner| inner.get(&left_id))
                    {
                        tocost(weight, self.cost_factor)
                    } else {
                        i16::MAX
                    };
                    writeln!(writer, "{right_id} {left_id} {cost}")?;
                }
            }
        } else {
            writeln!(writer, "0 0")?;
        }

        Ok(())
    }

    /// Update metadata.json with trained model values.
    ///
    /// The emitted `model_info` section deliberately carries no timestamp, so
    /// exporting the same model twice produces byte-identical output (#981);
    /// serde_json's map is ordered, so key order is stable too.
    ///
    /// # 引数
    ///
    /// * `base_metadata_path` - Path of the base `metadata.json` to update.
    /// * `writer` - Write sink for the updated JSON.
    pub fn update_metadata_json<W: std::io::Write>(
        &self,
        base_metadata_path: &std::path::Path,
        writer: &mut W,
    ) -> anyhow::Result<()> {
        // Read the base metadata.json file
        let base_content = std::fs::read_to_string(base_metadata_path)?;
        let mut metadata: serde_json::Value = serde_json::from_str(&base_content)?;

        // Calculate updated values based on trained model
        let updated_default_cost = if !self.feature_weights.is_empty() {
            // Calculate median feature weight for default cost
            let mut weights = self.feature_weights.clone();
            weights.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
            let median_weight = if weights.is_empty() {
                0.0
            } else {
                weights[weights.len() / 2]
            };
            // Convert to appropriate cost range for practical use
            (median_weight * 500.0).abs() as i32 + 1500
        } else {
            // Keep existing value if no trained weights
            metadata
                .get("default_word_cost")
                .and_then(|v| v.as_i64())
                .unwrap_or(2000) as i32
        };

        // Update metadata with trained model values
        metadata["default_word_cost"] =
            serde_json::Value::Number(serde_json::Number::from(updated_default_cost));

        // Add model_info section with training statistics
        let max_context_id = std::cmp::max(self.max_left_id, self.max_right_id);
        metadata["model_info"] = serde_json::json!({
            "feature_count": self.feature_weights.len(),
            "label_count": self.labels.len(),
            "max_left_context_id": self.max_left_id,
            "max_right_context_id": self.max_right_id,
            "connection_matrix_size": format!("{}x{}", max_context_id + 1, max_context_id + 1),
            "version": self.metadata.version,
            "training_iterations": self.metadata.iterations,
            "regularization": self.metadata.regularization,
        });

        // Write updated metadata
        let formatted = serde_json::to_string_pretty(&metadata)?;
        writer.write_all(formatted.as_bytes())?;

        Ok(())
    }

    /// Write unknown word definitions
    pub fn write_unknown_dictionary<W: std::io::Write>(
        &self,
        writer: &mut W,
    ) -> anyhow::Result<()> {
        let unk_start_idx = self
            .labels
            .len()
            .saturating_sub(self.unk_category_names.len());

        for (i, category_name) in self.unk_category_names.iter().enumerate() {
            let feature_idx = unk_start_idx + i;

            if feature_idx < self.feature_sets.len() {
                let fs = &self.feature_sets[feature_idx];

                let features = self
                    .unk_categories
                    .get(category_name)
                    .cloned()
                    .unwrap_or_else(|| self.generate_default_features());

                let cost = tocost(fs.weight, self.cost_factor);

                writeln!(
                    writer,
                    "{},{},{},{},{}",
                    category_name, fs.left_id, fs.right_id, cost, features
                )?;
            }
        }

        Ok(())
    }

    /// Writes the character definition file (char.def) content preserved from training.
    pub fn write_char_def<W: std::io::Write>(&self, writer: &mut W) -> anyhow::Result<()> {
        writer.write_all(self.char_def_content.as_bytes())?;
        Ok(())
    }

    /// Writes the feature definition file (feature.def) content preserved from training.
    pub fn write_feature_def<W: std::io::Write>(&self, writer: &mut W) -> anyhow::Result<()> {
        writer.write_all(self.feature_def_content.as_bytes())?;
        Ok(())
    }

    /// Writes the rewrite rule definition file (rewrite.def) content preserved from training.
    pub fn write_rewrite_def<W: std::io::Write>(&self, writer: &mut W) -> anyhow::Result<()> {
        writer.write_all(self.rewrite_def_content.as_bytes())?;
        Ok(())
    }

    /// Writes left-id.def: maps left context IDs to their feature strings.
    pub fn write_left_id_def<W: std::io::Write>(&self, writer: &mut W) -> anyhow::Result<()> {
        writeln!(writer, "0 BOS/EOS")?;
        for (id, feat_str) in &self.left_id_map {
            writeln!(writer, "{id} {feat_str}")?;
        }
        Ok(())
    }

    /// Writes right-id.def: maps right context IDs to their feature strings.
    pub fn write_right_id_def<W: std::io::Write>(&self, writer: &mut W) -> anyhow::Result<()> {
        writeln!(writer, "0 BOS/EOS")?;
        for (id, feat_str) in &self.right_id_map {
            writeln!(writer, "{id} {feat_str}")?;
        }
        Ok(())
    }
}

impl Model {
    /// Writes left-id.def: maps left context IDs to their feature strings.
    ///
    /// Format: `id feature_string` (one per line), with BOS/EOS at id=0.
    pub fn write_left_id_def<W: Write>(&self, writer: &mut W) -> Result<()> {
        // id=0 is reserved for BOS/EOS
        writeln!(writer, "0 BOS/EOS")?;

        // Build reverse mapping: NonZeroU32 -> feature string
        let left_ids = &self.config.feature_extractor.left_feature_ids;
        let mut entries: Vec<(u32, &str)> = left_ids
            .iter()
            .map(|(feat_str, &id)| (id.get(), feat_str.as_str()))
            .collect();
        entries.sort_by_key(|&(id, _)| id);

        for (id, feat_str) in entries {
            writeln!(writer, "{id} {feat_str}")?;
        }

        Ok(())
    }

    /// Writes right-id.def: maps right context IDs to their feature strings.
    ///
    /// Format: `id feature_string` (one per line), with BOS/EOS at id=0.
    pub fn write_right_id_def<W: Write>(&self, writer: &mut W) -> Result<()> {
        // id=0 is reserved for BOS/EOS
        writeln!(writer, "0 BOS/EOS")?;

        // Build reverse mapping: NonZeroU32 -> feature string
        let right_ids = &self.config.feature_extractor.right_feature_ids;
        let mut entries: Vec<(u32, &str)> = right_ids
            .iter()
            .map(|(feat_str, &id)| (id.get(), feat_str.as_str()))
            .collect();
        entries.sort_by_key(|&(id, _)| id);

        for (id, feat_str) in entries {
            writeln!(writer, "{id} {feat_str}")?;
        }

        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use crate::{Trainer, TrainerConfig};
    use std::io::Cursor;

    #[test]
    fn test_trainer_creation() {
        // Test that Trainer can be created from a valid config
        let lexicon_data = "外国,0,0,5000,名詞,一般,*,*,*,*,外国,ガイコク,ガイコク\n";
        let char_data = "# char.def placeholder\n";
        let unk_data = "# unk.def placeholder\n";
        let feature_data = "UNIGRAM:%F[0]\nLEFT:%L[0]\nRIGHT:%R[0]\n";
        let rewrite_data = "# rewrite.def placeholder\n";

        let config_result = TrainerConfig::from_readers(
            Cursor::new(lexicon_data.as_bytes()),
            Cursor::new(char_data.as_bytes()),
            Cursor::new(unk_data.as_bytes()),
            Cursor::new(feature_data.as_bytes()),
            Cursor::new(rewrite_data.as_bytes()),
        );

        assert!(config_result.is_ok());
        let config = config_result.unwrap();

        // Test trainer creation with builder pattern
        let trainer = Trainer::new(config)
            .unwrap()
            .regularization_cost(0.01)
            .max_iter(10)
            .num_threads(1);

        // Verify trainer settings using the getters
        assert_eq!(trainer.get_regularization_cost(), 0.01);
        assert_eq!(trainer.get_max_iter(), 10);
        assert_eq!(trainer.get_num_threads(), 1);
    }

    #[test]
    fn test_tocost() {
        use super::tocost;

        // MeCab: tocost(d, n) = clamp(-n * d, -32767, 32767)
        // Positive weight → negative cost
        assert_eq!(tocost(1.0, 700), -700);
        // Negative weight → positive cost
        assert_eq!(tocost(-1.0, 700), 700);
        // Zero weight → zero cost
        assert_eq!(tocost(0.0, 700), 0);
        // Clamp to i16::MAX
        assert_eq!(tocost(-100.0, 700), i16::MAX);
        // Clamp to i16::MIN
        assert_eq!(tocost(100.0, 700), i16::MIN);
        // Fractional weight
        assert_eq!(tocost(0.5, 700), -350);
    }
}

// Regression tests for #974: `model.dat` must be byte-reproducible.
//
// Feature ids and connection costs are stable across runs, but the archived
// bytes were not: rkyv lays an archived `HashMap` out as a SwissTable whose
// slot assignment resolves collisions in source iteration order, and
// `HashMap` iteration order is seeded per instance.
#[cfg(test)]
mod determinism_tests {
    use super::{FeatureSetInfo, ModelMetadata, SerializableModel};
    use std::collections::BTreeMap;

    /// Builds a model whose two order-sensitive maps are populated by walking
    /// `keys` in the given order. Everything else is fixed.
    fn model_with_maps(keys: &[usize]) -> SerializableModel {
        let mut connection_matrix = BTreeMap::new();
        let mut unk_categories = BTreeMap::new();
        for &k in keys {
            // The inner key set is derived from `k`, not from the traversal
            // order, so two builds over the same keys in different orders
            // produce logically identical maps -- which is exactly what this
            // test needs to isolate ordering from content.
            let mut inner = BTreeMap::new();
            for j in 0..4 {
                inner.insert(k + j, (k * 64 + j) as f64);
            }
            connection_matrix.insert(k, inner);
            unk_categories.insert(format!("CAT{k:03}"), format!("feat{k},*,*"));
        }
        SerializableModel {
            feature_weights: vec![0.5; 4],
            labels: vec!["a".to_string()],
            pos_info: vec!["名詞,一般".to_string()],
            metadata: ModelMetadata {
                version: "1.0.0".to_string(),
                regularization: 0.01,
                iterations: 10,
                feature_count: 4,
                label_count: 1,
            },
            connection_matrix,
            max_left_id: 63,
            max_right_id: 63,
            feature_sets: vec![FeatureSetInfo {
                left_id: 1,
                right_id: 1,
                weight: 0.25,
            }],
            unk_category_names: vec!["CAT000".to_string()],
            unk_categories,
            char_def_content: String::new(),
            feature_def_content: String::new(),
            rewrite_def_content: String::new(),
            cost_factor: 700,
            left_id_map: vec![(1, "*".to_string())],
            right_id_map: vec![(1, "*".to_string())],
        }
    }

    /// The archived bytes must depend only on the logical contents of the two
    /// maps, never on the order they were populated in.
    ///
    /// 32 keys is deliberate: rkyv sizes the SwissTable at ~89% load and its
    /// probe window is 16 slots, so under `HashMap` the ascending and
    /// descending builds necessarily resolve collisions differently. Two
    /// separately constructed `HashMap`s also get different `RandomState`
    /// seeds, so this is not a coin flip.
    #[test]
    fn serialized_model_is_independent_of_map_insertion_order() {
        let forward: Vec<usize> = (0..32).collect();
        let reverse: Vec<usize> = (0..32).rev().collect();

        let a = match rkyv::to_bytes::<rkyv::rancor::Error>(&model_with_maps(&forward)) {
            Ok(bytes) => bytes,
            Err(err) => panic!("failed to serialize ascending model: {err}"),
        };
        let b = match rkyv::to_bytes::<rkyv::rancor::Error>(&model_with_maps(&reverse)) {
            Ok(bytes) => bytes,
            Err(err) => panic!("failed to serialize descending model: {err}"),
        };

        assert_eq!(
            a.as_slice(),
            b.as_slice(),
            "archived bytes must not depend on map insertion order"
        );
    }

    /// Serializing the very same value twice must also be stable -- a weaker
    /// property than the one above, but it is the one a caller writing the
    /// same model to two files relies on.
    #[test]
    fn serializing_the_same_model_twice_is_stable() {
        let keys: Vec<usize> = (0..32).collect();
        let model = model_with_maps(&keys);

        let a = match rkyv::to_bytes::<rkyv::rancor::Error>(&model) {
            Ok(bytes) => bytes,
            Err(err) => panic!("first serialization failed: {err}"),
        };
        let b = match rkyv::to_bytes::<rkyv::rancor::Error>(&model) {
            Ok(bytes) => bytes,
            Err(err) => panic!("second serialization failed: {err}"),
        };

        assert_eq!(a.as_slice(), b.as_slice());
    }

    /// The connection matrix's type, spelled out so the compile-time guard
    /// below names it exactly.
    type OrderedConnectionMatrix = BTreeMap<usize, BTreeMap<usize, f64>>;

    /// The unknown-word category map's type, for the same reason.
    type OrderedUnkCategories = BTreeMap<String, String>;

    /// Compile-time guard. `model.dat` is byte-reproducible only while these
    /// two fields are ordered maps. These functions are never called; they
    /// exist so that switching either field back to `HashMap` fails to
    /// compile rather than silently regressing (#974).
    #[allow(dead_code)]
    fn assert_connection_matrix_is_ordered(model: &SerializableModel) -> &OrderedConnectionMatrix {
        &model.connection_matrix
    }

    /// See [`assert_connection_matrix_is_ordered`].
    #[allow(dead_code)]
    fn assert_unk_categories_is_ordered(model: &SerializableModel) -> &OrderedUnkCategories {
        &model.unk_categories
    }

    /// A payload that is neither valid rkyv nor JSON must say so and tell the
    /// user to re-train, rather than surfacing an unrelated UTF-8 error.
    #[test]
    fn read_model_rejects_unrecognized_payload_with_an_actionable_error() {
        let garbage = vec![0xffu8; 64];

        let err = match super::Model::read_model(garbage.as_slice()) {
            Ok(_) => panic!("garbage must not deserialize"),
            Err(err) => err.to_string(),
        };

        assert!(err.contains("lindera train"), "unhelpful message: {err}");
    }

    /// A payload that looks like JSON but is malformed is reported as a JSON
    /// parse failure, so the legacy path stays diagnosable.
    #[test]
    fn read_model_reports_malformed_legacy_json() {
        let err = match super::Model::read_model(&b"{ not json"[..]) {
            Ok(_) => panic!("malformed JSON must not deserialize"),
            Err(err) => err.to_string(),
        };

        assert!(err.contains("legacy JSON"), "unhelpful message: {err}");
    }

    /// An empty file is its own case, and used to reach the JSON reader.
    #[test]
    fn read_model_rejects_an_empty_file() {
        let err = match super::Model::read_model(&b""[..]) {
            Ok(_) => panic!("an empty file must not deserialize"),
            Err(err) => err.to_string(),
        };

        assert!(err.contains("empty"), "unhelpful message: {err}");
    }

    /// `update_metadata_json` must be byte-reproducible and must not emit a
    /// timestamp; its output must round-trip through
    /// `lindera_dictionary`'s `Metadata` loader.
    ///
    /// Before #981 it stamped `updated_at` from `SystemTime::now()` (and
    /// `unwrap()`ed in production code), so two exports of the same model
    /// could never compare equal.
    #[test]
    fn metadata_json_is_reproducible_and_timestamp_free() {
        use lindera_dictionary::dictionary::metadata::Metadata;

        let model = model_with_maps(&[0, 7, 21]);

        let dir = match tempfile::tempdir() {
            Ok(dir) => dir,
            Err(err) => panic!("failed to create temp dir: {err}"),
        };
        let base_path = dir.path().join("metadata.json");
        let base_json = match serde_json::to_string_pretty(&Metadata::default()) {
            Ok(json) => json,
            Err(err) => panic!("failed to serialize base metadata: {err}"),
        };
        if let Err(err) = std::fs::write(&base_path, base_json) {
            panic!("failed to write base metadata: {err}");
        }

        let update = || -> Vec<u8> {
            let mut buf = Vec::new();
            if let Err(err) = model.update_metadata_json(&base_path, &mut buf) {
                panic!("update_metadata_json failed: {err}");
            }
            buf
        };

        let first = update();
        let second = update();
        assert_eq!(first, second, "metadata.json output must be reproducible");

        let value: serde_json::Value = match serde_json::from_slice(&first) {
            Ok(value) => value,
            Err(err) => panic!("output is not valid JSON: {err}"),
        };
        let model_info = match value.get("model_info") {
            Some(serde_json::Value::Object(map)) => map,
            other => panic!("model_info must be an object, got {other:?}"),
        };
        assert!(
            !model_info.contains_key("updated_at"),
            "model_info must not carry a timestamp"
        );
        assert!(model_info.contains_key("feature_count"));
        assert!(model_info.contains_key("version"));

        // The emitted file must load through the dictionary-side struct,
        // pinning writer/struct consistency end to end.
        if let Err(err) = Metadata::load(&first) {
            panic!("exported metadata.json must load as Metadata: {err}");
        }
    }
}