lindera_dictionary/loader/
connection_cost_matrix.rs

1use std::path::Path;
2
3use crate::LinderaResult;
4use crate::dictionary::connection_cost_matrix::ConnectionCostMatrix;
5#[cfg(feature = "mmap")]
6use crate::util::mmap_file;
7use crate::util::read_file;
8
9/// Loader for connection cost matrix data from disk files.
10pub struct ConnectionCostMatrixLoader {}
11
12impl ConnectionCostMatrixLoader {
13    /// Load connection cost matrix from a file in the specified directory.
14    ///
15    /// # Arguments
16    ///
17    /// * `input_dir` - Path to the directory containing matrix.mtx.
18    ///
19    /// # Returns
20    ///
21    /// A `ConnectionCostMatrix` loaded from the file.
22    pub fn load(input_dir: &Path) -> LinderaResult<ConnectionCostMatrix> {
23        let data = read_file(input_dir.join("matrix.mtx").as_path())?;
24
25        ConnectionCostMatrix::load(data)
26    }
27
28    /// Load connection cost matrix using memory-mapped file.
29    ///
30    /// This is the zero-copy path: an mmap base is page-aligned and
31    /// `matrix.mtx` already stores its costs as little-endian `i16` in the
32    /// in-memory layout, so [`ConnectionCostMatrix::load`] views the payload
33    /// in place instead of decoding it into an owned `Vec<i16>`. Loading is
34    /// O(1) in the matrix size and costs no anonymous memory; the pages are
35    /// faulted in lazily as tokenization touches them. UniDic's matrix alone
36    /// is 71.5 MB, so this is the bulk of that dictionary's load cost.
37    ///
38    /// # Arguments
39    ///
40    /// * `input_dir` - Path to the directory containing matrix.mtx.
41    ///
42    /// # Returns
43    ///
44    /// A `ConnectionCostMatrix` loaded via memory mapping.
45    #[cfg(feature = "mmap")]
46    pub fn load_mmap(input_dir: &Path) -> LinderaResult<ConnectionCostMatrix> {
47        let data = mmap_file(input_dir.join("matrix.mtx").as_path())?;
48
49        ConnectionCostMatrix::load(data)
50    }
51}
lindera_dictionary/loader/connection_cost_matrix.rs

lindera_dictionary/loader/
connection_cost_matrix.rs
