lindera_dictionary/
macros.rs

1//! Shared macros for the per-dictionary crates (`lindera-ipadic`,
2//! `lindera-ko-dic`, `lindera-unidic`, `lindera-cc-cedict`, `lindera-jieba`,
3//! `lindera-ipadic-neologd`).
4//!
5//! Each of those crates' `embedded` module used to contain ~90 lines of
6//! identical boilerplate that differed only in the dictionary subdirectory
7//! name and the loader struct name. [`embedded_dictionary!`] generates that
8//! boilerplate from those two inputs.
9
10/// Generates the embedded-dictionary loader for a dictionary crate.
11///
12/// The dictionary data is baked into the binary with `include_bytes!`,
13/// reading from the `LINDERA_WORKDIR` directory populated by the crate's
14/// build script.
15///
16/// The data is bound to `static`s rather than `const`s deliberately. A `const`
17/// body is encoded into the crate's metadata, so a `const` here would put a copy
18/// of every dictionary byte into `lib.rmeta` at roughly 4x its size — several
19/// hundred megabytes per dictionary crate, re-read by every downstream crate.
20/// The data is private to `load()` below and never const-evaluated, so a `static`
21/// is all that is needed.
22///
23/// * `$dir` — the dictionary subdirectory inside `LINDERA_WORKDIR`
24///   (e.g. `"/lindera-ipadic"`).
25/// * `$loader` — the public loader struct name (e.g. `EmbeddedIPADICLoader`).
26///
27/// # Example
28///
29/// ```ignore
30/// lindera_dictionary::embedded_dictionary!("/lindera-ipadic", EmbeddedIPADICLoader);
31/// ```
32#[macro_export]
33macro_rules! embedded_dictionary {
34    ($dir:literal, $loader:ident) => {
35        static CHAR_DEFINITION_DATA: &[u8] =
36            include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/char_def.bin"));
37        static CONNECTION_DATA: &[u8] =
38            include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/matrix.mtx"));
39        static DA_DATA: &[u8] = include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.da"));
40        static VALS_DATA: &[u8] =
41            include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.vals"));
42        static UNKNOWN_DATA: &[u8] =
43            include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/unk.bin"));
44        static WORDS_IDX_DATA: &[u8] =
45            include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.wordsidx"));
46        static WORDS_DATA: &[u8] =
47            include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.words"));
48        static METADATA_DATA: &[u8] =
49            include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/metadata.json"));
50
51        /// Loads the embedded dictionary from data baked into the binary.
52        pub fn load() -> $crate::LinderaResult<$crate::dictionary::Dictionary> {
53            let metadata = $crate::dictionary::metadata::Metadata::load(METADATA_DATA)?;
54            let prefix_dictionary = $crate::dictionary::prefix_dictionary::PrefixDictionary::load(
55                DA_DATA,
56                VALS_DATA,
57                WORDS_IDX_DATA,
58                WORDS_DATA,
59                true,
60                $crate::dictionary::prefix_dictionary::DaTrust::Trusted,
61            )?;
62            let connection_cost_matrix =
63                $crate::dictionary::connection_cost_matrix::ConnectionCostMatrix::load(
64                    CONNECTION_DATA,
65                )?;
66            let character_definition =
67                $crate::dictionary::character_definition::CharacterDefinition::load(
68                    CHAR_DEFINITION_DATA,
69                )?;
70            let unknown_dictionary =
71                $crate::dictionary::unknown_dictionary::UnknownDictionary::load(UNKNOWN_DATA)?;
72
73            Ok($crate::dictionary::Dictionary {
74                prefix_dictionary: ::std::sync::Arc::new(prefix_dictionary),
75                connection_cost_matrix: ::std::sync::Arc::new(connection_cost_matrix),
76                character_definition: ::std::sync::Arc::new(character_definition),
77                unknown_dictionary: ::std::sync::Arc::new(unknown_dictionary),
78                metadata: ::std::sync::Arc::new(metadata),
79            })
80        }
81
82        /// Loader that returns the dictionary embedded in the binary.
83        pub struct $loader;
84
85        impl Default for $loader {
86            fn default() -> Self {
87                Self::new()
88            }
89        }
90
91        impl $loader {
92            pub fn new() -> Self {
93                Self
94            }
95        }
96
97        impl $crate::loader::DictionaryLoader for $loader {
98            fn load(&self) -> $crate::LinderaResult<$crate::dictionary::Dictionary> {
99                load()
100            }
101        }
102    };
103}
lindera_dictionary/macros.rs

lindera_dictionary/
macros.rs
