lindera_dictionary/macros.rs
1//! Shared macros for the per-dictionary crates (`lindera-ipadic`,
2//! `lindera-ko-dic`, `lindera-unidic`, `lindera-cc-cedict`, `lindera-jieba`,
3//! `lindera-ipadic-neologd`).
4//!
5//! Each of those crates' `embedded` module used to contain ~90 lines of
6//! identical boilerplate that differed only in the dictionary subdirectory
7//! name and the loader struct name. [`embedded_dictionary!`] generates that
8//! boilerplate from those two inputs.
9
10/// Includes a file's bytes as a 16-byte-aligned `&'static [u8]`.
11///
12/// `include_bytes!` yields a `[u8; N]`, whose alignment guarantee is 1. The
13/// connection cost matrix payload starts at byte 6 of `matrix.mtx`, so a
14/// 1-aligned base can leave it at an odd address, which pushes
15/// [`crate::dictionary::connection_cost_matrix::ConnectionCostMatrix::load`]
16/// into its owning fallback and reintroduces exactly the copy that this crate
17/// avoids elsewhere (71.5 MB for UniDic). Wrapping the array in a
18/// `#[repr(C, align(16))]` struct raises the `static`'s alignment so the
19/// zero-copy path is taken deterministically rather than by the linker's
20/// goodwill.
21///
22/// Like [`embedded_dictionary!`], the data is bound to a `static` rather than
23/// a `const` so it is not copied into the crate's metadata.
24///
25/// # Arguments
26///
27/// * `$path` - Path forwarded to `include_bytes!`.
28///
29/// # Returns
30///
31/// A `&'static [u8]` whose first byte is 16-byte aligned.
32#[macro_export]
33macro_rules! include_bytes_aligned {
34 ($path:expr) => {{
35 /// Raises the alignment of the wrapped bytes to 16.
36 #[repr(C, align(16))]
37 struct Aligned16<T: ?Sized> {
38 /// The included bytes.
39 bytes: T,
40 }
41
42 static ALIGNED: &Aligned16<[u8]> = &Aligned16 {
43 bytes: *::core::include_bytes!($path),
44 };
45
46 &ALIGNED.bytes
47 }};
48}
49
50/// Generates the embedded-dictionary loader for a dictionary crate.
51///
52/// The dictionary data is baked into the binary with `include_bytes!`,
53/// reading from the `LINDERA_WORKDIR` directory populated by the crate's
54/// build script.
55///
56/// The data is bound to `static`s rather than `const`s deliberately. A `const`
57/// body is encoded into the crate's metadata, so a `const` here would put a copy
58/// of every dictionary byte into `lib.rmeta` at roughly 4x its size — several
59/// hundred megabytes per dictionary crate, re-read by every downstream crate.
60/// The data is private to `load()` below and never const-evaluated, so a `static`
61/// is all that is needed.
62///
63/// * `$dir` — the dictionary subdirectory inside `LINDERA_WORKDIR`
64/// (e.g. `"/lindera-ipadic"`).
65/// * `$loader` — the public loader struct name (e.g. `EmbeddedIPADICLoader`).
66///
67/// # Example
68///
69/// ```ignore
70/// lindera_dictionary::embedded_dictionary!("/lindera-ipadic", EmbeddedIPADICLoader);
71/// ```
72#[macro_export]
73macro_rules! embedded_dictionary {
74 ($dir:literal, $loader:ident) => {
75 static CHAR_DEFINITION_DATA: &[u8] =
76 include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/char_def.bin"));
77 // Aligned so `ConnectionCostMatrix::load` can view the payload as
78 // `[i16]` in place instead of decoding it into an owned buffer.
79 static CONNECTION_DATA: &[u8] =
80 $crate::include_bytes_aligned!(concat!(env!("LINDERA_WORKDIR"), $dir, "/matrix.mtx"));
81 static TRIE_DATA: &[u8] =
82 include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.trie"));
83 static VALS_IDX_DATA: &[u8] =
84 include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.valsidx"));
85 static VALS_DATA: &[u8] =
86 include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.vals"));
87 static UNKNOWN_DATA: &[u8] =
88 include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/unk.bin"));
89 static WORDS_IDX_DATA: &[u8] =
90 include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.wordsidx"));
91 static WORDS_DATA: &[u8] =
92 include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/dict.words"));
93 static METADATA_DATA: &[u8] =
94 include_bytes!(concat!(env!("LINDERA_WORKDIR"), $dir, "/metadata.json"));
95
96 /// Loads the embedded dictionary from data baked into the binary.
97 pub fn load() -> $crate::LinderaResult<$crate::dictionary::Dictionary> {
98 let metadata = $crate::dictionary::metadata::Metadata::load(METADATA_DATA)?;
99 // Guards against a stale build cache: `include_bytes!` bakes in
100 // whatever the build script produced, so a cache directory left
101 // behind by another format version must be caught here.
102 metadata.validate_format_version()?;
103 // The trie is walked in place over these bytes with
104 // bounds-checked reads, so unlike the retired daachorse
105 // representation there is no unchecked-deserialize fast path to
106 // guard -- embedded and filesystem data take the same safe code.
107 let prefix_dictionary = $crate::dictionary::prefix_dictionary::PrefixDictionary::load(
108 TRIE_DATA,
109 VALS_IDX_DATA,
110 VALS_DATA,
111 WORDS_IDX_DATA,
112 WORDS_DATA,
113 )?;
114 let connection_cost_matrix =
115 $crate::dictionary::connection_cost_matrix::ConnectionCostMatrix::load(
116 CONNECTION_DATA,
117 )?;
118 let character_definition =
119 $crate::dictionary::character_definition::CharacterDefinition::load(
120 CHAR_DEFINITION_DATA,
121 )?;
122 let unknown_dictionary =
123 $crate::dictionary::unknown_dictionary::UnknownDictionary::load(UNKNOWN_DATA)?;
124
125 Ok($crate::dictionary::Dictionary {
126 prefix_dictionary: ::std::sync::Arc::new(prefix_dictionary),
127 connection_cost_matrix: ::std::sync::Arc::new(connection_cost_matrix),
128 character_definition: ::std::sync::Arc::new(character_definition),
129 unknown_dictionary: ::std::sync::Arc::new(unknown_dictionary),
130 metadata: ::std::sync::Arc::new(metadata),
131 })
132 }
133
134 /// Loader that returns the dictionary embedded in the binary.
135 pub struct $loader;
136
137 impl Default for $loader {
138 fn default() -> Self {
139 Self::new()
140 }
141 }
142
143 impl $loader {
144 pub fn new() -> Self {
145 Self
146 }
147 }
148
149 impl $crate::loader::DictionaryLoader for $loader {
150 fn load(&self) -> $crate::LinderaResult<$crate::dictionary::Dictionary> {
151 load()
152 }
153 }
154 };
155}