msime_engine/multilingual/
dictionaries.rs

1use std::ops::Range;
2use std::path::Path;
3
4use super::{decode, Conversion, Language, Lexeme, MAX_INPUT_BYTES, MAX_LEXEMES};
5use crate::assets;
6use crate::dictionary::english::EnglishDictionary;
7use crate::japanese::{JapaneseProvider, JapaneseRomajiScheme};
8use crate::pinyin::segment::cut_one_piece_min_segments;
9use crate::quanpin::QuanpinEngine;
10use crate::types::QueryRequest;
11use crate::RuntimePaths;
12
13/// 每次输入变化最多查询的原始跨度数，每个跨度最多查询三个所选提供者。
14const MAX_SPAN_QUERIES: usize = 4096;
15/// 单个提供者词面的字节上限；合并后的同语言片段可包含多个词面。
16const MAX_SURFACE_BYTES: usize = 256;
17/// 与现有中文词网格的最长词一致，更长中文片段由多条词条连接。
18const MAX_CHINESE_SYLLABLES: usize = 7;
19
20/// 使用所选语言的现有词库提供者组成完整混合路径。构造应在会话准备阶段完成，
21/// 不要在每次按键时重建。它持有独立查询状态，中文 n 元表和日文模型仍由现有缓存共享。
22/// 语言集合或模型路径改变时重建；词典修订到达时调用 [`Self::clear`]。
23pub struct DictionaryComposer {
24    languages: Vec<Language>,
25    chinese: Option<QuanpinEngine>,
26    english: Option<EnglishDictionary>,
27    japanese: Option<JapaneseProvider>,
28    japanese_ready: bool,
29    japanese_scheme: JapaneseRomajiScheme,
30    japanese_request: QueryRequest,
31    input: String,
32    lexemes: Vec<Lexeme>,
33    answer: Option<Conversion>,
34}
35
36impl DictionaryComposer {
37    /// 只打开显式选择的语言，缺失资源按原有提供者语义返回没有词条，不下载或补写资源。
38    pub fn new(paths: &RuntimePaths, languages: &[Language]) -> Self {
39        Self::with_japanese_model(paths, languages, &paths.resource(assets::JAPANESE_MODEL))
40    }
41
42    /// 使用宿主已解析的日文资源包路径，其它词库仍取同一份运行时目录。
43    pub fn with_japanese_model(paths: &RuntimePaths, languages: &[Language], model: &Path) -> Self {
44        let languages: Vec<_> = [Language::Chinese, Language::English, Language::Japanese]
45            .into_iter()
46            .filter(|language| languages.contains(language))
47            .collect();
48        let active = languages.len() >= 2;
49        Self {
50            chinese: (active && languages.contains(&Language::Chinese))
51                .then(|| QuanpinEngine::new(paths)),
52            english: (active && languages.contains(&Language::English)).then(|| {
53                EnglishDictionary::open(&paths.dictionary(assets::ENGLISH_DICTIONARY), None, None)
54            }),
55            japanese: (active && languages.contains(&Language::Japanese))
56                .then(|| JapaneseProvider::new(model)),
57            languages,
58            japanese_ready: false,
59            japanese_scheme: JapaneseRomajiScheme::new(),
60            japanese_request: QueryRequest::default(),
61            input: String::new(),
62            lexemes: Vec::new(),
63            answer: None,
64        }
65    }
66
67    /// 会话准备时沿现有提供者预热；未选日文时不读模型，不等待后台完成。
68    pub fn warm_up_in_background(&self) {
69        if let Some(japanese) = &self.japanese {
70            japanese.warm_up_in_background();
71        }
72    }
73
74    /// 只返回全串覆盖的混合备选，完整中文的存在性由 `chinese_complete` 保留。
75    /// 当前仅接受小写拉丁编码；非法输入或预算不足时不返回部分路径。
76    pub fn convert(&mut self, input: &str) -> Option<Conversion> {
77        let japanese_ready = self
78            .japanese
79            .as_mut()
80            .is_some_and(JapaneseProvider::prepare_if_cached);
81        if japanese_ready != self.japanese_ready {
82            self.forget_input();
83            self.japanese_ready = japanese_ready;
84        }
85        if input == self.input {
86            return self.answer.clone();
87        }
88        if self.languages.len() < 2
89            || input.is_empty()
90            || input.len() > MAX_INPUT_BYTES
91            || !input.bytes().all(|byte| byte.is_ascii_lowercase())
92        {
93            self.forget_input();
94            return None;
95        }
96        let common = self
97            .input
98            .bytes()
99            .zip(input.bytes())
100            .take_while(|(left, right)| left == right)
101            .count();
102        self.lexemes.retain(|lexeme| lexeme.input.end <= common);
103        let mut queried = 0;
104        for end in common + 1..=input.len() {
105            for start in 0..end {
106                queried += 1;
107                if queried > MAX_SPAN_QUERIES {
108                    self.forget_input();
109                    return None;
110                }
111                if self.query_span(input, start..end).is_none() || self.lexemes.len() > MAX_LEXEMES
112                {
113                    self.forget_input();
114                    return None;
115                }
116            }
117        }
118        let mut answer = decode(input, &self.lexemes, &self.languages);
119        if let Some(conversion) = &mut answer {
120            self.rerank_chinese(input, conversion);
121        }
122        self.input.clear();
123        self.input.push_str(input);
124        self.answer = answer;
125        self.answer.clone()
126    }
127
128    /// 词典修订后丢弃已确认及未命中的跨度。不可变模型更换需重建本适配器。
129    pub fn clear(&mut self) {
130        self.forget_input();
131        if let Some(chinese) = &mut self.chinese {
132            chinese.reset_cache();
133        }
134        if let Some(japanese) = &mut self.japanese {
135            japanese.reset_cache();
136        }
137    }
138
139    fn forget_input(&mut self) {
140        self.input.clear();
141        self.lexemes.clear();
142        self.answer = None;
143    }
144
145    fn push(&mut self, input: Range<usize>, language: Language, text: String) -> Option<()> {
146        if text.len() > MAX_SURFACE_BYTES {
147            return None;
148        }
149        if !text.is_empty() {
150            self.lexemes.push(Lexeme {
151                input,
152                language,
153                text,
154                // 同范围、同语言的次选没有其它后续状态，首选以外的边不会胜出。
155                rank: 0,
156            });
157        }
158        Some(())
159    }
160
161    fn query_span(&mut self, input: &str, range: Range<usize>) -> Option<()> {
162        let raw = &input[range.clone()];
163        if let Some(chinese) = &mut self.chinese {
164            let syllables = cut_one_piece_min_segments(raw, true);
165            if !syllables.is_empty() && syllables.len() <= MAX_CHINESE_SYLLABLES {
166                if let Some(row) = chinese.conversion_rows(&syllables, 1).into_iter().next() {
167                    self.push(range.clone(), Language::Chinese, row.value)?;
168                }
169            }
170        }
171        if let Some(row) = self
172            .english
173            .as_ref()
174            .and_then(|english| english.query_exact(raw))
175        {
176            self.push(range.clone(), Language::English, row.word.to_string())?;
177        }
178        if let Some(japanese) = self.japanese.as_mut().filter(|_| self.japanese_ready) {
179            self.japanese_scheme.set_raw_input(raw, "");
180            self.japanese_scheme
181                .build_request_into(&mut self.japanese_request);
182            let exact = japanese
183                .whole_reading_lemma(&self.japanese_request)
184                .is_some_and(|lemma| lemma.exact && lemma.morae >= 3);
185            if exact {
186                if let Some(path) = japanese
187                    .whole_reading_best_path(&self.japanese_request)
188                    .filter(|path| path.unknown_kana == 0)
189                {
190                    self.push(range, Language::Japanese, path.text)?;
191                }
192            }
193        }
194        Some(())
195    }
196
197    fn rerank_chinese(&mut self, input: &str, conversion: &mut Conversion) {
198        let Some(chinese) = &mut self.chinese else {
199            return;
200        };
201        for segment in &mut conversion.segments {
202            if segment.language != Language::Chinese {
203                continue;
204            }
205            let syllables = cut_one_piece_min_segments(&input[segment.input.clone()], true);
206            let spans = chinese.convert_pinned(&syllables, &[]);
207            if spans.first().is_some_and(|span| span.start == 0)
208                && spans.last().is_some_and(|span| span.end == syllables.len())
209                && spans.windows(2).all(|pair| pair[0].end == pair[1].start)
210            {
211                segment.text = spans.into_iter().map(|span| span.word).collect();
212            }
213        }
214    }
215}
216
217#[cfg(test)]
218#[path = "dictionary_tests.rs"]
219mod tests;
msime_engine/multilingual/dictionaries.rs

msime_engine/multilingual/
dictionaries.rs
