msime_engine/stroke/
scheme.rs

1//! 笔画组合：键入的笔画与通配符，以及 `msime-stroke.db` 对它们给出的单字候选。选中任一候选都结束整个组合，没有分段，也不保留词组进度。
2
3use super::{glyph, is_key, is_stroke, key_of_glyph, MAX_STROKES, WILDCARD};
4use crate::error::Result;
5use crate::language_dictionary::{LanguageDictionary, LanguageEntry};
6use crate::types::{QueryRequest, SchemeKey, SchemeType};
7
8/// 笔画码与键入完全相同（含通配匹配到同样长度）的字最多读这么多。
9pub const EXACT_LIMIT: usize = 200;
10/// 以键入笔画开头、笔画更多的字最多读这么多。
11pub const COMPLETION_LIMIT: usize = 100;
12
13/// 一个候选字和它在 `msime-stroke.db` 里的笔画码。
14#[derive(Debug, Clone, PartialEq, Eq)]
15pub struct StrokeCandidate {
16    pub text: String,
17    pub weight: i64,
18    /// 该字完整的笔画码（`entries.key`）。
19    pub key: String,
20}
21
22#[derive(Debug, Clone, Default)]
23pub struct StrokeScheme {
24    /// `hspnz` 与 `x`，至多 `MAX_STROKES` 个。键入时 `x` 不会出现在开头；宿主编辑可能把它留在开头。
25    input: String,
26}
27
28/// 会话持有的字典查询行，重复文字也保留，以便下次查询复用它们的字符串。
29#[derive(Default)]
30pub(crate) struct StrokeQueryBuffer {
31    entries: Vec<LanguageEntry>,
32    exact_pattern: Vec<(String, LanguageEntry)>,
33    completions: Vec<(String, LanguageEntry)>,
34}
35
36impl StrokeQueryBuffer {
37    fn query(&mut self, dictionary: &LanguageDictionary, input: &str) -> Result<bool> {
38        let wildcard = char::from(WILDCARD);
39        let has_wildcard = input.contains(wildcard);
40        if has_wildcard {
41            dictionary.lookup_pattern_into(
42                input,
43                wildcard,
44                false,
45                EXACT_LIMIT,
46                &mut self.exact_pattern,
47            )?;
48            dictionary.lookup_pattern_into(
49                input,
50                wildcard,
51                true,
52                COMPLETION_LIMIT,
53                &mut self.completions,
54            )?;
55        } else {
56            dictionary.lookup_into(input, EXACT_LIMIT, &mut self.entries)?;
57            dictionary.lookup_completions_into(input, COMPLETION_LIMIT, &mut self.completions)?;
58        }
59        Ok(has_wildcard)
60    }
61}
62
63impl StrokeScheme {
64    pub fn new() -> Self {
65        Self::default()
66    }
67
68    pub fn reset(&mut self) {
69        self.input.clear();
70    }
71
72    /// 笔画键追加一笔；通配符只在已有笔画时追加；Backspace 删最后一笔。达到 `MAX_STROKES` 后不再追加。其他键（别的字母、大写、符号）都不改变组合，是否吞掉由会话决定。
73    pub fn handle_key(&mut self, key: SchemeKey) -> bool {
74        match key {
75            SchemeKey::Letter(letter) if is_stroke(letter) => self.push(letter),
76            SchemeKey::Letter(WILDCARD) if !self.input.is_empty() => self.push(WILDCARD),
77            SchemeKey::Backspace => self.input.pop().is_some(),
78            SchemeKey::Letter(_)
79            | SchemeKey::Apostrophe
80            | SchemeKey::Semicolon
81            | SchemeKey::Minus
82            | SchemeKey::Symbol(_)
83            | SchemeKey::Requery => false,
84        }
85    }
86
87    fn push(&mut self, key: u8) -> bool {
88        if self.input.len() < MAX_STROKES {
89            self.input.push(char::from(key));
90            true
91        } else {
92            false
93        }
94    }
95
96    /// 用宿主编辑后的文本替换组合：笔画键和通配符原样保留，笔画字形换回对应的键，其他字符丢弃，超出 `MAX_STROKES` 的部分截掉。
97    pub fn set_raw_input(&mut self, raw: &str) {
98        self.input.clear();
99        self.input.reserve(raw.len().min(MAX_STROKES));
100        for character in raw.chars() {
101            let key = match u8::try_from(character) {
102                Ok(byte) if is_key(byte) => Some(byte),
103                _ => key_of_glyph(character),
104            };
105            if let Some(key) = key {
106                self.push(key);
107            }
108        }
109    }
110
111    /// 键入的笔画字母（`hspnzx`），Enter 上屏的就是它。
112    pub fn input(&self) -> &str {
113        &self.input
114    }
115
116    pub fn is_empty(&self) -> bool {
117        self.input.is_empty()
118    }
119
120    /// 组合显示的笔画字形，例如 `一丨＊`。
121    pub fn glyphs(&self) -> String {
122        let mut output = String::new();
123        self.glyphs_into(&mut output);
124        output
125    }
126
127    pub fn glyphs_into(&self, output: &mut String) {
128        output.clear();
129        output.extend(self.input.bytes().filter_map(glyph));
130    }
131
132    /// 会话刷新用的请求。`raw_input` 是键入的字母，也是宿主编辑的文本；`normalized_segmentation` 是快照 `reading` 带给宿主绘制的笔画字形。
133    pub fn build_request(&self) -> QueryRequest {
134        let mut request = QueryRequest::default();
135        self.build_request_into(&mut request);
136        request
137    }
138
139    pub fn build_request_into(&self, request: &mut QueryRequest) {
140        request.scheme = SchemeType::Stroke;
141        request.raw_input.clone_from(&self.input);
142        request.raw_input_with_cases.clone_from(&self.input);
143        request.normalized_input.clone_from(&self.input);
144        request.raw_segmentation.clone_from(&self.input);
145        self.glyphs_into(&mut request.normalized_segmentation);
146        request.segmentation.clone_from(&self.input);
147        request.valid = !self.input.is_empty();
148    }
149
150    /// 会话显示并据以判断是否在组合的文本：笔画字形，没有组合时为空。
151    pub fn preedit(&self) -> String {
152        self.glyphs()
153    }
154
155    /// 组合的候选，每个字只列一次：
156    /// 1. 有字频、笔画码与键入完全相同的字（通配符匹配任意一笔），重的在前；
157    /// 2. 有字频、以键入笔画开头、笔画更多的字，重的在前；
158    /// 3. 没有字频的字（权重为 0 的生僻字），同样精确匹配在前、补全在后。
159    ///
160    /// 笔数恰好打满时，生僻字的精确匹配不再挡在常用字的补全前面。
161    pub fn candidates(&self, dictionary: &LanguageDictionary) -> Result<Vec<StrokeCandidate>> {
162        let input = self.input.as_str();
163        if input.is_empty() {
164            return Ok(Vec::new());
165        }
166        let mut buffer = StrokeQueryBuffer::default();
167        buffer.query(dictionary, input)?;
168        let unique_len = {
169            let mut seen: [Option<&str>; EXACT_LIMIT + COMPLETION_LIMIT] =
170                [None; EXACT_LIMIT + COMPLETION_LIMIT];
171            let mut unique_len = 0;
172            for text in buffer
173                .entries
174                .iter()
175                .map(|entry| entry.text.as_str())
176                .chain(
177                    buffer
178                        .exact_pattern
179                        .iter()
180                        .map(|(_, entry)| entry.text.as_str()),
181                )
182                .chain(
183                    buffer
184                        .completions
185                        .iter()
186                        .map(|(_, entry)| entry.text.as_str()),
187                )
188            {
189                if !seen[..unique_len]
190                    .iter()
191                    .flatten()
192                    .any(|seen| *seen == text)
193                {
194                    seen[unique_len] = Some(text);
195                    unique_len += 1;
196                }
197            }
198            unique_len
199        };
200        let mut candidates = Vec::with_capacity(unique_len);
201        for entry in buffer.entries {
202            push_owned_candidate(&mut candidates, input, entry);
203        }
204        for (key, entry) in buffer.exact_pattern.into_iter().chain(buffer.completions) {
205            push_owned_candidate(&mut candidates, key, entry);
206        }
207        candidates.sort_by_key(|candidate| candidate.weight <= 0);
208        Ok(candidates)
209    }
210
211    /// 复用查询行与候选字符串；查询失败时清空候选，不暴露上次结果。
212    pub(crate) fn candidates_into(
213        &self,
214        dictionary: &LanguageDictionary,
215        buffer: &mut StrokeQueryBuffer,
216        candidates: &mut Vec<StrokeCandidate>,
217    ) -> Result<()> {
218        let input = self.input.as_str();
219        if input.is_empty() {
220            candidates.clear();
221            return Ok(());
222        }
223        let has_wildcard = match buffer.query(dictionary, input) {
224            Ok(has_wildcard) => has_wildcard,
225            Err(error) => {
226                candidates.clear();
227                return Err(error);
228            }
229        };
230        let exact_len = if has_wildcard {
231            buffer.exact_pattern.len()
232        } else {
233            buffer.entries.len()
234        };
235        let row_count = exact_len + buffer.completions.len();
236        candidates.reserve(row_count.saturating_sub(candidates.len()));
237        // 先按查询顺序去重，保留首次行；不能让高权重补全替代零或负权重精确行。
238        let mut unique: [Option<(&str, &LanguageEntry)>; EXACT_LIMIT + COMPLETION_LIMIT] =
239            [None; EXACT_LIMIT + COMPLETION_LIMIT];
240        debug_assert!(row_count <= unique.len());
241        let mut unique_length = 0;
242        for index in 0..row_count {
243            let (key, entry) = if index >= exact_len {
244                let (key, entry) = &buffer.completions[index - exact_len];
245                (key.as_str(), entry)
246            } else if has_wildcard {
247                let (key, entry) = &buffer.exact_pattern[index];
248                (key.as_str(), entry)
249            } else {
250                (input, &buffer.entries[index])
251            };
252            if unique[..unique_length]
253                .iter()
254                .flatten()
255                .any(|(_, previous)| previous.text == entry.text)
256            {
257                continue;
258            }
259            unique[unique_length] = Some((key, entry));
260            unique_length += 1;
261        }
262        // 直接按最终稳定顺序覆写，避免排序移动字符串容量后下次热查询又扩容。
263        let mut length = 0;
264        for non_positive in [false, true] {
265            for &(key, entry) in unique[..unique_length].iter().flatten() {
266                if (entry.weight <= 0) == non_positive {
267                    push_candidate(candidates, &mut length, key, entry);
268                }
269            }
270        }
271        candidates.truncate(length);
272        Ok(())
273    }
274}
275
276fn contains_text(candidates: &[StrokeCandidate], text: &str) -> bool {
277    candidates.iter().any(|candidate| candidate.text == text)
278}
279
280fn push_owned_candidate(
281    candidates: &mut Vec<StrokeCandidate>,
282    key: impl Into<String>,
283    entry: LanguageEntry,
284) {
285    if contains_text(candidates, &entry.text) {
286        return;
287    }
288    candidates.push(StrokeCandidate {
289        key: key.into(),
290        text: entry.text,
291        weight: entry.weight,
292    });
293}
294
295fn push_candidate(
296    candidates: &mut Vec<StrokeCandidate>,
297    length: &mut usize,
298    key: &str,
299    entry: &LanguageEntry,
300) {
301    if let Some(candidate) = candidates.get_mut(*length) {
302        candidate.key.clear();
303        candidate.key.push_str(key);
304        candidate.text.clear();
305        candidate.text.push_str(&entry.text);
306        candidate.weight = entry.weight;
307    } else {
308        candidates.push(StrokeCandidate {
309            key: key.to_owned(),
310            text: entry.text.clone(),
311            weight: entry.weight,
312        });
313    }
314    *length += 1;
315}
316
317#[cfg(test)]
318mod tests {
319    use super::*;
320    use crate::language_dictionary::open_read_only;
321    use crate::stroke::fixture;
322
323    struct Fixture {
324        _dir: tempfile::TempDir,
325        dictionary: LanguageDictionary,
326    }
327
328    fn fixture() -> Fixture {
329        let dir = tempfile::tempdir().unwrap();
330        let path = dir.path().join("msime-stroke.db");
331        fixture::build(&path);
332        Fixture {
333            dictionary: open_read_only(&path).unwrap(),
334            _dir: dir,
335        }
336    }
337
338    fn typed(keys: &str) -> StrokeScheme {
339        let mut scheme = StrokeScheme::new();
340        for key in keys.bytes() {
341            scheme.handle_key(SchemeKey::Letter(key));
342        }
343        scheme
344    }
345
346    fn texts(scheme: &StrokeScheme, dictionary: &LanguageDictionary) -> Vec<String> {
347        scheme
348            .candidates(dictionary)
349            .unwrap()
350            .into_iter()
351            .map(|candidate| candidate.text)
352            .collect()
353    }
354
355    #[test]
356    fn strokes_append_and_show_as_glyphs() {
357        let scheme = typed("hspnz");
358        assert_eq!(scheme.input(), "hspnz");
359        assert_eq!(scheme.preedit(), "一丨丿丶乛");
360        let request = scheme.build_request();
361        assert_eq!(request.scheme, SchemeType::Stroke);
362        assert_eq!(request.raw_input, "hspnz");
363        assert_eq!(request.raw_input_with_cases, "hspnz");
364        assert_eq!(request.normalized_segmentation, "一丨丿丶乛");
365        assert!(request.valid);
366        assert!(!StrokeScheme::new().build_request().valid);
367        assert_eq!(StrokeScheme::new().preedit(), "");
368    }
369
370    #[test]
371    fn other_keys_change_nothing() {
372        let mut scheme = typed("h");
373        for key in [
374            SchemeKey::Letter(b'a'),
375            SchemeKey::Letter(b'H'),
376            SchemeKey::Letter(b'q'),
377            SchemeKey::Apostrophe,
378            SchemeKey::Semicolon,
379            SchemeKey::Minus,
380            SchemeKey::Symbol(b'*'),
381            SchemeKey::Requery,
382        ] {
383            scheme.handle_key(key);
384        }
385        assert_eq!(scheme.input(), "h");
386    }
387
388    #[test]
389    fn the_wildcard_needs_a_stroke_before_it() {
390        let mut scheme = typed("x");
391        assert!(scheme.is_empty());
392        scheme.handle_key(SchemeKey::Letter(b'h'));
393        scheme.handle_key(SchemeKey::Letter(b'x'));
394        scheme.handle_key(SchemeKey::Letter(b'x'));
395        assert_eq!(scheme.input(), "hxx");
396        assert_eq!(scheme.preedit(), "一＊＊");
397    }
398
399    #[test]
400    fn backspace_removes_the_last_stroke_and_reset_clears() {
401        let mut scheme = typed("hsx");
402        scheme.handle_key(SchemeKey::Backspace);
403        assert_eq!(scheme.input(), "hs");
404        scheme.reset();
405        assert!(scheme.is_empty());
406        scheme.handle_key(SchemeKey::Backspace);
407        assert!(scheme.is_empty());
408    }
409
410    #[test]
411    fn the_composition_stops_at_the_stroke_limit() {
412        let mut scheme = typed(&"h".repeat(MAX_STROKES));
413        scheme.handle_key(SchemeKey::Letter(b's'));
414        scheme.handle_key(SchemeKey::Letter(b'x'));
415        assert_eq!(scheme.input().len(), MAX_STROKES);
416        assert!(scheme.input().bytes().all(|key| key == b'h'));
417        scheme.set_raw_input(&"s".repeat(MAX_STROKES + 5));
418        assert_eq!(scheme.input().len(), MAX_STROKES);
419    }
420
421    #[test]
422    fn host_edits_keep_keys_and_read_glyphs_back() {
423        let mut scheme = StrokeScheme::new();
424        scheme.set_raw_input("h丨a丿x1乛 ＊");
425        assert_eq!(scheme.input(), "hspxzx");
426        assert_eq!(scheme.preedit(), "一丨丿＊乛＊");
427        // 宿主编辑留下的开头通配符保留，查询照样能用。
428        scheme.set_raw_input("xs");
429        assert_eq!(scheme.input(), "xs");
430        scheme.set_raw_input("");
431        assert!(scheme.is_empty());
432    }
433
434    #[test]
435    fn exact_codes_come_before_longer_ones_and_texts_are_listed_once() {
436        let fixture = fixture();
437        let dictionary = &fixture.dictionary;
438        // 十 的笔画码正好是 hs，排在更重的长码字前面；土 有两个笔画码，只列一次。
439        assert_eq!(texts(&typed("hs"), dictionary), ["十", "土"]);
440        assert_eq!(
441            texts(&typed("h"), dictionary),
442            ["一", "大", "二", "十", "三", "王", "土", "干"]
443        );
444        let candidates = typed("hs").candidates(dictionary).unwrap();
445        assert_eq!(
446            candidates[0],
447            StrokeCandidate {
448                text: "十".to_owned(),
449                weight: 4500,
450                key: "hs".to_owned(),
451            }
452        );
453        assert_eq!(candidates[1].key, "hsh");
454        assert!(texts(&typed("zzzz"), dictionary).is_empty());
455        assert!(texts(&StrokeScheme::new(), dictionary).is_empty());
456    }
457
458    #[test]
459    fn owned_candidates_do_not_reserve_duplicate_query_rows() {
460        let fixture = fixture();
461        let candidates = typed("h").candidates(&fixture.dictionary).unwrap();
462
463        assert_eq!(candidates.len(), 8);
464        assert_eq!(
465            candidates.capacity(),
466            candidates.len(),
467            "重复笔画码不应继续占用候选容量"
468        );
469    }
470
471    #[test]
472    fn empty_owned_query_does_not_allocate_page_buffers() {
473        let fixture = fixture();
474        for (input, expected_allocations) in [("zzzz", 3), ("zzzx", 6)] {
475            let scheme = typed(input);
476            scheme.candidates(&fixture.dictionary).unwrap();
477            let (candidates, allocations) = crate::ime::personal_rerank::allocations::count(|| {
478                scheme.candidates(&fixture.dictionary).unwrap()
479            });
480            assert!(candidates.is_empty());
481            assert_eq!(
482                allocations, expected_allocations,
483                "空笔画查询 {input} 申请了 {allocations} 个缓冲"
484            );
485        }
486    }
487
488    #[test]
489    fn reused_candidates_keep_first_duplicate_before_stable_frequency_partition() {
490        let fixture = fixture();
491        let connection =
492            rusqlite::Connection::open(fixture._dir.path().join("msime-stroke.db")).unwrap();
493        connection
494            .execute("DELETE FROM entries WHERE key NOT IN ('hs', 'hsh')", [])
495            .unwrap();
496        for (key, text, weight) in [
497            ("hs", "重複零", 0),
498            ("hsh", "重複零", 10000),
499            ("hs", "重複負", -1),
500            ("hsh", "重複負", 9000),
501            ("hsh", "補全零", 0),
502            ("hsh", "補全負", -2),
503        ] {
504            connection
505                .execute(
506                    "INSERT INTO entries VALUES (?1, ?2, ?3)",
507                    (key, text, weight),
508                )
509                .unwrap();
510        }
511        drop(connection);
512        let mut buffer = StrokeQueryBuffer::default();
513        let mut candidates = Vec::new();
514        for input in ["hs", "hx", "hs"] {
515            let scheme = typed(input);
516            scheme
517                .candidates_into(&fixture.dictionary, &mut buffer, &mut candidates)
518                .unwrap();
519            assert_eq!(
520                candidates
521                    .iter()
522                    .map(|row| (row.text.as_str(), row.key.as_str(), row.weight))
523                    .collect::<Vec<_>>(),
524                [
525                    ("十", "hs", 4500),
526                    ("土", "hsh", 2000),
527                    ("重複零", "hs", 0),
528                    ("重複負", "hs", -1),
529                    ("補全零", "hsh", 0),
530                    ("補全負", "hsh", -2)
531                ]
532            );
533            assert_eq!(candidates, scheme.candidates(&fixture.dictionary).unwrap());
534        }
535        typed("zzzz")
536            .candidates_into(&fixture.dictionary, &mut buffer, &mut candidates)
537            .unwrap();
538        assert!(candidates.is_empty());
539        typed("h")
540            .candidates_into(&fixture.dictionary, &mut buffer, &mut candidates)
541            .unwrap();
542        assert!(!candidates.is_empty());
543        StrokeScheme::new()
544            .candidates_into(&fixture.dictionary, &mut buffer, &mut candidates)
545            .unwrap();
546        assert!(candidates.is_empty());
547    }
548
549    #[test]
550    fn characters_without_a_frequency_follow_every_weighted_one() {
551        let fixture = fixture();
552        let dictionary = &fixture.dictionary;
553        // 乚 的笔画码正好是 sz，但没有字频，排在笔画更多、有字频的 口 后面。
554        assert_eq!(texts(&typed("sz"), dictionary), ["口", "乚"]);
555        // 通配符同样如此。
556        assert_eq!(texts(&typed("sx"), dictionary), ["口", "乚"]);
557    }
558
559    #[test]
560    fn the_wildcard_matches_any_one_stroke() {
561        let fixture = fixture();
562        let dictionary = &fixture.dictionary;
563        // hx：两笔的 二 十 在前，再是 h 开头、笔画更多的字。
564        assert_eq!(
565            texts(&typed("hx"), dictionary),
566            ["二", "十", "大", "三", "王", "土", "干"]
567        );
568        assert_eq!(texts(&typed("hxh"), dictionary), ["三", "土"]);
569        let mut scheme = StrokeScheme::new();
570        scheme.set_raw_input("xn");
571        assert_eq!(texts(&scheme, dictionary), ["人"]);
572    }
573}
msime_engine/stroke/scheme.rs

msime_engine/stroke/
scheme.rs
