msime-engine 0.1.1

Input method engine for Chinese and other languages: quanpin, shuangpin, nine-key, wubi, zhuyin, Cantonese, Japanese, Korean and more, ranked over a word lattice with typo correction and learning from what the user picks. The engine of 水杉输入法 (Metasequoia IME).
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
//! 五笔按键处理(`R/schemes/wubi_scheme.*`):字母 `a..=y`,`z` 在允许混输拼音或打开通配时收;最多四个字母,混输的拼音回退正在延长组字时最多 32 个。混输关闭且通配关掉时,组字开头的 `z` 打开临时拼音反查:之后的字母都是拼音,最多 32 个。

use crate::types::{QueryRequest, SchemeKey, SchemeType};

pub const MAX_CODE_LENGTH: usize = 4;
/// Long enough for any pinyin spelling, short enough to stay a composition rather than a paragraph.
pub const MAX_MIXED_CODE_LENGTH: usize = 32;

#[derive(Debug, Clone, Default)]
pub struct WubiScheme {
    raw: String,
    /// No wubi code uses `z`, but mixed input has to accept it: dropping it would not refuse a pinyin spelling, it would silently turn `zhongguo` into `hongguo`. It follows the setting rather than the last query because the letter can open a composition.
    mixed_pinyin_allowed: bool,
    /// A fifth letter is refused unless mixed input found that the table cannot answer the code in hand, so `nihao` can be finished while a fluent wubi typist sees no change.
    extended_length_allowed: bool,
    /// 混输关闭时由会话打开:组字以 `z` 开头就是临时拼音反查。五笔码不用 `z`,混输关闭时这个键原本被拒绝、交回宿主,所以占用它不会挡住任何五笔码。混输打开时 `z` 已经是拼音字母,会话会把这里关掉。
    pinyin_lookup_allowed: bool,
    /// `z` 通配:`z` 是未知码位,匹配码表该位置的任意一个字母。与混输无关,两条路都能让 `z` 进码;打开通配时会话关掉临时拼音反查,两条路不能同时占用这个键。
    z_wildcard: bool,
    /// 这次组字是 14 键打的:`raw` 里每一位是一组字母的代表字母([`super::grid`]),不是确定的字母。由第一个键决定,组字清空时复位;14 键组字中不收确定的字母,26 键组字中也不收 14 键的组码,两种码不混在一串里。
    grid: bool,
}

/// 临时拼音反查的引导键。
pub const PINYIN_LOOKUP_PREFIX: u8 = b'z';

fn is_wubi_letter(lower: u8, mixed_pinyin_allowed: bool, z_wildcard: bool) -> bool {
    (b'a'..=b'y').contains(&lower) || (lower == b'z' && (mixed_pinyin_allowed || z_wildcard))
}

impl WubiScheme {
    pub fn new() -> Self {
        Self::default()
    }

    /// Clears the code and the extended-length allowance.
    pub fn reset(&mut self) {
        self.raw.clear();
        self.extended_length_allowed = false;
        self.grid = false;
    }

    /// 14 键的一键:`representative` 是这一组的代表字母([`super::grid::representative`] 的结果)。26 键组字中、码已满四位或不是代表字母时不收。14 键组码不进混输拼音,也不开临时拼音反查,所以上限永远是四位。
    pub fn handle_grid_key(&mut self, representative: u8) -> bool {
        if super::grid::letters(representative).is_empty()
            || (!self.raw.is_empty() && !self.grid)
            || self.raw.len() >= MAX_CODE_LENGTH
        {
            return false;
        }
        self.grid = true;
        self.extended_length_allowed = false;
        self.raw.push(char::from(representative));
        true
    }

    /// 当前组字是 14 键打的组码。
    pub fn grid_composing(&self) -> bool {
        self.grid && !self.raw.is_empty()
    }

    /// Keys beyond the length limit or outside the alphabet are ignored, which the session reports as unhandled.
    pub fn handle_key(&mut self, key: SchemeKey) -> bool {
        match key {
            SchemeKey::Backspace => {
                let removed = self.raw.pop().is_some();
                if self.raw.is_empty() {
                    self.grid = false;
                }
                removed
            }
            // 14 键组字中硬件键盘的字母不并进组码:组码的每一位是一组,字母是确定的一个。
            SchemeKey::Letter(_) if self.grid_composing() => false,
            SchemeKey::Letter(letter) if letter.is_ascii_alphabetic() => {
                let lower = letter.to_ascii_lowercase();
                let opens_lookup = self.pinyin_lookup_allowed
                    && self.raw.is_empty()
                    && lower == PINYIN_LOOKUP_PREFIX;
                // 反查里的字母是拼音,`z` 和其余字母一样收下。
                let accepted = opens_lookup
                    || self.in_pinyin_lookup()
                    || is_wubi_letter(lower, self.mixed_pinyin_allowed, self.z_wildcard);
                if accepted && self.raw.len() < self.max_code_length() {
                    self.raw.push(char::from(lower));
                    true
                } else {
                    false
                }
            }
            SchemeKey::Letter(_)
            | SchemeKey::Apostrophe
            | SchemeKey::Semicolon
            | SchemeKey::Minus
            | SchemeKey::Symbol(_)
            | SchemeKey::Requery => false,
        }
    }

    /// Every string field is the raw code; valid when non-empty.
    pub fn build_request(&self) -> QueryRequest {
        let mut request = QueryRequest::default();
        self.build_request_into(&mut request);
        request
    }

    /// 将五笔请求写入已有存储,避免逐键刷新重复克隆同一码字符串。
    pub fn build_request_into(&self, request: &mut QueryRequest) {
        request.scheme = SchemeType::Wubi;
        request.raw_input.clone_from(&self.raw);
        request.raw_input_with_cases.clone_from(&self.raw);
        request.normalized_input.clone_from(&self.raw);
        request.raw_segmentation.clone_from(&self.raw);
        request.normalized_segmentation.clone_from(&self.raw);
        request.segmentation.clone_from(&self.raw);
        // 通配只在码里真有 `z` 时改走模式匹配;混输单独打开时永远走前缀范围扫描。14 键组码不含 `z`。
        request.wubi_z_wildcard = self.z_wildcard && self.raw.contains('z');
        request.wubi_grid = self.grid_composing();
        request.valid = !self.raw.is_empty();
    }

    pub fn preedit(&self) -> String {
        self.raw.clone()
    }

    /// 宿主编辑:转小写、只留字母表里的字母、按当前上限截断。C++ 在 `raw_input_with_cases` 非空时读它;转小写后两种拼写给出同一个码,传哪个都行。反查允许时,第一个字母是 `z` 的文本整段按反查收:字母全部保留,上限 32。
    pub fn set_raw_input(&mut self, raw: &str) {
        // 宿主编辑写回的是确定的字母,不是 14 键组码。
        self.grid = false;
        let lookup = self.pinyin_lookup_allowed
            && raw
                .bytes()
                .find(u8::is_ascii_alphabetic)
                .is_some_and(|first| first.to_ascii_lowercase() == PINYIN_LOOKUP_PREFIX);
        // 上限由新文本决定,不能读旧组字:旧组字是反查时 `max_code_length` 会给出 32。
        let limit = if lookup || self.extended_length_allowed {
            MAX_MIXED_CODE_LENGTH
        } else {
            MAX_CODE_LENGTH
        };
        let mixed_pinyin_allowed = self.mixed_pinyin_allowed;
        let z_wildcard = self.z_wildcard;
        self.raw.clear();
        self.raw.reserve(limit.min(raw.len()));
        self.raw.extend(
            raw.bytes()
                .map(|byte| byte.to_ascii_lowercase())
                .filter(|&lower| {
                    if lookup {
                        lower.is_ascii_lowercase()
                    } else {
                        is_wubi_letter(lower, mixed_pinyin_allowed, z_wildcard)
                    }
                })
                .take(limit)
                .map(char::from),
        );
    }

    /// 正好四个字母。更长的混输拼音是回退查询,不是完整的五笔码;反查里的字母是拼音,四个字母也不是完整五笔码。
    pub fn has_complete_code(&self) -> bool {
        self.raw.len() == MAX_CODE_LENGTH && !self.in_pinyin_lookup()
    }

    pub fn set_mixed_pinyin_allowed(&mut self, allowed: bool) {
        self.mixed_pinyin_allowed = allowed;
    }

    /// 打开或关闭 `z` 通配。只影响下一次按键与宿主编辑,已在组字里的文本不变。
    pub fn set_z_wildcard(&mut self, enabled: bool) {
        self.z_wildcard = enabled;
    }

    /// 打开或关闭 `z` 临时拼音反查。只决定下一次组字能否以 `z` 开头,已在组字中的文本不变。
    pub fn set_pinyin_lookup_allowed(&mut self, allowed: bool) {
        self.pinyin_lookup_allowed = allowed;
    }

    /// 当前组字是临时拼音反查:反查允许且组字以 `z` 开头。
    pub fn in_pinyin_lookup(&self) -> bool {
        self.pinyin_lookup_allowed && self.raw.as_bytes().first() == Some(&PINYIN_LOOKUP_PREFIX)
    }

    pub fn set_extended_length_allowed(&mut self, allowed: bool) {
        self.extended_length_allowed = allowed;
    }

    pub fn extended_length_allowed(&self) -> bool {
        self.extended_length_allowed
    }

    fn max_code_length(&self) -> usize {
        if self.extended_length_allowed || self.in_pinyin_lookup() {
            MAX_MIXED_CODE_LENGTH
        } else {
            MAX_CODE_LENGTH
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn typed(scheme: &mut WubiScheme, text: &str) {
        for byte in text.bytes() {
            let key = match byte {
                b'\'' => SchemeKey::Apostrophe,
                _ => SchemeKey::Letter(byte),
            };
            scheme.handle_key(key);
        }
    }

    // test_input_session.cpp:775-782: z, a fifth letter and an apostrophe are refused.
    #[test]
    fn refuses_z_a_fifth_letter_and_apostrophes() {
        let mut scheme = WubiScheme::new();
        typed(&mut scheme, "z");
        assert_eq!(scheme.preedit(), "");
        typed(&mut scheme, "abcd");
        typed(&mut scheme, "e");
        assert_eq!(scheme.preedit(), "abcd");
        typed(&mut scheme, "'");
        assert_eq!(scheme.preedit(), "abcd");
        assert!(scheme.has_complete_code());
    }

    #[test]
    fn symbol_keys_are_ignored() {
        let mut scheme = WubiScheme::new();
        typed(&mut scheme, "ab");
        scheme.handle_key(SchemeKey::Symbol(b';'));
        scheme.handle_key(SchemeKey::Symbol(b' '));
        assert_eq!(scheme.preedit(), "ab");
    }

    #[test]
    fn uppercase_letters_are_lowercased() {
        let mut scheme = WubiScheme::new();
        typed(&mut scheme, "WQ");
        assert_eq!(scheme.preedit(), "wq");
    }

    #[test]
    fn mixed_pinyin_accepts_z_and_extended_length_lifts_the_limit() {
        let mut scheme = WubiScheme::new();
        scheme.set_mixed_pinyin_allowed(true);
        typed(&mut scheme, "zhongguo");
        assert_eq!(scheme.preedit(), "zhon");
        assert!(scheme.has_complete_code());

        scheme.set_extended_length_allowed(true);
        typed(&mut scheme, "gguo");
        assert_eq!(scheme.preedit(), "zhongguo");
        assert!(!scheme.has_complete_code());

        // Reset drops the allowance but keeps the setting-driven z.
        scheme.reset();
        assert_eq!(scheme.preedit(), "");
        typed(&mut scheme, "zhongguo");
        assert_eq!(scheme.preedit(), "zhon");
    }

    #[test]
    fn a_leading_z_opens_the_pinyin_lookup() {
        let mut scheme = WubiScheme::new();
        scheme.set_pinyin_lookup_allowed(true);
        typed(&mut scheme, "zzhongguo");
        assert_eq!(scheme.preedit(), "zzhongguo");
        assert!(scheme.in_pinyin_lookup());
        assert!(!scheme.has_complete_code());

        // 反查最多 32 个字母,与混输拼音的上限相同。
        scheme.reset();
        typed(&mut scheme, &"z".repeat(40));
        assert_eq!(scheme.preedit().len(), MAX_MIXED_CODE_LENGTH);

        // 四个字母的反查不是完整五笔码。
        scheme.reset();
        typed(&mut scheme, "zhao");
        assert!(!scheme.has_complete_code());

        // 只有组字开头的 `z` 引导反查;五笔码中间的 `z` 照旧拒绝。
        scheme.reset();
        typed(&mut scheme, "wqz");
        assert_eq!(scheme.preedit(), "wq");
        assert!(!scheme.in_pinyin_lookup());

        // 退格删掉 `z` 就离开反查。
        scheme.reset();
        typed(&mut scheme, "zh");
        scheme.handle_key(SchemeKey::Backspace);
        assert!(scheme.in_pinyin_lookup());
        scheme.handle_key(SchemeKey::Backspace);
        assert_eq!(scheme.preedit(), "");
        assert!(!scheme.in_pinyin_lookup());
        typed(&mut scheme, "abcde");
        assert_eq!(scheme.preedit(), "abcd");
    }

    #[test]
    fn the_pinyin_lookup_stays_closed_unless_allowed() {
        let mut scheme = WubiScheme::new();
        typed(&mut scheme, "zhao");
        assert_eq!(scheme.preedit(), "hao");
        assert!(!scheme.in_pinyin_lookup());

        // 混输里的 `z` 是拼音字母,不是反查。
        let mut mixed = WubiScheme::new();
        mixed.set_mixed_pinyin_allowed(true);
        typed(&mut mixed, "zhao");
        assert_eq!(mixed.preedit(), "zhao");
        assert!(!mixed.in_pinyin_lookup());
        assert!(mixed.has_complete_code());
    }

    #[test]
    fn set_raw_input_keeps_a_lookup_spelling_whole() {
        let mut scheme = WubiScheme::new();
        scheme.set_pinyin_lookup_allowed(true);
        scheme.set_raw_input("Z'zhong-guo");
        assert_eq!(scheme.preedit(), "zzhongguo");
        assert!(scheme.in_pinyin_lookup());

        // 不以 `z` 开头的文本仍是五笔码:`z` 被滤掉,四码截断。
        scheme.set_raw_input("nihaoz");
        assert_eq!(scheme.preedit(), "niha");
        assert!(!scheme.in_pinyin_lookup());
    }

    #[test]
    fn backspace_and_return() {
        let mut scheme = WubiScheme::new();
        typed(&mut scheme, "wq");
        scheme.handle_key(SchemeKey::Backspace);
        assert_eq!(scheme.preedit(), "w");
        scheme.handle_key(SchemeKey::Backspace);
        scheme.handle_key(SchemeKey::Backspace);
        assert_eq!(scheme.preedit(), "");
        typed(&mut scheme, "ab");
        scheme.reset();
        assert_eq!(scheme.preedit(), "");
    }

    #[test]
    fn set_raw_input_normalises_and_clips() {
        let mut scheme = WubiScheme::new();
        scheme.set_raw_input("A'b-Z9cdef");
        assert_eq!(scheme.preedit(), "abcd");

        scheme.set_mixed_pinyin_allowed(true);
        scheme.set_extended_length_allowed(true);
        scheme.set_raw_input("Ni'Hao");
        assert_eq!(scheme.preedit(), "nihao");
    }

    #[test]
    fn set_raw_input_reuses_existing_storage() {
        let mut scheme = WubiScheme::new();
        scheme.set_mixed_pinyin_allowed(true);
        scheme.set_extended_length_allowed(true);
        scheme.set_raw_input("abcdefghijklmnopqrstuvwxyzabcdef");
        let capacity = scheme.raw.capacity();

        scheme.set_raw_input("ab");

        assert_eq!(scheme.preedit(), "ab");
        assert!(scheme.raw.capacity() >= capacity);
    }

    #[test]
    fn grid_keys_compose_group_codes_up_to_four() {
        let mut scheme = WubiScheme::new();
        // 混输和通配都开着也不影响 14 键:组码最多四位,不延长。
        scheme.set_mixed_pinyin_allowed(true);
        scheme.set_z_wildcard(true);
        scheme.set_pinyin_lookup_allowed(true);
        for group in *b"qqut" {
            assert!(scheme.handle_grid_key(group));
        }
        assert!(scheme.grid_composing());
        assert!(!scheme.handle_grid_key(b'q'));
        assert_eq!(scheme.preedit(), "qqut");
        assert!(scheme.has_complete_code());
        assert!(!scheme.in_pinyin_lookup());
        let request = scheme.build_request();
        assert!(request.wubi_grid);
        assert!(!request.wubi_z_wildcard);
        assert_eq!(request.normalized_input, "qqut");
    }

    #[test]
    fn grid_codes_and_letters_never_share_a_composition() {
        let mut scheme = WubiScheme::new();
        assert!(scheme.handle_grid_key(b'q'));
        // 14 键组字中硬件键盘的确定字母不收。
        assert!(!scheme.handle_key(SchemeKey::Letter(b'a')));
        assert_eq!(scheme.preedit(), "q");
        // 退格到空就离开 14 键组字,下一个字母照常是 26 键的码。
        assert!(scheme.handle_key(SchemeKey::Backspace));
        assert!(!scheme.grid_composing());
        assert!(!scheme.build_request().wubi_grid);
        typed(&mut scheme, "w");
        assert!(!scheme.handle_grid_key(b'q'));
        assert_eq!(scheme.preedit(), "w");
        assert!(!scheme.build_request().wubi_grid);
        // `w`、`z` 不是代表字母。
        scheme.reset();
        assert!(!scheme.handle_grid_key(b'w'));
        assert!(!scheme.handle_grid_key(b'z'));
        assert!(scheme.handle_grid_key(b'x'));
        // 宿主编辑写回的是字母,回到 26 键的码。
        scheme.set_raw_input("wq");
        assert!(!scheme.grid_composing());
        scheme.reset();
        assert!(scheme.handle_grid_key(b'm'));
        scheme.reset();
        assert!(!scheme.grid_composing());
        assert!(scheme.preedit().is_empty());
    }

    #[test]
    fn request_carries_the_code_in_every_field() {
        let mut scheme = WubiScheme::new();
        assert!(!scheme.build_request().valid);
        typed(&mut scheme, "wq");
        let request = scheme.build_request();
        assert_eq!(request.scheme, SchemeType::Wubi);
        assert!(request.valid);
        for field in [
            &request.raw_input,
            &request.raw_input_with_cases,
            &request.normalized_input,
            &request.raw_segmentation,
            &request.normalized_segmentation,
            &request.segmentation,
        ] {
            assert_eq!(field, "wq");
        }
    }
}