1use std::borrow::Cow;
6use std::cmp::{Ordering, Reverse};
7use std::collections::{HashMap, HashSet};
8use std::hash::{BuildHasherDefault, Hasher};
9use std::path::PathBuf;
10use std::sync::OnceLock;
11
12use crate::assets;
13use crate::diagnostics;
14use crate::dictionary::english::{append_keypad_ranked, EnglishDictionary, KeypadRow};
15use crate::error::EngineError;
16use crate::ime::queries::{
17 insert_expressive_rows, keep_expressive_rows_off_the_lead, MIXED_EXPRESSIVE_MINIMUM_INPUT,
18};
19use crate::language_dictionary::{self, LanguageDictionary};
20use crate::lattice::decode::PHRASE_LENGTH_BONUS;
21use crate::lattice::neural::{
22 shared_sentence_model, NeuralReranker, CONTEXT_CHARACTERS, MAX_RERANK_PATHS,
23};
24use crate::lattice::personal::PersonalTransition;
25use crate::lattice::SentencePath;
26use crate::local::date_time::{inline_date_time_keyword, insert_inline_date_time};
27use crate::local::emoji::{query_emoji_readings, query_kaomoji_readings, ExpressiveRow};
28use crate::local::symbol_candidates::{query_symbol_readings, SymbolRow, MIXED_SYMBOL_LIMIT};
29use crate::nine_key_shuangpin::NineKeyShuangpin;
30use crate::paths::RuntimePaths;
31use crate::pinyin::segment::{is_complete_pinyin_input, split_segments};
32use crate::pinyin::syllables::intact_pinyin_list;
33use crate::quanpin::QuanpinDictionary;
34use crate::session::{LocalClock, SessionSnapshot};
35use crate::stroke;
36use crate::text::{count_utf8_chars, is_all_han, is_han_phrase, last_characters};
37use crate::types::{
38 CandidateSource, Command, EnglishInputOptions, FirstPageRows, FrequencyAdjustmentMode,
39 FrequencyAdjustmentOptions, FuzzyPinyinOptions, KeyResult, LocalInputMode,
40 MixedExpressiveOptions, PersonalDictionaryKind, SchemeType, SentenceAssociationOptions,
41 SharedString, WordItem,
42};
43use crate::user_dictionary::positions;
44use crate::user_dictionary::ranking::{self, RankingRequest};
45use crate::user_dictionary::removal;
46
47pub const PATH_LIMIT: usize = 48;
48pub const DIGIT_LIMIT: usize = 48;
49pub const CANDIDATE_LIMIT: usize = 128;
50const SENTENCE_SEED_LIMIT: usize = 4;
52const SEED_RETAIN_LIMIT: usize = 8;
54const SEED_TAIL_LIMIT: usize = 3;
56const SEEDED_TRUNCATE_DIGITS: usize = 12;
58const SEEDED_PATH_LIMIT: usize = 24;
59const QUERY_PATH_LIMIT: usize = 24;
61const SYLLABLE_FREQUENCY: &str = include_str!("nine_key/syllable_frequency.tsv");
63const SMALL_QUERY_KEY_BATCH: usize = 64;
65const SMALL_CANDIDATE_DEDUP: usize = 64;
67const SMALL_CANDIDATE_SORT: usize = 32;
69
70const KEYPAD: &[u8; 26] = b"22233344455566677778889999";
72const DIGIT_LETTERS: [&str; 10] = [
74 "", "", "abc", "def", "ghi", "jkl", "mno", "pqrs", "tuv", "wxyz",
75];
76const FOURTEEN_KEYPAD: &[u8; 26] = b"abcdedggujjlmbooqeatucqztz";
78const FOURTEEN_KEY_GROUPS: [&str; 14] = [
80 "qw", "er", "ty", "ui", "op", "as", "df", "gh", "jk", "l", "zx", "cv", "bn", "m",
81];
82const EIGHTEEN_KEYPAD: &[u8; 26] = b"abxswffhijjlmbipqrsruvwxyz";
84const EIGHTEEN_KEY_GROUPS: [&str; 18] = [
85 "q", "we", "rt", "y", "u", "io", "p", "a", "sd", "fg", "h", "jk", "l", "z", "xc", "v", "bn",
86 "m",
87];
88
89pub(crate) fn eighteen_key_code(letter: u8) -> Option<u8> {
91 if letter == b';' {
92 Some(b'l')
93 } else {
94 KeyGrid::EighteenKey.code_of(letter)
95 }
96}
97
98const ENGLISH_PREFIX_BUDGET: usize = 64;
100const ENGLISH_LIMIT: usize = 5;
101const ENGLISH_CANDIDATE_CAPACITY: usize = ENGLISH_PREFIX_BUDGET * ENGLISH_LIMIT;
102const ENGLISH_CACHED_PREFIX_ROWS: usize = 2048;
104const INITIALS_CODE_LIMIT: usize = 1024;
106const INITIALS_ROW_LIMIT: usize = 64;
108const SHUANGPIN_WORD_MIN_SYLLABLES: usize = 3;
110const SHUANGPIN_WORD_SCAN_LIMIT: usize = 1024;
112const INITIALS_SCAN_LIMIT: usize = 512;
114pub(crate) const PERSONAL_PICK_TIMES: u32 = 2;
116const RERANK_CACHE_ENTRIES: usize = 16;
118const SYLLABLE_ROWS_BEFORE_INITIALS: usize = 3;
120const COMPLETION_WEIGHT_RATIO: i64 = 6;
122const EXPRESSIVE_READING_LIMIT: usize = 8;
124pub(crate) const MAX_LEARNED_SENTENCE_SYLLABLES: usize = 7;
126
127#[derive(Clone, Copy, Debug)]
131pub enum KeyGrid {
132 NineKey,
134 FourteenKey,
136 EighteenKey,
138 NineKeyShuangpin(&'static NineKeyShuangpin),
140 FourteenKeyShuangpin(&'static NineKeyShuangpin),
142 EighteenKeyShuangpin(&'static NineKeyShuangpin),
144}
145
146impl PartialEq for KeyGrid {
147 fn eq(&self, other: &Self) -> bool {
148 match (self, other) {
149 (Self::NineKey, Self::NineKey)
150 | (Self::FourteenKey, Self::FourteenKey)
151 | (Self::EighteenKey, Self::EighteenKey) => true,
152 (Self::NineKeyShuangpin(left), Self::NineKeyShuangpin(right))
153 | (Self::FourteenKeyShuangpin(left), Self::FourteenKeyShuangpin(right))
154 | (Self::EighteenKeyShuangpin(left), Self::EighteenKeyShuangpin(right)) => {
155 std::ptr::eq(*left, *right)
156 }
157 _ => false,
158 }
159 }
160}
161
162impl Eq for KeyGrid {}
163
164impl std::hash::Hash for KeyGrid {
165 fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
166 std::mem::discriminant(self).hash(state);
167 if let Some(table) = self.shuangpin_table() {
168 std::ptr::hash(table, state);
169 }
170 }
171}
172
173impl KeyGrid {
174 fn shuangpin_table(self) -> Option<&'static NineKeyShuangpin> {
175 match self {
176 Self::NineKeyShuangpin(table)
177 | Self::FourteenKeyShuangpin(table)
178 | Self::EighteenKeyShuangpin(table) => Some(table),
179 _ => None,
180 }
181 }
182
183 fn keypad(self) -> Option<&'static [u8; 26]> {
185 match self {
186 Self::NineKey => Some(KEYPAD),
187 Self::FourteenKey => Some(FOURTEEN_KEYPAD),
188 Self::EighteenKey => Some(EIGHTEEN_KEYPAD),
189 Self::NineKeyShuangpin(_)
190 | Self::FourteenKeyShuangpin(_)
191 | Self::EighteenKeyShuangpin(_) => None,
192 }
193 }
194
195 pub fn code_of(self, letter: u8) -> Option<u8> {
197 if matches!(self, Self::EighteenKeyShuangpin(_)) {
198 if letter.is_ascii_uppercase() {
199 return Some(letter);
200 }
201 return eighteen_key_code(letter);
202 }
203 if matches!(self, Self::FourteenKeyShuangpin(_)) {
204 return if letter == b';' {
205 Some(b'l')
206 } else {
207 Self::FourteenKey.code_of(letter)
208 };
209 }
210 let keypad = self.keypad()?;
211 letter
212 .is_ascii_lowercase()
213 .then(|| keypad[usize::from(letter - b'a')])
214 }
215
216 pub fn letters(self, code: u8) -> &'static str {
218 match self {
219 Self::NineKey if (b'2'..=b'9').contains(&code) => {
220 DIGIT_LETTERS[usize::from(code - b'0')]
221 }
222 Self::NineKey => "",
223 Self::FourteenKey => FOURTEEN_KEY_GROUPS
224 .iter()
225 .find(|group| group.as_bytes().first() == Some(&code))
226 .copied()
227 .unwrap_or(""),
228 Self::EighteenKey => EIGHTEEN_KEY_GROUPS
229 .iter()
230 .find(|group| group.as_bytes().first() == Some(&code))
231 .copied()
232 .unwrap_or(""),
233 Self::NineKeyShuangpin(table)
234 | Self::FourteenKeyShuangpin(table)
235 | Self::EighteenKeyShuangpin(table) => table.letters(code),
236 }
237 }
238
239 pub fn is_code(self, byte: u8) -> bool {
241 match self {
242 Self::NineKeyShuangpin(table)
243 | Self::FourteenKeyShuangpin(table)
244 | Self::EighteenKeyShuangpin(table) => table.uses_key(byte),
245 _ => !self.letters(byte).is_empty(),
246 }
247 }
248
249 pub fn encode(self, pinyin: &str) -> String {
251 let Some(keypad) = self.keypad() else {
252 return match self {
253 Self::NineKeyShuangpin(table)
254 | Self::FourteenKeyShuangpin(table)
255 | Self::EighteenKeyShuangpin(table) => table.encode(pinyin),
256 _ => String::new(),
257 };
258 };
259 pinyin
260 .bytes()
261 .filter(u8::is_ascii_lowercase)
262 .map(|letter| char::from(keypad[usize::from(letter - b'a')]))
263 .collect()
264 }
265
266 fn matching_code_prefix_len(self, pinyin: &str, digits: &str) -> Option<usize> {
268 let Some(keypad) = self.keypad() else {
269 return self.shuangpin_table()?.matching_prefix_len(pinyin, digits);
270 };
271 let digits = digits.as_bytes();
272 let mut covered = None;
273 for (index, letter) in pinyin.bytes().filter(u8::is_ascii_lowercase).enumerate() {
274 let Some(&digit) = digits.get(index) else {
275 return Some(index);
276 };
277 if keypad[usize::from(letter - b'a')] != digit {
278 return None;
279 }
280 covered = Some(index + 1);
281 }
282 covered
283 }
284
285 pub(crate) fn syllable_code_len(self, syllable: &str) -> usize {
287 match self {
288 Self::NineKeyShuangpin(_)
289 | Self::FourteenKeyShuangpin(_)
290 | Self::EighteenKeyShuangpin(_) => 2,
291 _ => syllable.len(),
292 }
293 }
294
295 fn reading_code_len(self, canonical_pinyin: &str) -> usize {
297 match self {
298 Self::NineKeyShuangpin(_)
299 | Self::FourteenKeyShuangpin(_)
300 | Self::EighteenKeyShuangpin(_) => 2 * reading_syllable_count(canonical_pinyin),
301 _ => canonical_pinyin
302 .bytes()
303 .filter(u8::is_ascii_lowercase)
304 .count(),
305 }
306 }
307
308 pub fn offers_raw_key(self) -> bool {
310 match self {
311 Self::NineKey | Self::NineKeyShuangpin(_) => true,
312 Self::FourteenKey
313 | Self::EighteenKey
314 | Self::FourteenKeyShuangpin(_)
315 | Self::EighteenKeyShuangpin(_) => false,
316 }
317 }
318
319 pub fn uses_digits(self) -> bool {
321 matches!(self, Self::NineKey | Self::NineKeyShuangpin(_))
322 }
323
324 fn spells_english(self) -> bool {
326 self.shuangpin_table().is_none()
327 }
328
329 fn spelling_table(self) -> &'static SpellingTable {
331 match self {
332 Self::NineKey => {
333 static TABLE: OnceLock<SpellingTable> = OnceLock::new();
334 TABLE.get_or_init(|| SpellingTable::build(self, intact_pinyin_list()))
335 }
336 Self::FourteenKey => {
337 static TABLE: OnceLock<SpellingTable> = OnceLock::new();
338 TABLE.get_or_init(|| SpellingTable::build(self, intact_pinyin_list()))
339 }
340 Self::EighteenKey => {
341 static TABLE: OnceLock<SpellingTable> = OnceLock::new();
342 TABLE.get_or_init(|| SpellingTable::build(self, intact_pinyin_list()))
343 }
344 Self::NineKeyShuangpin(table)
345 | Self::FourteenKeyShuangpin(table)
346 | Self::EighteenKeyShuangpin(table) => table
347 .spelling
348 .get_or_init(|| SpellingTable::build_shuangpin(self, table)),
349 }
350 }
351}
352
353type Path = Vec<String>;
354
355pub struct NineKeySession {
356 paths: RuntimePaths,
357 learning: bool,
358 frequency: FrequencyAdjustmentOptions,
359 fuzzy: FuzzyPinyinOptions,
360 english_options: EnglishInputOptions,
361 expressive: MixedExpressiveOptions,
363 mixed_candidates_enabled: bool,
365 inline_date_time: Option<LocalClock>,
367 sentence_association: SentenceAssociationOptions,
369 sentence_alternatives: bool,
371 first_page_rows: FirstPageRows,
373 rescoring_context: String,
375 keyboard: Option<Option<NeuralReranker>>,
377 rerank_cache: Vec<RerankEntry>,
379 personal_context: bool,
381 remember_recent: bool,
383 completions_discounted: bool,
385 digits: String,
386 locked: Vec<String>,
387 splits: Vec<usize>,
389 spellings: Vec<String>,
390 spelling_match_indices: Vec<usize>,
392 reading: String,
394 candidates: Vec<WordItem>,
395 sentence_seeds: Vec<String>,
397 seed_buffer: Vec<String>,
399 path_buffer: Vec<Path>,
401 path_scratch: PathScratch<'static>,
403 english_only: bool,
404 phrase_pinyin: String,
406 phrase_word: String,
407 phrase_storable: bool,
408 caret: Option<usize>,
410 pinyin: bool,
412 single_character_only: bool,
414 dictionary: Option<QuanpinDictionary>,
416 prior: Option<SyllablePrior<'static>>,
418 english: Option<EnglishDictionary>,
419 english_pattern: String,
421 english_rows: Option<EnglishKeypadCache>,
423 lock_undo: Vec<Option<LockUndo>>,
425 initial: Option<Initial>,
427 single_character: bool,
429 strokes: String,
431 stroke_dictionary: PathBuf,
433 stroke: Option<LanguageDictionary>,
435 stroke_texts: HashSet<char>,
437 grid: KeyGrid,
439}
440
441struct LockUndo {
443 replaced: String,
445 splits: [usize; DIGIT_LIMIT],
447 split_count: usize,
448 initial: Option<Initial>,
450 choices: Vec<String>,
452}
453
454#[derive(Clone, Copy)]
455struct Initial {
456 letter: u8,
457 typed: usize,
459 splits: usize,
460}
461
462enum Choice {
464 Syllable,
465 Letter(u8),
466 Digit(u8),
467}
468
469fn choice_kind(choice: &str) -> Choice {
470 match choice.as_bytes() {
471 [letter] if letter.is_ascii_uppercase() => Choice::Letter(letter.to_ascii_lowercase()),
472 [digit] if digit.is_ascii_digit() => Choice::Digit(*digit),
473 _ => Choice::Syllable,
474 }
475}
476
477impl NineKeySession {
478 pub fn new(
480 paths: &RuntimePaths,
481 learning: bool,
482 frequency: FrequencyAdjustmentOptions,
483 fuzzy: FuzzyPinyinOptions,
484 english: EnglishInputOptions,
485 pinyin: bool,
486 single_character_only: bool,
487 ) -> Self {
488 Self {
489 paths: paths.clone(),
490 learning,
491 frequency,
492 fuzzy,
493 english_options: english,
494 expressive: MixedExpressiveOptions::default(),
495 mixed_candidates_enabled: true,
496 inline_date_time: None,
497 sentence_association: SentenceAssociationOptions::default(),
498 sentence_alternatives: false,
499 first_page_rows: FirstPageRows::default(),
500 rescoring_context: String::new(),
501 keyboard: None,
502 rerank_cache: Vec::new(),
503 personal_context: true,
504 remember_recent: true,
505 completions_discounted: false,
506 digits: String::new(),
507 locked: Vec::new(),
508 splits: Vec::new(),
509 spellings: Vec::new(),
510 spelling_match_indices: Vec::new(),
511 reading: String::new(),
512 candidates: Vec::new(),
513 sentence_seeds: Vec::with_capacity(SEED_RETAIN_LIMIT),
515 seed_buffer: Vec::with_capacity(SEED_RETAIN_LIMIT),
516 path_buffer: Vec::new(),
517 path_scratch: PathScratch::default(),
518 english_only: false,
519 phrase_pinyin: String::new(),
520 phrase_word: String::new(),
521 phrase_storable: true,
522 caret: None,
523 pinyin,
524 single_character_only,
525 dictionary: None,
526 prior: None,
527 english: None,
528 english_pattern: String::new(),
529 english_rows: None,
530 lock_undo: Vec::new(),
531 initial: None,
532 single_character: false,
533 strokes: String::new(),
534 stroke_dictionary: PathBuf::new(),
535 stroke: None,
536 stroke_texts: HashSet::new(),
537 grid: KeyGrid::NineKey,
538 }
539 }
540
541 pub fn set_stroke_dictionary(&mut self, path: PathBuf) {
543 if self.stroke_dictionary != path {
544 self.stroke = None;
545 }
546 self.stroke_dictionary = path;
547 }
548
549 pub fn set_mixed_expressive(&mut self, options: MixedExpressiveOptions) {
551 self.expressive = options;
552 }
553
554 pub fn set_frequency(&mut self, frequency: FrequencyAdjustmentOptions) {
556 self.frequency = frequency;
557 }
558
559 pub fn set_mixed_candidates_enabled(&mut self, enabled: bool) {
561 self.mixed_candidates_enabled = enabled;
562 }
563
564 pub fn set_inline_date_time(&mut self, clock: Option<LocalClock>) {
566 self.inline_date_time = clock;
567 }
568
569 pub fn set_sentence_options(
571 &mut self,
572 association: SentenceAssociationOptions,
573 alternatives: bool,
574 ) {
575 if self.sentence_association == association && self.sentence_alternatives == alternatives {
576 return;
577 }
578 self.sentence_association = association.clone();
579 self.sentence_alternatives = alternatives;
580 if !association.neural_keyboard {
581 self.keyboard = None;
582 self.rerank_cache.clear();
583 }
584 let dictionary_association = self.dictionary_association();
585 if let Some(dictionary) = self.dictionary.as_mut() {
586 dictionary.set_sentence_association(dictionary_association);
587 dictionary.set_sentence_alternatives(alternatives);
588 }
589 if self.active() {
590 self.refresh();
591 }
592 }
593
594 pub fn set_first_page_rows(&mut self, rows: FirstPageRows) {
596 if self.first_page_rows == rows {
597 return;
598 }
599 self.first_page_rows = rows;
600 if let Some(dictionary) = self.dictionary.as_mut() {
601 dictionary.set_first_page_rows(rows);
602 }
603 if self.active() {
604 self.refresh();
605 }
606 }
607
608 pub fn set_personal_context_enabled(&mut self, enabled: bool) {
610 self.personal_context = enabled;
611 }
612
613 pub fn set_rescoring_context(&mut self, context: &str) {
615 let trimmed = last_characters(context, CONTEXT_CHARACTERS);
616 if self.rescoring_context != trimmed {
617 self.rescoring_context.clear();
618 self.rescoring_context.push_str(trimmed);
619 }
620 }
621
622 fn dictionary_association(&self) -> SentenceAssociationOptions {
624 let association = self.sentence_association.clone();
625 SentenceAssociationOptions {
626 word_lattice: association.word_lattice || association.neural_keyboard,
627 neural_keyboard: false,
628 show_next_on_duplicate: association.show_next_on_duplicate,
629 ..association
630 }
631 }
632
633 fn dictionary_options(&self) -> DictionaryOptions {
636 DictionaryOptions {
637 association: self.dictionary_association(),
638 alternatives: self.sentence_alternatives,
639 first_page_rows: self.first_page_rows,
640 }
641 }
642
643 pub fn active(&self) -> bool {
645 !self.digits.is_empty()
646 }
647
648 pub fn set_english_only(&mut self, english_only: bool) {
650 if self.english_only == english_only {
651 return;
652 }
653 self.english_only = english_only;
654 self.refresh();
655 }
656
657 pub fn set_grid(&mut self, grid: KeyGrid) {
659 self.command(Command::Cancel);
660 self.sentence_seeds.clear();
661 self.grid = grid;
662 }
663
664 pub fn character(&mut self, digit: u8) -> KeyResult {
666 if digit == b'\'' {
667 if !self.active() || self.english_only || !self.pinyin {
669 return KeyResult::unhandled();
670 }
671 let caret = self.caret_position();
672 if self.on_lock_boundary(caret) || self.splits.contains(&caret) {
674 return KeyResult::handled();
675 }
676 self.unlock_from(caret);
677 let at = self.splits.partition_point(|&split| split < caret);
678 self.splits.insert(at, caret);
679 self.refresh();
680 return KeyResult::handled();
681 }
682 if !self.grid.is_code(digit) {
683 return KeyResult::unhandled();
684 }
685 if self.digits.len() >= DIGIT_LIMIT {
686 return KeyResult::handled()
687 .with_diagnostic(Some(diagnostics::NINE_KEY_DIGIT_LIMIT.to_string()));
688 }
689 let caret = self.caret_position();
690 self.unlock_from(caret);
691 if caret == self.locked_length() {
692 self.initial = None;
694 }
695 self.digits.insert(caret, char::from(digit));
696 for split in &mut self.splits {
698 if *split > caret {
699 *split += 1;
700 }
701 }
702 self.set_caret(caret + 1);
703 self.refresh();
704 KeyResult::handled()
705 }
706
707 pub fn limit_pick_ready(&self, digit: u8) -> bool {
709 self.active()
710 && self.grid.is_code(digit)
711 && self.digits.len() >= DIGIT_LIMIT
712 && self.caret_position() == self.digits.len()
713 && self
714 .candidates
715 .first()
716 .is_some_and(|candidate| !candidate.pinyin.is_empty())
717 }
718
719 fn caret_position(&self) -> usize {
721 let length = self.digits.len();
722 self.caret.map_or(length, |caret| caret.min(length))
723 }
724
725 fn set_caret(&mut self, caret: usize) {
726 self.caret = (caret < self.digits.len()).then_some(caret);
727 }
728
729 fn on_lock_boundary(&self, position: usize) -> bool {
731 let mut end = 0;
732 position == 0
733 || self.locked.iter().any(|spelling| {
734 end += self.grid.syllable_code_len(spelling);
735 end == position
736 })
737 }
738
739 fn unlock_from(&mut self, position: usize) {
741 while self.locked_length() > position {
742 self.locked.pop();
743 self.lock_undo.pop();
744 self.initial = None;
745 }
746 }
747
748 pub fn choose_spelling(&mut self, index: usize) -> KeyResult {
750 let Some(choice) = self.spellings.get(index).cloned() else {
751 return KeyResult::unhandled();
752 };
753 let reselecting = self.reselecting();
754 let choices = if reselecting {
755 match self.undo_last_lock() {
756 Some(choices) => choices,
757 None => return KeyResult::unhandled(),
758 }
759 } else {
760 Vec::new()
761 };
762 match choice_kind(&choice) {
763 Choice::Digit(digit) => {
764 self.consume(1);
766 if self.active() {
768 self.phrase_storable = false;
769 self.phrase_word.push(char::from(digit));
770 } else {
771 self.reset_phrase();
772 }
773 self.refresh();
774 return KeyResult::committed(char::from(digit).to_string());
775 }
776 Choice::Letter(letter) => {
777 self.initial = Some(Initial {
778 letter,
779 typed: self.digits.len(),
780 splits: self.splits.len(),
781 });
782 }
783 Choice::Syllable => {
784 let choices = if reselecting {
785 choices
786 } else {
787 std::mem::take(&mut self.spellings)
788 };
789 let offset = self.locked_length();
790 let end = offset
792 + self
793 .grid
794 .syllable_code_len(&choice)
795 .min(self.digits.len() - offset);
796 let replaced = self.digits[offset..end].to_string();
797 let before = self.digits.len();
798 self.digits
799 .replace_range(offset..end, &self.grid.encode(&choice));
800 if let Some(caret) = self.caret {
802 if caret >= end {
803 self.set_caret(caret + self.digits.len() - before);
804 }
805 }
806 self.locked.push(choice);
807 let locked_length = self.locked_length();
808 let mut dropped = [0; DIGIT_LIMIT];
809 let mut split_count = 0;
810 self.splits.retain(|&split| {
811 if split <= locked_length {
812 dropped[split_count] = split;
813 split_count += 1;
814 false
815 } else {
816 true
817 }
818 });
819 self.lock_undo.push(Some(LockUndo {
820 replaced,
821 splits: dropped,
822 split_count,
823 initial: self.initial.take(),
824 choices,
825 }));
826 }
827 }
828 self.refresh();
829 KeyResult::handled()
830 }
831
832 fn reselecting(&self) -> bool {
834 self.active()
835 && self.locked_length() >= self.digits.len()
836 && self.lock_undo.last().is_some_and(Option::is_some)
837 }
838
839 fn undo_last_lock(&mut self) -> Option<Vec<String>> {
841 let undo = self.lock_undo.last_mut()?.take()?;
842 self.lock_undo.pop();
843 let spelling = self.locked.pop()?;
844 let offset = self.locked_length();
845 let end = (offset + self.grid.syllable_code_len(&spelling)).min(self.digits.len());
846 self.digits.replace_range(offset..end, &undo.replaced);
847 self.splits
848 .extend(undo.splits[..undo.split_count].iter().copied());
849 self.splits.sort_unstable();
850 self.splits.dedup();
851 self.initial = undo.initial.map(|initial| Initial {
852 typed: self.digits.len(),
853 splits: self.splits.len(),
854 ..initial
855 });
856 if let Some(caret) = self.caret {
858 self.set_caret(caret);
859 }
860 Some(undo.choices)
861 }
862
863 pub fn select(&mut self, index: usize) -> KeyResult {
864 let Some(selected) = self.candidates.get(index) else {
865 return KeyResult::unhandled();
866 };
867 let selected_source = selected.source;
868 let selected_fixed_position = selected.fixed_position;
869 let selected_pinyin_length = selected.pinyin.len();
870 let selected_word = selected.word.clone();
871 let selected_is_initials = is_initials_row(selected);
872 let selected_canonical_pinyin = if self.learning
873 && (selected_source.is_dictionary() || selected_source.is_sentence_learning())
874 {
875 selected.canonical_pinyin.to_string()
876 } else {
877 String::new()
878 };
879 let recorded_initials =
881 self.records_personal_use(selected_source, &selected_word) && selected_is_initials;
882 let mut diagnostic = if self.learning
883 && self.frequency.mode != FrequencyAdjustmentMode::Disabled
884 && index != 0
885 && selected_fixed_position == 0
886 && !recorded_initials
887 && self.editable(index)
888 {
889 self.adjust_frequency(index, false)
890 } else {
891 None
892 };
893 if self.learning
895 && selected_source.is_dictionary()
896 && !selected_canonical_pinyin.is_empty()
897 && !selected_is_initials
898 && selected_pinyin_length == self.digits.len()
899 && self.remember_recent
900 {
901 if index != 0 && selected_fixed_position == 0 {
902 diagnostic = diagnostic.or(self.remember_recent_candidate(
903 &self.ranking_context(),
904 &selected_canonical_pinyin,
905 &selected_word,
906 ));
907 }
908 if !self.locked.is_empty() {
909 let unlocked_context = format!("nine-key:{}:", self.digits);
910 if !positions::is_fixed(
912 &self.paths.user(assets::USER_JOURNAL),
913 &unlocked_context,
914 &selected_canonical_pinyin,
915 &selected_word,
916 ) {
917 diagnostic = diagnostic.or(self.remember_recent_candidate(
918 &unlocked_context,
919 &selected_canonical_pinyin,
920 &selected_word,
921 ));
922 }
923 }
924 }
925 self.consume(selected_pinyin_length);
926 if self.learning {
927 let (learned, stored) =
928 self.learn_selection(selected_source, &selected_canonical_pinyin, &selected_word);
929 diagnostic = diagnostic.or(learned);
930 let recorded = self.record_personal_use(selected_source, &selected_word, stored);
931 diagnostic = diagnostic.or(recorded);
932 } else {
933 self.reset_phrase();
934 }
935 self.refresh();
936 KeyResult::committed(selected_word).with_diagnostic(diagnostic)
937 }
938
939 fn learn_selection(
941 &mut self,
942 selected_source: CandidateSource,
943 selected_canonical_pinyin: &str,
944 selected_word: &str,
945 ) -> (Option<String>, Option<String>) {
946 let reading = if selected_source.is_dictionary() || selected_source.is_sentence_learning() {
947 selected_canonical_pinyin
948 } else {
949 ""
950 };
951 if self.active() {
952 if reading.is_empty() {
953 self.phrase_storable = false;
954 } else if self.phrase_storable {
955 if !self.phrase_pinyin.is_empty() {
956 self.phrase_pinyin.push('\'');
957 }
958 self.phrase_pinyin.push_str(reading);
959 }
960 self.phrase_word.push_str(selected_word);
961 return (None, None);
962 }
963 let phrase = !self.phrase_word.is_empty();
964 let stored = if phrase {
965 (self.phrase_storable && !reading.is_empty()).then(|| {
966 (
967 format!("{}'{reading}", self.phrase_pinyin),
968 format!("{}{}", self.phrase_word, selected_word),
969 )
970 })
971 } else {
972 (selected_source.is_sentence_learning()
973 && !reading.is_empty()
974 && split_segments(reading).len() <= MAX_LEARNED_SENTENCE_SYLLABLES)
975 .then(|| (reading.to_owned(), selected_word.to_owned()))
976 };
977 self.reset_phrase();
978 let Some((pinyin, word)) = stored else {
979 return (None, None);
980 };
981 let options = self.dictionary_options();
983 match self
984 .dictionary
985 .get_or_insert_with(|| open_dictionary(&self.paths, options))
986 .create_word_from_canonical_pinyin(&pinyin, &word)
987 {
988 Ok(()) => (None, Some(word)),
989 Err(EngineError::InvalidArgument(_)) => (None, None),
990 Err(_) if phrase => (Some(diagnostics::PHRASE_NOT_PERSISTED.to_string()), None),
991 Err(_) => (Some(diagnostics::SENTENCE_NOT_PERSISTED.to_string()), None),
992 }
993 }
994
995 fn record_personal_use(
997 &self,
998 selected_source: CandidateSource,
999 selected_word: &str,
1000 stored: Option<String>,
1001 ) -> Option<String> {
1002 if !self.personal_context {
1003 return None;
1004 }
1005 let mut transitions: Vec<PersonalTransition> = Vec::with_capacity(2);
1006 let words = selected_source
1007 .is_dictionary()
1008 .then_some(selected_word)
1009 .into_iter()
1010 .chain(stored.as_deref());
1011 for word in words {
1012 if is_personal_word(word)
1013 && !transitions.iter().any(|transition| transition.word == word)
1014 {
1015 transitions.push(PersonalTransition {
1016 earlier: String::new(),
1017 previous: String::new(),
1018 word: word.to_owned(),
1019 times: PERSONAL_PICK_TIMES,
1020 });
1021 }
1022 }
1023 if transitions.is_empty() {
1024 return None;
1025 }
1026 let dictionary = self.dictionary.as_ref()?;
1028 dictionary
1029 .record_personal_use(&transitions)
1030 .err()
1031 .map(|_| diagnostics::PERSONAL_CONTEXT_NOT_PERSISTED.to_string())
1032 }
1033
1034 fn records_personal_use(&self, selected_source: CandidateSource, selected_word: &str) -> bool {
1036 self.learning
1037 && self.personal_context
1038 && selected_source.is_dictionary()
1039 && is_personal_word(selected_word)
1040 }
1041
1042 fn reset_phrase(&mut self) {
1043 self.phrase_pinyin.clear();
1044 self.phrase_word.clear();
1045 self.phrase_storable = true;
1046 }
1047
1048 pub fn finish(&mut self, first_index: usize) -> KeyResult {
1050 if !self.active() {
1051 return KeyResult::unhandled();
1052 }
1053 if first_index >= self.candidates.len() {
1054 return self.command(Command::CommitRaw);
1055 }
1056 let mut commit = String::new();
1057 let mut diagnostic = None;
1058 let mut index = first_index;
1059 while self.active() {
1060 if self.candidates.is_empty() {
1061 if matches!(self.grid, KeyGrid::EighteenKeyShuangpin(_)) {
1062 self.digits.make_ascii_lowercase();
1063 }
1064 commit.push_str(&self.digits);
1065 break;
1066 }
1067 let previous_remember_recent = self.remember_recent;
1068 self.remember_recent = false;
1069 let result = self.select(index);
1070 self.remember_recent = previous_remember_recent;
1071 if diagnostic.is_none() {
1072 diagnostic = result.diagnostic;
1073 }
1074 if let Some(text) = result.commit {
1075 if commit.is_empty() {
1076 commit = text;
1077 } else {
1078 commit.push_str(&text);
1079 }
1080 }
1081 index = 0;
1082 }
1083 self.command(Command::Cancel);
1084 KeyResult::committed(commit).with_diagnostic(diagnostic)
1085 }
1086
1087 pub fn command(&mut self, command: Command) -> KeyResult {
1088 if !self.active() {
1089 return KeyResult::unhandled();
1090 }
1091 match command {
1092 Command::CommitCandidate => return self.select(0),
1093 Command::CommitRaw => {
1094 if !self.grid.offers_raw_key() && !self.candidates.is_empty() {
1096 return self.finish(0);
1097 }
1098 let mut raw = std::mem::take(&mut self.digits);
1099 if matches!(self.grid, KeyGrid::EighteenKeyShuangpin(_)) {
1101 raw.make_ascii_lowercase();
1102 }
1103 self.clear_composition();
1104 return KeyResult::committed(raw);
1105 }
1106 Command::Cancel => self.clear_composition(),
1107 Command::Backspace => {
1110 let caret = self.caret_position();
1111 let at_end = caret == self.digits.len();
1112 let initial_untouched = at_end
1113 && self.initial.is_some_and(|initial| {
1114 initial.typed == self.digits.len() && initial.splits == self.splits.len()
1115 });
1116 if initial_untouched {
1117 self.initial = None;
1118 } else if at_end && self.reselecting() {
1119 self.undo_last_lock();
1120 } else if let Some(at) = self.splits.iter().position(|&split| split == caret) {
1121 self.splits.remove(at);
1122 } else if caret > 0 {
1123 self.remove_digit(caret - 1);
1124 self.set_caret(caret - 1);
1125 }
1126 }
1127 Command::DeleteForward => {
1128 let caret = self.caret_position();
1129 if caret < self.digits.len() {
1130 self.remove_digit(caret);
1131 self.set_caret(caret);
1132 }
1133 }
1134 Command::MoveLeft | Command::MoveRight | Command::MoveHome | Command::MoveEnd => {
1136 let caret = self.caret_position();
1137 let target = match command {
1138 Command::MoveLeft => caret.saturating_sub(1),
1139 Command::MoveRight => caret + 1,
1140 Command::MoveHome => 0,
1141 _ => self.digits.len(),
1142 };
1143 self.set_caret(target);
1144 return KeyResult::handled();
1145 }
1146 _ => return KeyResult::unhandled(),
1147 }
1148 if !self.active() {
1149 self.clear_composition();
1151 }
1152 self.refresh();
1153 KeyResult::handled()
1154 }
1155
1156 fn clear_composition(&mut self) {
1158 self.path_scratch.pages.clear();
1159 self.english_rows = None;
1161 self.digits.clear();
1162 self.locked.clear();
1163 self.lock_undo.clear();
1164 self.splits.clear();
1165 self.initial = None;
1166 self.caret = None;
1167 self.reset_phrase();
1168 }
1169
1170 fn remove_digit(&mut self, index: usize) {
1172 let anchor = self.locked_length();
1173 self.unlock_from(index);
1174 self.digits.remove(index);
1175 for split in &mut self.splits {
1176 if *split > index {
1177 *split -= 1;
1178 }
1179 }
1180 let locked_length = self.locked_length();
1181 self.splits.retain(|&split| split > locked_length);
1182 self.splits.dedup();
1183 let length = self.digits.len();
1184 if index <= anchor || length <= locked_length {
1185 self.initial = None;
1186 } else if let Some(initial) = self.initial.as_mut() {
1187 if initial.typed > length {
1188 initial.typed = usize::MAX;
1189 }
1190 }
1191 }
1192
1193 pub fn pin(&mut self, index: usize) -> KeyResult {
1194 if !self.editable(index) {
1195 return KeyResult::unhandled();
1196 }
1197 let selected = self.candidates[index].clone();
1198 let replaces_recent = selected.fixed_position == 0
1199 && self
1200 .candidates
1201 .iter()
1202 .any(|candidate| candidate.fixed_position == 1);
1203 let diagnostic = self.adjust_frequency(index, true);
1204 let position_diagnostic = replaces_recent.then(|| {
1205 positions::set_fixed_position(
1206 &self.paths.user(assets::USER_JOURNAL),
1207 &self.ranking_context(),
1208 &selected.canonical_pinyin,
1209 &selected.word,
1210 1,
1211 )
1212 .err()
1213 .map(|_| diagnostics::NINE_KEY_POSITION_NOT_PERSISTED.to_string())
1214 });
1215 self.refresh();
1216 let mut diagnostic = diagnostic.or(position_diagnostic.flatten());
1217 if diagnostic.is_none()
1219 && self.completions_discounted
1220 && self
1221 .candidates
1222 .first()
1223 .is_some_and(|first| first.word != selected.word)
1224 && self
1225 .candidates
1226 .iter()
1227 .any(|candidate| candidate.word == selected.word)
1228 {
1229 diagnostic = positions::set_fixed_position(
1230 &self.paths.user(assets::USER_JOURNAL),
1231 &self.ranking_context(),
1232 &selected.canonical_pinyin,
1233 &selected.word,
1234 1,
1235 )
1236 .err()
1237 .map(|_| diagnostics::NINE_KEY_POSITION_NOT_PERSISTED.to_string());
1238 self.refresh();
1239 }
1240 KeyResult::handled().with_diagnostic(diagnostic)
1241 }
1242
1243 pub fn remove(&mut self, index: usize) -> KeyResult {
1244 if !self.editable(index) || count_utf8_chars(&self.candidates[index].word) <= 1 {
1245 return KeyResult::unhandled();
1246 }
1247 let item = &self.candidates[index];
1248 let deleted = removal::delete_dictionary_candidate(
1249 &self.paths.dictionary(assets::MAIN_DICTIONARY),
1250 &self.paths.user(assets::USER_JOURNAL),
1251 PersonalDictionaryKind::Pinyin,
1252 &item.canonical_pinyin,
1253 &item.word,
1254 );
1255 if deleted.is_err() {
1257 return KeyResult::handled().with_diagnostic(Some(
1258 diagnostics::NINE_KEY_REMOVAL_NOT_PERSISTED.to_string(),
1259 ));
1260 }
1261 if let Some(dictionary) = self.dictionary.as_mut() {
1262 dictionary.reset_cache();
1263 }
1264 self.refresh();
1265 KeyResult::handled()
1266 }
1267
1268 pub fn set_position(&mut self, index: usize, position: i32) -> KeyResult {
1270 if !self.editable(index) || !(0..=5).contains(&position) {
1271 return KeyResult::unhandled();
1272 }
1273 let item = &self.candidates[index];
1274 let journal = self.paths.user(assets::USER_JOURNAL);
1275 let context = self.ranking_context();
1276 let saved = if position == 0 {
1277 positions::clear_fixed_position(&journal, &context, &item.canonical_pinyin, &item.word)
1278 } else {
1279 positions::set_fixed_position(
1280 &journal,
1281 &context,
1282 &item.canonical_pinyin,
1283 &item.word,
1284 position,
1285 )
1286 };
1287 if saved.is_err() {
1289 return KeyResult::handled().with_diagnostic(Some(
1290 diagnostics::NINE_KEY_POSITION_NOT_PERSISTED.to_string(),
1291 ));
1292 }
1293 self.refresh();
1294 KeyResult::handled()
1295 }
1296
1297 pub fn snapshot(&self) -> SessionSnapshot {
1298 let locked_length = self.locked_length();
1299 let locked_preedit_length = self
1300 .locked
1301 .iter()
1302 .map(String::len)
1303 .sum::<usize>()
1304 .saturating_add(self.locked.len().saturating_sub(1));
1305 let mut preedit = String::with_capacity(
1306 locked_preedit_length
1307 .saturating_add(self.digits.len().saturating_sub(locked_length))
1308 .saturating_add(self.splits.len())
1309 .saturating_add(usize::from(
1310 self.active() && locked_length < self.digits.len() && !self.locked.is_empty(),
1311 )),
1312 );
1313 for (index, spelling) in self.locked.iter().enumerate() {
1314 if index != 0 {
1315 preedit.push('\'');
1316 }
1317 preedit.push_str(spelling);
1318 }
1319 if self.active() && locked_length < self.digits.len() {
1320 if !preedit.is_empty() {
1321 preedit.push('\'');
1322 }
1323 let mut start = locked_length;
1324 for &split in &self.splits {
1325 preedit.push_str(&self.digits[start..split]);
1326 preedit.push('\'');
1327 start = split;
1328 }
1329 preedit.push_str(&self.digits[start..]);
1330 if let Some(initial) = self.initial {
1331 let at = locked_preedit_length + usize::from(!self.locked.is_empty());
1333 let mut initial_text = [0; 4];
1334 let initial_text = char::from(initial.letter).encode_utf8(&mut initial_text);
1335 preedit.replace_range(at..=at, initial_text);
1336 }
1337 }
1338 SessionSnapshot {
1339 prediction: false,
1340 scheme: if matches!(
1341 self.grid,
1342 KeyGrid::FourteenKeyShuangpin(_) | KeyGrid::EighteenKeyShuangpin(_)
1343 ) {
1344 SchemeType::Shuangpin
1345 } else {
1346 SchemeType::Quanpin
1347 },
1348 local_mode: LocalInputMode::None,
1349 dedicated_english: self.english_only,
1351 preedit,
1352 candidates: self.candidates.clone(),
1353 editing_text: self.digits.clone(),
1354 caret_position: self.caret_position(),
1355 nine_key_spellings: self.spellings.clone(),
1356 nine_key_reading: self.reading.clone(),
1357 nine_key_single_character: self.single_character,
1358 nine_key_strokes: self.strokes.clone(),
1359 candidate_sources: self.candidates.iter().map(|item| item.source).collect(),
1360 candidate_annotations: self
1361 .candidates
1362 .iter()
1363 .map(|item| item.corrected_from.to_string())
1364 .collect(),
1365 candidate_answers_key: self
1367 .candidates
1368 .iter()
1369 .map(|item| item.pinyin.len() >= self.digits.len())
1370 .collect(),
1371 ..SessionSnapshot::default()
1372 }
1373 }
1374
1375 fn refresh(&mut self) {
1376 self.candidates.clear();
1377 self.reading.clear();
1378 self.completions_discounted = false;
1379 if !self.active() {
1380 self.path_scratch.pages.clear();
1381 self.spellings.clear();
1382 self.sentence_seeds.clear();
1384 self.lock_undo.clear();
1385 self.initial = None;
1386 self.single_character = false;
1387 self.strokes.clear();
1388 return;
1389 }
1390 if self.english_only || !self.pinyin {
1391 self.path_scratch.pages.clear();
1392 self.spellings.clear();
1393 self.candidates = self.english_candidates(false);
1395 return;
1396 }
1397 self.lock_undo.resize_with(self.locked.len(), || None);
1399 let grid = self.grid;
1400 let table = grid.spelling_table();
1401 let locked_length = self.locked_length();
1402 let remaining = remaining_digits(&self.digits, locked_length);
1403 let unlocked_digit_count = remaining.len();
1404 let mut split_offsets = [0usize; DIGIT_LIMIT];
1405 for (index, split) in self.splits.iter().enumerate() {
1406 split_offsets[index] = split - locked_length;
1407 }
1408 let splits = &split_offsets[..self.splits.len()];
1409 let initial = self.initial.map(|initial| initial.letter);
1410 let starts_right =
1411 |piece: &str| initial.is_none_or(|letter| piece.as_bytes().first() == Some(&letter));
1412 let dictionary_options = self.dictionary_options();
1413 let mut alternatives = std::mem::take(&mut self.path_buffer);
1414 let alternatives = if remaining.is_empty() {
1415 self.spellings.clear();
1416 alternatives.clear();
1417 alternatives.push(Vec::new());
1418 alternatives
1419 } else {
1420 let prior = self
1421 .prior
1422 .get_or_insert_with(|| SyllablePrior::shipped(table));
1423 table.spellings_for_into(
1424 remaining,
1425 locked_length,
1426 splits.first().copied(),
1427 prior,
1428 &mut self.spellings,
1429 &mut self.spelling_match_indices,
1430 );
1431 self.spellings.retain(|syllable| starts_right(syllable));
1432 table.split_paths_with_scratch(
1433 remaining,
1434 splits,
1435 prior,
1436 QUERY_PATH_LIMIT,
1437 &mut self.path_scratch,
1438 &mut alternatives,
1439 );
1440 alternatives.retain(|path| path.first().is_none_or(|piece| starts_right(piece)));
1441 if self.locked.is_empty() && splits.is_empty() && initial.is_none() {
1443 if !self.sentence_seeds.is_empty() && remaining.len() >= SEEDED_TRUNCATE_DIGITS {
1444 alternatives.truncate(SEEDED_PATH_LIMIT);
1445 }
1446 seeded_paths(
1447 &mut alternatives,
1448 &self.sentence_seeds,
1449 remaining,
1450 table,
1451 prior,
1452 );
1453 }
1454 alternatives.extend(
1456 self.spellings
1457 .iter()
1458 .filter(|spelling| grid.syllable_code_len(spelling) <= remaining.len())
1459 .map(|spelling| vec![spelling.clone()]),
1460 );
1461 alternatives
1462 };
1463
1464 let mut locked_key = String::new();
1465 append_path_key(&mut locked_key, "", &self.locked);
1466 let mut dictionary = self
1467 .dictionary
1468 .get_or_insert_with(|| open_dictionary(&self.paths, dictionary_options.clone()))
1469 .row_cache_batch();
1470 let mut queried = (alternatives.len() > SMALL_QUERY_KEY_BATCH)
1471 .then(|| HashSet::with_capacity(alternatives.len()));
1472 let mut candidates = std::mem::take(&mut self.candidates);
1474 candidates.clear();
1475 let mut key = String::with_capacity(locked_key.len() + self.digits.len() * 4 + 1);
1476 let mut leading = LeadingMap::default();
1478 let prefer_exact = !self.splits.is_empty();
1480 let initials =
1483 self.locked.is_empty() && initial.is_none() && initials_apply(remaining.len(), splits);
1484 let initials_lead = initials && !splits.is_empty();
1485 let filter_dominated = self.fuzzy.rules == 0
1487 && matches!(
1488 grid,
1489 KeyGrid::NineKey | KeyGrid::FourteenKey | KeyGrid::EighteenKey
1490 );
1491 for (index, path) in alternatives.iter().enumerate() {
1492 append_path_key(&mut key, &locked_key, path);
1493 if key.is_empty() || !query_key_is_new(&alternatives, index, &key, queried.as_ref()) {
1494 continue;
1495 }
1496 let full_len = self.locked.len() + path.len();
1497 let accepts_reading = |candidate: &WordItem, cached: bool| {
1498 let word = candidate.word.as_str();
1499 let pinyin = candidate.pinyin.as_str();
1500 let canonical = candidate.canonical_pinyin.as_str();
1501 let fuzzy = candidate.fuzzy;
1502 if self.single_character_only && is_han_phrase(word) {
1503 return false;
1504 }
1505 let canonical = if canonical.is_empty() {
1506 pinyin
1507 } else {
1508 canonical
1509 };
1510 if reading_syllable_count(canonical) > full_len {
1512 return false;
1513 }
1514 let matched = if fuzzy { pinyin } else { canonical };
1515 if !agrees_with_locked(matched, &locked_key) {
1516 return false;
1517 }
1518 if let Some(letter) = initial {
1519 let next = matched
1521 .split('\'')
1522 .nth(self.locked.len())
1523 .and_then(|syllable| syllable.as_bytes().first());
1524 if next != Some(&letter) {
1525 return false;
1526 }
1527 }
1528 if cached && filter_dominated {
1529 if let Some(best) = leading_rank(&leading, &candidates, word) {
1530 let Some(covered) = grid.matching_code_prefix_len(matched, &self.digits)
1531 else {
1532 return false;
1533 };
1534 if best
1536 <= rank_key_for_reading(
1537 candidate,
1538 covered,
1539 canonical,
1540 prefer_exact,
1541 initials_lead,
1542 grid,
1543 )
1544 {
1545 return false;
1546 }
1547 }
1548 }
1549 true
1550 };
1551 for mut candidate in
1552 dictionary.query_filtered_rows(&key, &key, self.fuzzy, accepts_reading)
1553 {
1554 let canonical = take_candidate_canonical_pinyin(&mut candidate);
1555 let matched = if candidate.fuzzy {
1556 candidate.pinyin.as_str()
1557 } else {
1558 canonical.as_str()
1559 };
1560 let Some(covered) = grid.matching_code_prefix_len(matched, &self.digits) else {
1562 continue;
1563 };
1564 candidate.pinyin = SharedString::from(&self.digits[..covered]);
1566 candidate.canonical_pinyin = canonical;
1567 push_ranked(
1568 &mut candidates,
1569 &mut leading,
1570 candidate,
1571 prefer_exact,
1572 initials_lead,
1573 grid,
1574 );
1575 }
1576 if let Some(seen) = queried.as_mut() {
1577 seen.insert(key.clone());
1578 }
1579 }
1580 if let Some(table) = grid.shuangpin_table() {
1582 let codes = (self.locked.is_empty()
1583 && initial.is_none()
1584 && splits.is_empty()
1585 && !self.single_character_only)
1586 .then(|| shuangpin_word_codes(grid, remaining))
1587 .flatten();
1588 if let Some(codes) = codes {
1589 for mut candidate in
1590 dictionary.query_jianpin_codes_per_table(&codes, SHUANGPIN_WORD_SCAN_LIMIT)
1591 {
1592 if !shuangpin_word_spells(table, &candidate.canonical_pinyin, remaining) {
1593 continue;
1594 }
1595 candidate.pinyin.clear();
1596 candidate.pinyin.push_str(&self.digits);
1597 push_ranked(
1598 &mut candidates,
1599 &mut leading,
1600 candidate,
1601 prefer_exact,
1602 initials_lead,
1603 grid,
1604 );
1605 }
1606 }
1607 }
1608 let mut boost = None;
1609 if let Some(codes) = initials
1610 .then(|| initials_codes(grid, remaining, INITIALS_CODE_LIMIT))
1611 .flatten()
1612 {
1613 let personal =
1615 self.learning && self.personal_context && !dictionary.personal_model().is_empty();
1616 let shuangpin = match grid {
1618 KeyGrid::NineKeyShuangpin(table)
1619 | KeyGrid::FourteenKeyShuangpin(table)
1620 | KeyGrid::EighteenKeyShuangpin(table) => Some(table),
1621 _ => None,
1622 };
1623 let mut rows = if personal {
1624 dictionary.query_jianpin_codes_per_table(&codes, INITIALS_SCAN_LIMIT)
1625 } else if shuangpin.is_some() {
1626 dictionary.query_jianpin_codes(&codes, INITIALS_SCAN_LIMIT)
1627 } else {
1628 dictionary.query_jianpin_codes(&codes, INITIALS_ROW_LIMIT)
1629 };
1630 if let Some(table) = shuangpin {
1631 rows.retain(|row| table.spells_initials(&row.canonical_pinyin, remaining));
1632 if !personal {
1633 rows.truncate(INITIALS_ROW_LIMIT);
1634 }
1635 }
1636 let mut used = HashMap::new();
1637 if personal {
1638 let model = dictionary.personal_model();
1640 for row in &rows {
1641 let count = model.word_count(&row.word);
1642 if count > 0 {
1643 used.insert(row.word.clone(), count);
1644 }
1645 }
1646 drop(model);
1647 if rows.len() > INITIALS_ROW_LIMIT {
1648 rows.sort_by_key(|row| !used.contains_key(&row.word));
1650 rows.truncate(INITIALS_ROW_LIMIT);
1651 }
1652 }
1653 if self.frequency.mode != FrequencyAdjustmentMode::Disabled && !used.is_empty() {
1655 boost = Some(InitialsBoost {
1656 used,
1657 frequency: self.frequency,
1658 });
1659 }
1660 for mut candidate in rows {
1661 candidate.pinyin.clear();
1663 candidate.pinyin.push_str(&self.digits);
1664 push_ranked(
1665 &mut candidates,
1666 &mut leading,
1667 candidate,
1668 prefer_exact,
1669 initials_lead,
1670 grid,
1671 );
1672 }
1673 }
1674 drop(dictionary);
1675 let filtering = self.single_character || !self.strokes.is_empty();
1676 if filtering {
1677 let strokes = (!self.strokes.is_empty()).then_some(&self.stroke_texts);
1678 candidates.retain(|item| passes_filter(&item.word, self.single_character, strokes));
1679 }
1680 rank_candidates(
1681 &mut candidates,
1682 prefer_exact,
1683 initials_lead,
1684 boost.as_ref(),
1685 grid,
1686 );
1687 self.rerank_sentences(&mut candidates);
1688 let mut seeds = std::mem::take(&mut self.seed_buffer);
1689 next_seeds(
1690 grid,
1691 &candidates,
1692 &self.digits,
1693 &mut self.sentence_seeds,
1694 &mut seeds,
1695 );
1696 self.seed_buffer = std::mem::replace(&mut self.sentence_seeds, seeds);
1697 let (emoji, kaomoji, symbols) = if filtering {
1699 (Vec::new(), Vec::new(), Vec::new())
1700 } else {
1701 self.expressive_candidates(&alternatives, &candidates)
1702 };
1703
1704 let unanswered = candidates.is_empty();
1706 let mut english = if initial.is_some() || filtering {
1708 Vec::new()
1709 } else {
1710 self.english_candidates(unanswered)
1711 };
1712 if !english.is_empty() {
1713 let first = english.remove(0);
1715 let slot = if first.weight > 0 {
1716 candidates.len().min(1)
1717 } else {
1718 candidates.len()
1719 };
1720 candidates.insert(slot, first);
1721 let extra = self
1722 .english_options
1723 .first_page_count
1724 .saturating_sub(1)
1725 .min(self.first_page_rows.get().saturating_sub(slot + 1))
1726 .min(english.len());
1727 if extra > 0 {
1728 candidates.splice(slot + 1..slot + 1, english.drain(..extra));
1729 }
1730 candidates.extend(english);
1731 }
1732 let mut candidates = insert_expressive_rows(candidates, emoji, kaomoji, symbols);
1733 if let Some(clock) = self
1735 .inline_date_time
1736 .as_ref()
1737 .filter(|_| self.mixed_candidates_enabled && initial.is_none() && !filtering)
1738 {
1739 let digits = self.digits.as_str();
1740 if let Some((_keyword, anchor, kind)) =
1741 inline_date_time_keyword(|keyword| keyword_spells_digits(grid, keyword, digits))
1742 {
1743 insert_inline_date_time(&mut candidates, anchor, kind, digits, || clock(), None);
1745 }
1746 }
1747 positions::apply_fixed_positions(
1748 &self.paths.user(assets::USER_JOURNAL),
1749 &format!("nine-key:{}:{locked_key}", self.digits),
1750 &mut candidates,
1751 false,
1752 None,
1753 false,
1754 );
1755 keep_expressive_rows_off_the_lead(&mut candidates);
1756 self.completions_discounted = !prefer_exact
1758 && discount_completions_on_first_page(
1759 &mut candidates,
1760 self.first_page_rows.get(),
1761 locked_syllables(&locked_key),
1762 grid,
1763 );
1764 if let Some(front) = candidates.first() {
1768 let offset = if locked_key.is_empty() {
1769 0
1770 } else {
1771 locked_key.len() + 1
1772 };
1773 if let Some(rest) = front.canonical_pinyin.get(offset..) {
1774 let preferred = rest.split('\'').next().unwrap_or_default();
1775 if unlocked_digit_count < 2 || preferred.len() <= unlocked_digit_count {
1776 if let Some(found) = self.spellings.iter().position(|s| s == preferred) {
1777 self.spellings[..=found].rotate_right(1);
1778 }
1779 }
1780 }
1781 }
1782 let mut reading = std::mem::take(&mut self.reading);
1783 self.write_reading(candidates.first(), &mut reading);
1784 self.reading = reading;
1785 self.candidates = candidates;
1786 self.path_buffer = alternatives;
1787 let remaining = remaining_digits(&self.digits, locked_length);
1788 if remaining.is_empty() {
1789 if let Some(Some(undo)) = self.lock_undo.last() {
1791 self.spellings = undo.choices.clone();
1792 }
1793 } else {
1794 self.append_key_choices(locked_length);
1795 }
1796 }
1797
1798 fn rerank_sentences(&mut self, candidates: &mut Vec<WordItem>) {
1800 if !self.sentence_association.neural_keyboard {
1801 return;
1802 }
1803 let paths = &self.paths;
1804 let keyboard = self.keyboard.get_or_insert_with(|| {
1805 shared_sentence_model(&paths.resource(assets::NEURAL_MODEL_KEYBOARD))
1806 .map(|model| NeuralReranker::new(CandidateSource::NeuralKeyboard, model))
1807 });
1808 let ranked: &[String] = if let Some(keyboard) = keyboard.as_mut() {
1809 let (scored, scored_len) = collect_lattice_sentences(candidates);
1810 let scored = &scored[..scored_len];
1811 let context = &self.rescoring_context;
1812 let ranked = if let Some(index) = self
1813 .rerank_cache
1814 .iter()
1815 .position(|entry| rerank_cache_matches(entry, context, scored))
1816 {
1817 self.rerank_cache[index].ranked.as_slice()
1819 } else {
1820 let signature: Vec<(String, i64)> = scored
1822 .iter()
1823 .flatten()
1824 .map(|item| (item.word.to_string(), comparable_weight(item)))
1825 .collect();
1826 let mut sentences: Vec<SentencePath> = scored
1827 .iter()
1828 .flatten()
1829 .map(|item| SentencePath {
1830 sentence: item.word.to_string(),
1831 key: String::new(),
1834 log_prob: comparable_weight(item) as f64 / 1000.0,
1835 words: Vec::new(),
1836 typo_edges: 0,
1837 })
1838 .collect();
1839 let ranked = if keyboard.rerank(&mut sentences, context) {
1840 sentences.into_iter().map(|path| path.sentence).collect()
1841 } else {
1842 Vec::new()
1843 };
1844 if self.rerank_cache.len() >= RERANK_CACHE_ENTRIES {
1845 self.rerank_cache.remove(0);
1846 }
1847 self.rerank_cache.push(RerankEntry {
1848 context: context.clone(),
1849 sentences: signature,
1850 ranked,
1851 });
1852 self.rerank_cache.last().unwrap().ranked.as_slice()
1853 };
1854 ranked
1855 } else {
1856 &[]
1857 };
1858 place_keyboard_pick(
1859 candidates,
1860 ranked,
1861 self.sentence_association.show_next_on_duplicate,
1862 self.sentence_association.word_lattice,
1863 );
1864 }
1865
1866 fn expressive_candidates(
1870 &self,
1871 alternatives: &[Path],
1872 candidates: &[WordItem],
1873 ) -> (Vec<ExpressiveRow>, Vec<ExpressiveRow>, Vec<SymbolRow>) {
1874 let enabled = self.mixed_candidates_enabled
1875 && (self.expressive.emoji_candidates
1876 || self.expressive.kaomoji_candidates
1877 || self.expressive.symbol_candidates);
1878 if !enabled || self.digits.len() < MIXED_EXPRESSIVE_MINIMUM_INPUT {
1879 return (Vec::new(), Vec::new(), Vec::new());
1880 }
1881 let mut boundaries: Vec<usize> = self
1882 .locked
1883 .iter()
1884 .scan(0, |end, spelling| {
1885 *end += self.grid.syllable_code_len(spelling);
1886 Some(*end)
1887 })
1888 .chain(self.splits.iter().copied())
1889 .collect();
1890 boundaries.sort_unstable();
1891 boundaries.dedup();
1892 let mut groups: Vec<(Vec<usize>, Vec<String>)> = Vec::new();
1894 for (reading, letters) in self.expressive_readings(alternatives, &boundaries, candidates) {
1895 match groups
1896 .iter_mut()
1897 .find(|(known, _)| known.as_slice() == letters.as_ref())
1898 {
1899 Some((_, readings)) => readings.push(reading),
1900 None => groups.push((letters.into_owned(), vec![reading])),
1901 }
1902 }
1903 let others = (self.expressive.emoji_candidates || self.expressive.kaomoji_candidates)
1904 .then(|| self.paths.resource(assets::OTHER_DICTIONARY));
1905 let mut emoji = Vec::new();
1906 let mut kaomoji = Vec::new();
1907 let mut symbols: Vec<SymbolRow> = Vec::new();
1908 for (letters, readings) in &groups {
1909 let accept = |key: &str| splits_into_syllables_at(key, letters);
1910 if self.expressive.emoji_candidates {
1912 if let Some(others) = others.as_deref() {
1913 append_unique_rows(&mut emoji, query_emoji_readings(readings, others, &accept));
1914 }
1915 }
1916 if self.expressive.kaomoji_candidates {
1917 if let Some(others) = others.as_deref() {
1918 append_unique_rows(
1919 &mut kaomoji,
1920 query_kaomoji_readings(readings, others, &accept),
1921 );
1922 }
1923 }
1924 if self.expressive.symbol_candidates {
1925 for row in query_symbol_readings(readings, &accept) {
1926 if symbols.len() < MIXED_SYMBOL_LIMIT
1927 && !symbols.iter().any(|known| known.item.word == row.item.word)
1928 {
1929 symbols.push(row);
1930 }
1931 }
1932 }
1933 }
1934 for item in emoji
1936 .iter_mut()
1937 .chain(kaomoji.iter_mut())
1938 .map(|row| &mut row.item)
1939 .chain(symbols.iter_mut().map(|row| &mut row.item))
1940 {
1941 item.pinyin = SharedString::from(&self.digits);
1942 }
1943 (emoji, kaomoji, symbols)
1944 }
1945
1946 fn expressive_readings<'a>(
1948 &self,
1949 alternatives: &[Path],
1950 boundaries: &'a [usize],
1951 candidates: &[WordItem],
1952 ) -> Vec<(String, Cow<'a, [usize]>)> {
1953 let mut readings: Vec<(String, usize)> = Vec::new();
1954 for (path_index, path) in alternatives.iter().enumerate() {
1955 if !self.reading_fits(path) {
1956 continue;
1957 }
1958 let mut reading = String::with_capacity(self.digits.len());
1959 for spelling in &self.locked {
1960 reading.push_str(spelling);
1961 }
1962 path.iter().for_each(|piece| reading.push_str(piece));
1963 if !readings.iter().any(|(known, _)| *known == reading) {
1964 readings.push((reading, path_index));
1965 }
1966 }
1967 let support = |reading: &String| {
1968 candidates
1969 .iter()
1970 .position(|item| {
1971 !item.fuzzy
1972 && item.pinyin.len() == self.digits.len()
1973 && letters_start_with(&item.canonical_pinyin, reading)
1974 })
1975 .unwrap_or(usize::MAX)
1976 };
1977 readings.sort_by_cached_key(|(reading, _)| support(reading));
1979 readings.truncate(EXPRESSIVE_READING_LIMIT);
1980 readings
1981 .into_iter()
1982 .filter_map(|(reading, path_index)| {
1983 self.reading_boundaries(&alternatives[path_index], boundaries)
1984 .map(|letters| (reading, letters))
1985 })
1986 .collect()
1987 }
1988
1989 fn reading_fits(&self, path: &Path) -> bool {
1991 let remaining = remaining_digits(&self.digits, self.locked_length());
1992 let Some(table) = self.grid.shuangpin_table() else {
1993 return path.iter().map(String::len).sum::<usize>() == remaining.len();
1994 };
1995 let mut offset = 0;
1996 for piece in path {
1997 let Some(length) = table.piece_code_len(piece, &remaining[offset..]) else {
1998 return false;
1999 };
2000 offset += length;
2001 }
2002 offset == remaining.len()
2003 }
2004
2005 fn reading_boundaries<'a>(
2007 &self,
2008 path: &Path,
2009 boundaries: &'a [usize],
2010 ) -> Option<Cow<'a, [usize]>> {
2011 let remaining = remaining_digits(&self.digits, self.locked_length());
2012 let Some(table) = self.grid.shuangpin_table() else {
2013 return (path.iter().map(String::len).sum::<usize>() == remaining.len())
2014 .then_some(Cow::Borrowed(boundaries));
2015 };
2016 let mut letters_at_boundaries = Vec::with_capacity(boundaries.len());
2018 let mut next_boundary = 0;
2019 let (mut digit, mut letter) = (0, 0);
2020 for spelling in &self.locked {
2021 digit += self.grid.syllable_code_len(spelling);
2022 letter += spelling.len();
2023 while next_boundary < boundaries.len() && boundaries[next_boundary] <= digit {
2024 if boundaries[next_boundary] == digit {
2025 letters_at_boundaries.push(letter);
2026 }
2027 next_boundary += 1;
2028 }
2029 }
2030 let mut offset = 0;
2031 for piece in path {
2032 let length = table.piece_code_len(piece, &remaining[offset..])?;
2033 offset += length;
2034 digit += length;
2035 letter += piece.len();
2036 while next_boundary < boundaries.len() && boundaries[next_boundary] <= digit {
2037 if boundaries[next_boundary] == digit {
2038 letters_at_boundaries.push(letter);
2039 }
2040 next_boundary += 1;
2041 }
2042 }
2043 (offset == remaining.len()).then_some(Cow::Owned(letters_at_boundaries))
2044 }
2045
2046 fn append_key_choices(&mut self, locked_length: usize) {
2048 let remaining = remaining_digits(&self.digits, locked_length);
2049 let Some(&digit) = remaining.as_bytes().first() else {
2050 return;
2051 };
2052 let table = self.grid.spelling_table();
2053 let letters = self.grid.letters(digit);
2054 let raw_key = self.locked.is_empty() && self.grid.offers_raw_key();
2055 self.spellings.reserve(letters.len() + usize::from(raw_key));
2056 for letter in letters
2057 .bytes()
2058 .filter(|&letter| table.starts_syllable(letter))
2059 {
2060 self.spellings
2061 .push(char::from(letter.to_ascii_uppercase()).to_string());
2062 }
2063 if raw_key {
2064 self.spellings.push(char::from(digit).to_string());
2065 }
2066 }
2067
2068 #[cfg(test)]
2069 fn key_choices(&self, remaining: &str) -> Vec<String> {
2070 let Some(&digit) = remaining.as_bytes().first() else {
2071 return Vec::new();
2072 };
2073 let table = self.grid.spelling_table();
2074 let mut choices: Vec<String> = self
2075 .grid
2076 .letters(digit)
2077 .bytes()
2078 .filter(|&letter| table.starts_syllable(letter))
2079 .map(|letter| char::from(letter.to_ascii_uppercase()).to_string())
2080 .collect();
2081 if self.locked.is_empty() && self.grid.offers_raw_key() {
2082 choices.push(char::from(digit).to_string());
2083 }
2084 choices
2085 }
2086
2087 pub fn set_filter(&mut self, single_character: bool, strokes: &str) -> KeyResult {
2089 if !self.active() || self.english_only || !self.pinyin {
2090 return KeyResult::unhandled();
2091 }
2092 if strokes.len() > stroke::MAX_STROKES || !strokes.bytes().all(stroke::is_stroke) {
2093 return KeyResult::unhandled();
2094 }
2095 if !strokes.is_empty() && strokes != self.strokes {
2096 let Some(texts) = self.stroke_texts_for(strokes) else {
2097 return KeyResult::handled().with_diagnostic(Some(
2098 diagnostics::LANGUAGE_DICTIONARY_UNAVAILABLE.to_string(),
2099 ));
2100 };
2101 self.stroke_texts = texts;
2102 }
2103 if strokes.is_empty() {
2104 self.stroke_texts.clear();
2105 }
2106 self.single_character = single_character;
2107 self.strokes = strokes.to_string();
2108 self.refresh();
2109 KeyResult::handled()
2110 }
2111
2112 fn stroke_texts_for(&mut self, strokes: &str) -> Option<HashSet<char>> {
2113 if self.stroke.is_none() {
2114 self.stroke = language_dictionary::open_read_only(&self.stroke_dictionary).ok();
2115 }
2116 let texts = self.stroke.as_ref()?.texts_with_key_prefix(strokes).ok()?;
2117 Some(
2118 texts
2119 .iter()
2120 .filter_map(|text| {
2121 let mut characters = text.chars();
2122 let first = characters.next()?;
2123 characters.next().is_none().then_some(first)
2124 })
2125 .collect(),
2126 )
2127 }
2128
2129 #[cfg(test)]
2131 fn reading_for(&self, front: Option<&WordItem>) -> String {
2132 let mut reading = String::new();
2133 self.write_reading(front, &mut reading);
2134 reading
2135 }
2136
2137 fn write_reading(&self, front: Option<&WordItem>, reading: &mut String) {
2138 reading.clear();
2139 let Some(front) = front else {
2140 return;
2141 };
2142 if front.fuzzy {
2144 return;
2145 }
2146 if front.source == CandidateSource::EnglishDictionary
2147 && matches!(self.grid, KeyGrid::FourteenKey | KeyGrid::EighteenKey)
2148 {
2149 let typed = front.pinyin.len().min(self.digits.len());
2151 for byte in front.pinyin[..typed].bytes() {
2152 reading.push(char::from(byte.to_ascii_lowercase()));
2153 }
2154 return;
2155 }
2156 if front.canonical_pinyin.is_empty() || !reads_pinyin(front.source) {
2158 return;
2159 }
2160 if is_initials_row(front) {
2162 let mut first = true;
2163 for syllable in front.canonical_pinyin.split('\'') {
2164 let Some(letter) = syllable.get(..1) else {
2165 continue;
2166 };
2167 if !first {
2168 reading.push('\'');
2169 }
2170 reading.push_str(letter);
2171 first = false;
2172 }
2173 return;
2174 }
2175 let covered = front.pinyin.len();
2176 if let Some(table) = self.grid.shuangpin_table() {
2177 let mut digits = covered;
2179 for syllable in front.canonical_pinyin.split('\'') {
2180 if digits == 0 {
2181 break;
2182 }
2183 if !reading.is_empty() {
2184 reading.push('\'');
2185 }
2186 if digits >= 2 {
2187 reading.push_str(syllable);
2188 digits -= 2;
2189 } else {
2190 reading.push_str(table.initial_part(syllable));
2191 digits = 0;
2192 }
2193 }
2194 } else {
2195 let mut letters = 0;
2196 for character in front.canonical_pinyin.chars() {
2197 if letters == covered {
2198 break;
2199 }
2200 if character.is_ascii_lowercase() {
2201 letters += 1;
2202 }
2203 reading.push(character);
2204 }
2205 }
2206 let trimmed_len = reading.trim_end_matches('\'').len();
2207 reading.truncate(trimmed_len);
2208 if reading.is_empty() {
2209 return;
2210 }
2211 let mut start = covered.min(self.digits.len());
2212 if start < self.digits.len() {
2213 reading.push('\'');
2214 for &split in &self.splits {
2215 if split > start && split < self.digits.len() {
2216 reading.push_str(&self.digits[start..split]);
2217 reading.push('\'');
2218 start = split;
2219 }
2220 }
2221 reading.push_str(&self.digits[start..]);
2222 }
2223 }
2224
2225 fn english_candidates(&mut self, unanswered: bool) -> Vec<WordItem> {
2227 if !self.grid.spells_english() {
2229 return Vec::new();
2230 }
2231 if self.english_only {
2233 if self.digits.is_empty() {
2234 return Vec::new();
2235 }
2236 } else if unanswered {
2237 if self.digits.is_empty() || !self.locked.is_empty() {
2238 return Vec::new();
2239 }
2240 } else if !self.mixed_candidates_enabled
2241 || !self.english_options.mixed_candidates
2242 || !self.locked.is_empty()
2243 || self.digits.len() < self.english_options.minimum_prefix
2244 {
2245 return Vec::new();
2246 }
2247 let grid = self.grid;
2248 if self.open_english().is_none() {
2249 return Vec::new();
2250 }
2251 let Some(english) = self.english.as_ref() else {
2252 return Vec::new();
2253 };
2254 let digits = self.digits.as_str();
2255 self.english_pattern.clear();
2256 for digit in digits.bytes() {
2257 let letters = grid.letters(digit);
2258 if letters.is_empty() {
2259 return Vec::new();
2260 }
2261 self.english_pattern.push('[');
2262 self.english_pattern.push_str(letters);
2263 for letter in letters.bytes() {
2264 self.english_pattern
2265 .push(char::from(letter.to_ascii_uppercase()));
2266 }
2267 self.english_pattern.push(']');
2268 }
2269 self.english_pattern.push('*');
2270 let depth = letter_prefix_depth(grid, digits, ENGLISH_PREFIX_BUDGET);
2271 let reusable = self.english_rows.as_mut().filter(|cache| {
2272 cache.grid == grid && cache.depth == depth && digits.starts_with(&cache.digits)
2273 });
2274 match reusable {
2275 Some(cache) => {
2276 if let Some(rows) = &mut cache.rows {
2277 let from = cache.digits.len();
2278 for prefix_rows in rows.iter_mut() {
2279 prefix_rows.retain(|row| word_spells_from(grid, &row.word, digits, from));
2280 }
2281 } else {
2282 cache.rows = letter_prefixes(grid, digits, ENGLISH_PREFIX_BUDGET)
2284 .iter()
2285 .map(|prefix| {
2286 english.keypad_prefix_rows(
2287 prefix,
2288 &self.english_pattern,
2289 ENGLISH_CACHED_PREFIX_ROWS,
2290 )
2291 })
2292 .collect();
2293 }
2294 cache.digits.clear();
2295 cache.digits.push_str(digits);
2296 }
2297 None => {
2298 let cache = self.english_rows.get_or_insert_with(|| EnglishKeypadCache {
2299 grid,
2300 depth,
2301 digits: String::new(),
2302 rows: None,
2303 });
2304 cache.grid = grid;
2305 cache.depth = depth;
2306 cache.rows = None;
2307 cache.digits.clear();
2308 cache.digits.push_str(digits);
2309 }
2310 }
2311 let mut words = Vec::new();
2312 if let Some(rows) = self
2313 .english_rows
2314 .as_ref()
2315 .and_then(|cache| cache.rows.as_ref())
2316 {
2317 let capacity = rows.len().saturating_mul(ENGLISH_LIMIT);
2318 for prefix_rows in rows {
2319 append_keypad_ranked(
2320 prefix_rows,
2321 digits.len(),
2322 ENGLISH_LIMIT,
2323 &mut words,
2324 capacity,
2325 );
2326 }
2327 } else {
2328 let prefixes = letter_prefixes(grid, digits, ENGLISH_PREFIX_BUDGET);
2329 let capacity = prefixes.len().saturating_mul(ENGLISH_LIMIT);
2330 for prefix in prefixes {
2331 english.append_keypad_prefix(
2332 &prefix,
2333 &self.english_pattern,
2334 digits.len(),
2335 ENGLISH_LIMIT,
2336 &mut words,
2337 capacity,
2338 );
2339 }
2340 }
2341 deduplicate_english_words(&mut words);
2342 rank_english(&mut words, digits.len());
2343 words
2344 }
2345
2346 fn open_english(&mut self) -> Option<&EnglishDictionary> {
2348 if self.english.is_none() {
2349 let path = self.paths.dictionary(assets::ENGLISH_DICTIONARY);
2350 if !path.exists() {
2351 return None;
2352 }
2353 self.english = Some(EnglishDictionary::open(&path, None, None));
2354 }
2355 self.english.as_ref()
2356 }
2357
2358 fn locked_length(&self) -> usize {
2359 self.locked
2360 .iter()
2361 .map(|spelling| self.grid.syllable_code_len(spelling))
2362 .sum()
2363 }
2364
2365 fn consume(&mut self, count: usize) {
2367 let count = count.min(self.digits.len());
2368 self.digits.drain(..count);
2369 self.english_rows = None;
2371 self.caret = None;
2373 self.splits.retain_mut(|split| {
2374 if *split > count {
2375 *split -= count;
2376 true
2377 } else {
2378 false
2379 }
2380 });
2381 let mut consumed = count;
2382 while let Some(front) = self.locked.first() {
2383 let length = self.grid.syllable_code_len(front);
2384 if consumed < length {
2385 break;
2386 }
2387 consumed -= length;
2388 self.locked.remove(0);
2389 if !self.lock_undo.is_empty() {
2390 self.lock_undo.remove(0);
2391 }
2392 }
2393 self.lock_undo.iter_mut().for_each(|undo| *undo = None);
2395 self.initial = None;
2396 }
2397
2398 fn ranking_context(&self) -> String {
2399 format!("nine-key:{}:{}", self.digits, self.locked.join("'"))
2400 }
2401
2402 fn editable(&self, index: usize) -> bool {
2403 self.candidates
2404 .get(index)
2405 .is_some_and(|item| !item.canonical_pinyin.is_empty() && item.source.is_dictionary())
2406 }
2407
2408 fn adjust_frequency(&mut self, index: usize, force_top: bool) -> Option<String> {
2410 let item = &self.candidates[index];
2411 let main_db = self.paths.dictionary(assets::MAIN_DICTIONARY);
2412 let user_db = self.paths.user(assets::USER_JOURNAL);
2413 let context = self.ranking_context();
2414 let request = RankingRequest {
2415 main_db: &main_db,
2416 user_db: &user_db,
2417 context_key: &context,
2418 ordered: &self.candidates,
2419 entry_key: &item.canonical_pinyin,
2420 value: &item.word,
2421 mode: if force_top {
2422 FrequencyAdjustmentMode::Pin
2423 } else {
2424 self.frequency.mode
2425 },
2426 linear_step: self.frequency.linear_step,
2427 trigger_count: self.frequency.trigger_count,
2428 force_top,
2429 kind: PersonalDictionaryKind::Pinyin,
2430 };
2431 match ranking::adjust_candidate_ranking(&request) {
2432 Ok(changed) => {
2433 if changed {
2434 if let Some(dictionary) = self.dictionary.as_mut() {
2435 dictionary.reset_cache();
2436 }
2437 }
2438 None
2439 }
2440 Err(_) => Some(diagnostics::NINE_KEY_FREQUENCY_NOT_PERSISTED.to_string()),
2442 }
2443 }
2444
2445 fn remember_recent_candidate(
2447 &self,
2448 context: &str,
2449 canonical_pinyin: &str,
2450 word: &str,
2451 ) -> Option<String> {
2452 positions::set_fixed_position(
2453 &self.paths.user(assets::USER_JOURNAL),
2454 context,
2455 canonical_pinyin,
2456 word,
2457 1,
2458 )
2459 .err()
2460 .map(|_| diagnostics::NINE_KEY_POSITION_NOT_PERSISTED.to_string())
2461 }
2462}
2463
2464struct RerankEntry {
2466 context: String,
2467 sentences: Vec<(String, i64)>,
2468 ranked: Vec<String>,
2469}
2470
2471fn rerank_cache_matches(entry: &RerankEntry, context: &str, scored: &[Option<&WordItem>]) -> bool {
2473 entry.context == context
2474 && entry.sentences.len() == scored.len()
2475 && entry
2476 .sentences
2477 .iter()
2478 .zip(scored)
2479 .all(|((word, weight), item)| {
2480 item.is_some_and(|item| word == &item.word && *weight == comparable_weight(item))
2481 })
2482}
2483
2484#[derive(Clone)]
2487struct DictionaryOptions {
2488 association: SentenceAssociationOptions,
2490 alternatives: bool,
2491 first_page_rows: FirstPageRows,
2492}
2493
2494fn open_dictionary(paths: &RuntimePaths, options: DictionaryOptions) -> QuanpinDictionary {
2496 let mut dictionary = QuanpinDictionary::new(paths);
2497 dictionary.set_sentence_association(options.association);
2498 dictionary.set_sentence_alternatives(options.alternatives);
2499 dictionary.set_first_page_rows(options.first_page_rows);
2500 dictionary
2501}
2502
2503fn is_lattice_sentence(item: &WordItem) -> bool {
2505 matches!(
2506 item.source,
2507 CandidateSource::Generated | CandidateSource::Collocation
2508 ) && item.sentence_association
2509}
2510
2511fn collect_lattice_sentences(
2513 candidates: &[WordItem],
2514) -> ([Option<&WordItem>; MAX_RERANK_PATHS], usize) {
2515 let mut slots = [None; MAX_RERANK_PATHS];
2516 let mut len = 0;
2517 for item in candidates {
2518 if !is_lattice_sentence(item) {
2519 continue;
2520 }
2521 slots[len] = Some(item);
2522 len += 1;
2523 if len == MAX_RERANK_PATHS {
2524 break;
2525 }
2526 }
2527 (slots, len)
2528}
2529
2530fn place_keyboard_pick(
2532 candidates: &mut Vec<WordItem>,
2533 ranked: &[String],
2534 show_next_on_duplicate: bool,
2535 word_lattice: bool,
2536) {
2537 let take_pick = |candidates: &mut Vec<WordItem>, sentence: &str, at_position: usize| {
2538 if let Some(at) = candidates
2540 .iter()
2541 .position(|item| is_lattice_sentence(item) && item.word == sentence)
2542 {
2543 let mut pick = candidates.remove(at);
2544 pick.source = CandidateSource::NeuralKeyboard;
2545 candidates.insert(at_position, pick);
2546 }
2547 };
2548 if let Some(best) = candidates.iter().position(is_lattice_sentence) {
2549 if !word_lattice {
2550 if let Some(sentence) = ranked.first() {
2551 take_pick(candidates, sentence, best);
2552 }
2553 } else {
2554 for sentence in ranked {
2555 if candidates[best].word != *sentence {
2556 take_pick(candidates, sentence, best + 1);
2558 break;
2559 }
2560 if !show_next_on_duplicate {
2561 break;
2562 }
2563 }
2564 }
2565 }
2566 if !word_lattice {
2567 candidates.retain(|item| !is_lattice_sentence(item));
2568 }
2569}
2570
2571fn seed_prefix(seed: &str) -> &str {
2573 seed.rfind('\'').map_or("", |end| &seed[..end])
2574}
2575
2576fn fitting_syllables(grid: KeyGrid, prefix: &str, digits: &str) -> (usize, usize, bool) {
2578 let mut syllables = 0;
2579 let mut covered = 0;
2580 if prefix.is_empty() {
2581 return (0, 0, true);
2582 }
2583 for syllable in prefix.split('\'') {
2584 let end = covered + grid.syllable_code_len(syllable);
2585 if !digits
2586 .get(covered..end)
2587 .is_some_and(|code| letters_spell_code(grid, syllable, code))
2588 {
2589 return (syllables, covered, false);
2590 }
2591 syllables += 1;
2592 covered = end;
2593 }
2594 (syllables, covered, true)
2595}
2596
2597fn letters_spell_code(grid: KeyGrid, letters: &str, code: &str) -> bool {
2599 let Some(keypad) = grid.keypad() else {
2600 return match grid {
2601 KeyGrid::NineKeyShuangpin(table)
2602 | KeyGrid::FourteenKeyShuangpin(table)
2603 | KeyGrid::EighteenKeyShuangpin(table) => table
2604 .syllable_codes(letters)
2605 .any(|pair| pair.as_slice() == code.as_bytes()),
2606 _ => false,
2607 };
2608 };
2609 letters.len() == code.len()
2610 && letters.bytes().zip(code.bytes()).all(|(letter, digit)| {
2611 letter.is_ascii_lowercase() && keypad[usize::from(letter - b'a')] == digit
2612 })
2613}
2614
2615fn next_seeds(
2617 grid: KeyGrid,
2618 candidates: &[WordItem],
2619 digits: &str,
2620 previous: &mut Vec<String>,
2621 seeds: &mut Vec<String>,
2622) {
2623 seeds.clear();
2624 let is_new = |seeds: &[String], seed: &str| {
2625 seeds
2626 .iter()
2627 .all(|existing| seed_prefix(existing) != seed_prefix(seed))
2628 };
2629 for item in candidates {
2630 if seeds.len() == SENTENCE_SEED_LIMIT {
2631 break;
2632 }
2633 if item.pinyin.len() != digits.len()
2634 || item.canonical_pinyin.is_empty()
2635 || item.fuzzy
2636 || !is_new(seeds, &item.canonical_pinyin)
2637 {
2638 continue;
2639 }
2640 let seed = match previous
2641 .iter()
2642 .position(|old| *old == item.canonical_pinyin)
2643 {
2644 Some(index) => previous.remove(index),
2645 None => item.canonical_pinyin.to_string(),
2646 };
2647 seeds.push(seed);
2648 }
2649 for seed in previous.drain(..) {
2650 if seeds.len() < SEED_RETAIN_LIMIT
2651 && fitting_syllables(grid, seed_prefix(&seed), digits).2
2652 && is_new(seeds, &seed)
2653 {
2654 seeds.push(seed);
2655 }
2656 }
2657}
2658
2659fn seeded_paths(
2661 alternatives: &mut Vec<Path>,
2662 seeds: &[String],
2663 remaining: &str,
2664 table: &SpellingTable,
2665 prior: &SyllablePrior,
2666) {
2667 for seed in seeds {
2668 let (kept, covered, _) = fitting_syllables(table.grid, seed_prefix(seed), remaining);
2669 if kept == 0 {
2670 continue;
2671 }
2672 let mut pieces = [""; DIGIT_LIMIT];
2674 if kept <= pieces.len() {
2675 for (slot, piece) in pieces.iter_mut().zip(seed.split('\'').take(kept)) {
2676 *slot = piece;
2677 }
2678 }
2679 let mut append = |tail_path| {
2680 if kept <= pieces.len() {
2681 append_seeded_path(
2682 alternatives,
2683 pieces[..kept].iter().copied(),
2684 kept,
2685 tail_path,
2686 );
2687 } else {
2688 append_seeded_path(alternatives, seed.split('\'').take(kept), kept, tail_path);
2689 }
2690 };
2691 let tail = &remaining[covered..];
2692 if tail.is_empty() {
2693 append(Vec::new());
2694 } else {
2695 for tail_path in table
2696 .split_paths_limited(tail, &[], prior, SEED_TAIL_LIMIT)
2697 .into_iter()
2698 .take(SEED_TAIL_LIMIT)
2699 {
2700 append(tail_path);
2701 }
2702 let (_, full_covered, full_fits) = fitting_syllables(table.grid, seed, remaining);
2704 if full_fits {
2705 let last = &seed[seed_prefix(seed).len() + 1..];
2706 let rest = &remaining[full_covered..];
2707 let rest_path = if rest.is_empty() {
2708 Some(Vec::new())
2709 } else {
2710 table
2711 .split_paths_limited(rest, &[], prior, 1)
2712 .into_iter()
2713 .next()
2714 };
2715 if let Some(rest_path) = rest_path {
2716 let mut anchored = Vec::with_capacity(1 + rest_path.len());
2717 anchored.push(last.to_owned());
2718 anchored.extend(rest_path);
2719 append(anchored);
2720 }
2721 }
2722 }
2723 }
2724}
2725
2726fn append_seeded_path<'a>(
2728 alternatives: &mut Vec<Path>,
2729 prefix: impl Iterator<Item = &'a str> + Clone,
2730 kept: usize,
2731 tail_path: Path,
2732) {
2733 if alternatives.iter().any(|path| {
2734 path.len() == kept + tail_path.len()
2735 && path[..kept].iter().map(String::as_str).eq(prefix.clone())
2736 && path[kept..] == tail_path
2737 }) {
2738 return;
2739 }
2740 let mut path = Vec::with_capacity(kept + tail_path.len());
2741 path.extend(prefix.map(str::to_owned));
2742 path.extend(tail_path);
2743 alternatives.push(path);
2744}
2745
2746fn remaining_digits(digits: &str, locked_length: usize) -> &str {
2747 &digits[locked_length..]
2748}
2749
2750fn append_path_key(output: &mut String, prefix: &str, path: &[String]) {
2751 output.clear();
2752 output.push_str(prefix);
2753 for (index, part) in path.iter().enumerate() {
2754 if !prefix.is_empty() || index != 0 {
2755 output.push('\'');
2756 }
2757 output.push_str(part);
2758 }
2759}
2760
2761fn agrees_with_locked(matched: &str, locked_key: &str) -> bool {
2763 if locked_key.is_empty() || matched == locked_key {
2764 return true;
2765 }
2766 let under = matched
2767 .strip_prefix(locked_key)
2768 .is_some_and(|rest| rest.starts_with('\''));
2769 let over = locked_key
2770 .strip_prefix(matched)
2771 .is_some_and(|rest| rest.starts_with('\''));
2772 under || over
2773}
2774
2775fn append_unique_rows(rows: &mut Vec<ExpressiveRow>, more: Vec<ExpressiveRow>) {
2777 for row in more {
2778 if !rows.iter().any(|known| known.item.word == row.item.word) {
2779 rows.push(row);
2780 }
2781 }
2782}
2783
2784fn letters_start_with(pinyin: &str, reading: &str) -> bool {
2786 let mut letters = pinyin.bytes().filter(|&byte| byte != b'\'');
2787 reading.bytes().all(|byte| letters.next() == Some(byte))
2788}
2789
2790fn splits_into_syllables_at(key: &str, boundaries: &[usize]) -> bool {
2794 let mut start = 0;
2795 for &end in boundaries.iter().chain(std::iter::once(&key.len())) {
2796 let Some(piece) = key.get(start..end) else {
2797 return false;
2798 };
2799 if !piece.is_empty() && (piece.contains('\'') || !is_complete_pinyin_input(piece)) {
2801 return false;
2802 }
2803 start = end;
2804 }
2805 true
2806}
2807
2808fn passes_filter(word: &str, single_character: bool, strokes: Option<&HashSet<char>>) -> bool {
2810 let mut characters = word.chars();
2811 let Some(first) = characters.next() else {
2812 return false;
2813 };
2814 if single_character && characters.next().is_some() {
2815 return false;
2816 }
2817 strokes.is_none_or(|texts| texts.contains(&first))
2818}
2819
2820#[cfg(test)]
2821fn has_candidate_word(candidates: &[WordItem], word: &str) -> bool {
2822 candidates.iter().any(|candidate| candidate.word == word)
2823}
2824
2825fn query_key_is_new(
2826 alternatives: &[Path],
2827 index: usize,
2828 key: &str,
2829 queried: Option<&HashSet<String>>,
2830) -> bool {
2831 match queried {
2832 Some(queried) => !queried.contains(key),
2833 None => !alternatives[..index]
2834 .iter()
2835 .any(|alternative| alternative == &alternatives[index]),
2836 }
2837}
2838
2839type RankKey = (Reverse<usize>, bool, bool, bool, bool, Reverse<i64>);
2841
2842const LEADING_FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
2843const LEADING_FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
2844
2845#[derive(Default)]
2846struct LeadingHasher(u64);
2847
2848impl Hasher for LeadingHasher {
2849 fn finish(&self) -> u64 {
2850 self.0
2851 }
2852
2853 fn write(&mut self, bytes: &[u8]) {
2854 self.0 = bytes.iter().fold(LEADING_FNV_OFFSET, |state, &byte| {
2855 (state ^ u64::from(byte)).wrapping_mul(LEADING_FNV_PRIME)
2856 });
2857 }
2858
2859 fn write_u64(&mut self, value: u64) {
2860 self.0 = value;
2861 }
2862}
2863
2864#[derive(Clone, Copy)]
2865struct LeadingEntry {
2866 candidate: usize,
2867 key: RankKey,
2868}
2869
2870enum LeadingSlot {
2871 One(LeadingEntry),
2872 Many(Vec<LeadingEntry>),
2873}
2874
2875impl LeadingSlot {
2876 fn find(&self, candidates: &[WordItem], word: &str) -> Option<(usize, RankKey)> {
2877 let entries: &[LeadingEntry] = match self {
2878 Self::One(entry) => std::slice::from_ref(entry),
2879 Self::Many(entries) => entries,
2880 };
2881 entries.iter().enumerate().find_map(|(slot, entry)| {
2882 (candidates[entry.candidate].word == word).then_some((slot, entry.key))
2883 })
2884 }
2885
2886 fn update(&mut self, slot: usize, candidate: usize, key: RankKey) {
2887 let entries: &mut [LeadingEntry] = match self {
2888 Self::One(entry) => std::slice::from_mut(entry),
2889 Self::Many(entries) => entries,
2890 };
2891 entries[slot] = LeadingEntry { candidate, key };
2892 }
2893
2894 fn insert(&mut self, entry: LeadingEntry) {
2895 match self {
2896 Self::One(previous) => {
2897 *self = Self::Many(vec![*previous, entry]);
2898 }
2899 Self::Many(entries) => entries.push(entry),
2900 }
2901 }
2902}
2903
2904type LeadingMap = HashMap<u64, LeadingSlot, BuildHasherDefault<LeadingHasher>>;
2905
2906fn leading_hash(word: &str) -> u64 {
2907 word.bytes().fold(LEADING_FNV_OFFSET, |state, byte| {
2908 (state ^ u64::from(byte)).wrapping_mul(LEADING_FNV_PRIME)
2909 })
2910}
2911
2912fn leading_rank(leading: &LeadingMap, candidates: &[WordItem], word: &str) -> Option<RankKey> {
2913 leading
2914 .get(&leading_hash(word))
2915 .and_then(|slot| slot.find(candidates, word).map(|(_, key)| key))
2916}
2917
2918fn rank_key(item: &WordItem, prefer_exact: bool, initials_lead: bool, grid: KeyGrid) -> RankKey {
2923 rank_key_for_reading(
2924 item,
2925 item.pinyin.len(),
2926 &item.canonical_pinyin,
2927 prefer_exact,
2928 initials_lead,
2929 grid,
2930 )
2931}
2932
2933fn rank_key_for_reading(
2935 item: &WordItem,
2936 covered: usize,
2937 canonical: &str,
2938 prefer_exact: bool,
2939 initials_lead: bool,
2940 grid: KeyGrid,
2941) -> RankKey {
2942 #[cfg(all(test, debug_assertions))]
2943 cached_rank_tests::KEY_EVALUATIONS.with(|count| count.set(count.get() + 1));
2944 let completion = prefer_exact && grid.reading_code_len(canonical) > covered;
2945 let initials = is_initials_reading(item, covered, canonical);
2946 (
2947 Reverse(covered),
2948 item.source.is_generated_or_fallback(),
2949 initials != initials_lead,
2950 completion,
2951 item.fuzzy,
2952 Reverse(comparable_weight_for_reading(item, canonical)),
2953 )
2954}
2955
2956fn discount_completions_on_first_page(
2960 candidates: &mut [WordItem],
2961 rows: usize,
2962 locked_syllables: usize,
2963 grid: KeyGrid,
2964) -> bool {
2965 let rows = rows.min(candidates.len());
2966 let completion =
2967 |item: &WordItem| grid.reading_code_len(&item.canonical_pinyin) > item.pinyin.len();
2968 let mut coverage = None;
2969 let mut end = rows;
2970 let mut spelled_phrase = false;
2971 for (seat, item) in candidates[..rows].iter().enumerate() {
2972 match discount_seat(item, &mut coverage) {
2973 DiscountSeat::Stop => {
2974 end = seat;
2975 break;
2976 }
2977 DiscountSeat::Row => {
2978 spelled_phrase |= !item.fuzzy
2979 && !completion(item)
2980 && item
2981 .canonical_pinyin
2982 .split('\'')
2983 .nth(locked_syllables + 1)
2984 .is_some();
2985 }
2986 DiscountSeat::Skip | DiscountSeat::Boundary => {}
2987 }
2988 }
2989 if !spelled_phrase {
2990 return false;
2991 }
2992 let ratio = COMPLETION_WEIGHT_RATIO;
2993 let key = |item: &WordItem| {
2994 let discounted = !item.fuzzy && completion(item);
2995 let scaled = match (discounted, item.weight >= 0) {
2996 (false, _) => item.weight.saturating_mul(ratio),
2997 (true, true) => item.weight,
2998 (true, false) => item.weight.saturating_mul(ratio).saturating_mul(ratio),
2999 };
3000 (item.fuzzy, Reverse(scaled), discounted)
3001 };
3002 for seat in 0..end {
3004 if discount_seat(&candidates[seat], &mut coverage) != DiscountSeat::Row {
3005 continue;
3006 }
3007 let mut current = seat;
3008 loop {
3009 let mut previous = None;
3010 for earlier in (0..current).rev() {
3011 match discount_seat(&candidates[earlier], &mut coverage) {
3012 DiscountSeat::Row => {
3013 previous = Some(earlier);
3014 break;
3015 }
3016 DiscountSeat::Skip => {}
3017 DiscountSeat::Boundary | DiscountSeat::Stop => break,
3018 }
3019 }
3020 match previous {
3021 Some(earlier) if key(&candidates[earlier]) > key(&candidates[current]) => {
3022 candidates.swap(earlier, current);
3023 current = earlier;
3024 }
3025 _ => break,
3026 }
3027 }
3028 }
3029 true
3030}
3031
3032#[derive(Clone, Copy, PartialEq, Eq)]
3034enum DiscountSeat {
3035 Row,
3037 Boundary,
3039 Skip,
3041 Stop,
3043}
3044
3045fn discount_seat(item: &WordItem, coverage: &mut Option<usize>) -> DiscountSeat {
3046 if item.fixed_position != 0 {
3047 return DiscountSeat::Skip;
3048 }
3049 if !item.source.is_dictionary() {
3050 return match item.source {
3051 CandidateSource::Generated
3052 | CandidateSource::Fallback
3053 | CandidateSource::Collocation
3054 | CandidateSource::NeuralKeyboard
3055 | CandidateSource::Emoji
3056 | CandidateSource::Kaomoji => DiscountSeat::Stop,
3057 _ => DiscountSeat::Skip,
3058 };
3059 }
3060 if *coverage.get_or_insert(item.pinyin.len()) != item.pinyin.len() {
3061 DiscountSeat::Stop
3062 } else if is_initials_row(item) {
3063 DiscountSeat::Boundary
3064 } else {
3065 DiscountSeat::Row
3066 }
3067}
3068
3069fn locked_syllables(locked_key: &str) -> usize {
3071 if locked_key.is_empty() {
3072 0
3073 } else {
3074 locked_key.split('\'').count()
3075 }
3076}
3077
3078fn comparable_weight(item: &WordItem) -> i64 {
3080 comparable_weight_for_reading(item, &item.canonical_pinyin)
3081}
3082
3083fn reading_syllable_count(canonical: &str) -> usize {
3085 canonical.bytes().filter(|&byte| byte == b'\'').count() + 1
3086}
3087
3088fn comparable_weight_for_reading(item: &WordItem, canonical: &str) -> i64 {
3089 if item.source != CandidateSource::Generated || canonical.is_empty() {
3090 return item.weight;
3091 }
3092 let syllables = reading_syllable_count(canonical);
3093 item.weight - (PHRASE_LENGTH_BONUS * 1000.0 * syllables as f64) as i64
3095}
3096
3097fn is_personal_word(word: &str) -> bool {
3099 count_utf8_chars(word) >= 2 && is_all_han(word)
3100}
3101
3102fn reads_pinyin(source: CandidateSource) -> bool {
3104 source.is_dictionary()
3105 || source.is_generated_or_fallback()
3106 || source == CandidateSource::NeuralKeyboard
3107}
3108
3109fn is_initials_row(item: &WordItem) -> bool {
3111 is_initials_reading(item, item.pinyin.len(), &item.canonical_pinyin)
3112}
3113
3114fn is_initials_reading(item: &WordItem, digits: usize, canonical: &str) -> bool {
3115 digits >= 2
3116 && !item.source.is_generated_or_fallback()
3117 && reading_syllable_count(canonical) == digits
3118 && canonical.bytes().filter(u8::is_ascii_lowercase).count() > digits
3119}
3120
3121fn initials_apply(digits: usize, splits: &[usize]) -> bool {
3123 if digits < 2 {
3124 return false;
3125 }
3126 let mut start = 0;
3127 for &split in splits {
3128 if split - start > 1 {
3129 return false;
3130 }
3131 start = split;
3132 }
3133 splits.is_empty() || digits - start <= 1
3134}
3135
3136fn shuangpin_word_codes(grid: KeyGrid, digits: &str) -> Option<Vec<String>> {
3138 if digits.len().div_ceil(2) < SHUANGPIN_WORD_MIN_SYLLABLES {
3139 return None;
3140 }
3141 let initials: String = digits.bytes().step_by(2).map(char::from).collect();
3142 initials_codes(grid, &initials, INITIALS_CODE_LIMIT)
3143}
3144
3145fn shuangpin_word_spells(table: &NineKeyShuangpin, canonical_pinyin: &str, digits: &str) -> bool {
3147 reading_syllable_count(canonical_pinyin) == digits.len().div_ceil(2)
3148 && table.matching_prefix_len(canonical_pinyin, digits) == Some(digits.len())
3149}
3150
3151fn initials_for_digit(grid: KeyGrid, digit: u8) -> impl Iterator<Item = u8> {
3153 grid.letters(digit)
3154 .bytes()
3155 .filter(|letter| !matches!(letter, b'i' | b'u' | b'v'))
3156}
3157
3158fn initials_codes(grid: KeyGrid, digits: &str, limit: usize) -> Option<Vec<String>> {
3160 let mut total = 1usize;
3162 for digit in digits.bytes() {
3163 let count = initials_for_digit(grid, digit).count();
3164 total = total.saturating_mul(count);
3165 if count == 0 || total > limit {
3166 return None;
3167 }
3168 }
3169 let mut codes = vec![String::with_capacity(digits.len())];
3170 for digit in digits.bytes() {
3171 codes = codes
3172 .iter()
3173 .flat_map(|code| {
3174 initials_for_digit(grid, digit).map(move |letter| {
3175 let mut next = String::with_capacity(digits.len());
3176 next.push_str(code);
3177 next.push(char::from(letter));
3178 next
3179 })
3180 })
3181 .collect();
3182 }
3183 Some(codes)
3184}
3185
3186fn take_candidate_canonical_pinyin(candidate: &mut WordItem) -> SharedString {
3187 if candidate.canonical_pinyin.is_empty() {
3188 candidate.pinyin.clone()
3189 } else {
3190 std::mem::take(&mut candidate.canonical_pinyin)
3191 }
3192}
3193
3194fn deduplicate_english_words(words: &mut Vec<WordItem>) {
3196 assert!(words.len() <= ENGLISH_CANDIDATE_CAPACITY);
3197 let mut duplicates = [0usize; ENGLISH_CANDIDATE_CAPACITY];
3198 let mut duplicate_len = 0;
3199 {
3200 let mut seen: [Option<&str>; ENGLISH_CANDIDATE_CAPACITY] =
3201 [None; ENGLISH_CANDIDATE_CAPACITY];
3202 let mut seen_len = 0;
3203 for (index, word) in words.iter().enumerate() {
3204 if seen[..seen_len]
3205 .iter()
3206 .flatten()
3207 .any(|existing| *existing == word.word.as_str())
3208 {
3209 duplicates[duplicate_len] = index;
3210 duplicate_len += 1;
3211 } else {
3212 seen[seen_len] = Some(word.word.as_str());
3213 seen_len += 1;
3214 }
3215 }
3216 }
3217 let mut next_duplicate = 0;
3218 let mut write = 0;
3219 for read in 0..words.len() {
3220 if next_duplicate < duplicate_len && duplicates[next_duplicate] == read {
3221 next_duplicate += 1;
3222 continue;
3223 }
3224 if write != read {
3225 words.swap(write, read);
3226 }
3227 write += 1;
3228 }
3229 words.truncate(write);
3230}
3231
3232fn push_ranked(
3234 candidates: &mut Vec<WordItem>,
3235 leading: &mut LeadingMap,
3236 candidate: WordItem,
3237 prefer_exact: bool,
3238 initials_lead: bool,
3239 grid: KeyGrid,
3240) {
3241 let key = rank_key(&candidate, prefer_exact, initials_lead, grid);
3242 let hash = leading_hash(&candidate.word);
3243 let existing = leading
3244 .get(&hash)
3245 .and_then(|slot| slot.find(candidates, &candidate.word));
3246 if let Some((slot, best)) = existing {
3247 if best <= key {
3248 return;
3249 }
3250 let candidate_index = candidates.len();
3251 candidates.push(candidate);
3252 leading
3253 .get_mut(&hash)
3254 .expect("leading hash entry remains present")
3255 .update(slot, candidate_index, key);
3256 return;
3257 }
3258 if leading.is_empty() {
3259 leading.reserve(CANDIDATE_LIMIT);
3260 }
3261 let candidate_index = candidates.len();
3262 candidates.push(candidate);
3263 let entry = LeadingEntry {
3264 candidate: candidate_index,
3265 key,
3266 };
3267 if let Some(slot) = leading.get_mut(&hash) {
3268 slot.insert(entry);
3269 } else {
3270 leading.insert(hash, LeadingSlot::One(entry));
3271 }
3272}
3273
3274fn rank_candidates(
3276 candidates: &mut Vec<WordItem>,
3277 prefer_exact: bool,
3278 initials_lead: bool,
3279 boost: Option<&InitialsBoost>,
3280 grid: KeyGrid,
3281) {
3282 if candidates.len() <= SMALL_CANDIDATE_SORT {
3283 candidates.sort_by_key(|item| rank_key(item, prefer_exact, initials_lead, grid));
3284 } else {
3285 candidates.sort_by_cached_key(|item| rank_key(item, prefer_exact, initials_lead, grid));
3286 }
3287 retain_unique_words(candidates);
3288 if !initials_lead {
3289 interleave_initials(candidates);
3290 }
3291 if let Some(boost) = boost {
3292 boost_used_initials(candidates, initials_lead, boost);
3293 }
3294 candidates.truncate(CANDIDATE_LIMIT);
3295}
3296
3297struct InitialsBoost {
3299 used: HashMap<SharedString, u32>,
3301 frequency: FrequencyAdjustmentOptions,
3302}
3303
3304fn boost_used_initials(candidates: &mut Vec<WordItem>, initials_lead: bool, boost: &InitialsBoost) {
3306 let Some(first) = candidates.first() else {
3307 return;
3308 };
3309 let coverage = first.pinyin.len();
3310 let end = candidates
3311 .iter()
3312 .take_while(|item| item.pinyin.len() == coverage && !item.source.is_generated_or_fallback())
3313 .count();
3314 let floor = if initials_lead {
3315 0
3316 } else {
3317 candidates[..end]
3318 .iter()
3319 .take(SYLLABLE_ROWS_BEFORE_INITIALS)
3320 .take_while(|item| !is_initials_row(item))
3321 .count()
3322 };
3323 let trigger_count = u32::try_from(boost.frequency.trigger_count.clamp(1, 10)).unwrap_or(1);
3324 let per_trigger = PERSONAL_PICK_TIMES * trigger_count;
3325 let mut moves = Vec::new();
3327 for (index, item) in candidates[..end].iter().enumerate() {
3328 if !is_initials_row(item) {
3329 continue;
3330 }
3331 let Some(&count) = boost.used.get(&item.word) else {
3332 continue;
3333 };
3334 let mut rank = index;
3335 let triggers = usize::try_from(count / per_trigger).unwrap_or(usize::MAX);
3337 for _ in 0..triggers.min(index) {
3338 if rank == 0 {
3339 break;
3340 }
3341 rank = ranking::ranking_target(
3342 rank,
3343 boost.frequency.mode,
3344 boost.frequency.linear_step,
3345 false,
3346 );
3347 }
3348 let target = rank.max(floor);
3349 if target < index {
3350 moves.push((target, index));
3351 }
3352 }
3353 if moves.is_empty() {
3354 return;
3355 }
3356 let mut rows: Vec<(usize, usize, WordItem)> = moves
3358 .iter()
3359 .rev()
3360 .map(|&(target, index)| (target, index, candidates.remove(index)))
3361 .collect();
3362 rows.sort_by_key(|&(target, index, _)| (target, index));
3363 let mut next = 0;
3364 for (target, _, row) in rows {
3365 let at = target.max(next);
3366 candidates.insert(at, row);
3367 next = at + 1;
3368 }
3369}
3370
3371fn interleave_initials(candidates: &mut Vec<WordItem>) {
3373 let mut start = 0;
3374 while start < candidates.len() {
3375 let coverage = candidates[start].pinyin.len();
3376 let generated = candidates[start].source.is_generated_or_fallback();
3377 let end = start
3378 + candidates[start..]
3379 .iter()
3380 .take_while(|item| {
3381 item.pinyin.len() == coverage
3382 && item.source.is_generated_or_fallback() == generated
3383 })
3384 .count();
3385 if !generated {
3386 let middle = start
3388 + candidates[start..end]
3389 .iter()
3390 .take_while(|item| !is_initials_row(item))
3391 .count();
3392 let kept = start + SYLLABLE_ROWS_BEFORE_INITIALS;
3393 if kept < middle && middle < end {
3394 let mut syllables: Vec<WordItem> = candidates.drain(kept..end).collect();
3395 let initials = syllables.split_off(middle - kept);
3396 let mut syllables = syllables.into_iter().peekable();
3397 let mut initials = initials.into_iter().peekable();
3398 let mut merged = Vec::with_capacity(end - kept);
3399 loop {
3400 let take_initial = match (syllables.peek(), initials.peek()) {
3401 (Some(syllable), Some(initial)) => initial.weight > syllable.weight,
3402 (None, Some(_)) => true,
3403 (_, None) => false,
3404 };
3405 let next = if take_initial {
3406 initials.next()
3407 } else {
3408 syllables.next()
3409 };
3410 match next {
3411 Some(item) => merged.push(item),
3412 None => break,
3413 }
3414 }
3415 candidates.splice(kept..kept, merged);
3416 }
3417 }
3418 start = end;
3419 }
3420}
3421
3422fn retain_unique_words(candidates: &mut Vec<WordItem>) {
3423 if candidates.len() <= SMALL_CANDIDATE_DEDUP {
3424 let mut write = 0;
3425 for read in 0..candidates.len() {
3426 if candidates[..write]
3427 .iter()
3428 .any(|existing| existing.word == candidates[read].word)
3429 {
3430 continue;
3431 }
3432 if write != read {
3433 candidates.swap(write, read);
3434 }
3435 write += 1;
3436 }
3437 candidates.truncate(write);
3438 return;
3439 }
3440 let mut seen = HashSet::with_capacity(candidates.len());
3441 let duplicates = candidates
3442 .iter()
3443 .enumerate()
3444 .filter_map(|(index, item)| (!seen.insert(item.word.as_str())).then_some(index))
3445 .collect::<Vec<_>>();
3446 drop(seen);
3447 let mut duplicates = duplicates.into_iter().peekable();
3448 let mut write = 0;
3449 for read in 0..candidates.len() {
3450 if duplicates.peek() == Some(&read) {
3451 duplicates.next();
3452 continue;
3453 }
3454 if write != read {
3455 candidates.swap(write, read);
3456 }
3457 write += 1;
3458 }
3459 candidates.truncate(write);
3460}
3461
3462fn rank_english(words: &mut Vec<WordItem>, typed: usize) {
3464 let exact = |item: &WordItem| item.word.len() == typed && item.weight > 0;
3465 words.sort_by(|a, b| {
3466 exact(b)
3467 .cmp(&exact(a))
3468 .then_with(|| b.weight.cmp(&a.weight))
3469 .then_with(|| a.word.len().cmp(&b.word.len()))
3470 });
3471 words.truncate(ENGLISH_LIMIT);
3472}
3473
3474#[cfg(test)]
3476fn digits_for_word(word: &str) -> String {
3477 let lowered = word.to_ascii_lowercase();
3478 if !lowered.bytes().all(|byte| byte.is_ascii_lowercase()) {
3479 return String::new();
3480 }
3481 KeyGrid::NineKey.encode(&lowered)
3482}
3483
3484#[cfg(test)]
3485fn word_matches_digits(grid: KeyGrid, word: &str, digits: &str) -> bool {
3486 let Some(keypad) = grid.keypad() else {
3487 return false;
3488 };
3489 let mut matched = 0;
3490 for byte in word.bytes() {
3491 let letter = byte.to_ascii_lowercase();
3492 if !letter.is_ascii_lowercase() {
3493 return false;
3494 }
3495 if let Some(&digit) = digits.as_bytes().get(matched) {
3496 if keypad[usize::from(letter - b'a')] != digit {
3497 return false;
3498 }
3499 matched += 1;
3500 }
3501 }
3502 matched == digits.len()
3503}
3504
3505fn keyword_spells_digits(grid: KeyGrid, keyword: &str, digits: &str) -> bool {
3507 let Some(keypad) = grid.keypad() else {
3508 return match grid {
3509 KeyGrid::NineKeyShuangpin(table)
3510 | KeyGrid::FourteenKeyShuangpin(table)
3511 | KeyGrid::EighteenKeyShuangpin(table) => table.spells_keyword(keyword, digits),
3512 _ => false,
3513 };
3514 };
3515 keyword.len() == digits.len()
3516 && keyword.bytes().zip(digits.bytes()).all(|(letter, digit)| {
3517 letter.is_ascii_lowercase() && keypad[usize::from(letter - b'a')] == digit
3518 })
3519}
3520
3521fn letter_prefix_span(grid: KeyGrid, digits: &str, budget: usize) -> (usize, usize) {
3523 let mut count = 1;
3524 let mut depth = 0;
3525 for digit in digits.bytes() {
3526 let letters = grid.letters(digit);
3527 let width = letters.chars().count();
3528 if width == 0 || count > budget / width {
3529 break;
3530 }
3531 count *= width;
3532 depth += 1;
3533 }
3534 (depth, count)
3535}
3536
3537fn letter_prefix_depth(grid: KeyGrid, digits: &str, budget: usize) -> usize {
3538 letter_prefix_span(grid, digits, budget).0
3539}
3540
3541fn word_spells_from(grid: KeyGrid, word: &str, digits: &str, from: usize) -> bool {
3543 word.len() >= digits.len()
3544 && word.as_bytes()[from..digits.len()]
3545 .iter()
3546 .zip(&digits.as_bytes()[from..])
3547 .all(|(&letter, &digit)| {
3548 grid.letters(digit)
3549 .as_bytes()
3550 .contains(&letter.to_ascii_lowercase())
3551 })
3552}
3553
3554fn letter_prefixes(grid: KeyGrid, digits: &str, budget: usize) -> Vec<String> {
3556 let (depth, count) = letter_prefix_span(grid, digits, budget);
3557 if depth == 0 {
3558 Vec::new()
3559 } else {
3560 fn visit(grid: KeyGrid, digits: &[u8], prefix: &mut String, result: &mut Vec<String>) {
3561 let Some((&digit, rest)) = digits.split_first() else {
3562 result.push(prefix.clone());
3563 return;
3564 };
3565 for letter in grid.letters(digit).chars() {
3566 prefix.push(letter);
3567 visit(grid, rest, prefix, result);
3568 prefix.pop();
3569 }
3570 }
3571
3572 let mut prefixes = Vec::with_capacity(count);
3573 let mut prefix = String::with_capacity(depth);
3574 visit(
3575 grid,
3576 &digits.as_bytes()[..depth],
3577 &mut prefix,
3578 &mut prefixes,
3579 );
3580 prefixes
3581 }
3582}
3583
3584pub(crate) struct SpellingTable {
3586 grid: KeyGrid,
3588 syllables: Vec<(String, String)>,
3590 by_code: HashMap<String, Vec<(String, bool)>>,
3592 longest_code: usize,
3593}
3594
3595struct EnglishKeypadCache {
3597 grid: KeyGrid,
3598 depth: usize,
3600 digits: String,
3602 rows: Option<Vec<Vec<KeypadRow>>>,
3604}
3605
3606#[derive(Default)]
3608struct PathScratch<'a> {
3609 pages: Vec<Vec<SpellingPath<'a>>>,
3610}
3611
3612#[derive(Clone, Copy)]
3614struct SpellingPath<'a> {
3615 score: f64,
3616 complete: bool,
3617 piece: &'a str,
3618 next: (usize, usize),
3619 length: usize,
3620}
3621
3622impl<'a> SpellingPath<'a> {
3623 fn pieces<'s>(self, suffix: &'s [Vec<Self>]) -> impl Iterator<Item = &'a str> + 's
3624 where
3625 'a: 's,
3626 {
3627 let mut first = (self.length != 0).then_some(self.piece);
3628 let mut next = (self.length > 1).then_some(self.next);
3629 std::iter::from_fn(move || {
3630 if let Some(piece) = first.take() {
3631 return Some(piece);
3632 }
3633 let (column, row) = next?;
3634 let node = &suffix[column][row];
3635 next = (node.length > 1).then_some(node.next);
3636 Some(node.piece)
3637 })
3638 }
3639}
3640
3641impl SpellingTable {
3642 fn build(grid: KeyGrid, syllables: &[&str]) -> Self {
3643 let prefix_capacity = syllables.iter().map(|syllable| syllable.len()).sum();
3644 let intact: HashSet<&str> = syllables.iter().copied().collect();
3645 let mut seen = HashSet::with_capacity(prefix_capacity);
3646 let mut by_code: HashMap<String, Vec<(String, bool)>> =
3647 HashMap::with_capacity(prefix_capacity);
3648 let mut longest_code = 0;
3649 for syllable in syllables {
3650 for end in 1..=syllable.len() {
3651 let Some(prefix) = syllable.get(..end) else {
3652 continue;
3653 };
3654 if seen.insert(prefix.to_string()) {
3655 let code = grid.encode(prefix);
3656 longest_code = longest_code.max(code.len());
3657 by_code
3658 .entry(code)
3659 .or_default()
3660 .push((prefix.to_string(), intact.contains(prefix)));
3661 }
3662 }
3663 }
3664 Self {
3665 grid,
3666 syllables: syllables
3667 .iter()
3668 .map(|syllable| (syllable.to_string(), grid.encode(syllable)))
3669 .collect(),
3670 by_code,
3671 longest_code,
3672 }
3673 }
3674
3675 fn build_shuangpin(grid: KeyGrid, table: &NineKeyShuangpin) -> Self {
3677 let mut syllables = Vec::new();
3678 let mut by_code: HashMap<String, Vec<(String, bool)>> = HashMap::new();
3679 for (syllable, code) in table.coded_syllables() {
3680 let code: String = code.iter().map(|&digit| char::from(digit)).collect();
3681 by_code
3682 .entry(code.clone())
3683 .or_default()
3684 .push((syllable.to_owned(), true));
3685 syllables.push((syllable.to_owned(), code));
3686 }
3687 for (piece, key) in table.one_key_pieces() {
3688 by_code
3689 .entry(char::from(key).to_string())
3690 .or_default()
3691 .push((piece.to_owned(), false));
3692 }
3693 Self {
3694 grid,
3695 syllables,
3696 by_code,
3697 longest_code: 2,
3698 }
3699 }
3700
3701 fn starts_syllable(&self, letter: u8) -> bool {
3703 self.syllables
3704 .iter()
3705 .any(|(syllable, _)| syllable.as_bytes().first() == Some(&letter))
3706 }
3707
3708 #[cfg(test)]
3713 fn spellings_for(
3714 &self,
3715 remaining: &str,
3716 locked_length: usize,
3717 split: Option<usize>,
3718 prior: &SyllablePrior,
3719 ) -> Vec<String> {
3720 let mut spellings = Vec::new();
3721 let mut matches = Vec::new();
3722 self.spellings_for_into(
3723 remaining,
3724 locked_length,
3725 split,
3726 prior,
3727 &mut spellings,
3728 &mut matches,
3729 );
3730 spellings
3731 }
3732
3733 fn spellings_for_into(
3734 &self,
3735 remaining: &str,
3736 locked_length: usize,
3737 split: Option<usize>,
3738 prior: &SyllablePrior,
3739 destination: &mut Vec<String>,
3740 matches: &mut Vec<usize>,
3741 ) {
3742 matches.clear();
3743 if remaining.is_empty() {
3744 destination.clear();
3745 return;
3746 }
3747 matches.extend(
3748 self.syllables
3749 .iter()
3750 .enumerate()
3751 .filter_map(|(index, (_, code))| {
3752 let matches = match split {
3753 Some(split) => code.len() <= split && remaining.starts_with(code.as_str()),
3754 None => {
3755 locked_length + remaining.len().max(code.len()) <= DIGIT_LIMIT
3756 && (remaining.starts_with(code.as_str())
3757 || code.starts_with(remaining))
3758 }
3759 };
3760 matches.then_some(index)
3761 }),
3762 );
3763 let covered = |code: &str| code.len().min(remaining.len());
3764 matches.sort_by(|&a, &b| {
3765 let (a, a_code) = &self.syllables[a];
3766 let (b, b_code) = &self.syllables[b];
3767 covered(b_code)
3768 .cmp(&covered(a_code))
3769 .then_with(|| {
3770 if remaining.len() > 1 {
3771 a_code.len().cmp(&b_code.len())
3772 } else {
3773 Ordering::Equal
3774 }
3775 })
3776 .then_with(|| prior.syllable_score(b).total_cmp(&prior.syllable_score(a)))
3777 .then_with(|| a.cmp(b))
3778 });
3779 matches.dedup_by(|a, b| self.syllables[*a].0 == self.syllables[*b].0);
3781 destination.truncate(matches.len());
3782 for (position, &index) in matches.iter().enumerate() {
3783 if let Some(spelling) = destination.get_mut(position) {
3784 spelling.clear();
3785 spelling.push_str(&self.syllables[index].0);
3786 } else {
3787 destination.push(self.syllables[index].0.clone());
3788 }
3789 }
3790 }
3791
3792 #[cfg(test)]
3794 fn paths(&self, digits: &str) -> Vec<Path> {
3795 self.split_paths(digits, &[], &SyllablePrior::default())
3796 }
3797
3798 #[cfg(test)]
3802 fn split_paths(&self, digits: &str, splits: &[usize], prior: &SyllablePrior) -> Vec<Path> {
3803 self.split_paths_limited(digits, splits, prior, PATH_LIMIT)
3804 }
3805
3806 fn split_paths_limited(
3808 &self,
3809 digits: &str,
3810 splits: &[usize],
3811 prior: &SyllablePrior,
3812 output_limit: usize,
3813 ) -> Vec<Path> {
3814 let mut paths = Vec::new();
3815 self.split_paths_into(digits, splits, prior, output_limit, &mut paths);
3816 paths
3817 }
3818
3819 fn split_paths_into(
3821 &self,
3822 digits: &str,
3823 splits: &[usize],
3824 prior: &SyllablePrior,
3825 output_limit: usize,
3826 destination: &mut Vec<Path>,
3827 ) {
3828 self.split_paths_with_scratch(
3829 digits,
3830 splits,
3831 prior,
3832 output_limit,
3833 &mut PathScratch::default(),
3834 destination,
3835 );
3836 }
3837
3838 fn split_paths_with_scratch<'a>(
3839 &'a self,
3840 digits: &str,
3841 splits: &[usize],
3842 prior: &SyllablePrior,
3843 output_limit: usize,
3844 scratch: &mut PathScratch<'a>,
3845 destination: &mut Vec<Path>,
3846 ) {
3847 let length = digits.len();
3848 if scratch.pages.is_empty() {
3849 scratch.pages = vec![Vec::new(); length + 1];
3850 } else {
3851 scratch.pages.resize_with(length + 1, Vec::new);
3852 }
3853 let suffix = &mut scratch.pages;
3854 if destination.is_empty() {
3855 suffix[0] = Vec::new();
3857 }
3858 suffix[length].clear();
3859 suffix[length].push(SpellingPath {
3860 score: 0.0,
3861 complete: false,
3862 piece: "",
3863 next: (length, 0),
3864 length: 0,
3865 });
3866 for offset in (0..length).rev() {
3867 let mut result = std::mem::take(&mut suffix[offset]);
3868 result.clear();
3869 for end in offset + 1..=length.min(offset + self.longest_code) {
3870 if splits.iter().any(|&split| offset < split && split < end) {
3871 break;
3872 }
3873 let Some(pieces) = self.by_code.get(&digits[offset..end]) else {
3874 continue;
3875 };
3876 let must_complete = end != length || splits.contains(&end);
3877 for (piece, complete) in pieces {
3878 if must_complete && !complete {
3879 continue;
3880 }
3881 let score = prior.score(piece);
3882 for (row, tail) in suffix[end].iter().enumerate() {
3883 if result.is_empty() {
3884 result.reserve_exact(PATH_LIMIT);
3885 }
3886 result.push(SpellingPath {
3887 score: score + tail.score,
3888 complete: if tail.length == 0 {
3889 *complete
3890 } else {
3891 tail.complete
3892 },
3893 piece,
3894 next: (end, row),
3895 length: tail.length + 1,
3896 });
3897 }
3898 }
3899 }
3900 let compare = |a: &SpellingPath<'_>, b: &SpellingPath<'_>| {
3902 a.length
3903 .cmp(&b.length)
3904 .then_with(|| b.complete.cmp(&a.complete))
3905 .then_with(|| b.score.total_cmp(&a.score))
3906 .then_with(|| a.pieces(suffix).cmp(b.pieces(suffix)))
3907 };
3908 let keep = if offset == 0 {
3909 output_limit.min(PATH_LIMIT)
3910 } else {
3911 PATH_LIMIT
3912 };
3913 if keep < PATH_LIMIT && result.len() > keep {
3915 result.select_nth_unstable_by(keep, compare);
3916 result.truncate(keep);
3917 }
3918 result.sort_by(compare);
3919 result.truncate(keep);
3920 suffix[offset] = result;
3921 }
3922 if destination.is_empty() {
3924 let roots = std::mem::take(&mut suffix[0]);
3925 *destination = roots
3926 .into_iter()
3927 .take(output_limit)
3928 .map(|root| {
3929 let mut path = Vec::with_capacity(root.length);
3930 path.extend(root.pieces(suffix).map(str::to_owned));
3931 path
3932 })
3933 .collect();
3934 return;
3935 }
3936 let roots = &suffix[0];
3937 let output_len = roots.len().min(output_limit);
3938 destination.reserve(output_len.saturating_sub(destination.len()));
3939 destination.truncate(output_len);
3940 for (index, root) in roots.iter().copied().take(output_limit).enumerate() {
3941 let path = if let Some(path) = destination.get_mut(index) {
3942 path
3943 } else {
3944 destination.push(Vec::with_capacity(root.length));
3945 destination.last_mut().expect("new path")
3946 };
3947 path.truncate(root.length);
3948 path.reserve(root.length.saturating_sub(path.len()));
3949 for (piece_index, piece) in root.pieces(suffix).enumerate() {
3950 if piece_index == path.len() {
3951 path.push(String::new());
3952 }
3953 path[piece_index].clear();
3954 path[piece_index].push_str(piece);
3955 }
3956 path.truncate(root.length);
3957 }
3958 }
3959}
3960
3961#[derive(Default)]
3965struct SyllablePrior<'a> {
3966 scores: HashMap<&'a str, f64>,
3967 syllables: HashMap<&'a str, f64>,
3969}
3970
3971impl<'a> SyllablePrior<'a> {
3972 fn shipped(table: &'a SpellingTable) -> SyllablePrior<'a> {
3974 Self::from_weights(&table.syllables, shipped_syllable_frequency())
3975 }
3976
3977 fn from_weights(syllables: &'a [(String, String)], weights: &HashMap<String, i64>) -> Self {
3978 let total: f64 = syllables
3979 .iter()
3980 .map(|(syllable, _)| weights.get(syllable).copied().unwrap_or(0).max(1) as f64)
3981 .sum();
3982 let prefix_capacity = syllables.iter().map(|(syllable, _)| syllable.len()).sum();
3983 let mut scores: HashMap<&str, f64> = HashMap::with_capacity(prefix_capacity);
3984 let mut own: HashMap<&str, f64> = HashMap::with_capacity(syllables.len());
3985 for (syllable, _) in syllables {
3986 let weight = weights.get(syllable).copied().unwrap_or(0).max(1) as f64;
3988 let score = (weight / total).ln();
3989 own.insert(syllable.as_str(), score);
3990 for end in 1..=syllable.len() {
3991 let Some(prefix) = syllable.get(..end) else {
3992 continue;
3993 };
3994 let slot = scores.entry(prefix).or_insert(score);
3995 if *slot < score {
3996 *slot = score;
3997 }
3998 }
3999 }
4000 Self {
4001 scores,
4002 syllables: own,
4003 }
4004 }
4005
4006 fn score(&self, piece: &str) -> f64 {
4007 self.scores.get(piece).copied().unwrap_or(0.0)
4008 }
4009
4010 fn syllable_score(&self, syllable: &str) -> f64 {
4011 self.syllables.get(syllable).copied().unwrap_or(0.0)
4012 }
4013}
4014
4015fn shipped_syllable_frequency() -> &'static HashMap<String, i64> {
4017 static TABLE: OnceLock<HashMap<String, i64>> = OnceLock::new();
4018 TABLE.get_or_init(|| {
4019 SYLLABLE_FREQUENCY
4020 .lines()
4021 .filter(|line| !line.starts_with('#'))
4022 .map(|line| {
4023 let (syllable, weight) = line
4024 .split_once('\t')
4025 .expect("syllable_frequency.tsv: syllable<TAB>weight");
4026 let weight = weight
4027 .parse()
4028 .expect("syllable_frequency.tsv: integer weight");
4029 (syllable.to_owned(), weight)
4030 })
4031 .collect()
4032 })
4033}
4034
4035#[cfg(test)]
4036mod dominated_projection_tests;
4037#[cfg(test)]
4038mod filter_projection_tests;
4039#[cfg(test)]
4040mod isolated_test;
4041#[cfg(test)]
4042mod leading_entry_refresh_tests;
4043#[cfg(test)]
4044mod leading_entry_tests;
4045
4046#[cfg(test)]
4047mod cached_rank_tests;
4048#[cfg(test)]
4049mod english_cache_tests;
4050#[cfg(test)]
4051mod syllable_count_tests;
4052
4053#[cfg(test)]
4054mod initials_code_tests;
4055
4056#[cfg(test)]
4057mod path_chain_tests;
4058
4059#[cfg(test)]
4060mod path_buffer_tests;
4061
4062#[cfg(test)]
4063mod path_scratch_tests;
4064
4065#[cfg(test)]
4066mod seed_tail_tests;
4067
4068#[cfg(test)]
4069mod seed_dedup_tests;
4070
4071#[cfg(test)]
4072mod code_prefix_tests;
4073
4074#[cfg(test)]
4075mod spelling_buffer_tests;
4076
4077#[cfg(test)]
4078mod tests {
4079 use super::*;
4080 use crate::types::{CandidateSource, NineKeyShuangpinKind, DEFAULT_FIRST_PAGE_ROWS};
4081 use rusqlite::Connection;
4082
4083 fn encode(pinyin: &str) -> String {
4085 KeyGrid::NineKey.encode(pinyin)
4086 }
4087
4088 fn word_matches_digits(word: &str, digits: &str) -> bool {
4089 super::word_matches_digits(KeyGrid::NineKey, word, digits)
4090 }
4091
4092 fn initials_codes(digits: &str, limit: usize) -> Option<Vec<String>> {
4093 super::initials_codes(KeyGrid::NineKey, digits, limit)
4094 }
4095
4096 fn letter_prefixes(digits: &str, budget: usize) -> Vec<String> {
4097 super::letter_prefixes(KeyGrid::NineKey, digits, budget)
4098 }
4099
4100 fn spelling_table() -> &'static SpellingTable {
4101 KeyGrid::NineKey.spelling_table()
4102 }
4103
4104 #[test]
4105 fn spelling_match_indices_reuse_storage() {
4106 let table = spelling_table();
4107 let prior = SyllablePrior::default();
4108 let mut destination = Vec::with_capacity(128);
4109 let mut matches = Vec::new();
4110 table.spellings_for_into("64", 0, None, &prior, &mut destination, &mut matches);
4111 let (_, allocations) = crate::ime::personal_rerank::allocations::count(|| {
4112 table.spellings_for_into("64", 0, None, &prior, &mut destination, &mut matches);
4113 });
4114 assert!(
4115 allocations <= destination.len() + 1,
4116 "拼写匹配刷新不应重新申请索引工作区:{allocations} 次分配,{} 个读音",
4117 destination.len()
4118 );
4119 }
4120
4121 #[test]
4122 fn syllable_prior_does_not_clone_static_spelling_keys() {
4123 let table = spelling_table();
4124 let weights = HashMap::new();
4125 let (_, allocations) = crate::ime::personal_rerank::allocations::count(|| {
4126 SyllablePrior::from_weights(&table.syllables, &weights)
4127 });
4128 assert!(
4129 allocations <= 4,
4130 "静态九键音节表的频度索引不应复制音节和前缀键:{allocations} 次分配"
4131 );
4132 }
4133
4134 fn fitting_syllables(prefix: &str, digits: &str) -> (usize, usize, bool) {
4135 super::fitting_syllables(KeyGrid::NineKey, prefix, digits)
4136 }
4137
4138 fn next_seeds(
4139 candidates: &[WordItem],
4140 digits: &str,
4141 previous: &mut Vec<String>,
4142 seeds: &mut Vec<String>,
4143 ) {
4144 super::next_seeds(KeyGrid::NineKey, candidates, digits, previous, seeds);
4145 }
4146
4147 impl SpellingTable {
4148 fn new(syllables: &[&str]) -> Self {
4149 Self::build(KeyGrid::NineKey, syllables)
4150 }
4151 }
4152
4153 #[test]
4156 fn codes_follow_the_keypad() {
4157 assert_eq!(encode("ni'hao"), "64426");
4158 assert_eq!(encode("zhuang"), "948264");
4159 assert_eq!(digits_for_word("OK"), "65");
4160 assert_eq!(digits_for_word("ogham"), "64426");
4161 assert_eq!(digits_for_word("don't"), "");
4162 assert_eq!(digits_for_word("café"), "");
4163 assert!(word_matches_digits("OK", "65"));
4164 assert!(word_matches_digits("ogham", "64426"));
4165 assert!(!word_matches_digits("ogham", "64427"));
4166 assert!(!word_matches_digits("don't", "3668"));
4167 assert!(!word_matches_digits("ogham", "644260"));
4168 }
4169
4170 #[test]
4171 fn initials_codes_take_one_initial_per_digit() {
4172 assert_eq!(initials_codes("68", 1024).unwrap(), ["mt", "nt", "ot"]);
4173 assert_eq!(initials_codes("48", 1024).unwrap(), ["gt", "ht"]);
4175 assert_eq!(initials_codes("2356", 1024).unwrap().len(), 3 * 3 * 3 * 3);
4176 assert!(initials_codes("2356", 1024)
4177 .unwrap()
4178 .contains(&"cflm".to_string()));
4179 assert_eq!(initials_codes("99999", 1024).unwrap().len(), 1024);
4180 assert!(initials_codes("999999", 1024).is_none(), "past the limit");
4181 assert!(initials_codes("61", 1024).is_none(), "1 spells no letter");
4182 }
4183
4184 #[test]
4185 fn initials_apply_only_when_every_digit_can_be_a_syllable() {
4186 assert!(
4187 !initials_apply(1, &[]),
4188 "one digit already completes by its initial"
4189 );
4190 assert!(initials_apply(2, &[]));
4191 assert!(initials_apply(2, &[1]));
4192 assert!(initials_apply(5, &[1, 2, 3, 4]));
4193 assert!(
4194 initials_apply(3, &[1, 2, 3]),
4195 "a trailing split after one digit"
4196 );
4197 assert!(!initials_apply(3, &[2]), "94'2: 94 is one syllable");
4198 assert!(!initials_apply(2, &[2]), "68': 68 is one syllable");
4199 assert!(!initials_apply(3, &[1]), "6'84: 84 is one syllable");
4200 }
4201
4202 #[test]
4203 fn letter_prefixes_stop_at_the_budget() {
4204 assert_eq!(letter_prefixes("6", 64), ["m", "n", "o"]);
4205 let prefixes = letter_prefixes("64426", 64);
4206 assert_eq!(prefixes.len(), 27);
4207 assert_eq!(prefixes.first().map(String::as_str), Some("mgg"));
4208 assert_eq!(prefixes.last().map(String::as_str), Some("oii"));
4209 assert_eq!(letter_prefixes("79", 64).len(), 16);
4210 assert!(letter_prefixes("1", 64).is_empty());
4211 assert!(letter_prefixes("", 64).is_empty());
4212 }
4213
4214 #[test]
4215 fn letter_prefixes_do_not_allocate_discarded_intermediate_levels() {
4216 let (_, allocations) =
4217 crate::ime::personal_rerank::allocations::count(|| letter_prefixes("64426", 64));
4218 assert!(
4219 allocations <= 30,
4220 "letter prefix expansion allocated {allocations} buffers"
4221 );
4222 }
4223
4224 #[test]
4225 fn paths_prefer_fewer_syllables_then_a_complete_tail() {
4226 let table = SpellingTable::new(&["ni", "mi", "o", "ma", "hao", "gao"]);
4227 assert_eq!(
4229 table.paths("64"),
4230 [vec!["mi"], vec!["ni"], vec!["o", "g"], vec!["o", "h"]]
4231 );
4232 assert_eq!(table.paths("6"), [vec!["o"], vec!["m"], vec!["n"]]);
4234 let paths = table.paths("64426");
4235 assert_eq!(
4236 paths[..4],
4237 [
4238 vec!["mi", "gao"],
4239 vec!["mi", "hao"],
4240 vec!["ni", "gao"],
4241 vec!["ni", "hao"]
4242 ]
4243 );
4244 assert_eq!(paths.len(), 4);
4246 assert!(table.paths("11").is_empty());
4247 }
4248
4249 #[test]
4250 fn unreadable_paths_do_not_reserve_each_offset_page() {
4251 let table = SpellingTable::new(&["ni", "mi", "o"]);
4252 let digits = "1".repeat(32);
4253 let (paths, allocations) =
4254 crate::ime::personal_rerank::allocations::count(|| table.paths(&digits));
4255 assert!(paths.is_empty());
4256 assert!(
4257 allocations <= 2,
4258 "死路偏移不应逐偏移申请路径页:{allocations}"
4259 );
4260 }
4261
4262 #[test]
4263 fn paths_keep_at_most_the_limit_per_offset() {
4264 let table = SpellingTable::new(&["m", "n", "o"]);
4265 let paths = table.paths("66666");
4266 assert_eq!(paths.len(), PATH_LIMIT);
4267 assert_eq!(paths[0], ["m", "m", "m", "m", "m"]);
4268 assert!(paths.iter().all(|path| path.len() == 5));
4269 }
4270
4271 #[test]
4272 fn the_limit_keeps_frequent_syllables_rather_than_early_letters() {
4273 let table = SpellingTable::new(&["xi", "yi", "dian"]);
4274 let weights = HashMap::from([
4275 ("xi".to_string(), 1_000),
4276 ("yi".to_string(), 1_000_000),
4277 ("dian".to_string(), 10_000),
4278 ]);
4279 let prior = SyllablePrior::from_weights(&table.syllables, &weights);
4280 let digits = "943426".repeat(7);
4282 let paths = table.split_paths(&digits, &[], &prior);
4283 assert_eq!(paths.len(), PATH_LIMIT);
4284 assert_eq!(paths[0], ["yi", "dian"].repeat(7));
4285 assert!(table
4286 .split_paths(&digits, &[], &SyllablePrior::default())
4287 .iter()
4288 .all(|path| path[0] == "xi"));
4289 }
4290
4291 #[test]
4292 fn a_prefix_scores_as_its_best_syllable() {
4293 let table = SpellingTable::new(&["shi", "si", "ran"]);
4294 let weights = HashMap::from([("shi".to_string(), 5_000), ("si".to_string(), 50)]);
4295 let prior = SyllablePrior::from_weights(&table.syllables, &weights);
4296 let total = 5_051f64;
4298 assert_eq!(prior.score("s"), (5_000f64 / total).ln());
4299 assert_eq!(prior.score("sh"), (5_000f64 / total).ln());
4300 assert_eq!(prior.score("si"), (50f64 / total).ln());
4301 assert_eq!(prior.score("ran"), (1f64 / total).ln());
4302 }
4303
4304 #[test]
4305 fn sentences_from_different_paths_compare_without_the_phrase_bonus() {
4306 let bonus = (PHRASE_LENGTH_BONUS * 1000.0) as i64;
4307 let mut longer = item(
4309 "洗噩梦算吗",
4310 "9436364782662",
4311 5 * bonus - 30_000,
4312 CandidateSource::Generated,
4313 );
4314 longer.canonical_pinyin = "xi'e'meng'suan'ma".into();
4315 let mut shorter = item(
4316 "这能算吗",
4317 "9436364782662",
4318 4 * bonus - 20_000,
4319 CandidateSource::Generated,
4320 );
4321 shorter.canonical_pinyin = "zhe'neng'suan'ma".into();
4322 assert!(longer.weight > shorter.weight);
4323 assert_eq!(
4324 ranked(vec![longer, shorter], false),
4325 ["这能算吗", "洗噩梦算吗"]
4326 );
4327 let mut word = item("西安", "9426", 500, CandidateSource::Database);
4329 word.canonical_pinyin = "xi'an".into();
4330 assert_eq!(comparable_weight(&word), 500);
4331 }
4332
4333 #[test]
4335 fn phrase_insertion_cost_is_not_deducted_again_across_paths() {
4336 let bonus = (PHRASE_LENGTH_BONUS * 1000.0) as i64;
4337 let sentence = |word: &str, pinyin: &str, words: &[&str], weight: i64| {
4338 let mut row = item(
4339 word,
4340 "64646484268966566262",
4341 weight,
4342 CandidateSource::Generated,
4343 );
4344 row.canonical_pinyin = pinyin.into();
4345 row.sentence_words = words.iter().map(|word| (*word).to_owned()).collect();
4346 row
4347 };
4348 let phrases = sentence(
4350 "匿名天佑龙马",
4351 "ni'ming'tian'you'long'ma",
4352 &["匿名", "天佑", "龙马"],
4353 40_000 - 3 * bonus,
4354 );
4355 let plain = sentence(
4356 "你明天有空吗",
4357 "ni'ming'tian'you'kong'ma",
4358 &["你", "明天", "有空", "吗"],
4359 30_000 - 2 * bonus,
4360 );
4361 assert_eq!(comparable_weight(&phrases), 40_000 - 9 * bonus);
4362 assert_eq!(comparable_weight(&plain), 30_000 - 8 * bonus);
4363 assert_eq!(
4364 ranked(vec![phrases, plain], false),
4365 ["你明天有空吗", "匿名天佑龙马"]
4366 );
4367 }
4368
4369 fn path(syllables: &[&str]) -> Path {
4370 syllables
4371 .iter()
4372 .map(|syllable| (*syllable).to_owned())
4373 .collect()
4374 }
4375
4376 #[test]
4377 fn seeds_take_distinct_prefixes_then_keep_old_ones_that_still_fit() {
4378 let digits = encode("wo'men'yi'qi");
4379 let full = |word: &str, pinyin: &str| {
4380 let mut row = item(word, &digits, 100, CandidateSource::Generated);
4381 row.canonical_pinyin = pinyin.into();
4382 row
4383 };
4384 let mut fuzzy = full("我们一起", "wo'men'yi'qi");
4385 fuzzy.fuzzy = true;
4386 let candidates = vec![
4387 fuzzy,
4388 full("我们一起", "wo'men'yi'qi"),
4389 full("我们一批", "wo'men'yi'pi"),
4391 item("我们", "96636", 100, CandidateSource::Database),
4393 full("我们洗漆", "wo'men'xi'qi"),
4394 ];
4395 let previous = vec![
4396 "wo'men'yi'qi".to_owned(),
4398 "ni'men".to_owned(),
4400 "wo'men'yi'pi'a".to_owned(),
4401 ];
4402 let mut previous = previous;
4403 let reused = previous[0].as_ptr();
4404 let mut seeds = vec!["stale".to_owned()];
4405 next_seeds(&candidates, &digits, &mut previous, &mut seeds);
4406 assert_eq!(seeds, ["wo'men'yi'qi", "wo'men'xi'qi", "wo'men'yi'pi'a"]);
4407 assert_eq!(seeds[0].as_ptr(), reused);
4408 assert!(previous.is_empty());
4409 next_seeds(&[], &digits, &mut vec!["wo'men'yi".to_owned()], &mut seeds);
4411 assert_eq!(seeds, ["wo'men'yi"]);
4412 }
4413
4414 #[test]
4415 fn seed_prefixes_fit_digits_syllable_by_syllable() {
4416 assert_eq!(seed_prefix("wo'men'yi"), "wo'men");
4417 assert_eq!(seed_prefix("wo"), "");
4418 let digits = encode("wo'men'yi'qi");
4419 assert_eq!(fitting_syllables("wo'men", &digits), (2, 5, true));
4420 assert_eq!(fitting_syllables("wo'ne'yi", &digits), (2, 4, false));
4422 assert_eq!(fitting_syllables("wo'mo", &digits), (1, 2, false));
4423 assert_eq!(fitting_syllables("", &digits), (0, 0, true));
4424 assert_eq!(fitting_syllables("wo'men'yi'qi'a", &digits), (4, 9, false));
4425 }
4426
4427 #[test]
4428 fn seeded_paths_extend_the_fitting_prefix_with_tail_splits() {
4429 let table = SpellingTable::new(&["wo", "men", "yi", "qi", "xi", "pi"]);
4430 let prior = SyllablePrior::default();
4431 let remaining = encode("wo'men'yi'qi");
4432 let mut alternatives = vec![path(&["wo", "men", "xi", "pi"])];
4433 seeded_paths(
4434 &mut alternatives,
4435 &["wo'men'yi".to_owned(), "xi'men'yi".to_owned()],
4437 &remaining,
4438 &table,
4439 &prior,
4440 );
4441 assert_eq!(
4443 alternatives,
4444 [
4445 path(&["wo", "men", "xi", "pi"]),
4446 path(&["wo", "men", "xi", "qi"]),
4447 path(&["wo", "men", "yi", "pi"]),
4448 ]
4449 );
4450 let mut whole = Vec::new();
4452 seeded_paths(
4453 &mut whole,
4454 &["wo'men'yi'qi'a".to_owned()],
4455 &remaining,
4456 &table,
4457 &prior,
4458 );
4459 assert_eq!(whole, [path(&["wo", "men", "yi", "qi"])]);
4460 }
4461
4462 #[test]
4464 fn seed_tails_keep_the_seed_last_syllable_when_it_is_spelled_in_full() {
4465 let table = KeyGrid::NineKey.spelling_table();
4466 let prior = SyllablePrior::shipped(table);
4467 let remaining = encode("chi'fan'ba");
4468 let seeds = ["chi'fan".to_owned()];
4469 let tail = &remaining[encode("chi").len()..];
4470 assert!(!table
4471 .split_paths_limited(tail, &[], &prior, SEED_TAIL_LIMIT)
4472 .contains(&path(&["fan", "ba"])));
4473 let mut alternatives = Vec::new();
4474 seeded_paths(&mut alternatives, &seeds, &remaining, table, &prior);
4475 assert!(alternatives.contains(&path(&["chi", "fan", "ba"])));
4476 assert_eq!(alternatives.len(), SEED_TAIL_LIMIT + 1);
4477 }
4478
4479 #[test]
4481 fn seed_tails_keep_a_reading_whose_best_character_barely_loses() {
4482 let table = KeyGrid::NineKey.spelling_table();
4483 let remaining = encode("wo'men'jin'tian'qu'na'li");
4484 let seeds = ["wo'men'jin'tian'qu'na".to_owned()];
4485 let wanted = path(&["wo", "men", "jin", "tian", "qu", "na", "li"]);
4486 let tails = |prior: &SyllablePrior| {
4487 let mut alternatives = Vec::new();
4488 seeded_paths(&mut alternatives, &seeds, &remaining, table, prior);
4489 alternatives
4490 };
4491 let best: HashMap<String, i64> = [
4493 ("na", 5_580_163),
4494 ("ma", 5_664_547),
4495 ("li", 7_995_093),
4496 ("ji", 2_726_986),
4497 ("jiu", 8_500_000),
4498 ]
4499 .into_iter()
4500 .map(|(syllable, weight)| (syllable.to_owned(), weight))
4501 .collect();
4502 assert!(!tails(&SyllablePrior::from_weights(&table.syllables, &best)).contains(&wanted));
4503 assert!(tails(&SyllablePrior::shipped(table)).contains(&wanted));
4504 }
4505
4506 #[test]
4507 fn shipped_syllable_frequency_covers_every_spelling_and_names_its_source() {
4508 let source = SYLLABLE_FREQUENCY
4510 .lines()
4511 .next()
4512 .and_then(|line| line.strip_prefix("# msime-pinyin.db sha256="))
4513 .expect("source line");
4514 assert!(source.len() == 64 && source.bytes().all(|byte| byte.is_ascii_hexdigit()));
4515 let weights = shipped_syllable_frequency();
4517 let mut missing: Vec<&str> = KeyGrid::NineKey
4518 .spelling_table()
4519 .syllables
4520 .iter()
4521 .map(|(syllable, _)| syllable.as_str())
4522 .filter(|syllable| weights.get(*syllable).is_none_or(|weight| *weight <= 0))
4523 .collect();
4524 missing.sort_unstable();
4525 assert_eq!(
4526 missing,
4527 ["jv", "jve", "lue", "nue", "qv", "qve", "xv", "xve", "yv", "yve"]
4528 );
4529 }
4530
4531 #[test]
4533 fn seed_tails_keep_a_syllable_that_mostly_occurs_inside_words() {
4534 let table =
4536 SpellingTable::new(&["xia", "zha", "wu", "yu", "yue", "xu", "xue", "zu", "zuo"]);
4537 let tail = encode("xia'wu");
4538 let wanted = path(&["xia", "wu"]);
4539 let characters: HashMap<String, i64> = [
4541 ("xia", 4_490_788),
4542 ("zha", 175_682),
4543 ("wu", 1_131_635),
4544 ("yu", 18_021_628),
4545 ("yue", 6_441_425),
4546 ("xu", 271_070),
4547 ("xue", 1_673_519),
4548 ("zu", 685_682),
4549 ("zuo", 7_592_605),
4550 ]
4551 .into_iter()
4552 .map(|(syllable, weight)| (syllable.to_owned(), weight))
4553 .collect();
4554 let tails = |weights: &HashMap<String, i64>| {
4555 let prior = SyllablePrior::from_weights(&table.syllables, weights);
4556 table.split_paths_limited(&tail, &[], &prior, SEED_TAIL_LIMIT)
4557 };
4558 assert!(!tails(&characters).contains(&wanted));
4560 assert!(tails(shipped_syllable_frequency()).contains(&wanted));
4561 }
4562
4563 #[test]
4565 fn seeds_survive_a_partial_syllable_and_end_with_the_composition() {
4566 let fixture = fixture();
4567 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
4568 type_digits(&mut session, "64426");
4569 assert!(session.sentence_seeds.contains(&"ni'hao".to_owned()));
4570 type_digits(&mut session, "3");
4571 assert!(session.sentence_seeds.contains(&"ni'hao".to_owned()));
4572 session.command(Command::Cancel);
4573 assert!(session.sentence_seeds.is_empty());
4574 }
4575
4576 #[test]
4577 fn query_key_scan_uses_no_temporary_heap_state() {
4578 let alternatives = vec![vec!["ni".to_owned(), "hao".to_owned()]];
4579 let (found, allocations) = crate::ime::personal_rerank::allocations::count(|| {
4580 query_key_is_new(&alternatives, 0, "ni'hao", None)
4581 });
4582
4583 assert!(found);
4584 assert_eq!(allocations, 0);
4585 }
4586
4587 #[test]
4588 fn spellings_sort_by_digits_covered_within_the_limit() {
4589 let table = SpellingTable::new(&[
4590 "ga", "gan", "gang", "gao", "ha", "han", "hang", "hao", "ni", "a", "ai",
4591 ]);
4592 let prior = SyllablePrior::default();
4593 assert_eq!(
4595 table.spellings_for("426", 2, None, &prior),
4596 ["gan", "gao", "han", "hao", "gang", "hang", "ga", "ha"]
4597 );
4598 assert_eq!(
4600 table.spellings_for("2", DIGIT_LIMIT - 1, None, &prior),
4601 ["a"]
4602 );
4603 assert!(table.spellings_for("", 0, None, &prior).is_empty());
4604 assert_eq!(table.spellings_for("426", 0, Some(2), &prior), ["ga", "ha"]);
4606 }
4607
4608 #[test]
4609 fn spellings_put_exact_syllables_first_then_frequent_ones() {
4610 let table = SpellingTable::new(&[
4612 "pi", "pian", "piao", "pie", "pin", "ping", "qi", "qia", "qian", "ri", "shi", "si",
4613 ]);
4614 let weights: HashMap<String, i64> = [
4615 ("shi", 9_000),
4616 ("qi", 5_000),
4617 ("si", 3_000),
4618 ("ri", 2_000),
4619 ("pi", 1_000),
4620 ("pian", 800),
4621 ("qian", 700),
4622 ("ping", 600),
4623 ]
4624 .into_iter()
4625 .map(|(syllable, weight)| (syllable.to_owned(), weight))
4626 .collect();
4627 let prior = SyllablePrior::from_weights(&table.syllables, &weights);
4628 assert_eq!(
4629 table.spellings_for("74", 0, None, &prior),
4630 ["qi", "si", "ri", "pi", "shi", "pie", "pin", "qia", "pian", "qian", "ping", "piao"]
4631 );
4632 }
4633
4634 #[test]
4635 fn complete_two_digit_readings_precede_longer_completions() {
4636 let table = SpellingTable::new(&["pi", "pian", "piao", "pie", "qi", "ri", "shi", "si"]);
4637 let choices = table.spellings_for("74", 0, None, &SyllablePrior::default());
4638 assert_eq!(&choices[..4], ["pi", "qi", "ri", "si"]);
4639 }
4640
4641 #[test]
4642 fn one_digit_keeps_predictive_readings_ahead_of_single_letter_syllables() {
4643 let table = SpellingTable::new(&["ma", "mi", "o"]);
4644 assert_eq!(
4645 table.spellings_for("6", 0, None, &SyllablePrior::default()),
4646 ["ma", "mi", "o"]
4647 );
4648 }
4649
4650 fn item(word: &str, digits: &str, weight: i64, source: CandidateSource) -> WordItem {
4651 WordItem::new(digits, word, weight, source, "")
4652 }
4653
4654 #[test]
4655 fn refresh_tail_is_borrowed_from_digits() {
4656 assert_eq!(remaining_digits("64426", 2), "426");
4657 }
4658
4659 #[test]
4660 fn path_key_builder_reuses_existing_string_storage() {
4661 let path = vec!["hao".to_owned(), "ma".to_owned()];
4662 let mut key = String::with_capacity(32);
4663 append_path_key(&mut key, "ni", &path);
4664 assert_eq!(key, "ni'hao'ma");
4665
4666 let ((), allocations) = crate::ime::personal_rerank::allocations::count(|| {
4667 append_path_key(&mut key, "ni", &path);
4668 });
4669
4670 assert_eq!(key, "ni'hao'ma");
4671 assert_eq!(allocations, 0);
4672 }
4673
4674 #[test]
4675 fn candidate_word_lookup_scans_existing_rows() {
4676 let candidates = vec![item("old", "653", 1, CandidateSource::EnglishDictionary)];
4677 assert!(has_candidate_word(&candidates, "old"));
4678 assert!(!has_candidate_word(&candidates, "older"));
4679 }
4680
4681 #[test]
4682 fn push_ranked_defers_leading_capacity_until_a_candidate_exists() {
4683 let mut candidates = Vec::new();
4684 let mut leading = LeadingMap::default();
4685 assert_eq!(leading.capacity(), 0);
4686
4687 push_ranked(
4688 &mut candidates,
4689 &mut leading,
4690 item("你", "64", 1, CandidateSource::Database),
4691 false,
4692 false,
4693 KeyGrid::NineKey,
4694 );
4695
4696 assert_eq!(candidates.len(), 1);
4697 assert!(leading.capacity() >= CANDIDATE_LIMIT);
4698 }
4699
4700 #[test]
4701 fn push_ranked_leading_map_allocation_profile() {
4702 let rows: Vec<_> = (0..256)
4703 .map(|index| {
4704 item(
4705 &format!("合成词{index}"),
4706 "64426",
4707 index,
4708 CandidateSource::Database,
4709 )
4710 })
4711 .collect();
4712 let ((), allocations) = crate::ime::personal_rerank::allocations::count(|| {
4713 let mut candidates = Vec::new();
4714 let mut leading = LeadingMap::default();
4715 for row in rows.iter().cloned() {
4716 push_ranked(
4717 &mut candidates,
4718 &mut leading,
4719 row,
4720 false,
4721 false,
4722 KeyGrid::NineKey,
4723 );
4724 }
4725 });
4726 eprintln!("push-ranked-leading allocations: {allocations}");
4727 assert!(allocations < 700, "leading map allocations: {allocations}");
4728 }
4729
4730 #[test]
4731 fn leading_slot_keeps_distinct_words_when_hashes_collide() {
4732 let candidates = vec![
4733 item("甲", "64", 1, CandidateSource::Database),
4734 item("乙", "64", 2, CandidateSource::Database),
4735 ];
4736 let first = LeadingEntry {
4737 candidate: 0,
4738 key: rank_key(&candidates[0], false, false, KeyGrid::NineKey),
4739 };
4740 let second = LeadingEntry {
4741 candidate: 1,
4742 key: rank_key(&candidates[1], false, false, KeyGrid::NineKey),
4743 };
4744 let mut slot = LeadingSlot::One(first);
4745 slot.insert(second);
4746 assert!(slot.find(&candidates, "甲").is_some());
4747 assert!(slot.find(&candidates, "乙").is_some());
4748 assert!(slot.find(&candidates, "丙").is_none());
4749 }
4750
4751 #[test]
4752 fn unique_word_retain_keeps_the_first_sorted_row() {
4753 let mut candidates = vec![
4754 item("你", "644", 10, CandidateSource::Database),
4755 item("你", "64", 1, CandidateSource::Generated),
4756 item("泥", "64", 2, CandidateSource::Database),
4757 ];
4758 retain_unique_words(&mut candidates);
4759 assert_eq!(
4760 candidates
4761 .iter()
4762 .map(|candidate| candidate.word.as_str())
4763 .collect::<Vec<_>>(),
4764 ["你", "泥"]
4765 );
4766 }
4767
4768 #[test]
4769 fn short_unique_word_retain_avoids_temporary_heap_state() {
4770 let mut candidates = vec![
4771 item("你", "644", 10, CandidateSource::Database),
4772 item("你", "64", 1, CandidateSource::Generated),
4773 item("泥", "64", 2, CandidateSource::Database),
4774 ];
4775 let ((), allocations) = crate::ime::personal_rerank::allocations::count(|| {
4776 retain_unique_words(&mut candidates);
4777 });
4778
4779 assert_eq!(allocations, 0);
4780 assert_eq!(
4781 candidates
4782 .iter()
4783 .map(|candidate| candidate.word.as_str())
4784 .collect::<Vec<_>>(),
4785 ["你", "泥"]
4786 );
4787 }
4788
4789 #[test]
4790 fn query_key_lookup_deduplicates_paths_and_hashed_keys() {
4791 let alternatives = vec![
4792 vec!["ni".to_owned(), "hao".to_owned()],
4793 vec!["ni".to_owned(), "hao".to_owned()],
4794 ];
4795 assert!(query_key_is_new(&alternatives, 0, "ni'hao", None));
4796 assert!(!query_key_is_new(&alternatives, 1, "ni'hao", None));
4797
4798 let mut queried = HashSet::new();
4799 assert!(query_key_is_new(&alternatives, 0, "ni'hao", Some(&queried)));
4800 queried.insert("ni'hao".to_owned());
4801 assert!(!query_key_is_new(
4802 &alternatives,
4803 1,
4804 "ni'hao",
4805 Some(&queried)
4806 ));
4807 }
4808
4809 #[test]
4810 fn query_key_dedup_switches_to_hashing_after_small_batch_boundary() {
4811 let alternatives = (0..=SMALL_QUERY_KEY_BATCH)
4812 .map(|index| vec![format!("syllable-{}", index % 32)])
4813 .collect::<Vec<_>>();
4814 let mut queried = (alternatives.len() > SMALL_QUERY_KEY_BATCH)
4815 .then(|| HashSet::with_capacity(alternatives.len()));
4816 let mut unique = 0;
4817 for (index, path) in alternatives.iter().enumerate() {
4818 let key = path[0].as_str();
4819 if query_key_is_new(&alternatives, index, key, queried.as_ref()) {
4820 unique += 1;
4821 if let Some(seen) = queried.as_mut() {
4822 seen.insert(key.to_owned());
4823 }
4824 }
4825 }
4826 assert_eq!(unique, 32);
4827 assert!(queried.is_some());
4828 }
4829
4830 #[test]
4831 fn candidates_rank_by_coverage_then_dictionary_rows() {
4832 let mut fuzzy = item("泥", "64", 900, CandidateSource::Database);
4833 fuzzy.fuzzy = true;
4834 let mut candidates = vec![
4835 item("你", "64", 100, CandidateSource::Database),
4836 item("米好", "64426", -19113, CandidateSource::Generated),
4837 fuzzy,
4838 item("米", "64", 50, CandidateSource::UserDatabase),
4839 item("你好", "64426", 1000, CandidateSource::Database),
4840 item("你", "64", 10, CandidateSource::Database),
4841 ];
4842 rank_candidates(&mut candidates, false, false, None, KeyGrid::NineKey);
4843 let words: Vec<_> = candidates.iter().map(|item| item.word.as_str()).collect();
4844 assert_eq!(words, ["你好", "米好", "你", "米", "泥"]);
4845 assert_eq!(candidates[2].weight, 100);
4846 }
4847
4848 fn spelled(
4849 word: &str,
4850 digits: &str,
4851 canonical: &str,
4852 weight: i64,
4853 source: CandidateSource,
4854 ) -> WordItem {
4855 let mut row = item(word, digits, weight, source);
4856 row.canonical_pinyin = canonical.into();
4857 row
4858 }
4859
4860 fn ranked(mut candidates: Vec<WordItem>, prefer_exact: bool) -> Vec<String> {
4861 rank_candidates(&mut candidates, prefer_exact, false, None, KeyGrid::NineKey);
4862 candidates
4863 .into_iter()
4864 .map(|item| item.word.into_string())
4865 .collect()
4866 }
4867
4868 #[test]
4869 fn without_a_split_a_completion_keeps_its_weight_over_an_exact_syllable() {
4870 let rows = vec![
4872 spelled("额", "3", "e", 10, CandidateSource::Database),
4873 spelled("的", "3", "de", 1000, CandidateSource::Database),
4874 ];
4875 assert_eq!(ranked(rows, false), ["的", "额"]);
4876 }
4877
4878 #[test]
4879 fn a_synthesised_exact_row_never_leads_a_dictionary_completion() {
4880 for prefer_exact in [false, true] {
4881 let rows = vec![
4882 spelled("额", "3", "e", 99_999, CandidateSource::Generated),
4883 spelled("的", "3", "de", 1000, CandidateSource::Database),
4884 ];
4885 assert_eq!(
4886 ranked(rows, prefer_exact),
4887 ["的", "额"],
4888 "prefer_exact {prefer_exact}"
4889 );
4890 }
4891 }
4892
4893 #[test]
4894 fn after_a_split_an_exact_reading_leads_a_completion() {
4895 let rows = vec![
4897 spelled("自从", "9426", "zi'cong", 5000, CandidateSource::Database),
4898 spelled("西安", "9426", "xi'an", 100, CandidateSource::Database),
4899 ];
4900 assert_eq!(ranked(rows.clone(), true), ["西安", "自从"]);
4901 assert_eq!(ranked(rows, false), ["自从", "西安"]);
4902 }
4903
4904 fn ranked_on_page(mut candidates: Vec<WordItem>, prefer_exact: bool) -> Vec<String> {
4906 rank_candidates(&mut candidates, prefer_exact, false, None, KeyGrid::NineKey);
4907 if !prefer_exact {
4908 discount_completions_on_first_page(
4909 &mut candidates,
4910 DEFAULT_FIRST_PAGE_ROWS,
4911 0,
4912 KeyGrid::NineKey,
4913 );
4914 }
4915 candidates
4916 .into_iter()
4917 .map(|item| item.word.into_string())
4918 .collect()
4919 }
4920
4921 #[test]
4922 fn without_a_split_a_completion_competes_with_a_spelled_phrase_at_a_fraction_of_its_weight() {
4923 let ratio = COMPLETION_WEIGHT_RATIO;
4925 let rows = |chifan: i64| {
4926 vec![
4927 spelled("吃饭", "2443", "chi'fan", chifan, CandidateSource::Database),
4928 spelled("挨个", "2443", "ai'ge", 1000, CandidateSource::Database),
4929 ]
4930 };
4931 assert_eq!(
4932 ranked_on_page(rows(1000 * ratio - 1), false),
4933 ["挨个", "吃饭"]
4934 );
4935 assert_eq!(ranked_on_page(rows(1000 * ratio), false), ["挨个", "吃饭"]);
4936 assert_eq!(
4937 ranked_on_page(rows(1000 * ratio + 1), false),
4938 ["吃饭", "挨个"]
4939 );
4940 assert_eq!(
4941 ranked_on_page(rows(1000 * ratio * 1000), true),
4942 ["挨个", "吃饭"]
4943 );
4944 }
4945
4946 #[test]
4947 fn without_a_spelled_phrase_on_the_first_page_nothing_is_discounted() {
4948 let rows = vec![
4950 spelled("额", "3", "e", 99, CandidateSource::Database),
4951 spelled("的", "3", "de", 100, CandidateSource::Database),
4952 ];
4953 assert_eq!(ranked_on_page(rows, false), ["的", "额"]);
4954 let rows = vec![
4955 spelled("肯", "536", "ken", 40_000, CandidateSource::Database),
4956 spelled("冷", "536", "leng", 113_000, CandidateSource::Database),
4957 spelled(
4958 "可能",
4959 "536",
4960 "ke'neng",
4961 5_500_000,
4962 CandidateSource::Database,
4963 ),
4964 ];
4965 assert_eq!(ranked_on_page(rows, false), ["可能", "冷", "肯"]);
4966 }
4967
4968 #[test]
4969 fn completions_compare_with_each_other_at_their_own_weights() {
4970 let rows = vec![
4972 spelled(
4973 "主编",
4974 "9482",
4975 "zhu'bian",
4976 10_000_000,
4977 CandidateSource::Database,
4978 ),
4979 spelled("装", "9482", "zhuang", 3_000_000, CandidateSource::Database),
4980 spelled(
4981 "抓",
4982 "9482",
4983 "zhua",
4984 10_000_000 / COMPLETION_WEIGHT_RATIO + 1,
4985 CandidateSource::Database,
4986 ),
4987 spelled("猪啊", "9482", "zhu'a", 500, CandidateSource::Database),
4988 ];
4989 assert_eq!(ranked_on_page(rows, false), ["抓", "主编", "装", "猪啊"]);
4990 }
4991
4992 #[test]
4993 fn the_discount_keeps_the_first_page_and_the_rest_of_the_list() {
4994 let mut rows = vec![
4996 spelled("从", "266", "cong", 30_000_000, CandidateSource::Database),
4997 spelled("聪", "266", "cong", 130_000, CandidateSource::Database),
4998 spelled(
4999 "阿莫",
5000 "266",
5001 "a'mo",
5002 30_000_000 / COMPLETION_WEIGHT_RATIO - 1,
5003 CandidateSource::Database,
5004 ),
5005 spelled("匆", "266", "cong", 60_000, CandidateSource::Database),
5006 spelled("丛", "266", "cong", 40_000, CandidateSource::Database),
5007 spelled("按钮", "266", "an'niu", 500_000, CandidateSource::Database),
5008 spelled("澳门", "266", "ao'men", 300_000, CandidateSource::Database),
5009 spelled(
5010 "不那么",
5011 "266",
5012 "bu'na'me",
5013 20_000,
5014 CandidateSource::Database,
5015 ),
5016 ];
5017 for offset in 0..200 {
5018 let word = char::from_u32(0x3400 + offset).unwrap().to_string();
5019 rows.push(spelled(&word, "266", "cong", 0, CandidateSource::Database));
5020 }
5021 let before = ranked(rows.clone(), false);
5022 let shown = ranked_on_page(rows, false);
5023 let page = DEFAULT_FIRST_PAGE_ROWS;
5024 assert_eq!(shown[..page], ["从", "阿莫", "按钮", "澳门", "聪", "匆"]);
5025 let mut sorted_shown = shown[..page].to_vec();
5026 let mut sorted_before = before[..page].to_vec();
5027 sorted_shown.sort();
5028 sorted_before.sort();
5029 assert_eq!(sorted_shown, sorted_before);
5030 assert_eq!(shown[page..], before[page..]);
5031 }
5032
5033 #[test]
5034 fn initials_rows_keep_their_seats_and_no_row_passes_them() {
5035 let rows = vec![
5037 spelled(
5038 "吃饭",
5039 "2443",
5040 "chi'fan",
5041 1_100_000,
5042 CandidateSource::Database,
5043 ),
5044 spelled(
5045 "挨个",
5046 "2443",
5047 "ai'ge",
5048 1_000_000,
5049 CandidateSource::Database,
5050 ),
5051 spelled("爱", "2443", "ai", 800_000, CandidateSource::Database),
5052 spelled("碧荷", "2443", "bi'he", 700_000, CandidateSource::Database),
5053 spelled(
5054 "不会过的",
5055 "2443",
5056 "bu'hui'guo'de",
5057 500_000,
5058 CandidateSource::Database,
5059 ),
5060 spelled(
5061 "吃过",
5062 "2443",
5063 "chi'guo",
5064 400_000,
5065 CandidateSource::Database,
5066 ),
5067 ];
5068 assert_eq!(
5069 ranked(rows.clone(), false),
5070 ["吃饭", "挨个", "爱", "碧荷", "不会过的", "吃过"]
5071 );
5072 assert_eq!(
5073 ranked_on_page(rows, false),
5074 ["挨个", "爱", "碧荷", "吃饭", "不会过的", "吃过"]
5075 );
5076 }
5077
5078 #[test]
5079 fn rows_mixed_in_after_ranking_keep_their_seats() {
5080 let mut rows = vec![
5082 spelled("吃饭", "2443", "chi'fan", 1100, CandidateSource::Database),
5083 spelled(
5084 "chief",
5085 "2443",
5086 "",
5087 1000,
5088 CandidateSource::EnglishDictionary,
5089 ),
5090 spelled("挨个", "2443", "ai'ge", 1000, CandidateSource::Database),
5091 spelled("爱河", "2443", "ai'he", 900, CandidateSource::Database),
5092 spelled("碧荷", "2443", "bi'he", 800, CandidateSource::Database),
5093 spelled("次格", "2443", "ci'ge", 700, CandidateSource::Database),
5094 spelled("笔格", "2443", "bi'ge", 600, CandidateSource::Database),
5095 ];
5096 let mut shown = rows.clone();
5097 assert!(discount_completions_on_first_page(
5098 &mut shown,
5099 6,
5100 0,
5101 KeyGrid::NineKey
5102 ));
5103 let words = |rows: &[WordItem]| {
5104 rows.iter()
5105 .map(|row| row.word.to_string())
5106 .collect::<Vec<_>>()
5107 };
5108 assert_eq!(
5109 words(&shown),
5110 ["挨个", "chief", "爱河", "碧荷", "次格", "吃饭", "笔格"]
5111 );
5112 rows[3].fixed_position = 4;
5113 rows.insert(1, spelled("🍚", "2443", "", 0, CandidateSource::Emoji));
5114 let mut shown = rows.clone();
5115 discount_completions_on_first_page(&mut shown, 6, 0, KeyGrid::NineKey);
5116 assert_eq!(shown, rows);
5117 rows.remove(1);
5118 let mut shown = rows.clone();
5119 discount_completions_on_first_page(&mut shown, 6, 0, KeyGrid::NineKey);
5120 assert_eq!(
5121 words(&shown),
5122 ["挨个", "chief", "碧荷", "爱河", "次格", "吃饭", "笔格"]
5123 );
5124 }
5125
5126 #[test]
5127 fn locked_syllables_do_not_count_toward_a_spelled_phrase() {
5128 let rows = || {
5130 vec![
5131 spelled("我的", "963", "wo'de", 150, CandidateSource::Database),
5132 spelled("我饿", "963", "wo'e", 100, CandidateSource::Database),
5133 ]
5134 };
5135 let mut shown = rows();
5136 assert!(!discount_completions_on_first_page(
5137 &mut shown,
5138 6,
5139 1,
5140 KeyGrid::NineKey
5141 ));
5142 assert_eq!(shown, rows());
5143 let mut shown = rows();
5144 assert!(discount_completions_on_first_page(
5145 &mut shown,
5146 6,
5147 0,
5148 KeyGrid::NineKey
5149 ));
5150 assert_eq!(shown[0].word, "我饿");
5151 }
5152
5153 #[test]
5154 fn the_discount_only_permutes_pinyin_rows_inside_the_first_page() {
5155 let readings = [
5156 "ai'ge",
5157 "bi'he",
5158 "chi'fan",
5159 "ai'hao",
5160 "bu'hui'guo'de",
5161 "chang",
5162 "ci",
5163 ];
5164 let mut state = 0x9E37_79B9_7F4A_7C15_u64;
5165 let mut next = |bound: u64| {
5166 state ^= state << 13;
5167 state ^= state >> 7;
5168 state ^= state << 17;
5169 state % bound
5170 };
5171 for round in 0..300 {
5172 let mut rows = Vec::new();
5173 for index in 0..(1 + next(40)) {
5174 let reading = readings[next(readings.len() as u64) as usize];
5175 let weight = [0, 1, 7, 100, 999, 1000, 64_000, 1_000_000][next(8) as usize];
5176 let mut row = spelled(
5177 &format!("w{round}-{index}"),
5178 "2443",
5179 reading,
5180 weight,
5181 CandidateSource::Database,
5182 );
5183 row.fuzzy = next(6) == 0;
5184 rows.push(row);
5185 }
5186 let page = 1 + next(9) as usize;
5187 let mut before = rows;
5188 rank_candidates(&mut before, false, false, None, KeyGrid::NineKey);
5189 for extra in 0..next(3) {
5191 let seat = next(before.len() as u64 + 1) as usize;
5192 let source = [
5193 CandidateSource::EnglishDictionary,
5194 CandidateSource::Generated,
5195 CandidateSource::Database,
5196 ][next(3) as usize];
5197 let mut row = spelled(&format!("x{round}-{extra}"), "2443", "ai'ge", 5, source);
5198 row.fixed_position = i32::from(source == CandidateSource::Database);
5199 before.insert(seat, row);
5200 }
5201 let mut after = before.clone();
5202 discount_completions_on_first_page(&mut after, page, 0, KeyGrid::NineKey);
5203 let cut = page.min(before.len());
5204 assert_eq!(after[cut..], before[cut..], "round {round}");
5205 let stop = before[..cut]
5206 .iter()
5207 .position(|row| row.source == CandidateSource::Generated)
5208 .unwrap_or(cut);
5209 assert_eq!(after[stop..], before[stop..], "round {round}");
5210 for index in 0..stop {
5211 let fixed = !before[index].source.is_dictionary()
5212 || before[index].fixed_position != 0
5213 || is_initials_row(&before[index]);
5214 if fixed {
5215 assert_eq!(after[index], before[index], "round {round}");
5216 } else {
5217 assert!(after[index].source.is_dictionary(), "round {round}");
5218 assert_eq!(after[index].fixed_position, 0, "round {round}");
5219 assert!(!is_initials_row(&after[index]), "round {round}");
5220 }
5221 }
5222 let mut words_before: Vec<_> = before[..cut].iter().map(|row| &row.word).collect();
5223 let mut words_after: Vec<_> = after[..cut].iter().map(|row| &row.word).collect();
5224 words_before.sort();
5225 words_after.sort();
5226 assert_eq!(words_after, words_before, "round {round}");
5227 }
5228 }
5229
5230 #[test]
5231 fn skipping_outranked_rows_ranks_like_pushing_everything() {
5232 let sources = [
5233 CandidateSource::Database,
5234 CandidateSource::UserDatabase,
5235 CandidateSource::Generated,
5236 CandidateSource::Fallback,
5237 ];
5238 let mut state = 0x2545_F491_4F6C_DD1D_u64;
5239 let mut next = |bound: u64| {
5240 state ^= state << 13;
5241 state ^= state >> 7;
5242 state ^= state << 17;
5243 state % bound
5244 };
5245 for round in 0..200 {
5246 let mut rows = Vec::new();
5247 for _ in 0..(1 + next(300)) {
5248 let digits = &"64426646"[..1 + next(8) as usize];
5249 let mut row = item(
5250 &format!("w{}", next(40)),
5251 digits,
5252 next(5) as i64 * 10,
5253 sources[next(4) as usize],
5254 );
5255 row.fuzzy = next(3) == 0;
5256 row.canonical_pinyin =
5257 ["e", "de", "xi'an", "zi'cong", "ni'hao'ma"][next(5) as usize].into();
5258 rows.push(row);
5259 }
5260 let prefer_exact = round % 2 == 0;
5261 let mut everything = rows.clone();
5262 rank_candidates(&mut everything, prefer_exact, false, None, KeyGrid::NineKey);
5263 let mut skipped = Vec::new();
5264 let mut leading = LeadingMap::default();
5265 for row in rows {
5266 push_ranked(
5267 &mut skipped,
5268 &mut leading,
5269 row,
5270 prefer_exact,
5271 false,
5272 KeyGrid::NineKey,
5273 );
5274 }
5275 rank_candidates(&mut skipped, prefer_exact, false, None, KeyGrid::NineKey);
5276 assert_eq!(skipped, everything, "round {round}");
5277 }
5278 }
5279
5280 #[test]
5281 fn english_exact_code_leads_only_with_a_weight() {
5282 let english =
5283 |word: &str, weight| item(word, word, weight, CandidateSource::EnglishDictionary);
5284 let mut words = vec![
5285 english("old", 1000),
5286 english("ok", 900),
5287 english("older", 800),
5288 ];
5289 rank_english(&mut words, 2);
5290 let order: Vec<_> = words.iter().map(|item| item.word.as_str()).collect();
5291 assert_eq!(order, ["ok", "old", "older"]);
5292 let mut words = vec![english("ogham", 0), english("oh", 5), english("ogre", 5)];
5293 rank_english(&mut words, 5);
5294 let order: Vec<_> = words.iter().map(|item| item.word.as_str()).collect();
5295 assert_eq!(order, ["oh", "ogre", "ogham"]);
5296 let mut many: Vec<_> = (0..8).map(|n| english(&format!("w{n}"), n)).collect();
5297 rank_english(&mut many, 2);
5298 assert_eq!(many.len(), ENGLISH_LIMIT);
5299 }
5300
5301 #[test]
5302 fn english_dedup_keeps_the_first_row_for_each_display_word() {
5303 let mut words = vec![
5304 item("one", "one", 100, CandidateSource::EnglishDictionary),
5305 item("two", "two", 90, CandidateSource::EnglishDictionary),
5306 item("one", "one", 80, CandidateSource::EnglishDictionary),
5307 ];
5308
5309 deduplicate_english_words(&mut words);
5310
5311 assert_eq!(
5312 words
5313 .iter()
5314 .map(|word| (word.word.as_str(), word.weight))
5315 .collect::<Vec<_>>(),
5316 [("one", 100), ("two", 90)]
5317 );
5318 }
5319
5320 #[test]
5321 fn locked_syllables_constrain_readings() {
5322 assert!(agrees_with_locked("mi", ""));
5323 assert!(agrees_with_locked("ni", "ni"));
5324 assert!(agrees_with_locked("ni'hao", "ni"));
5325 assert!(agrees_with_locked("ni", "ni'hao"));
5326 assert!(!agrees_with_locked("mi", "ni"));
5327 assert!(!agrees_with_locked("nian", "ni"));
5328 assert!(!agrees_with_locked("ni", "nian"));
5329 }
5330
5331 fn detached() -> NineKeySession {
5332 NineKeySession::new(
5333 &RuntimePaths::default(),
5334 false,
5335 FrequencyAdjustmentOptions::default(),
5336 FuzzyPinyinOptions::default(),
5337 EnglishInputOptions::default(),
5338 true,
5339 false,
5340 )
5341 }
5342
5343 #[test]
5344 fn consuming_digits_drops_covered_locked_syllables() {
5345 let mut session = detached();
5346 session.digits = "64426".into();
5347 session.locked = vec!["ni".into(), "hao".into()];
5348 session.consume(2);
5349 assert_eq!(session.digits, "426");
5350 assert_eq!(session.locked, ["hao"]);
5351 session.consume(10);
5353 assert!(session.digits.is_empty());
5354 assert!(session.locked.is_empty());
5355 }
5356
5357 #[test]
5358 fn consuming_digits_reuses_split_storage() {
5359 let mut session = detached();
5360 session.digits = "64426".into();
5361 session.splits = vec![2, 4];
5362 let (_, allocations) =
5363 crate::ime::personal_rerank::allocations::count(|| session.consume(2));
5364 assert_eq!(session.splits, [2]);
5365 assert_eq!(allocations, 0);
5366 }
5367
5368 #[test]
5369 fn snapshot_shows_locked_syllables_before_the_open_digits() {
5370 let mut session = detached();
5371 session.digits = "64426".into();
5372 assert_eq!(session.snapshot().preedit, "64426");
5373 session.locked = vec!["ni".into()];
5374 assert_eq!(session.snapshot().preedit, "ni'426");
5375 session.locked.push("hao".into());
5376 session.candidates = vec![
5377 item("你好", "64426", 1000, CandidateSource::Database),
5378 item("你", "64", 100, CandidateSource::Database),
5379 ];
5380 let view = session.snapshot();
5381 assert_eq!(view.preedit, "ni'hao");
5382 assert_eq!(view.editing_text, "64426");
5383 assert_eq!(view.caret_position, 5);
5384 assert_eq!(view.scheme, SchemeType::Quanpin);
5385 assert_eq!(view.candidate_answers_key, [true, false]);
5386 assert_eq!(view.candidate_sources, [CandidateSource::Database; 2]);
5387 assert_eq!(view.candidate_annotations, ["", ""]);
5388 assert_eq!(session.ranking_context(), "nine-key:64426:ni'hao");
5389 }
5390
5391 #[test]
5392 fn snapshot_reuses_locked_prefix_length_for_an_initial() {
5393 let mut session = detached();
5394 session.digits = "644269".into();
5395 session.locked = vec!["ni".into(), "hao".into()];
5396 session.initial = Some(Initial {
5397 letter: b'n',
5398 typed: 6,
5399 splits: 0,
5400 });
5401 let (_, allocations) =
5402 crate::ime::personal_rerank::allocations::count(|| session.snapshot());
5403 assert!(
5404 allocations <= 2,
5405 "九键快照预编辑缓冲扩容产生了 {allocations} 次分配"
5406 );
5407 }
5408
5409 const MAIN_FIXTURE: &str = "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_n VALUES('ni','n','你',100);CREATE TABLE tbl_1_m(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_m VALUES('mi','m','米',50);CREATE TABLE tbl_1_h(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_h VALUES('hao','h','好',100);CREATE TABLE tbl_2_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_n VALUES('ni''hao','nh','你好',1000);";
5412 const ENGLISH_FIXTURE: &str = "CREATE TABLE english_words(word TEXT,display TEXT,weight INTEGER);INSERT INTO english_words VALUES('ok','ok',900);INSERT INTO english_words VALUES('old','old',1000);INSERT INTO english_words VALUES('older','older',800);INSERT INTO english_words VALUES('ogham','ogham',0);INSERT INTO english_words VALUES('qq','QQ',500);";
5413
5414 struct Fixture {
5415 directory: tempfile::TempDir,
5416 paths: RuntimePaths,
5417 }
5418
5419 fn fixture() -> Fixture {
5420 fixture_with(MAIN_FIXTURE)
5421 }
5422
5423 fn fixture_with(main: &str) -> Fixture {
5424 let directory = tempfile::tempdir().expect("temporary directory");
5425 let root = directory.path().to_path_buf();
5426 Connection::open(root.join(assets::MAIN_DICTIONARY))
5427 .and_then(|db| db.execute_batch(main))
5428 .expect("main fixture");
5429 Connection::open(root.join(assets::ENGLISH_DICTIONARY))
5430 .and_then(|db| db.execute_batch(ENGLISH_FIXTURE))
5431 .expect("english fixture");
5432 let paths = RuntimePaths {
5433 resources: root.clone(),
5434 user_data: root.clone(),
5435 cache: root.clone(),
5436 dictionaries: root,
5437 };
5438 Fixture { directory, paths }
5439 }
5440
5441 fn open(paths: &RuntimePaths, learning: bool, english: EnglishInputOptions) -> NineKeySession {
5442 let frequency = FrequencyAdjustmentOptions {
5443 mode: if learning {
5444 FrequencyAdjustmentMode::Promote
5445 } else {
5446 FrequencyAdjustmentMode::Disabled
5447 },
5448 ..FrequencyAdjustmentOptions::default()
5449 };
5450 NineKeySession::new(
5451 paths,
5452 learning,
5453 frequency,
5454 FuzzyPinyinOptions::default(),
5455 english,
5456 true,
5457 false,
5458 )
5459 }
5460
5461 fn mixed() -> EnglishInputOptions {
5462 EnglishInputOptions {
5463 mixed_candidates: true,
5464 ..EnglishInputOptions::default()
5465 }
5466 }
5467
5468 fn type_digits(session: &mut NineKeySession, digits: &str) {
5469 for digit in digits.bytes() {
5470 assert!(session.character(digit).handled, "digit {digit} unhandled");
5471 }
5472 }
5473
5474 fn index_of(session: &NineKeySession, word: &str) -> usize {
5475 session
5476 .snapshot()
5477 .candidates
5478 .iter()
5479 .position(|item| item.word == word)
5480 .unwrap_or_else(|| panic!("missing candidate {word}"))
5481 }
5482
5483 fn words(session: &NineKeySession) -> Vec<String> {
5484 session
5485 .snapshot()
5486 .candidates
5487 .into_iter()
5488 .map(|item| item.word.into_string())
5489 .collect()
5490 }
5491
5492 #[test]
5493 fn digits_offer_readings_and_ranked_english() {
5494 let fixture = fixture();
5495 let mut session = open(&fixture.paths, false, mixed());
5496 assert!(!session.character(b'0').handled && !session.character(b'1').handled);
5497 type_digits(&mut session, "65");
5499 assert_eq!(words(&session), ["ok", "old", "older"]);
5500 assert_eq!(
5501 session.snapshot().nine_key_spellings,
5502 ["o", "M", "N", "O", "6"]
5503 );
5504 session.command(Command::Cancel);
5505
5506 type_digits(&mut session, "64426");
5508 let view = session.snapshot();
5509 assert_eq!(words(&session), ["你好", "米好", "你", "米", "ogham"]);
5510 assert_eq!(view.candidate_sources[1], CandidateSource::Generated);
5511 assert_eq!(
5512 view.nine_key_spellings,
5513 ["ni", "mi", "o", "M", "N", "O", "6"]
5514 );
5515 assert_eq!(view.candidate_answers_key[..4], [true, true, false, false]);
5516 for pair in view.candidates.windows(2) {
5518 if pair[0].pinyin.len() == pair[1].pinyin.len()
5519 && pair[0].source.is_generated_or_fallback()
5520 {
5521 assert!(
5522 !pair[1].source.is_dictionary(),
5523 "{} after {}",
5524 pair[1].word,
5525 pair[0].word
5526 );
5527 }
5528 }
5529 session.command(Command::Cancel);
5530
5531 type_digits(&mut session, "6");
5532 assert_eq!(
5533 words(&session),
5534 ["你", "米"],
5535 "mixed English waits for the minimum prefix"
5536 );
5537 assert_eq!(session.snapshot().candidates[0].pinyin, "6");
5538 }
5539
5540 #[test]
5541 fn unfinished_top_candidate_does_not_displace_complete_readings() {
5542 let fixture = fixture_with("CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_s VALUES('shi','s','是',10000);");
5543 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5544 type_digits(&mut session, "74");
5545 let view = session.snapshot();
5546 assert_eq!(
5547 view.candidates
5548 .first()
5549 .map(|candidate| candidate.word.as_str()),
5550 Some("是")
5551 );
5552 assert_eq!(&view.nine_key_spellings[..4], ["qi", "si", "ri", "pi"]);
5554 }
5555
5556 #[test]
5557 fn selecting_a_nine_key_candidate_does_not_clone_unused_row_fields() {
5558 let fixture = fixture();
5559 let mut session = open(&fixture.paths, false, mixed());
5560 type_digits(&mut session, "64426");
5561 let index = index_of(&session, "你好");
5562
5563 let (result, allocations) =
5564 crate::ime::personal_rerank::allocations::count(|| session.select(index));
5565
5566 assert_eq!(result.commit.as_deref(), Some("你好"));
5567 assert!(
5568 allocations <= 1,
5569 "九键选择候选复制无用行字段产生了 {allocations} 次分配"
5570 );
5571 }
5572
5573 #[test]
5574 fn choosing_a_nine_key_letter_does_not_clone_spelling_choices() {
5575 let fixture = fixture();
5576 let mut session = open(&fixture.paths, false, mixed());
5577 type_digits(&mut session, "64426");
5578 let index = session
5579 .snapshot()
5580 .nine_key_spellings
5581 .iter()
5582 .position(|spelling| spelling == "N")
5583 .unwrap();
5584
5585 let (result, allocations) =
5586 crate::ime::personal_rerank::allocations::count(|| session.choose_spelling(index));
5587
5588 assert!(result.handled);
5589 assert!(
5590 allocations <= 347,
5591 "九键选择字母复制音节列表产生了 {allocations} 次分配"
5592 );
5593 }
5594
5595 #[test]
5596 fn split_refresh_reuses_normalized_boundaries() {
5597 let fixture = fixture();
5598 let mut plain = open(&fixture.paths, false, EnglishInputOptions::default());
5599 plain.digits = "644".into();
5600 plain.refresh();
5601 let (_, plain_allocations) =
5602 crate::ime::personal_rerank::allocations::count(|| plain.refresh());
5603 let mut split = open(&fixture.paths, false, EnglishInputOptions::default());
5604 split.digits = "644".into();
5605 split.splits = vec![2];
5606 split.refresh();
5607 let (_, split_allocations) =
5608 crate::ime::personal_rerank::allocations::count(|| split.refresh());
5609
5610 assert!(
5611 split_allocations <= 151,
5612 "split refresh should keep normalized boundaries off the heap: {split_allocations}"
5613 );
5614 assert!(split_allocations < plain_allocations);
5615 }
5616
5617 #[test]
5618 fn refresh_reuses_candidate_pinyin_storage() {
5619 let fixture = fixture();
5620 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5621 session.digits = "64".into();
5622 session.refresh();
5623 let (_, allocations) =
5624 crate::ime::personal_rerank::allocations::count(|| session.refresh());
5625 assert!(
5626 allocations <= 149,
5627 "candidate pinyin normalization allocated {allocations} buffers"
5628 );
5629 }
5630
5631 #[test]
5632 fn refresh_appends_key_choices_without_a_temporary_vector() {
5633 let fixture = fixture();
5634 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5635 session.digits = "64".into();
5636 session.refresh();
5637 let (_, allocations) =
5638 crate::ime::personal_rerank::allocations::count(|| session.refresh());
5639 let spellings = session.snapshot().nine_key_spellings;
5640 assert!(spellings.ends_with(
5641 &["o", "M", "N", "O", "6"]
5642 .into_iter()
5643 .map(String::from)
5644 .collect::<Vec<_>>()
5645 ));
5646 assert!(
5647 allocations <= 110,
5648 "key choices refresh allocations: {allocations}"
5649 );
5650 }
5651
5652 #[test]
5653 fn refresh_reuses_reading_storage() {
5654 let fixture = fixture();
5655 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5656 session.digits = "64426".into();
5657 session.refresh();
5658
5659 let (_, allocations) =
5660 crate::ime::personal_rerank::allocations::count(|| session.refresh());
5661 assert!(allocations < 601, "九键读音热刷新分配了 {allocations} 次");
5662 }
5663
5664 #[test]
5665 fn refresh_reuses_search_pages_and_releases_them_outside_pinyin_composition() {
5666 let fixture = fixture();
5667 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5668 session.digits = "64426".into();
5669 session.refresh();
5670 session.refresh();
5671 let pages: Vec<_> = session
5672 .path_scratch
5673 .pages
5674 .iter()
5675 .map(|page| (page.as_ptr(), page.capacity()))
5676 .collect();
5677 assert!(pages.iter().any(|(_, capacity)| *capacity != 0));
5678 session.refresh();
5679 assert_eq!(
5680 pages,
5681 session
5682 .path_scratch
5683 .pages
5684 .iter()
5685 .map(|page| (page.as_ptr(), page.capacity()))
5686 .collect::<Vec<_>>()
5687 );
5688 session.english_only = true;
5689 session.refresh();
5690 assert!(session.path_scratch.pages.is_empty());
5691 session.english_only = false;
5692 session.refresh();
5693 assert!(!session.path_scratch.pages.is_empty());
5694 session.digits.clear();
5695 session.refresh();
5696 assert!(session.path_scratch.pages.is_empty());
5697 }
5698
5699 #[test]
5700 fn commit_raw_releases_search_pages_before_returning() {
5701 let fixture = fixture();
5702 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5703 session.digits = "64426".into();
5704 session.refresh();
5705 assert!(!session.path_scratch.pages.is_empty());
5706 let result = session.command(Command::CommitRaw);
5707 assert_eq!(result.commit.as_deref(), Some("64426"));
5708 assert!(session.path_scratch.pages.is_empty());
5709 }
5710
5711 #[test]
5712 fn refresh_reuses_split_path_storage() {
5713 let fixture = fixture();
5714 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5715 session.digits = "64426".into();
5716 session.refresh();
5717 let pointer = session.path_buffer[0][0].as_ptr();
5718 let capacity = session.path_buffer[0][0].capacity();
5719 session.refresh();
5720 assert_eq!(session.path_buffer[0][0].as_ptr(), pointer);
5721 assert_eq!(session.path_buffer[0][0].capacity(), capacity);
5722 }
5723
5724 #[test]
5725 fn refresh_reuses_candidate_vector_storage() {
5726 let fixture = fixture();
5727 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5728 session.digits = "64".into();
5729 session.refresh();
5730 let pointer = session.candidates.as_ptr();
5731 let capacity = session.candidates.capacity();
5732 assert!(capacity > 0);
5733 session.refresh();
5734 assert_eq!(session.candidates.as_ptr(), pointer);
5735 assert_eq!(session.candidates.capacity(), capacity);
5736 }
5737
5738 #[test]
5739 fn refresh_reuses_spelling_vector_storage() {
5740 let fixture = fixture();
5741 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5742 session.digits = "64426".into();
5743 session.refresh();
5744 let pointer = session.spellings.as_ptr();
5745 let capacity = session.spellings.capacity();
5746 assert!(capacity > 0);
5747 session.refresh();
5748 assert_eq!(session.spellings.as_ptr(), pointer);
5749 assert_eq!(session.spellings.capacity(), capacity);
5750 }
5751
5752 #[test]
5753 fn refresh_keeps_spelling_string_buffers() {
5754 let fixture = fixture();
5755 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5756 session.digits = "64".into();
5757 session.refresh();
5758 let expected = session.snapshot();
5759 let syllables = session.spellings.len() - session.key_choices("64").len();
5761 assert!(syllables > 0);
5762 for value in &mut session.spellings[..syllables] {
5763 value.reserve(64);
5764 }
5765 let mut before: Vec<_> = session.spellings[..syllables]
5766 .iter()
5767 .map(|value| value.as_ptr())
5768 .collect();
5769 before.sort_unstable();
5770 session.refresh();
5771 assert_eq!(session.snapshot(), expected);
5772 let mut after: Vec<_> = session.spellings[..syllables]
5773 .iter()
5774 .map(|value| value.as_ptr())
5775 .collect();
5776 after.sort_unstable();
5777 assert_eq!(after, before, "刷新不应清空已有音节字符串");
5778 assert!(session.spellings[..syllables]
5779 .iter()
5780 .all(|value| value.capacity() >= 64));
5781 }
5782
5783 #[test]
5784 fn refresh_discards_spellings_when_there_is_no_pinyin_target() {
5785 let fixture = fixture();
5786 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5787 session.digits = "64".into();
5788 session.refresh();
5789 assert!(!session.spellings.is_empty());
5790 session.set_english_only(true);
5791 assert!(session.spellings.is_empty());
5792 session.set_english_only(false);
5793 assert!(!session.spellings.is_empty());
5794 session.pinyin = false;
5795 session.refresh();
5796 assert!(session.spellings.is_empty());
5797 session.pinyin = true;
5798 session.refresh();
5799 assert!(!session.spellings.is_empty());
5800 session.locked = vec!["ni".to_owned()];
5802 session.refresh();
5803 assert!(session.spellings.is_empty());
5804 session.locked.clear();
5805 session.refresh();
5806 assert!(!session.spellings.is_empty());
5807 session.command(Command::Cancel);
5808 assert!(session.spellings.is_empty());
5809 }
5810
5811 #[test]
5812 fn locked_refresh_reuses_the_existing_ranking_context_prefix() {
5813 let fixture = fixture();
5814 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5815 type_digits(&mut session, "64426");
5816 assert!(
5817 session
5818 .choose_spelling(spelling_index(&session, "ni"))
5819 .handled
5820 );
5821 let (_, allocations) =
5822 crate::ime::personal_rerank::allocations::count(|| session.refresh());
5823 assert!(
5824 allocations <= 106,
5825 "locked refresh rebuilt the ranking context prefix: {allocations}"
5826 );
5827 }
5828
5829 #[test]
5830 fn finish_reuses_the_first_commit_buffer() {
5831 let fixture = fixture_with(INITIALS_FIXTURE);
5832 let mut session = open(&fixture.paths, true, EnglishInputOptions::default());
5833 type_digits(&mut session, "963443842662");
5834 assert!(session.select(index_of(&session, "我滴")).handled);
5835
5836 let (_, allocations) =
5837 crate::ime::personal_rerank::allocations::count(|| session.finish(0));
5838 assert!(
5839 allocations <= 32,
5840 "九键结束组字复制了首段提交文本:{allocations} 次分配"
5841 );
5842 }
5843
5844 #[test]
5845 fn choosing_a_spelling_with_a_split_does_not_allocate_a_second_partition() {
5846 let fixture = fixture();
5847 let mut plain = open(&fixture.paths, false, mixed());
5848 type_digits(&mut plain, "64426");
5849 let index = plain
5850 .spellings
5851 .iter()
5852 .position(|spelling| spelling == "ni")
5853 .unwrap();
5854 let (_, plain_allocations) =
5855 crate::ime::personal_rerank::allocations::count(|| plain.choose_spelling(index));
5856
5857 let mut session = open(&fixture.paths, false, mixed());
5858 type_digits(&mut session, "64426");
5859 session.splits = vec![2];
5860 session.refresh();
5861 let index = session
5862 .spellings
5863 .iter()
5864 .position(|spelling| spelling == "ni")
5865 .unwrap();
5866 let (_, allocations) =
5867 crate::ime::personal_rerank::allocations::count(|| session.choose_spelling(index));
5868
5869 assert!(
5870 allocations <= plain_allocations,
5871 "九键选择带切分的音节分配多于无切分路径:{allocations} > {plain_allocations}"
5872 );
5873 }
5874
5875 #[test]
5876 fn english_keypad_filters_full_code_before_prefix_limit() {
5877 for grid in [KeyGrid::NineKey, KeyGrid::FourteenKey] {
5878 for english_only in [false, true] {
5879 let fixture = fixture();
5880 Connection::open(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY))
5881 .unwrap()
5882 .execute_batch("INSERT INTO english_words VALUES('previewing','previewing',1),('previewsa','previewsa',100),('previewsb','previewsb',100),('previewsc','previewsc',100),('previewsd','previewsd',100),('previewse','previewse',100);")
5883 .unwrap();
5884 let mut session = open(&fixture.paths, false, mixed());
5885 session.set_grid(grid);
5886 session.set_english_only(english_only);
5887 type_digits(&mut session, &grid.encode("previewing"));
5888 assert!(
5889 words(&session).iter().any(|word| word == "previewing"),
5890 "完整组码不能被不匹配的高权重前缀词挤掉:{grid:?},专用英文={english_only}"
5891 );
5892 let result = session.select(index_of(&session, "previewing"));
5893 assert_eq!(result.commit.as_deref(), Some("previewing"));
5894 }
5895 }
5896 }
5897
5898 #[test]
5899 fn english_keypad_prefix_limit_uses_final_ranking() {
5900 let fixture = fixture();
5901 Connection::open(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY))
5902 .unwrap()
5903 .execute_batch("INSERT INTO english_words VALUES('commit','COMMIT',1),('commita','commita',100),('commitb','commitb',100),('commitc','commitc',100),('commitd','commitd',100),('commite','commite',100);")
5904 .unwrap();
5905 let mut session = open(&fixture.paths, false, mixed());
5906 session.set_english_only(true);
5907 type_digits(&mut session, "266648");
5908 assert_eq!(words(&session).first().map(String::as_str), Some("COMMIT"));
5909 assert_eq!(words(&session).len(), ENGLISH_LIMIT);
5910 }
5911
5912 #[test]
5913 fn english_keypad_invalid_rows_do_not_consume_prefix_limit() {
5914 let fixture = fixture();
5915 let db = Connection::open(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY)).unwrap();
5916 for lookup in [
5917 "commit!",
5918 "commit ",
5919 "commit9",
5920 "commit-é",
5921 "commit\0suffix",
5922 "commit",
5923 ] {
5924 let display = (lookup != "commit").then_some("合成无效词");
5925 db.execute(
5926 "INSERT INTO english_words VALUES(?1,?2,1000)",
5927 rusqlite::params![lookup, display],
5928 )
5929 .unwrap();
5930 }
5931 db.execute_batch("INSERT INTO english_words VALUES('commiT','don''t',1);")
5932 .unwrap();
5933 let mut session = open(&fixture.paths, false, mixed());
5934 session.set_english_only(true);
5935 type_digits(&mut session, "266648");
5936 assert_eq!(words(&session), ["don't"]);
5937 }
5938
5939 #[test]
5940 #[ignore = "需要通过 MSIME_TEST_ENGLISH_DICTIONARY 指定已校验的发布英文词库"]
5941 fn english_keypad_shipped_dictionary_reported_words() {
5942 let database =
5943 std::env::var_os("MSIME_TEST_ENGLISH_DICTIONARY").expect("指定已校验的发布英文词库");
5944 let fixture = fixture();
5945 std::fs::copy(
5946 database,
5947 fixture.paths.dictionary(assets::ENGLISH_DICTIONARY),
5948 )
5949 .unwrap();
5950 for grid in [KeyGrid::NineKey, KeyGrid::FourteenKey] {
5951 for english_only in [false, true] {
5952 for (lookup, display) in [
5953 ("github", "GitHub"),
5954 ("preview", "preview"),
5955 ("commit", "commit"),
5956 ("clash", "clash"),
5957 ] {
5958 let mut session = open(&fixture.paths, false, mixed());
5959 session.set_grid(grid);
5960 session.set_english_only(english_only);
5961 type_digits(&mut session, &grid.encode(lookup));
5962 assert!(
5963 words(&session).iter().any(|word| word == display),
5964 "发布词库缺少候选:{display},{grid:?},专用英文={english_only}"
5965 );
5966 assert_eq!(
5967 session
5968 .select(index_of(&session, display))
5969 .commit
5970 .as_deref(),
5971 Some(display)
5972 );
5973 }
5974 }
5975 }
5976 }
5977
5978 #[test]
5979 fn english_t9_empty_matches_do_not_reserve_prefix_capacity() {
5980 let fixture = fixture();
5981 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
5982 session.set_english_only(true);
5983 session.digits = "999999".into();
5984 let words = session.english_candidates(false);
5985 assert!(words.is_empty());
5986 assert_eq!(words.capacity(), 0);
5987 }
5988
5989 #[test]
5990 fn english_t9_filtered_matches_do_not_reserve_prefix_capacity() {
5991 let fixture = fixture();
5992 Connection::open(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY))
5993 .unwrap()
5994 .execute(
5995 "INSERT INTO english_words(word, display, weight) VALUES ('wwwa', '合成展示', 100)",
5996 [],
5997 )
5998 .unwrap();
5999 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
6000 session.set_english_only(true);
6001 session.digits = "999999".into();
6002 assert!(!session
6003 .open_english()
6004 .unwrap()
6005 .query_prefix("www", ENGLISH_LIMIT)
6006 .is_empty());
6007 let words = session.english_candidates(false);
6008 assert!(words.is_empty());
6009 assert_eq!(words.capacity(), 0);
6010 }
6011
6012 #[test]
6013 fn english_t9_hits_keep_the_original_prefix_capacity() {
6014 let fixture = fixture();
6015 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
6016 session.set_english_only(true);
6017 session.digits = "3668".into();
6018 let capacity =
6019 letter_prefixes(&session.digits, ENGLISH_PREFIX_BUDGET).len() * ENGLISH_LIMIT;
6020 Connection::open(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY))
6021 .unwrap()
6022 .execute(
6023 "INSERT INTO english_words(word, display, weight) VALUES ('dont', '合成展示', 100)",
6024 [],
6025 )
6026 .unwrap();
6027 let words = session.english_candidates(false);
6028 assert!(words
6029 .iter()
6030 .any(|word| word.word == "合成展示" && word.pinyin == "dont"));
6031 assert_eq!(words.capacity(), capacity);
6032 }
6033
6034 #[test]
6035 fn english_refresh_does_not_allocate_a_copy_of_typed_digits() {
6036 let fixture = fixture();
6037 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
6038 session.set_english_only(true);
6039 session.digits = "3668".into();
6040 session.english_candidates(false);
6041
6042 let (_, allocations) =
6043 crate::ime::personal_rerank::allocations::count(|| session.english_candidates(false));
6044 assert!(
6045 allocations < 157,
6046 "英文候选热刷新不应复制已输入的数字:{allocations} 次分配"
6047 );
6048 }
6049
6050 #[test]
6051 fn english_t9_uses_the_lookup_word_when_display_has_punctuation() {
6052 let fixture = fixture();
6053 Connection::open(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY))
6054 .unwrap()
6055 .execute(
6056 "INSERT INTO english_words(word, display, weight) VALUES ('dont', 'don''t', 100)",
6057 [],
6058 )
6059 .unwrap();
6060 let mut session = open(
6061 &fixture.paths,
6062 false,
6063 EnglishInputOptions {
6064 mixed_candidates: false,
6065 ..EnglishInputOptions::default()
6066 },
6067 );
6068 session.set_english_only(true);
6069 type_digits(&mut session, "3668");
6070 assert!(
6071 words(&session).contains(&"don't".to_owned()),
6072 "T9 should match the lookup key even when the displayed word contains punctuation"
6073 );
6074 }
6075
6076 #[test]
6077 fn english_digits_take_no_split() {
6078 let fixture = fixture();
6079 let mut english_only = open(&fixture.paths, false, mixed());
6080 english_only.set_english_only(true);
6081 let mut without_pinyin = NineKeySession::new(
6082 &fixture.paths,
6083 false,
6084 FrequencyAdjustmentOptions::default(),
6085 FuzzyPinyinOptions::default(),
6086 mixed(),
6087 false,
6088 false,
6089 );
6090 for session in [&mut english_only, &mut without_pinyin] {
6091 type_digits(session, "65");
6092 assert!(!session.character(b'\'').handled);
6093 assert!(session.splits.is_empty());
6094 assert_eq!(session.snapshot().preedit, "65");
6095 }
6096 }
6097
6098 #[test]
6099 fn a_fuzzy_leading_row_has_no_reading() {
6100 let mut session = detached();
6101 session.digits = "94".into();
6102 let xi = spelled("西", "94", "xi", 100, CandidateSource::Database);
6103 assert_eq!(session.reading_for(Some(&xi)), "xi");
6104 let mut zhi = spelled("知", "94", "zhi", 100, CandidateSource::Database);
6106 zhi.fuzzy = true;
6107 assert_eq!(session.reading_for(Some(&zhi)), "");
6108 }
6109
6110 #[test]
6111 fn split_keeps_both_sides_open_and_backspace_removes_it_first() {
6112 let table = SpellingTable::new(&["xi", "yi", "an", "xian", "yan"]);
6113 assert_eq!(
6114 table
6115 .split_paths("9426", &[], &SyllablePrior::default())
6116 .first(),
6117 Some(&vec!["xian".to_string()])
6118 );
6119 let split = table.split_paths("9426", &[2], &SyllablePrior::default());
6120 assert!(!split.is_empty());
6121 assert!(
6122 split.iter().all(|path| path.len() == 2 && path[1] == "an"),
6123 "{split:?}"
6124 );
6125
6126 let fixture = fixture();
6127 let mut session = open(&fixture.paths, false, mixed());
6128 assert!(!session.character(b'\'').handled, "nothing to split");
6129 type_digits(&mut session, "64");
6130 assert!(session.character(b'\'').handled);
6131 assert!(
6132 session.character(b'\'').handled,
6133 "a repeated split is absorbed"
6134 );
6135 assert_eq!(session.snapshot().preedit, "64'");
6136 assert!(
6137 session
6138 .snapshot()
6139 .nine_key_spellings
6140 .iter()
6141 .all(|s| s.len() <= 2),
6142 "no syllable runs past the split"
6143 );
6144 type_digits(&mut session, "426");
6145 assert_eq!(session.snapshot().preedit, "64'426");
6146 assert_eq!(words(&session).first().map(String::as_str), Some("你好"));
6147 assert_eq!(session.snapshot().nine_key_reading, "ni'hao");
6148 assert!(session.command(Command::Backspace).handled);
6149 assert_eq!(session.snapshot().preedit, "64'42");
6150 type_digits(&mut session, "6");
6151 let result = session.select(index_of(&session, "你"));
6152 assert_eq!(result.commit.as_deref(), Some("你"));
6153 assert_eq!(
6154 session.snapshot().preedit,
6155 "426",
6156 "the split went with the consumed digits"
6157 );
6158
6159 session.command(Command::Cancel);
6160 type_digits(&mut session, "64");
6161 session.character(b'\'');
6162 assert!(session.command(Command::Backspace).handled);
6163 assert_eq!(
6164 session.snapshot().preedit,
6165 "64",
6166 "Backspace takes the split before a digit"
6167 );
6168 }
6169
6170 #[test]
6171 fn spellings_lock_and_partial_selection_advances() {
6172 let fixture = fixture();
6173 let mut session = open(&fixture.paths, false, mixed());
6174 type_digits(&mut session, "64");
6175 let before = session.snapshot();
6176 assert!(!session.select(999).handled && !session.choose_spelling(999).handled);
6177 assert_eq!(session.snapshot(), before);
6178 assert_eq!(
6179 before.nine_key_spellings,
6180 [
6181 "ni", "mi", "min", "niu", "nin", "mie", "nie", "miu", "nian", "mian", "ming",
6182 "miao", "ning", "niao", "niang", "o", "M", "N", "O", "6"
6183 ],
6184 "the preferred spelling leads, the key's letters and digit follow"
6185 );
6186 assert!(session.choose_spelling(0).handled);
6187 assert_eq!(session.snapshot().preedit, "ni");
6188 assert_eq!(words(&session), ["你"], "the lock keeps 米 out");
6189 type_digits(&mut session, "426");
6190 let view = session.snapshot();
6191 assert_eq!(view.preedit, "ni'426");
6192 assert_eq!(
6193 view.nine_key_spellings,
6194 ["hao", "gao", "gan", "han", "hang", "gang", "ha", "ga", "G", "H"],
6195 "no i, which starts no syllable, and no digit behind a lock"
6196 );
6197 let result = session.select(index_of(&session, "你好"));
6198 assert_eq!(result.commit.as_deref(), Some("你好"));
6199 assert!(!session.active() && session.snapshot().preedit.is_empty());
6200
6201 type_digits(&mut session, "64426");
6202 let ni = session
6203 .snapshot()
6204 .nine_key_spellings
6205 .iter()
6206 .position(|s| s == "ni")
6207 .expect("ni offered");
6208 session.choose_spelling(ni);
6209 let hao_choices = session.snapshot().nine_key_spellings;
6210 session.choose_spelling(0);
6211 assert_eq!(session.snapshot().preedit, "ni'hao");
6212 assert_eq!(
6213 session.snapshot().nine_key_spellings,
6214 hao_choices,
6215 "every digit locked still offers the last lock's choices"
6216 );
6217 let result = session.select(index_of(&session, "你"));
6218 assert_eq!(result.commit.as_deref(), Some("你"));
6219 let view = session.snapshot();
6220 assert_eq!(view.editing_text, "426");
6221 assert_eq!(
6222 view.preedit, "hao",
6223 "partial selection lost the locked suffix"
6224 );
6225 let result = session.finish(0);
6226 assert_eq!(result.commit.as_deref(), Some("好"));
6227 assert!(!session.active());
6228
6229 type_digits(&mut session, "6");
6231 let ming = session
6232 .snapshot()
6233 .nine_key_spellings
6234 .iter()
6235 .position(|s| s == "ming")
6236 .expect("ming offered");
6237 session.choose_spelling(ming);
6238 assert_eq!(session.snapshot().editing_text, "6464");
6239 assert_eq!(session.snapshot().preedit, "ming");
6240 }
6241
6242 fn spelling_index(session: &NineKeySession, spelling: &str) -> usize {
6243 session
6244 .snapshot()
6245 .nine_key_spellings
6246 .iter()
6247 .position(|s| s == spelling)
6248 .unwrap_or_else(|| panic!("missing spelling {spelling}"))
6249 }
6250
6251 #[test]
6252 fn backspace_takes_back_the_last_lock_only_while_every_digit_is_locked() {
6253 let fixture = fixture();
6254 let mut session = open(&fixture.paths, false, mixed());
6255 type_digits(&mut session, "6464224");
6256 session.choose_spelling(spelling_index(&session, "ning"));
6257 assert_eq!(session.snapshot().preedit, "ning'224");
6258 let after_ning = session.snapshot().nine_key_spellings;
6259 assert!(after_ning.contains(&"bai".to_owned()) && after_ning.contains(&"cai".to_owned()));
6260 session.choose_spelling(spelling_index(&session, "bai"));
6261 assert_eq!(session.snapshot().preedit, "ning'bai");
6262 assert_eq!(session.snapshot().nine_key_spellings, after_ning);
6263
6264 session.choose_spelling(spelling_index(&session, "cai"));
6265 assert_eq!(
6266 session.snapshot().preedit,
6267 "ning'cai",
6268 "picking again swaps the last lock"
6269 );
6270 assert_eq!(session.locked, ["ning", "cai"]);
6271 assert_eq!(session.snapshot().nine_key_spellings, after_ning);
6272
6273 assert!(session.command(Command::Backspace).handled);
6274 assert_eq!(session.snapshot().preedit, "ning'224");
6275 assert_eq!(session.snapshot().nine_key_spellings, after_ning);
6276 assert!(session.command(Command::Backspace).handled);
6277 assert_eq!(
6278 session.locked,
6279 ["ning"],
6280 "with digits open a digit goes, not the lock"
6281 );
6282 assert_eq!(session.snapshot().preedit, "ning'22");
6283 }
6284
6285 #[test]
6286 fn taking_back_a_lock_restores_the_digits_it_extended() {
6287 let fixture = fixture();
6288 let mut session = open(&fixture.paths, false, mixed());
6289 type_digits(&mut session, "6");
6290 session.choose_spelling(spelling_index(&session, "ming"));
6291 assert_eq!(session.snapshot().editing_text, "6464");
6292 session.command(Command::Backspace);
6293 assert_eq!(session.snapshot().editing_text, "6");
6294 assert!(session.locked.is_empty());
6295 assert!(session
6296 .snapshot()
6297 .nine_key_spellings
6298 .contains(&"ming".to_owned()));
6299 }
6300
6301 #[test]
6302 fn a_split_dropped_by_a_lock_comes_back_with_it() {
6303 let fixture = fixture();
6304 let mut session = open(&fixture.paths, false, mixed());
6305 type_digits(&mut session, "64");
6306 session.character(b'\'');
6307 session.choose_spelling(spelling_index(&session, "ni"));
6308 assert!(session.splits.is_empty());
6309 assert_eq!(session.snapshot().preedit, "ni");
6310 session.command(Command::Backspace);
6311 assert_eq!(
6312 session.snapshot().preedit,
6313 "64'",
6314 "the split is back with the digits"
6315 );
6316 session.command(Command::Backspace);
6317 assert_eq!(session.snapshot().preedit, "64");
6318 }
6319
6320 #[test]
6321 fn a_key_letter_narrows_the_next_syllable_and_backspace_takes_it_back() {
6322 let fixture = fixture();
6323 let mut session = open(&fixture.paths, false, mixed());
6324 type_digits(&mut session, "64");
6325 assert_eq!(words(&session), ["你", "米", "你好", "ogham"]);
6327 assert!(
6328 session
6329 .choose_spelling(spelling_index(&session, "M"))
6330 .handled
6331 );
6332 let view = session.snapshot();
6333 assert_eq!(view.preedit, "m4");
6334 assert_eq!(words(&session), ["米"]);
6335 assert!(view
6336 .nine_key_spellings
6337 .iter()
6338 .filter(|s| s.len() > 1)
6339 .all(|s| s.starts_with('m')));
6340 assert!(view.nine_key_spellings.ends_with(&[
6341 "M".into(),
6342 "N".into(),
6343 "O".into(),
6344 "6".into()
6345 ]));
6346
6347 session.choose_spelling(spelling_index(&session, "mi"));
6348 assert_eq!(session.snapshot().preedit, "mi");
6349 session.command(Command::Backspace);
6350 assert_eq!(
6351 session.snapshot().preedit,
6352 "m4",
6353 "taking back the lock restores the letter"
6354 );
6355 session.command(Command::Backspace);
6356 assert_eq!(session.snapshot().preedit, "64");
6357 assert_eq!(words(&session), ["你", "米", "你好", "ogham"]);
6358 }
6359
6360 #[test]
6361 fn a_letter_survives_typing_but_not_deleting_past_it() {
6362 let fixture = fixture();
6363 let mut session = open(&fixture.paths, false, mixed());
6364 type_digits(&mut session, "64");
6365 session.choose_spelling(spelling_index(&session, "N"));
6366 type_digits(&mut session, "4");
6367 session.command(Command::Backspace);
6368 assert_eq!(
6369 session.snapshot().preedit,
6370 "n4",
6371 "the typed digit goes first"
6372 );
6373 session.command(Command::Backspace);
6374 assert_eq!(session.snapshot().preedit, "64");
6375
6376 type_digits(&mut session, "426");
6377 session.choose_spelling(spelling_index(&session, "N"));
6378 session.command(Command::Cancel);
6379 type_digits(&mut session, "64");
6380 assert_eq!(
6381 session.snapshot().preedit,
6382 "64",
6383 "a new composition starts without it"
6384 );
6385 }
6386
6387 #[test]
6388 fn the_key_digit_commits_itself_and_the_rest_keeps_composing() {
6389 let fixture = fixture();
6390 let mut session = open(&fixture.paths, false, mixed());
6391 type_digits(&mut session, "64");
6392 let result = session.choose_spelling(spelling_index(&session, "6"));
6393 assert_eq!(result.commit.as_deref(), Some("6"));
6394 assert_eq!(session.snapshot().editing_text, "4");
6395 }
6396
6397 #[test]
6399 fn a_key_digit_inside_a_phrase_keeps_it_from_being_stored() {
6400 let fixture = fixture();
6401 let mut session = open(&fixture.paths, true, mixed());
6402 type_digits(&mut session, "64426");
6403 session.select(index_of(&session, "你"));
6404 let result = session.choose_spelling(spelling_index(&session, "4"));
6405 assert_eq!(result.commit.as_deref(), Some("4"));
6406 assert!(!session.phrase_storable);
6407 assert_eq!(session.phrase_word, "你4");
6408 session.command(Command::Cancel);
6409
6410 type_digits(&mut session, "644");
6411 session.select(index_of(&session, "你"));
6412 session.choose_spelling(spelling_index(&session, "4"));
6413 assert!(!session.active());
6414 assert!(session.phrase_word.is_empty() && session.phrase_storable);
6415 }
6416
6417 #[test]
6418 fn single_character_filter_keeps_single_characters_until_the_composition_ends() {
6419 let fixture = fixture();
6420 let mut session = open(&fixture.paths, false, mixed());
6421 assert!(!session.set_filter(true, "").handled, "nothing composing");
6422 type_digits(&mut session, "64426");
6423 assert!(session.set_filter(true, "").handled);
6424 assert!(session.snapshot().nine_key_single_character);
6425 assert_eq!(words(&session), ["你", "米"]);
6426 assert!(session.set_filter(false, "").handled);
6427 assert!(words(&session).contains(&"你好".to_owned()));
6428 session.set_filter(true, "");
6429 session.command(Command::Cancel);
6430 type_digits(&mut session, "64426");
6431 assert!(!session.snapshot().nine_key_single_character);
6432 assert!(words(&session).contains(&"ogham".to_owned()));
6433 }
6434
6435 #[test]
6436 fn stroke_filter_matches_the_first_character_stroke_prefix() {
6437 let fixture = fixture();
6438 let strokes = fixture.directory.path().join("msime-stroke.db");
6439 crate::stroke::fixture::build(&strokes);
6440 Connection::open(&strokes)
6441 .unwrap()
6442 .execute_batch(
6443 "INSERT INTO entries VALUES ('pspzspn', '你', 100); INSERT INTO entries VALUES ('nphspn', '米', 100);",
6444 )
6445 .unwrap();
6446 let mut session = open(&fixture.paths, false, mixed());
6447 type_digits(&mut session, "64426");
6448 assert_eq!(
6449 session.set_filter(false, "p").diagnostic.as_deref(),
6450 Some(diagnostics::LANGUAGE_DICTIONARY_UNAVAILABLE),
6451 "no stroke dictionary yet"
6452 );
6453 assert!(session.snapshot().nine_key_strokes.is_empty());
6454
6455 session.set_stroke_dictionary(strokes);
6456 assert!(!session.set_filter(false, "q").handled, "not a stroke");
6457 assert!(session.set_filter(false, "ps").handled);
6458 assert_eq!(session.snapshot().nine_key_strokes, "ps");
6459 assert_eq!(words(&session), ["你好", "你"]);
6460 session.set_filter(true, "n");
6461 assert_eq!(words(&session), ["米"]);
6462 session.set_filter(false, "z");
6463 assert!(words(&session).is_empty());
6464 session.set_filter(false, "");
6465 assert!(words(&session).contains(&"米好".to_owned()));
6466 }
6467
6468 #[test]
6470 fn single_character_only_keeps_one_character_readings() {
6471 let fixture = fixture();
6472 let mut session = NineKeySession::new(
6473 &fixture.paths,
6474 false,
6475 FrequencyAdjustmentOptions::default(),
6476 FuzzyPinyinOptions::default(),
6477 mixed(),
6478 true,
6479 true,
6480 );
6481 type_digits(&mut session, "64426");
6482 assert_eq!(words(&session), ["你", "米", "ogham"]);
6483 let result = session.select(index_of(&session, "你"));
6484 assert_eq!(result.commit.as_deref(), Some("你"));
6485 assert_eq!(session.snapshot().editing_text, "426");
6486 assert_eq!(words(&session), ["好"]);
6487 let result = session.finish(0);
6488 assert_eq!(result.commit.as_deref(), Some("好"));
6489 assert!(!session.active());
6490 }
6491
6492 #[test]
6493 fn commands_edit_commit_and_cancel_the_digits() {
6494 let fixture = fixture();
6495 let mut session = open(&fixture.paths, false, mixed());
6496 assert!(!session.command(Command::Backspace).handled && !session.finish(0).handled);
6497 type_digits(&mut session, "64");
6498 session.command(Command::Backspace);
6499 assert_eq!(session.snapshot().editing_text, "6");
6500 session.command(Command::Backspace);
6501 assert!(
6502 !session.command(Command::Backspace).handled,
6503 "idle backspace"
6504 );
6505
6506 type_digits(&mut session, "64");
6507 session.choose_spelling(0);
6508 session.command(Command::Backspace);
6509 assert!(
6510 session.locked.is_empty(),
6511 "backspace first takes back a lock that covers every digit"
6512 );
6513 assert_eq!(session.snapshot().preedit, "64");
6514 session.command(Command::Backspace);
6515 assert_eq!(session.snapshot().preedit, "6");
6516 session.command(Command::Cancel);
6517
6518 type_digits(&mut session, "64");
6519 assert!(session.command(Command::MoveLeft).handled);
6521 assert_eq!(session.snapshot().caret_position, 1);
6522 assert_eq!(
6523 session.command(Command::CommitRaw).commit.as_deref(),
6524 Some("64")
6525 );
6526 assert_eq!(
6527 session.snapshot().caret_position,
6528 0,
6529 "a new composition starts at its end"
6530 );
6531 assert!(!session.active());
6532 type_digits(&mut session, "64");
6533 assert_eq!(session.finish(999).commit.as_deref(), Some("64"));
6534 type_digits(&mut session, "64426");
6535 assert_eq!(
6536 session.command(Command::CommitCandidate).commit.as_deref(),
6537 Some("你好")
6538 );
6539 type_digits(&mut session, "64");
6540 session.command(Command::Cancel);
6541 let view = session.snapshot();
6542 assert!(
6543 view.preedit.is_empty()
6544 && view.nine_key_spellings.is_empty()
6545 && view.candidates.is_empty()
6546 );
6547
6548 type_digits(&mut session, &"7".repeat(DIGIT_LIMIT));
6549 assert!(!session.limit_pick_ready(b'7'));
6551 let result = session.character(b'7');
6552 assert!(result.handled);
6553 assert_eq!(
6554 result.diagnostic.as_deref(),
6555 Some(diagnostics::NINE_KEY_DIGIT_LIMIT)
6556 );
6557 assert_eq!(
6558 result.diagnostic_code(),
6559 Some(diagnostics::DiagnosticCode::NineKeyDigitLimit)
6560 );
6561 assert_eq!(session.snapshot().editing_text.len(), DIGIT_LIMIT);
6562 assert!(session.snapshot().candidates.is_empty());
6563 assert_eq!(
6565 session.finish(0).commit.as_deref(),
6566 Some("7".repeat(DIGIT_LIMIT).as_str())
6567 );
6568
6569 let long = "64426".repeat(DIGIT_LIMIT / 5 + 1);
6571 type_digits(&mut session, &long[..DIGIT_LIMIT]);
6572 assert!(!session.snapshot().candidates.is_empty());
6573 assert!(session.limit_pick_ready(b'6'));
6574 assert!(!session.limit_pick_ready(b'1'));
6575 session.command(Command::MoveLeft);
6576 assert!(!session.limit_pick_ready(b'6'));
6577 session.command(Command::MoveEnd);
6578 assert!(session.limit_pick_ready(b'6'));
6579 assert!(diagnostics::NINE_KEY_DIGIT_LIMIT.contains(&DIGIT_LIMIT.to_string()));
6581 session.command(Command::Cancel);
6582 }
6583
6584 #[test]
6586 fn the_caret_edits_digits_in_the_middle() {
6587 let fixture = fixture();
6588 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
6589 type_digits(&mut session, "64526");
6591 assert_eq!(session.snapshot().caret_position, 5);
6592 for _ in 0..2 {
6593 assert!(session.command(Command::MoveLeft).handled);
6594 }
6595 assert_eq!(session.snapshot().caret_position, 3);
6596 assert!(session.command(Command::Backspace).handled);
6597 assert_eq!(session.snapshot().editing_text, "6426");
6598 assert_eq!(session.snapshot().caret_position, 2);
6599 type_digits(&mut session, "4");
6600 let view = session.snapshot();
6601 assert_eq!(view.editing_text, "64426");
6602 assert_eq!(view.caret_position, 3);
6603 assert_eq!(
6604 view.candidates[0].word, "你好",
6605 "the whole digits are decoded again"
6606 );
6607
6608 assert!(session.command(Command::MoveHome).handled);
6610 assert_eq!(session.snapshot().caret_position, 0);
6611 assert!(session.command(Command::MoveLeft).handled);
6612 assert_eq!(session.snapshot().caret_position, 0);
6613 assert!(
6614 session.command(Command::Backspace).handled,
6615 "backspace at the start does nothing"
6616 );
6617 assert_eq!(session.snapshot().editing_text, "64426");
6618 assert!(session.command(Command::DeleteForward).handled);
6619 assert_eq!(session.snapshot().editing_text, "4426");
6620 assert_eq!(session.snapshot().caret_position, 0);
6621 type_digits(&mut session, "6");
6622 assert!(session.command(Command::MoveEnd).handled);
6623 assert!(session.command(Command::MoveRight).handled);
6624 assert_eq!(session.snapshot().caret_position, 5);
6625 assert!(
6626 session.command(Command::DeleteForward).handled,
6627 "delete at the end does nothing"
6628 );
6629 assert_eq!(session.snapshot().editing_text, "64426");
6630 session.command(Command::Cancel);
6631
6632 type_digits(&mut session, "6426");
6634 session.command(Command::MoveLeft);
6635 session.command(Command::MoveLeft);
6636 assert!(session.character(b'\'').handled);
6637 assert_eq!(session.snapshot().preedit, "64'26");
6638 type_digits(&mut session, "4");
6639 assert_eq!(session.snapshot().preedit, "64'426");
6640 assert_eq!(session.snapshot().caret_position, 3);
6641 session.command(Command::MoveLeft);
6642 assert!(session.command(Command::Backspace).handled);
6643 assert_eq!(session.snapshot().preedit, "64426");
6644 assert_eq!(session.snapshot().caret_position, 2);
6645 session.command(Command::Cancel);
6646
6647 type_digits(&mut session, "64426");
6649 let ni = session
6650 .snapshot()
6651 .nine_key_spellings
6652 .iter()
6653 .position(|spelling| spelling == "ni")
6654 .expect("ni offered");
6655 session.choose_spelling(ni);
6656 assert_eq!(session.snapshot().preedit, "ni'426");
6657 session.command(Command::MoveHome);
6658 session.command(Command::MoveRight);
6659 type_digits(&mut session, "4");
6660 assert!(
6661 session.locked.is_empty(),
6662 "an edit inside a locked spelling unlocks it"
6663 );
6664 assert_eq!(session.snapshot().editing_text, "644426");
6665
6666 session.command(Command::Cancel);
6668 type_digits(&mut session, "64426");
6669 session.command(Command::MoveLeft);
6670 let ni = session.select(index_of(&session, "你"));
6671 assert_eq!(ni.commit.as_deref(), Some("你"));
6672 let view = session.snapshot();
6673 assert_eq!(view.editing_text, "426");
6674 assert_eq!(view.caret_position, 3);
6675 type_digits(&mut session, "6");
6676 assert_eq!(session.snapshot().editing_text, "4266");
6677 session.command(Command::Cancel);
6678
6679 type_digits(&mut session, "64426");
6681 for spelling in ["ni", "hao"] {
6682 let index = session
6683 .snapshot()
6684 .nine_key_spellings
6685 .iter()
6686 .position(|offered| offered == spelling)
6687 .unwrap_or_else(|| panic!("{spelling} offered"));
6688 session.choose_spelling(index);
6689 }
6690 assert_eq!(session.snapshot().preedit, "ni'hao");
6691 let before = session.snapshot();
6692 session.command(Command::MoveHome);
6693 assert!(session.character(b'\'').handled);
6694 session.command(Command::MoveRight);
6695 session.command(Command::MoveRight);
6696 assert!(session.character(b'\'').handled);
6697 assert_eq!(session.locked, ["ni", "hao"]);
6698 let after = session.snapshot();
6699 assert_eq!(after.preedit, before.preedit);
6700 assert_eq!(after.candidates, before.candidates);
6701 assert_eq!(after.nine_key_reading, before.nine_key_reading);
6702 session.command(Command::MoveRight);
6703 assert!(session.character(b'\'').handled);
6704 assert_eq!(session.locked, ["ni"]);
6705 assert_eq!(session.snapshot().preedit, "ni'4'26");
6706 }
6707
6708 #[test]
6710 fn the_caret_meets_key_letters_and_taking_back_locks() {
6711 let fixture = fixture();
6712 let mut session = open(&fixture.paths, false, mixed());
6713
6714 type_digits(&mut session, "644");
6716 session.choose_spelling(spelling_index(&session, "N"));
6717 assert_eq!(session.snapshot().preedit, "n44");
6718 session.command(Command::MoveLeft);
6719 assert!(session.command(Command::Backspace).handled);
6720 let view = session.snapshot();
6721 assert_eq!(view.preedit, "n4");
6722 assert_eq!(view.caret_position, 1);
6723 session.command(Command::MoveHome);
6725 type_digits(&mut session, "9");
6726 assert!(session.initial.is_none());
6727 assert_eq!(session.snapshot().preedit, "964");
6728 session.command(Command::Cancel);
6729 type_digits(&mut session, "644");
6730 session.choose_spelling(spelling_index(&session, "N"));
6731 session.command(Command::MoveHome);
6732 assert!(session.command(Command::DeleteForward).handled);
6733 assert!(session.initial.is_none());
6734 assert_eq!(session.snapshot().preedit, "44");
6735 session.command(Command::Cancel);
6736
6737 type_digits(&mut session, "6464224");
6739 session.choose_spelling(spelling_index(&session, "ning"));
6740 session.choose_spelling(spelling_index(&session, "bai"));
6741 assert_eq!(session.snapshot().preedit, "ning'bai");
6742 session.command(Command::MoveLeft);
6743 session.command(Command::MoveLeft);
6744 assert!(session.command(Command::Backspace).handled);
6745 assert_eq!(session.locked, ["ning"]);
6746 assert_eq!(session.lock_undo.len(), 1);
6747 let view = session.snapshot();
6748 assert_eq!(view.preedit, "ning'24");
6749 assert_eq!(view.caret_position, 4);
6750 session.command(Command::Cancel);
6751
6752 type_digits(&mut session, "64426");
6754 session.choose_spelling(spelling_index(&session, "ni"));
6755 session.choose_spelling(spelling_index(&session, "hao"));
6756 session.command(Command::MoveHome);
6757 session.command(Command::MoveRight);
6758 type_digits(&mut session, "4");
6759 assert!(session.locked.is_empty() && session.lock_undo.is_empty());
6760 session.command(Command::MoveEnd);
6761 session.command(Command::Backspace);
6762 assert_eq!(session.snapshot().editing_text, "64442");
6763 session.command(Command::Cancel);
6764
6765 type_digits(&mut session, "64");
6767 session.choose_spelling(spelling_index(&session, "ming"));
6768 assert_eq!(session.snapshot().editing_text, "6464");
6769 session.command(Command::MoveHome);
6770 session.command(Command::MoveRight);
6771 session.choose_spelling(spelling_index(&session, "ni"));
6772 let view = session.snapshot();
6773 assert_eq!(
6774 (view.preedit.as_str(), view.editing_text.as_str()),
6775 ("ni", "64")
6776 );
6777 assert_eq!(view.caret_position, 1);
6778 session.choose_spelling(spelling_index(&session, "ming"));
6779 assert_eq!(session.snapshot().caret_position, 1);
6780 session.command(Command::MoveEnd);
6781 session.command(Command::MoveLeft);
6782 assert_eq!(session.snapshot().caret_position, 3);
6783 session.choose_spelling(spelling_index(&session, "ni"));
6784 let view = session.snapshot();
6785 assert_eq!(view.editing_text, "64");
6786 assert_eq!(view.caret_position, 2);
6787 session.command(Command::Cancel);
6788
6789 type_digits(&mut session, "64426");
6791 session.choose_spelling(spelling_index(&session, "ni"));
6792 session.choose_spelling(spelling_index(&session, "H"));
6793 session.command(Command::MoveHome);
6794 assert!(session.command(Command::Cancel).handled);
6795 assert!(session.locked.is_empty() && session.lock_undo.is_empty());
6796 assert!(session.initial.is_none() && session.caret.is_none());
6797 type_digits(&mut session, "64");
6798 let view = session.snapshot();
6799 assert_eq!(view.preedit, "64");
6800 assert_eq!(view.caret_position, 2);
6801 session.command(Command::Cancel);
6802
6803 type_digits(&mut session, "64426");
6805 session.command(Command::MoveHome);
6806 session.command(Command::MoveRight);
6807 let result = session.choose_spelling(spelling_index(&session, "6"));
6808 assert_eq!(result.commit.as_deref(), Some("6"));
6809 let view = session.snapshot();
6810 assert_eq!(view.editing_text, "4426");
6811 assert_eq!(view.caret_position, 4);
6812 }
6813
6814 #[test]
6815 fn learning_and_management_use_canonical_keys() {
6816 let fixture = fixture();
6817 let mut session = open(&fixture.paths, false, mixed());
6818 type_digits(&mut session, "64");
6819 assert_eq!(
6820 session.select(index_of(&session, "米")).commit.as_deref(),
6821 Some("米")
6822 );
6823 let mut unchanged = open(&fixture.paths, false, mixed());
6824 type_digits(&mut unchanged, "64");
6825 assert_eq!(
6826 words(&unchanged)[0],
6827 "你",
6828 "disabled learning changed ranking"
6829 );
6830
6831 let mut learner = open(&fixture.paths, true, mixed());
6832 type_digits(&mut learner, "64");
6833 let learned = learner.select(index_of(&learner, "米"));
6834 assert_eq!(learned.commit.as_deref(), Some("米"));
6835 assert_eq!(learned.diagnostic, None);
6836
6837 let mut managed = open(&fixture.paths, false, mixed());
6838 type_digits(&mut managed, "64");
6839 assert_eq!(words(&managed)[0], "米", "learning did not persist");
6840 let pinned = managed.pin(index_of(&managed, "你"));
6841 assert_eq!(pinned, KeyResult::handled());
6842 assert_eq!(managed.snapshot().editing_text, "64");
6843 assert_eq!(words(&managed)[0], "你");
6844 assert!(
6845 !managed.remove(index_of(&managed, "你")).handled,
6846 "single character was removed"
6847 );
6848 let ogham = index_of(&managed, "ogham");
6849 assert!(
6850 !managed.pin(ogham).handled,
6851 "an English row is not a pinyin dictionary row"
6852 );
6853 assert!(!managed.pin(999).handled && !managed.remove(999).handled);
6854 assert!(!managed.set_position(0, 6).handled && !managed.set_position(0, -1).handled);
6855 assert!(!managed.set_position(999, 0).handled);
6856 assert_eq!(
6857 managed.set_position(index_of(&managed, "米"), 1),
6858 KeyResult::handled()
6859 );
6860 assert_eq!(words(&managed)[0], "米", "fixed position not applied");
6861
6862 let mut fixed = open(&fixture.paths, false, mixed());
6863 type_digits(&mut fixed, "64");
6864 assert_eq!(words(&fixed)[0], "米", "fixed position not persisted");
6865 let ni = fixed
6866 .snapshot()
6867 .nine_key_spellings
6868 .iter()
6869 .position(|s| s == "ni")
6870 .expect("ni offered");
6871 fixed.choose_spelling(ni);
6872 assert_eq!(
6873 words(&fixed)[0],
6874 "你",
6875 "fixed position escaped the spelling constraint"
6876 );
6877
6878 assert_eq!(
6879 managed.set_position(index_of(&managed, "米"), 0),
6880 KeyResult::handled()
6881 );
6882 assert_eq!(words(&managed)[0], "你", "clear fixed position failed");
6883 managed.command(Command::Cancel);
6884 type_digits(&mut managed, "64426");
6885 let removed = managed.remove(index_of(&managed, "你好"));
6886 assert_eq!(removed, KeyResult::handled());
6887 assert_eq!(managed.snapshot().editing_text, "64426");
6888
6889 let mut after_removal = open(&fixture.paths, false, mixed());
6890 type_digits(&mut after_removal, "64426");
6891 assert!(
6893 !after_removal
6894 .snapshot()
6895 .candidates
6896 .iter()
6897 .any(|item| item.word == "你好" && item.source.is_dictionary()),
6898 "phrase removal did not persist"
6899 );
6900 }
6901
6902 #[test]
6903 fn failed_learning_keeps_the_commit() {
6904 let fixture = fixture();
6905 let mut paths = fixture.paths.clone();
6906 paths.user_data = fixture.directory.path().join("blocked");
6907 std::fs::create_dir_all(paths.user(assets::USER_JOURNAL)).expect("block the journal");
6908 let mut failing = open(&paths, true, mixed());
6909 type_digits(&mut failing, "64");
6910 let result = failing.finish(index_of(&failing, "米"));
6911 assert_eq!(result.commit.as_deref(), Some("米"));
6912 assert_eq!(
6913 result.diagnostic.as_deref(),
6914 Some(diagnostics::NINE_KEY_FREQUENCY_NOT_PERSISTED)
6915 );
6916 assert!(!failing.active() && failing.snapshot().preedit.is_empty());
6917 }
6918
6919 #[test]
6920 fn mixed_english_first_page_count_respects_the_page_budget() {
6921 let fixture = fixture();
6922 Connection::open(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY))
6923 .unwrap()
6924 .execute_batch("INSERT INTO english_words VALUES('night','night',900),('nights','nights',800),('nightly','nightly',700);")
6925 .unwrap();
6926 for (count, page_rows, expected) in [
6927 (
6928 1,
6929 5,
6930 vec!["你", "night", "米", "你好", "nights", "nightly", "ogham"],
6931 ),
6932 (
6933 2,
6934 5,
6935 vec!["你", "night", "nights", "米", "你好", "nightly", "ogham"],
6936 ),
6937 (
6938 3,
6939 5,
6940 vec!["你", "night", "nights", "nightly", "米", "你好", "ogham"],
6941 ),
6942 (
6943 3,
6944 2,
6945 vec!["你", "night", "米", "你好", "nights", "nightly", "ogham"],
6946 ),
6947 ] {
6948 let mut session = open(
6949 &fixture.paths,
6950 false,
6951 EnglishInputOptions {
6952 mixed_candidates: true,
6953 minimum_prefix: 2,
6954 first_page_count: count,
6955 },
6956 );
6957 session.set_first_page_rows(FirstPageRows::new(page_rows));
6958 type_digits(&mut session, "64");
6959 assert_eq!(
6960 words(&session),
6961 expected,
6962 "首页英文数={count},首页行数={page_rows}"
6963 );
6964 assert_eq!(
6965 session
6966 .select(index_of(&session, "nights"))
6967 .commit
6968 .as_deref(),
6969 Some("nights")
6970 );
6971 }
6972 }
6973
6974 #[test]
6975 fn english_only_spells_words_from_the_first_digit() {
6976 let fixture = fixture();
6977 let options = EnglishInputOptions {
6978 mixed_candidates: false,
6979 minimum_prefix: 2,
6980 first_page_count: 1,
6981 };
6982 let mut words_only = open(&fixture.paths, false, options);
6983 words_only.set_english_only(true);
6984 type_digits(&mut words_only, "65");
6985 let view = words_only.snapshot();
6986 assert_eq!(words(&words_only), ["ok", "old", "older"]);
6987 assert!(
6988 view.nine_key_spellings.is_empty(),
6989 "english nine-key offered pinyin spellings"
6990 );
6991 assert!(
6992 view.dedicated_english,
6993 "the grid's snapshot dropped the English mode"
6994 );
6995 words_only.command(Command::Cancel);
6996 type_digits(&mut words_only, "653");
6997 assert_eq!(words(&words_only), ["old", "older"]);
6998 words_only.command(Command::Cancel);
6999
7000 type_digits(&mut words_only, "6");
7002 let result = words_only.select(index_of(&words_only, "old"));
7003 assert_eq!(result.commit.as_deref(), Some("old"));
7004 assert!(!words_only.active());
7005
7006 words_only.set_english_only(false);
7008 type_digits(&mut words_only, "64");
7009 assert_eq!(words(&words_only), ["你", "米", "你好"]);
7011 assert!(!words_only.snapshot().nine_key_spellings.is_empty());
7012 assert!(!words_only.snapshot().dedicated_english);
7013 words_only.set_english_only(true);
7015 assert_eq!(words(&words_only), ["ogham"]);
7016 }
7017
7018 fn sentence_words(session: &NineKeySession) -> Vec<String> {
7019 session
7020 .snapshot()
7021 .candidates
7022 .into_iter()
7023 .filter(is_lattice_sentence)
7024 .map(|item| item.word.into_string())
7025 .collect()
7026 }
7027
7028 #[test]
7030 fn sentence_rows_follow_the_sentence_association_options() {
7031 let fixture = fixture_with(&format!(
7032 "{MAIN_FIXTURE}INSERT INTO tbl_1_h VALUES('hao','h','号',50);"
7033 ));
7034 let mut session = open(&fixture.paths, false, mixed());
7035 type_digits(&mut session, "64426");
7036 assert_eq!(sentence_words(&session), ["米好"]);
7037
7038 session.set_sentence_options(
7039 SentenceAssociationOptions {
7040 word_lattice: false,
7041 ..SentenceAssociationOptions::default()
7042 },
7043 false,
7044 );
7045 assert!(sentence_words(&session).is_empty(), "{:?}", words(&session));
7046 assert_eq!(words(&session), ["你好", "你", "米", "ogham"]);
7047
7048 session.set_sentence_options(SentenceAssociationOptions::default(), true);
7050 let sentences = sentence_words(&session);
7051 for sentence in ["米好", "米号", "你号"] {
7052 assert!(
7053 sentences.iter().any(|word| word == sentence),
7054 "{sentences:?}"
7055 );
7056 }
7057 session.command(Command::Cancel);
7058
7059 let mut fresh = open(&fixture.paths, false, mixed());
7061 fresh.set_sentence_options(
7062 SentenceAssociationOptions {
7063 word_lattice: false,
7064 ..SentenceAssociationOptions::default()
7065 },
7066 false,
7067 );
7068 type_digits(&mut fresh, "64426");
7069 assert!(sentence_words(&fresh).is_empty(), "{:?}", words(&fresh));
7070 }
7071
7072 #[test]
7074 fn sentences_are_not_assembled_from_offensive_words() {
7075 let fixture = fixture_with(
7076 "CREATE TABLE tbl_1_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7077INSERT INTO tbl_1_w VALUES('wo','w','我',30000000);
7078CREATE TABLE tbl_1_m(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7079INSERT INTO tbl_1_m VALUES('men','m','们',3000000);
7080CREATE TABLE tbl_2_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7081INSERT INTO tbl_2_w VALUES('wo''men','wm','我们',13696325);
7082CREATE TABLE tbl_3_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7083INSERT INTO tbl_3_w VALUES('wo''men''you','wmy','我们有',83600);
7084INSERT INTO tbl_3_w VALUES('wo''men''zou','wmz','我们走',4361);
7085CREATE TABLE tbl_1_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7086INSERT INTO tbl_1_y VALUES('you','y','有',15615006);
7087INSERT INTO tbl_1_y VALUES('you','y','又',2794198);
7088CREATE TABLE tbl_1_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7089INSERT INTO tbl_1_z VALUES('zou','z','走',1052468);
7090CREATE TABLE tbl_1_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7091INSERT INTO tbl_1_j VALUES('ju','j','据',323831);
7092CREATE TABLE tbl_1_l(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7093INSERT INTO tbl_1_l VALUES('lu','l','路',476190);
7094CREATE TABLE tbl_1_q(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7095INSERT INTO tbl_1_q VALUES('qu','q','去',9884736);
7096CREATE TABLE tbl_1_r(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7097INSERT INTO tbl_1_r VALUES('ru','r','如',2514076);
7098INSERT INTO tbl_1_r VALUES('ru','r','乳',26706);
7099CREATE TABLE tbl_2_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7100INSERT INTO tbl_2_j VALUES('ju''ru','jr','巨乳',3174);
7101CREATE TABLE tbl_2_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7102INSERT INTO tbl_2_z VALUES('zou''lu','zl','走路',163972);
7103CREATE TABLE tbl_2_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7104INSERT INTO tbl_2_y VALUES('you''ju','yj','邮局',22016);",
7105 );
7106 let mut session = open(&fixture.paths, false, mixed());
7107 type_digits(&mut session, "966369685878");
7108 let sentences = sentence_words(&session);
7109 assert!(!sentences.is_empty(), "{:?}", words(&session));
7110 assert!(
7111 words(&session).iter().all(|word| !word.contains("巨乳")),
7112 "{:?}",
7113 words(&session)
7114 );
7115 session.command(Command::Cancel);
7116
7117 type_digits(&mut session, "5878");
7118 assert!(words(&session).contains(&"巨乳".to_owned()));
7119 }
7120
7121 #[test]
7123 fn sentences_are_not_assembled_from_words_containing_offensive_words() {
7124 let fixture = fixture_with(
7125 "CREATE TABLE tbl_1_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7126INSERT INTO tbl_1_w VALUES('wo','w','我',26297004);
7127CREATE TABLE tbl_1_m(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7128INSERT INTO tbl_1_m VALUES('men','m','们',544382);
7129CREATE TABLE tbl_2_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7130INSERT INTO tbl_2_w VALUES('wo''men','wm','我们',13696325);
7131CREATE TABLE tbl_3_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7132INSERT INTO tbl_3_w VALUES('wo''men''yao','wmy','我们要',330188);
7133CREATE TABLE tbl_1_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7134INSERT INTO tbl_1_y VALUES('yao','y','要',7812383);
7135CREATE TABLE tbl_1_f(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7136INSERT INTO tbl_1_f VALUES('fa','f','发',666088);
7137CREATE TABLE tbl_1_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7138INSERT INTO tbl_1_z VALUES('zhan','z','展',973557);
7139CREATE TABLE tbl_1_x(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7140INSERT INTO tbl_1_x VALUES('xian','x','现',299876);
7141CREATE TABLE tbl_2_f(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7142INSERT INTO tbl_2_f VALUES('fa''xian','fx','发现',3492384);
7143INSERT INTO tbl_2_f VALUES('fa''zhan','fz','发展',2158053);
7144CREATE TABLE tbl_1_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7145INSERT INTO tbl_1_j VALUES('jing','j','经',289401);
7146INSERT INTO tbl_1_j VALUES('jing','j','净',14695);
7147INSERT INTO tbl_1_j VALUES('ji','j','济',7885);
7148INSERT INTO tbl_1_j VALUES('ji','j','鸡',7677);
7149CREATE TABLE tbl_2_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7150INSERT INTO tbl_2_j VALUES('jing''ji','jj','经济',849252);
7151INSERT INTO tbl_2_j VALUES('ji''ba','jb','鸡巴',2190);
7152CREATE TABLE tbl_3_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7153INSERT INTO tbl_3_j VALUES('jing''ji''ba','jjb','净鸡巴',100);
7154CREATE TABLE tbl_1_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);
7155INSERT INTO tbl_1_b VALUES('ba','b','吧',8469475);
7156INSERT INTO tbl_1_b VALUES('ba','b','巴',16099);",
7157 );
7158 let mut session = open(&fixture.paths, false, mixed());
7159 type_digits(&mut session, "9663692632942654645422");
7160 assert!(
7161 !sentence_words(&session).is_empty(),
7162 "{:?}",
7163 words(&session)
7164 );
7165 assert!(
7166 words(&session).iter().all(|word| !word.contains("鸡巴")),
7167 "{:?}",
7168 words(&session)
7169 );
7170 }
7171
7172 #[test]
7174 fn a_missing_keyboard_model_leaves_the_lattice_rows_to_their_switch() {
7175 let fixture = fixture();
7176 let mut session = open(&fixture.paths, false, mixed());
7177 let neural = |word_lattice| SentenceAssociationOptions {
7178 word_lattice,
7179 neural_keyboard: true,
7180 show_next_on_duplicate: false,
7181 ..Default::default()
7182 };
7183 session.set_sentence_options(neural(true), false);
7184 type_digits(&mut session, "64426");
7185 assert_eq!(sentence_words(&session), ["米好"]);
7186 assert!(!session
7187 .snapshot()
7188 .candidates
7189 .iter()
7190 .any(|item| item.source == CandidateSource::NeuralKeyboard));
7191 session.set_sentence_options(neural(false), false);
7192 assert!(sentence_words(&session).is_empty(), "{:?}", words(&session));
7193 }
7194
7195 #[test]
7196 fn the_rescoring_context_keeps_the_characters_the_model_reads() {
7197 let fixture = fixture();
7198 let mut session = open(&fixture.paths, false, mixed());
7199 let long: String = "甲乙".repeat(CONTEXT_CHARACTERS);
7200 session.set_rescoring_context(&long);
7201 assert_eq!(
7202 session.rescoring_context.chars().count(),
7203 CONTEXT_CHARACTERS
7204 );
7205 assert!(long.ends_with(&session.rescoring_context));
7206 session.set_rescoring_context("今天");
7207 assert_eq!(session.rescoring_context, "今天");
7208 }
7209
7210 fn sentence(word: &str, weight: i64) -> WordItem {
7211 let mut item = item(word, "64426", weight, CandidateSource::Generated);
7212 item.sentence_association = true;
7213 item
7214 }
7215
7216 #[test]
7217 fn lattice_sentence_slots_filter_in_order_without_allocations() {
7218 let mut candidates = Vec::new();
7219 candidates.push(item("跳过来源", "64426", 0, CandidateSource::Database));
7220 candidates.push(sentence("整句 0", 0));
7221 candidates.push(sentence("整句 1", 0));
7222 let mut non_sentence = sentence("跳过标记", 0);
7223 non_sentence.sentence_association = false;
7224 candidates.push(non_sentence);
7225 for index in 2..14 {
7226 candidates.push(sentence(&format!("整句 {index}"), 0));
7227 }
7228
7229 let ((slots, len), allocations) = crate::ime::personal_rerank::allocations::count(|| {
7230 collect_lattice_sentences(&candidates)
7231 });
7232
7233 assert_eq!(len, MAX_RERANK_PATHS);
7234 assert_eq!(slots[0].map(|item| item.word.as_str()), Some("整句 0"));
7235 assert_eq!(slots[1].map(|item| item.word.as_str()), Some("整句 1"));
7236 assert_eq!(
7237 slots[MAX_RERANK_PATHS - 1].map(|item| item.word.as_str()),
7238 Some("整句 11")
7239 );
7240 assert_eq!(allocations, 0);
7241 }
7242
7243 #[test]
7244 fn rerank_cache_match_uses_borrowed_rows_without_allocating() {
7245 let rows = [sentence("整句 0", 17), sentence("整句 1", 23)];
7246 let scored = [Some(&rows[0]), Some(&rows[1])];
7247 let entry = RerankEntry {
7248 context: "上文".to_owned(),
7249 sentences: rows
7250 .iter()
7251 .map(|row| (row.word.to_string(), comparable_weight(row)))
7252 .collect(),
7253 ranked: vec!["整句 1".to_owned(), "整句 0".to_owned()],
7254 };
7255 let (matches, allocations) = crate::ime::personal_rerank::allocations::count(|| {
7256 rerank_cache_matches(&entry, "上文", &scored)
7257 });
7258 assert!(matches);
7259 assert_eq!(allocations, 0);
7260 assert!(!rerank_cache_matches(&entry, "别的上下文", &scored));
7261 }
7262
7263 fn row_sources(candidates: &[WordItem]) -> Vec<(&str, CandidateSource)> {
7264 candidates
7265 .iter()
7266 .map(|item| (item.word.as_str(), item.source))
7267 .collect()
7268 }
7269
7270 #[test]
7272 fn the_keyboard_pick_follows_the_best_lattice_sentence() {
7273 use CandidateSource::{Database, Generated, NeuralKeyboard};
7274 let list = || {
7275 vec![
7276 item("你好", "64426", 1000, Database),
7277 sentence("米好", -1000),
7278 sentence("你号", -2000),
7279 sentence("米号", -3000),
7280 ]
7281 };
7282 let ranked = |sentences: &[&str]| -> Vec<String> {
7283 sentences.iter().map(|word| (*word).to_owned()).collect()
7284 };
7285
7286 let mut candidates = list();
7287 place_keyboard_pick(
7288 &mut candidates,
7289 &ranked(&["米号", "米好", "你号"]),
7290 false,
7291 true,
7292 );
7293 assert_eq!(
7294 row_sources(&candidates),
7295 [
7296 ("你好", Database),
7297 ("米好", Generated),
7298 ("米号", NeuralKeyboard),
7299 ("你号", Generated)
7300 ]
7301 );
7302
7303 let mut candidates = list();
7305 place_keyboard_pick(
7306 &mut candidates,
7307 &ranked(&["米好", "你号", "米号"]),
7308 false,
7309 true,
7310 );
7311 assert_eq!(row_sources(&candidates), row_sources(&list()));
7312 let mut candidates = list();
7313 place_keyboard_pick(
7314 &mut candidates,
7315 &ranked(&["米好", "你号", "米号"]),
7316 true,
7317 true,
7318 );
7319 assert_eq!(candidates[2].word, "你号");
7320 assert_eq!(candidates[2].source, NeuralKeyboard);
7321
7322 let mut candidates = list();
7324 place_keyboard_pick(
7325 &mut candidates,
7326 &ranked(&["米号", "米好", "你号"]),
7327 false,
7328 false,
7329 );
7330 assert_eq!(
7331 row_sources(&candidates),
7332 [("你好", Database), ("米号", NeuralKeyboard)]
7333 );
7334 let mut candidates = list();
7336 place_keyboard_pick(
7337 &mut candidates,
7338 &ranked(&["米好", "你号", "米号"]),
7339 false,
7340 false,
7341 );
7342 assert_eq!(
7343 row_sources(&candidates),
7344 [("你好", Database), ("米好", NeuralKeyboard)]
7345 );
7346 let mut candidates = list();
7347 place_keyboard_pick(&mut candidates, &[], false, false);
7348 assert_eq!(row_sources(&candidates), [("你好", Database)]);
7349 }
7350
7351 #[test]
7353 fn digits_without_a_reading_offer_english_even_with_mixing_off() {
7354 let fixture = fixture();
7355 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
7356 type_digits(&mut session, "77");
7357 assert_eq!(words(&session), ["QQ"]);
7358 session.command(Command::Cancel);
7359 type_digits(&mut session, "64426");
7360 assert!(!words(&session).contains(&"ogham".to_owned()));
7361 }
7362
7363 #[test]
7364 fn a_missing_english_dictionary_means_no_english_rows() {
7365 let fixture = fixture();
7366 std::fs::remove_file(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY))
7367 .expect("remove msime-english.db");
7368 let mut session = open(&fixture.paths, false, mixed());
7369 type_digits(&mut session, "64");
7370 assert_eq!(words(&session), ["你", "米", "你好"]);
7371 session.set_english_only(true);
7372 assert!(words(&session).is_empty());
7373 }
7374
7375 const INITIALS_FIXTURE: &str = "CREATE TABLE tbl_1_m(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_m VALUES('mu','m','木',100);CREATE TABLE tbl_1_o(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_o VALUES('ou','o','欧',50);CREATE TABLE tbl_1_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_w VALUES('wo','w','我',100);CREATE TABLE tbl_1_d(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_d VALUES('di','d','滴',60);CREATE TABLE tbl_1_g(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_g VALUES('ge','g','个',100);CREATE TABLE tbl_1_t(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_t VALUES('tian','t','天',100);CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_n VALUES('na','n','呐',40);CREATE TABLE tbl_2_m(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_m VALUES('mei''tian','mt','每天',900);INSERT INTO tbl_2_m VALUES('ming''tian','mt','明天',800);CREATE TABLE tbl_2_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_n VALUES('na''tian','nt','那天',300);CREATE TABLE tbl_2_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_w VALUES('wo''di','wd','我滴',30);CREATE TABLE tbl_3_g(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_3_g VALUES('ge''tian''na','gtn','个天呐',20);CREATE TABLE tbl_4_c(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_4_c VALUES('chi''fan''le''mei','cflm','吃饭了没',100);CREATE TABLE tbl_4_g(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_4_g VALUES('guan''guan''ju''jiu','ggjj','关关雎鸠',1605);CREATE TABLE tbl_5_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);";
7376
7377 fn type_keys(session: &mut NineKeySession, keys: &str) {
7378 for key in keys.bytes() {
7379 assert!(session.character(key).handled, "key {key} unhandled");
7380 }
7381 }
7382
7383 const COMPLETION_FIXTURE: &str = "CREATE TABLE tbl_1_d(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_d VALUES('de','d','的',1000);CREATE TABLE tbl_1_e(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_e VALUES('e','e','额',900);CREATE TABLE tbl_1_c(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_c VALUES('chi','c','吃',500);CREATE TABLE tbl_1_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_a VALUES('ai','a','爱',800);CREATE TABLE tbl_1_g(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_g VALUES('ge','g','个',900);CREATE TABLE tbl_1_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_w VALUES('wo','w','我',700);CREATE TABLE tbl_2_c(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_c VALUES('chi''fan','cf','吃饭',1500);CREATE TABLE tbl_2_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_a VALUES('ai''ge','ag','挨个',1000);CREATE TABLE tbl_2_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_w VALUES('wo''de','wd','我的',150);INSERT INTO tbl_2_w VALUES('wo''e','we','我饿',100);";
7384
7385 #[test]
7387 fn without_a_split_a_completion_is_discounted_beside_a_spelled_phrase() {
7388 let fixture = fixture_with(COMPLETION_FIXTURE);
7389 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
7390
7391 type_digits(&mut session, "3");
7392 assert_eq!(words(&session)[..2], ["的", "额"]);
7393 session.command(Command::Cancel);
7394
7395 const { assert!(1500 < 1000 * COMPLETION_WEIGHT_RATIO) };
7397 type_digits(&mut session, "2443");
7398 let shown = words(&session);
7399 assert_eq!(shown[0], "挨个");
7400 assert!(
7401 shown
7402 .iter()
7403 .take(DEFAULT_FIRST_PAGE_ROWS)
7404 .any(|word| word == "吃饭"),
7405 "{shown:?}"
7406 );
7407 assert_eq!(session.snapshot().nine_key_reading, "ai'ge");
7408 session.command(Command::Cancel);
7409
7410 type_digits(&mut session, "2443");
7412 let chi = session
7413 .snapshot()
7414 .nine_key_spellings
7415 .iter()
7416 .position(|spelling| spelling == "chi")
7417 .expect("chi offered");
7418 session.choose_spelling(chi);
7419 assert_eq!(words(&session)[0], "吃饭");
7420 session.command(Command::Cancel);
7421
7422 const { assert!(150 < 100 * COMPLETION_WEIGHT_RATIO) };
7424 type_digits(&mut session, "963");
7425 assert_eq!(words(&session)[..2], ["我饿", "我的"]);
7426 let wo = session
7427 .snapshot()
7428 .nine_key_spellings
7429 .iter()
7430 .position(|spelling| spelling == "wo")
7431 .expect("wo offered");
7432 session.choose_spelling(wo);
7433 assert_eq!(words(&session)[..2], ["我的", "我饿"]);
7434 }
7435
7436 #[test]
7438 fn pinning_a_discounted_completion_puts_it_first() {
7439 for chifan in [1500, 500] {
7440 let fixture = fixture_with(
7441 &COMPLETION_FIXTURE.replace("'吃饭',1500", &format!("'吃饭',{chifan}")),
7442 );
7443 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
7444 type_digits(&mut session, "2443");
7445 assert_eq!(words(&session)[0], "挨个");
7446 let pinned = session.pin(index_of(&session, "吃饭"));
7447 assert!(pinned.handled);
7448 assert_eq!(pinned.diagnostic, None);
7449 assert_eq!(words(&session)[0], "吃饭", "weight {chifan}");
7450 session.command(Command::Cancel);
7451 let mut fresh = open(&fixture.paths, false, EnglishInputOptions::default());
7452 type_digits(&mut fresh, "2443");
7453 assert_eq!(words(&fresh)[0], "吃饭", "weight {chifan}");
7454 }
7455 }
7456
7457 #[test]
7459 fn a_mixed_english_row_does_not_push_a_discounted_completion_off_the_first_page() {
7460 let fixture = fixture_with("CREATE TABLE tbl_2_c(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_c VALUES('chi''fan','cf','吃饭',1100);INSERT INTO tbl_2_c VALUES('ci''ge','cg','次格',700);CREATE TABLE tbl_2_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_a VALUES('ai''ge','ag','挨个',1000);INSERT INTO tbl_2_a VALUES('ai''he','ah','爱河',900);CREATE TABLE tbl_2_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_b VALUES('bi''he','bh','碧荷',800);INSERT INTO tbl_2_b VALUES('bi''ge','bg','笔格',600);");
7461 Connection::open(fixture.directory.path().join(assets::ENGLISH_DICTIONARY))
7462 .and_then(|db| {
7463 db.execute_batch("INSERT INTO english_words VALUES('chief','chief',1000);")
7464 })
7465 .expect("english row");
7466 let mut session = open(&fixture.paths, false, mixed());
7467 type_digits(&mut session, "2443");
7468 let shown = words(&session);
7469 assert_eq!(
7470 shown[..DEFAULT_FIRST_PAGE_ROWS],
7471 ["挨个", "chief", "爱河", "碧荷", "次格", "吃饭"],
7472 "{shown:?}"
7473 );
7474 }
7475
7476 #[test]
7478 fn digits_spell_words_by_their_initials() {
7479 let fixture = fixture_with(INITIALS_FIXTURE);
7480 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
7481
7482 type_digits(&mut session, "68");
7484 assert_eq!(words(&session), ["木", "欧", "每天", "明天", "那天"]);
7485 assert_eq!(session.snapshot().nine_key_reading, "mu");
7486 session.command(Command::Cancel);
7487
7488 type_keys(&mut session, "6'8");
7490 assert_eq!(words(&session)[..3], ["每天", "明天", "那天"]);
7491 assert_eq!(session.snapshot().nine_key_reading, "m't");
7492 let chosen = session.select(index_of(&session, "明天"));
7493 assert_eq!(chosen.commit.as_deref(), Some("明天"));
7494 assert!(!session.active(), "an initials row consumes every digit");
7495
7496 type_digits(&mut session, "2356");
7498 assert_eq!(words(&session)[0], "吃饭了没");
7499 assert_eq!(session.snapshot().nine_key_reading, "c'f'l'm");
7500 session.command(Command::Cancel);
7501 type_digits(&mut session, "4455");
7502 assert_eq!(words(&session)[0], "关关雎鸠");
7503 session.command(Command::Cancel);
7504
7505 type_keys(&mut session, "68'");
7507 assert!(!words(&session).iter().any(|word| word == "每天"));
7508 session.command(Command::Cancel);
7509 type_keys(&mut session, "6'84");
7511 assert!(!words(&session).iter().any(|word| word.chars().count() == 3));
7512 session.command(Command::Cancel);
7513
7514 type_digits(&mut session, "68");
7516 let mu = session
7517 .snapshot()
7518 .nine_key_spellings
7519 .iter()
7520 .position(|spelling| spelling == "mu")
7521 .expect("mu offered");
7522 session.choose_spelling(mu);
7523 assert!(!words(&session).iter().any(|word| word == "每天"));
7524 }
7525
7526 #[test]
7528 fn recently_selected_candidate_leads_the_same_nine_key_context() {
7529 let fixture = fixture_with(
7530 "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);\
7531INSERT INTO tbl_1_n VALUES('neng','n','能',1),('neng','n','恁',100),('neng','n','嫩',90),('neng','n','嗯',80),('neng','n','𠮾',70),('neng','n','莻',60);",
7532 );
7533 let mut first = open(&fixture.paths, true, EnglishInputOptions::default());
7534 type_digits(&mut first, "6364");
7535 let selected = index_of(&first, "能");
7536 assert!(selected > 0, "fixture must require a non-first selection");
7537 assert_eq!(first.select(selected).commit.as_deref(), Some("能"));
7538
7539 let mut repeated = open(&fixture.paths, true, EnglishInputOptions::default());
7540 type_digits(&mut repeated, "6364");
7541 assert_eq!(words(&repeated)[0], "能");
7542 assert_eq!(repeated.snapshot().candidates[0].fixed_position, 1);
7543 }
7544
7545 const SPELLING_RECENT_FIXTURE: &str =
7546 "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);\
7547INSERT INTO tbl_1_n VALUES('ni','n','你',1000);\
7548CREATE TABLE tbl_1_m(key TEXT,jp TEXT,value TEXT,weight INTEGER);\
7549INSERT INTO tbl_1_m VALUES('mi','m','米',100),('mi','m','谜',10),('mo','m','墨',100);\
7550CREATE TABLE tbl_1_h(key TEXT,jp TEXT,value TEXT,weight INTEGER);\
7551INSERT INTO tbl_1_h VALUES('hao','h','好',100);";
7552
7553 fn lock_spelling(session: &mut NineKeySession, spelling: &str) {
7554 let index = session
7555 .snapshot()
7556 .nine_key_spellings
7557 .iter()
7558 .position(|choice| choice == spelling)
7559 .expect("synthetic spelling offered");
7560 assert!(session.choose_spelling(index).handled);
7561 }
7562
7563 #[test]
7565 fn spelling_selection_remembers_first_and_later_candidates_without_a_lock() {
7566 for (word, selected_index) in [("米", 0), ("谜", 1)] {
7567 let fixture = fixture_with(SPELLING_RECENT_FIXTURE);
7568 let mut session =
7569 open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Disabled, 1, 1);
7570 type_digits(&mut session, "64");
7571 assert_eq!(words(&session)[0], "你");
7572 lock_spelling(&mut session, "mi");
7573 assert_eq!(index_of(&session, word), selected_index);
7574 let selected = session.select(selected_index);
7575 assert_eq!(selected.commit.as_deref(), Some(word));
7576 assert_eq!(selected.diagnostic, None);
7577 drop(session);
7578
7579 let mut repeated = open(&fixture.paths, false, EnglishInputOptions::default());
7580 type_digits(&mut repeated, "64");
7581 assert_eq!(words(&repeated)[0], word);
7582 assert_eq!(repeated.snapshot().candidates[0].fixed_position, 1);
7583 lock_spelling(&mut repeated, "ni");
7584 assert_eq!(words(&repeated)[0], "你");
7585 }
7586 }
7587
7588 #[test]
7590 fn spelling_recent_requires_learning_and_an_explicit_whole_selection() {
7591 for case in ["learning_off", "finish", "partial", "unlocked_first"] {
7592 let fixture = fixture_with(SPELLING_RECENT_FIXTURE);
7593 let mut session =
7594 open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Disabled, 1, 1);
7595 session.learning = case != "learning_off";
7596 let digits = if case == "partial" { "64426" } else { "64" };
7597 type_digits(&mut session, digits);
7598 if case != "unlocked_first" {
7599 lock_spelling(&mut session, "mi");
7600 }
7601 let word = if case == "unlocked_first" {
7602 "你"
7603 } else {
7604 "米"
7605 };
7606 let index = index_of(&session, word);
7607 let result = if case == "finish" {
7608 session.finish(index)
7609 } else {
7610 session.select(index)
7611 };
7612 assert_eq!(result.commit.as_deref(), Some(word), "{case}");
7613 assert_eq!(result.diagnostic, None, "{case}");
7614 if case == "partial" {
7615 assert!(session.active());
7616 }
7617 let journal = fixture.paths.user(assets::USER_JOURNAL);
7618 for (key, spelling, value) in
7619 [("64", "mi", "米"), (digits, "mi", "米"), ("64", "ni", "你")]
7620 {
7621 assert!(
7622 !positions::is_fixed(&journal, &format!("nine-key:{key}:"), spelling, value),
7623 "{case}"
7624 );
7625 assert!(
7626 !positions::is_fixed(
7627 &journal,
7628 &format!("nine-key:{key}:{spelling}"),
7629 spelling,
7630 value
7631 ),
7632 "{case}"
7633 );
7634 }
7635 }
7636 }
7637
7638 #[test]
7640 fn spelling_recent_replaces_the_previous_unlocked_pick() {
7641 let fixture = fixture_with(SPELLING_RECENT_FIXTURE);
7642 let journal = fixture.paths.user(assets::USER_JOURNAL);
7643 positions::set_fixed_position(&journal, "nine-key:64:mi", "mi", "米", 1).unwrap();
7644 for word in ["米", "你", "米"] {
7645 let mut session =
7646 open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Disabled, 1, 1);
7647 type_digits(&mut session, "64");
7648 if word == "米" {
7649 lock_spelling(&mut session, "mi");
7650 assert_eq!(session.snapshot().candidates[0].fixed_position, 1);
7651 }
7652 let index = index_of(&session, word);
7653 assert_eq!(session.select(index).commit.as_deref(), Some(word));
7654 let mut repeated = open(&fixture.paths, false, EnglishInputOptions::default());
7655 type_digits(&mut repeated, "64");
7656 assert_eq!(words(&repeated)[0], word);
7657 lock_spelling(&mut repeated, "ni");
7658 assert_eq!(words(&repeated), ["你"]);
7659 repeated.command(Command::Cancel);
7660 type_digits(&mut repeated, "66");
7661 assert_eq!(words(&repeated)[0], "墨");
7662 assert_eq!(repeated.snapshot().candidates[0].fixed_position, 0);
7663 }
7664 }
7665
7666 #[test]
7668 fn spelling_recent_preserves_unlocked_manual_slots() {
7669 for selected_is_fixed in [false, true] {
7670 let fixture = fixture_with(SPELLING_RECENT_FIXTURE);
7671 let journal = fixture.paths.user(assets::USER_JOURNAL);
7672 positions::set_fixed_position(&journal, "nine-key:64:", "mi", "谜", 2).unwrap();
7673 if selected_is_fixed {
7674 positions::set_fixed_position(&journal, "nine-key:64:", "mi", "米", 3).unwrap();
7675 }
7676 let mut session =
7677 open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Disabled, 1, 1);
7678 type_digits(&mut session, "64");
7679 lock_spelling(&mut session, "mi");
7680 assert_eq!(session.select(0).commit.as_deref(), Some("米"));
7681 let mut repeated = open(&fixture.paths, false, EnglishInputOptions::default());
7682 type_digits(&mut repeated, "64");
7683 let snapshot = repeated.snapshot();
7684 let target = &snapshot.candidates[index_of(&repeated, "米")];
7685 assert_eq!(target.fixed_position, if selected_is_fixed { 3 } else { 1 });
7686 assert_eq!(snapshot.candidates[1].word, "谜");
7687 assert_eq!(snapshot.candidates[1].fixed_position, 2);
7688 }
7689 }
7690
7691 #[test]
7693 fn initials_rows_survive_more_syllable_rows_than_the_limit() {
7694 let mut main = INITIALS_FIXTURE.to_owned();
7695 for index in 0..CANDIDATE_LIMIT + 72 {
7696 let weight = 2000 - 10 * index as i64;
7697 main.push_str(&format!(
7698 "INSERT INTO tbl_1_m VALUES('mu','m','木{index}',{weight});"
7699 ));
7700 }
7701 let fixture = fixture_with(&main);
7702 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
7703 type_digits(&mut session, "68");
7704 let listed = words(&session);
7705 assert_eq!(listed.len(), CANDIDATE_LIMIT);
7706 assert_eq!(listed[..5], ["木0", "木1", "木2", "木3", "木4"]);
7708 let position = |word: &str| {
7709 listed
7710 .iter()
7711 .position(|candidate| candidate == word)
7712 .unwrap_or_else(|| panic!("{word} was cut from {listed:?}"))
7713 };
7714 assert_eq!(position("每天"), position("木110") + 1);
7716 assert_eq!(position("明天"), position("木120") + 1);
7717 assert_eq!(session.snapshot().nine_key_reading, "mu");
7718 session.command(Command::Cancel);
7719
7720 type_keys(&mut session, "6'8");
7722 assert_eq!(words(&session)[..3], ["每天", "明天", "那天"]);
7723 }
7724
7725 #[test]
7727 fn learned_phrases_come_back_by_their_initials() {
7728 let fixture = fixture_with(INITIALS_FIXTURE);
7729 let mut session = open(&fixture.paths, true, EnglishInputOptions::default());
7730 type_digits(&mut session, "963443842662");
7732 let first = session.select(index_of(&session, "我滴"));
7733 assert_eq!(first.commit.as_deref(), Some("我滴"));
7734 assert_eq!(first.diagnostic, None);
7735 assert!(session.active());
7736 let rest = session.select(index_of(&session, "个天呐"));
7737 assert_eq!(rest.commit.as_deref(), Some("个天呐"));
7738 assert_eq!(rest.diagnostic, None);
7739 assert!(!session.active());
7740
7741 let mut later = open(&fixture.paths, false, EnglishInputOptions::default());
7742 type_keys(&mut later, "9'3'4'8'6");
7743 assert_eq!(words(&later)[0], "我滴个天呐");
7744 assert_eq!(later.snapshot().nine_key_reading, "w'd'g't'n");
7745 later.command(Command::Cancel);
7746 type_digits(&mut later, "93486");
7748 assert!(words(&later).iter().any(|word| word == "我滴个天呐"));
7749 }
7750
7751 fn crowded_initials_fixture() -> Fixture {
7753 let mut main = String::from(
7754 "CREATE TABLE tbl_1_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_y VALUES('yin','y','因',100);\
7755CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_s VALUES('si','s','四',100);\
7756CREATE TABLE tbl_2_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_y VALUES('yin''si','ys','隐私',10);",
7757 );
7758 for index in 0..200 {
7759 main.push_str(&format!(
7760 "INSERT INTO tbl_2_y VALUES('ya''qi','yq','压{index}',{});",
7761 100_000 - index
7762 ));
7763 }
7764 fixture_with(&main)
7765 }
7766
7767 fn open_with_frequency(
7768 paths: &RuntimePaths,
7769 mode: FrequencyAdjustmentMode,
7770 trigger_count: i32,
7771 linear_step: i32,
7772 ) -> NineKeySession {
7773 NineKeySession::new(
7774 paths,
7775 true,
7776 FrequencyAdjustmentOptions {
7777 mode,
7778 trigger_count,
7779 linear_step,
7780 },
7781 FuzzyPinyinOptions::default(),
7782 EnglishInputOptions::default(),
7783 true,
7784 false,
7785 )
7786 }
7787
7788 fn record_use(paths: &RuntimePaths, word: &str) {
7790 crate::user_dictionary::ngram_store::PersonalNgramStore::for_journal(
7791 &paths.user(assets::USER_JOURNAL),
7792 )
7793 .record(&[PersonalTransition {
7794 earlier: String::new(),
7795 previous: String::new(),
7796 word: word.into(),
7797 times: PERSONAL_PICK_TIMES,
7798 }])
7799 .expect("record a use");
7800 }
7801
7802 fn personal_count(paths: &RuntimePaths, word: &str) -> u32 {
7803 crate::user_dictionary::ngram_store::PersonalNgramStore::for_journal(
7804 &paths.user(assets::USER_JOURNAL),
7805 )
7806 .model()
7807 .word_count(word)
7808 }
7809
7810 #[test]
7812 fn used_initials_words_survive_the_row_limit_and_move_up() {
7813 let fixture = crowded_initials_fixture();
7814 let mut session =
7815 open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Promote, 1, 1);
7816 type_keys(&mut session, "9'7");
7817 assert!(!words(&session).iter().any(|word| word == "隐私"));
7818 session.command(Command::Cancel);
7819
7820 type_digits(&mut session, "94674");
7822 assert_eq!(index_of(&session, "隐私"), 0);
7823 let chosen = session.select(0);
7824 assert_eq!(chosen.commit.as_deref(), Some("隐私"));
7825 assert_eq!(chosen.diagnostic, None);
7826 assert_eq!(personal_count(&fixture.paths, "隐私"), PERSONAL_PICK_TIMES);
7827 type_keys(&mut session, "9'7");
7828 assert_eq!(index_of(&session, "隐私"), 4);
7829 assert_eq!(words(&session)[..4], ["压0", "压1", "压2", "压3"]);
7830 session.command(Command::Cancel);
7831
7832 record_use(&fixture.paths, "隐私");
7833 type_keys(&mut session, "9'7");
7834 assert_eq!(index_of(&session, "隐私"), 3);
7835 session.command(Command::Cancel);
7836 type_digits(&mut session, "97");
7838 assert_eq!(index_of(&session, "隐私"), 3);
7839 }
7840
7841 #[test]
7843 fn picking_a_used_initials_word_moves_it_once() {
7844 let fixture = crowded_initials_fixture();
7845 record_use(&fixture.paths, "隐私");
7846 let mut session =
7847 open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Promote, 1, 1);
7848 type_keys(&mut session, "9'7");
7849 assert_eq!(index_of(&session, "隐私"), 4);
7850 let chosen = session.select(4);
7851 assert_eq!(chosen.commit.as_deref(), Some("隐私"));
7852 assert_eq!(chosen.diagnostic, None);
7853 assert_eq!(
7854 personal_count(&fixture.paths, "隐私"),
7855 2 * PERSONAL_PICK_TIMES
7856 );
7857 type_keys(&mut session, "9'7");
7858 assert_eq!(index_of(&session, "隐私"), 3);
7859 assert_eq!(words(&session)[..3], ["压0", "压1", "压2"]);
7860 }
7861
7862 #[test]
7864 fn a_used_word_tied_with_a_crowded_table_is_kept() {
7865 let mut main = String::from(
7866 "CREATE TABLE tbl_2_w(key TEXT,jp TEXT,value TEXT,weight INTEGER);\
7867CREATE TABLE tbl_2_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_y VALUES('yin''si','ys','隐私',100);",
7868 );
7869 for index in 0..INITIALS_SCAN_LIMIT + 10 {
7870 main.push_str(&format!(
7871 "INSERT INTO tbl_2_w VALUES('wa''pi','wp','瓦{index}',100);"
7872 ));
7873 }
7874 let fixture = fixture_with(&main);
7875 record_use(&fixture.paths, "隐私");
7876 let mut session =
7878 open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Disabled, 1, 1);
7879 type_keys(&mut session, "9'7");
7880 assert_eq!(index_of(&session, "隐私"), 0);
7881 assert_eq!(words(&session).len(), INITIALS_ROW_LIMIT);
7882 }
7883
7884 #[test]
7886 fn used_initials_words_follow_the_frequency_mode_and_trigger_count() {
7887 let position = |mode, trigger_count, linear_step, uses| {
7888 let fixture = crowded_initials_fixture();
7889 for _ in 0..uses {
7890 record_use(&fixture.paths, "隐私");
7891 }
7892 let mut session = open_with_frequency(&fixture.paths, mode, trigger_count, linear_step);
7893 type_keys(&mut session, "9'7");
7894 index_of(&session, "隐私")
7895 };
7896 assert_eq!(position(FrequencyAdjustmentMode::Pin, 1, 1, 1), 0);
7898 assert_eq!(position(FrequencyAdjustmentMode::Halve, 1, 1, 1), 31);
7899 assert_eq!(position(FrequencyAdjustmentMode::Halve, 1, 1, 2), 15);
7900 assert_eq!(position(FrequencyAdjustmentMode::Linear, 1, 3, 2), 57);
7901 assert_eq!(position(FrequencyAdjustmentMode::Promote, 2, 1, 1), 63);
7903 assert_eq!(position(FrequencyAdjustmentMode::Promote, 2, 1, 2), 4);
7904 }
7905
7906 #[test]
7908 fn used_initials_words_only_move_with_frequency_adjustment() {
7909 let fixture = crowded_initials_fixture();
7910 record_use(&fixture.paths, "隐私");
7911 let mut unadjusted =
7912 open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Disabled, 1, 1);
7913 type_keys(&mut unadjusted, "9'7");
7914 assert_eq!(index_of(&unadjusted, "隐私"), INITIALS_ROW_LIMIT - 1);
7915
7916 let mut quiet = open(&fixture.paths, false, EnglishInputOptions::default());
7917 type_keys(&mut quiet, "9'7");
7918 assert!(!words(&quiet).iter().any(|word| word == "隐私"));
7919 }
7920
7921 #[test]
7923 fn earlier_uses_are_ignored_with_personal_context_off() {
7924 let fixture = crowded_initials_fixture();
7925 record_use(&fixture.paths, "隐私");
7926 let mut session = open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Pin, 1, 1);
7927 type_keys(&mut session, "9'7");
7928 assert_eq!(index_of(&session, "隐私"), 0);
7929 session.command(Command::Cancel);
7930
7931 session.set_personal_context_enabled(false);
7932 type_keys(&mut session, "9'7");
7933 assert!(!words(&session).iter().any(|word| word == "隐私"));
7934 }
7935
7936 #[test]
7938 fn used_initials_words_stay_behind_the_leading_syllable_rows() {
7939 let fixture = fixture_with(INITIALS_FIXTURE);
7940 record_use(&fixture.paths, "那天");
7941 let mut session = open_with_frequency(&fixture.paths, FrequencyAdjustmentMode::Pin, 1, 1);
7942 type_digits(&mut session, "68");
7943 assert_eq!(words(&session), ["木", "欧", "那天", "每天", "明天"]);
7944 session.command(Command::Cancel);
7945 type_keys(&mut session, "6'8");
7946 assert_eq!(words(&session)[..3], ["那天", "每天", "明天"]);
7947 session.command(Command::Cancel);
7948
7949 record_use(&fixture.paths, "明天");
7951 type_keys(&mut session, "6'8");
7952 assert_eq!(words(&session)[..3], ["明天", "那天", "每天"]);
7953 }
7954
7955 #[test]
7957 fn nine_key_picks_are_recorded_for_the_initials() {
7958 let fixture = fixture_with(INITIALS_FIXTURE);
7959 let mut session = open(&fixture.paths, true, EnglishInputOptions::default());
7960 type_digits(&mut session, "963443842662");
7961 session.select(index_of(&session, "我滴"));
7962 let rest = session.select(index_of(&session, "个天呐"));
7963 assert_eq!(rest.diagnostic, None);
7964 for word in ["我滴", "个天呐", "我滴个天呐"] {
7965 assert_eq!(
7966 personal_count(&fixture.paths, word),
7967 PERSONAL_PICK_TIMES,
7968 "{word}"
7969 );
7970 }
7971 type_digits(&mut session, "68");
7972 session.select(index_of(&session, "木"));
7973 assert_eq!(personal_count(&fixture.paths, "木"), 0);
7974
7975 session.set_personal_context_enabled(false);
7976 type_keys(&mut session, "6'8");
7977 session.select(index_of(&session, "每天"));
7978 assert_eq!(personal_count(&fixture.paths, "每天"), 0);
7979 }
7980
7981 #[test]
7983 fn only_chosen_pieces_become_words() {
7984 let fixture = fixture_with(INITIALS_FIXTURE);
7985 let mut quiet = open(&fixture.paths, false, EnglishInputOptions::default());
7986 type_digits(&mut quiet, "963443842662");
7987 quiet.select(index_of(&quiet, "我滴"));
7988 quiet.select(index_of(&quiet, "个天呐"));
7989 type_digits(&mut quiet, "93486");
7990 assert!(
7991 !words(&quiet).iter().any(|word| word == "我滴个天呐"),
7992 "learning off still stored"
7993 );
7994 quiet.command(Command::Cancel);
7995
7996 let mut session = open(&fixture.paths, true, EnglishInputOptions::default());
7997 type_digits(&mut session, "963443842662");
7998 session.select(index_of(&session, "我滴"));
7999 session.command(Command::Cancel);
8000 type_digits(&mut session, "843");
8001 session.command(Command::Cancel);
8002 type_digits(&mut session, "93486");
8003 assert!(
8004 !words(&session).iter().any(|word| word == "我滴个天呐"),
8005 "a cancelled composition stored a phrase"
8006 );
8007 }
8008
8009 #[test]
8011 fn finishing_stores_the_chosen_pieces_with_the_rest() {
8012 let fixture = fixture_with(INITIALS_FIXTURE);
8013 let mut session = open(&fixture.paths, true, EnglishInputOptions::default());
8014 type_digits(&mut session, "963443842662");
8015 session.select(index_of(&session, "我滴"));
8016 assert_eq!(words(&session)[0], "个天呐");
8017 let rest = session.finish(0);
8018 assert_eq!(rest.commit.as_deref(), Some("个天呐"));
8019 assert_eq!(rest.diagnostic, None);
8020 assert!(!session.active());
8021 type_keys(&mut session, "9'3'4'8'6");
8022 assert_eq!(words(&session)[0], "我滴个天呐");
8023 }
8024
8025 #[test]
8027 fn without_quanpin_the_grid_spells_english_only() {
8028 let fixture = fixture();
8029 let mut session = NineKeySession::new(
8030 &fixture.paths,
8031 false,
8032 FrequencyAdjustmentOptions::default(),
8033 FuzzyPinyinOptions::default(),
8034 mixed(),
8035 false,
8036 false,
8037 );
8038 type_digits(&mut session, "64");
8039 assert!(session.snapshot().nine_key_spellings.is_empty());
8040 assert!(!words(&session).contains(&"你".to_owned()));
8041 assert!(session.dictionary.is_none());
8042
8043 let mut pinyin = open(&fixture.paths, false, mixed());
8044 type_digits(&mut pinyin, "64");
8045 assert!(words(&pinyin).contains(&"你".to_owned()));
8046 assert!(pinyin.dictionary.is_some());
8047 }
8048
8049 const EXPRESSIVE_MAIN_FIXTURE: &str = "CREATE TABLE tbl_1_m(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_m VALUES('mei','m','美',300);CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_n VALUES('nei','n','内',200);CREATE TABLE tbl_1_g(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_g VALUES('guo','g','国',300),('gao','g','高',200);CREATE TABLE tbl_2_m(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_m VALUES('mei''guo','mg','美国',1000);CREATE TABLE tbl_1_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_j VALUES('jing','j','警',100),('ji','j','鸡',300),('ji','j','几',200);CREATE TABLE tbl_1_l(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_l VALUES('li','l','里',500),('li','l','李',400);CREATE TABLE tbl_2_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_j VALUES('jing''gao','jg','警告',900);CREATE TABLE tbl_1_x(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_x VALUES('xian','x','先',300),('xiang','x','香',200),('xi','x','西',250);CREATE TABLE tbl_2_x(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_x VALUES('xiang''jiao','xj','香蕉',800);";
8053
8054 const EXPRESSIVE_OTHERS_FIXTURE: &str = "CREATE TABLE emoji_pinyin(key TEXT,emoji TEXT,sort_order INTEGER);INSERT INTO emoji_pinyin VALUES('meiguo','🇺🇸',1893),('meihuo','🇺🇸',1893),('meiguobentuwaixiaodaoyu','🇺🇲',1891),('jinggao','警告',1),('jinggao','⚠️',1433),('xiangjiao','🍌',725),('ji','🐔',626),('jitou','🐔',626),('laugh','😀',10),('mei','🌸',2000);CREATE TABLE emoji(emoji TEXT PRIMARY KEY,keywords TEXT);INSERT INTO emoji VALUES('🇺🇸','美国 美利坚 美利坚合众国 星条旗 flag: united states'),('🇺🇲','美国本土外小岛屿 flag: u.s. outlying islands'),('⚠️','警告 注意 危险 预警 warning'),('🍌','香蕉 banana'),('🐔','鸡 鸡头 chicken'),('😀','笑脸 laugh'),('🌸','樱花 梅花 Mei');CREATE TABLE kaomoji(pinyin TEXT,jianpin TEXT,kaomoji TEXT,sort_order INTEGER);INSERT INTO kaomoji VALUES('meiguo','mg','(•̀ᴗ•́)و',10),('jinggao','jg','(゚Д゚≡゚д゚)!?',20);CREATE TABLE kaomoji_catalog(kaomoji TEXT PRIMARY KEY,keywords TEXT);INSERT INTO kaomoji_catalog VALUES('(•̀ᴗ•́)و','mei guo'),('(゚Д゚≡゚д゚)!?','jing gao 警告');";
8056
8057 const BOTH_EXPRESSIVE: MixedExpressiveOptions = MixedExpressiveOptions {
8058 emoji_candidates: true,
8059 kaomoji_candidates: true,
8060 symbol_candidates: false,
8061 };
8062
8063 fn expressive_fixture() -> Fixture {
8064 let fixture = fixture_with(EXPRESSIVE_MAIN_FIXTURE);
8065 Connection::open(fixture.paths.resource(assets::OTHER_DICTIONARY))
8066 .and_then(|db| db.execute_batch(EXPRESSIVE_OTHERS_FIXTURE))
8067 .expect("others fixture");
8068 fixture
8069 }
8070
8071 fn open_expressive(
8072 fixture: &Fixture,
8073 english: EnglishInputOptions,
8074 expressive: MixedExpressiveOptions,
8075 ) -> NineKeySession {
8076 let mut session = open(&fixture.paths, true, english);
8077 session.set_mixed_expressive(expressive);
8078 session
8079 }
8080
8081 fn sources(session: &NineKeySession) -> Vec<CandidateSource> {
8082 session.snapshot().candidate_sources
8083 }
8084
8085 fn has_expressive_rows(session: &NineKeySession) -> bool {
8086 sources(session)
8087 .iter()
8088 .any(|source| matches!(source, CandidateSource::Emoji | CandidateSource::Kaomoji))
8089 }
8090
8091 #[test]
8092 fn digits_put_emoji_and_kaomoji_after_the_words_they_depict() {
8093 let fixture = expressive_fixture();
8094 let mut session =
8095 open_expressive(&fixture, EnglishInputOptions::default(), BOTH_EXPRESSIVE);
8096
8097 type_digits(&mut session, "634486");
8099 let view = session.snapshot();
8100 let listed = words(&session);
8101 assert_eq!(listed[..2], ["美国", "🇺🇸"]);
8102 assert_eq!(view.candidate_sources[1], CandidateSource::Emoji);
8103 assert_eq!(listed[listed.len() - 2..], ["🇺🇲", "(•̀ᴗ•́)و"]);
8104 assert_eq!(
8105 view.candidate_sources[listed.len() - 1],
8106 CandidateSource::Kaomoji
8107 );
8108 assert_eq!(listed.iter().filter(|word| *word == "🇺🇸").count(), 1);
8109 assert!(view.candidate_answers_key[1]);
8110 assert_eq!(view.candidates[1].pinyin, "634486");
8111 session.command(Command::Cancel);
8112
8113 type_digits(&mut session, "5464426");
8115 assert_eq!(words(&session)[..3], ["警告", "⚠️", "(゚Д゚≡゚д゚)!?"]);
8116 assert_eq!(
8117 words(&session)
8118 .iter()
8119 .filter(|word| *word == "警告")
8120 .count(),
8121 1
8122 );
8123 session.command(Command::Cancel);
8124
8125 type_digits(&mut session, "54");
8127 assert_eq!(words(&session)[..4], ["里", "李", "鸡", "🐔"]);
8128 session.command(Command::Cancel);
8129
8130 let mut emoji_only = open_expressive(
8132 &fixture,
8133 EnglishInputOptions::default(),
8134 MixedExpressiveOptions {
8135 emoji_candidates: true,
8136 kaomoji_candidates: false,
8137 symbol_candidates: false,
8138 },
8139 );
8140 type_digits(&mut emoji_only, "634486");
8141 assert_eq!(words(&emoji_only)[..2], ["美国", "🇺🇸"]);
8142 assert_eq!(words(&emoji_only).last().map(String::as_str), Some("🇺🇲"));
8143 assert!(!sources(&emoji_only).contains(&CandidateSource::Kaomoji));
8144 }
8145
8146 #[test]
8147 fn expressive_rows_follow_a_leading_english_word() {
8148 let fixture = expressive_fixture();
8149 Connection::open(fixture.paths.dictionary(assets::ENGLISH_DICTIONARY))
8150 .unwrap()
8151 .execute(
8152 "INSERT INTO english_words(word, display, weight) VALUES ('mei', 'Mei', 700)",
8153 [],
8154 )
8155 .unwrap();
8156 let mut session = open_expressive(&fixture, mixed(), BOTH_EXPRESSIVE);
8157 type_digits(&mut session, "634");
8159 let listed = words(&session);
8160 assert_eq!(listed[..3], ["美", "Mei", "🌸"]);
8161 assert_eq!(listed[listed.len() - 3..], ["🇺🇲", "🇺🇸", "(•̀ᴗ•́)و"]);
8162 }
8163
8164 #[test]
8165 fn both_switches_off_leave_the_list_unchanged() {
8166 let fixture = expressive_fixture();
8167 for digits in ["634486", "5464426", "94264", "63", "6"] {
8168 let mut off = open_expressive(&fixture, mixed(), MixedExpressiveOptions::default());
8169 let mut on = open_expressive(&fixture, mixed(), BOTH_EXPRESSIVE);
8170 type_digits(&mut off, digits);
8171 type_digits(&mut on, digits);
8172 assert!(!has_expressive_rows(&off), "{digits}");
8173 let mut without_expressive = on.snapshot().candidates;
8175 without_expressive.retain(|item| {
8176 !matches!(
8177 item.source,
8178 CandidateSource::Emoji | CandidateSource::Kaomoji
8179 )
8180 });
8181 assert_eq!(without_expressive, off.snapshot().candidates, "{digits}");
8182 }
8183 }
8184
8185 #[test]
8186 fn expressive_rows_need_two_digits_and_pinyin() {
8187 let fixture = expressive_fixture();
8188 let mut session =
8189 open_expressive(&fixture, EnglishInputOptions::default(), BOTH_EXPRESSIVE);
8190 type_digits(&mut session, "6");
8192 assert!(!has_expressive_rows(&session));
8193 type_digits(&mut session, "3");
8194 assert!(words(&session).contains(&"🇺🇸".to_owned()));
8195 session.command(Command::Cancel);
8196
8197 session.set_english_only(true);
8199 type_digits(&mut session, "634486");
8200 assert!(!has_expressive_rows(&session));
8201 session.set_english_only(false);
8202 session.command(Command::Cancel);
8203
8204 type_digits(&mut session, "634486");
8206 assert!(session.set_filter(true, "").handled);
8207 assert!(!has_expressive_rows(&session));
8208 session.command(Command::Cancel);
8209
8210 let mut english_grid = NineKeySession::new(
8212 &fixture.paths,
8213 false,
8214 FrequencyAdjustmentOptions::default(),
8215 FuzzyPinyinOptions::default(),
8216 mixed(),
8217 false,
8218 false,
8219 );
8220 english_grid.set_mixed_expressive(BOTH_EXPRESSIVE);
8221 type_digits(&mut english_grid, "634486");
8222 assert!(!has_expressive_rows(&english_grid));
8223 }
8224
8225 #[test]
8226 fn expressive_rows_follow_chosen_syllables_and_splits() {
8227 let fixture = expressive_fixture();
8228 let mut session =
8229 open_expressive(&fixture, EnglishInputOptions::default(), BOTH_EXPRESSIVE);
8230
8231 type_digits(&mut session, "634486");
8233 let nei = spelling_index(&session, "nei");
8234 assert!(session.choose_spelling(nei).handled);
8235 assert!(!words(&session).contains(&"🇺🇸".to_owned()));
8236 session.command(Command::Cancel);
8237 type_digits(&mut session, "634486");
8238 let mei = spelling_index(&session, "mei");
8239 assert!(session.choose_spelling(mei).handled);
8240 assert!(words(&session).contains(&"🇺🇸".to_owned()));
8241 session.command(Command::Cancel);
8242
8243 type_digits(&mut session, "94264");
8245 assert!(words(&session).contains(&"🍌".to_owned()));
8246 session.command(Command::Cancel);
8247 type_digits(&mut session, "9426");
8249 let xian = spelling_index(&session, "xian");
8250 assert!(session.choose_spelling(xian).handled);
8251 type_digits(&mut session, "4");
8252 assert!(!words(&session).contains(&"🍌".to_owned()));
8253 session.command(Command::Cancel);
8254 type_keys(&mut session, "9426'4");
8256 assert!(!words(&session).contains(&"🍌".to_owned()));
8257 }
8258
8259 #[test]
8260 fn choosing_an_expressive_row_commits_it_and_ends_the_composition() {
8261 let fixture = expressive_fixture();
8262 let mut session =
8263 open_expressive(&fixture, EnglishInputOptions::default(), BOTH_EXPRESSIVE);
8264 type_digits(&mut session, "634486");
8265 let chosen = session.select(index_of(&session, "🇺🇸"));
8266 assert_eq!(chosen.commit.as_deref(), Some("🇺🇸"));
8267 assert_eq!(chosen.diagnostic, None);
8268 assert!(!session.active());
8269 type_digits(&mut session, "634486");
8271 let index = index_of(&session, "🇺🇸");
8272 assert!(!session.pin(index).handled);
8273 assert!(!session.remove(index).handled);
8274 assert!(!session.set_position(index, 2).handled);
8275 assert_eq!(
8276 sources(&session)
8277 .iter()
8278 .zip(words(&session))
8279 .filter(|(_, word)| word == "🇺🇸")
8280 .map(|(source, _)| *source)
8281 .collect::<Vec<_>>(),
8282 [CandidateSource::Emoji]
8283 );
8284 }
8285
8286 #[test]
8287 fn expressive_readings_are_capped_and_led_by_the_candidates() {
8288 let fixture = expressive_fixture();
8289 let mut session =
8290 open_expressive(&fixture, EnglishInputOptions::default(), BOTH_EXPRESSIVE);
8291 type_digits(&mut session, "94264");
8292 let table = spelling_table();
8293 let mut every: Vec<String> = table
8294 .paths("94264")
8295 .into_iter()
8296 .map(|path| path.concat())
8297 .collect();
8298 every.sort();
8299 every.dedup();
8300 assert!(every.len() > EXPRESSIVE_READING_LIMIT, "{every:?}");
8301
8302 let readings: Vec<String> = session
8303 .expressive_readings(&table.paths("94264"), &[], &session.candidates)
8304 .into_iter()
8305 .map(|(reading, _)| reading)
8306 .collect();
8307 assert_eq!(readings.len(), EXPRESSIVE_READING_LIMIT);
8308 assert_eq!(session.candidates[0].word, "香");
8310 assert_eq!(readings[0], "xiang");
8311 assert!(readings.iter().all(|reading| reading.len() == 5));
8313 }
8314
8315 #[test]
8316 fn shuangpin_readings_do_not_allocate_an_intermediate_boundary_table() {
8317 let fixture = fixture_with(SHUANGPIN_FIXTURE);
8318 let mut session = shuangpin_session(&fixture.paths, NineKeyShuangpinKind::Migala3);
8319 session.digits = "6447".into();
8320 let alternatives = session.grid.spelling_table().paths("6447");
8321 let candidates = session.candidates.clone();
8322 let boundaries = [2, 4];
8323 let (_, allocations) = crate::ime::personal_rerank::allocations::count(|| {
8324 session.expressive_readings(&alternatives, &boundaries, &candidates)
8325 });
8326 assert!(
8327 allocations <= 100,
8328 "shuangpin readings allocated an intermediate boundary table: {allocations}"
8329 );
8330 }
8331
8332 #[test]
8333 fn shuangpin_readings_materialize_boundaries_only_for_retained_readings() {
8334 let fixture = fixture_with(SHUANGPIN_FIXTURE);
8335 let mut session = shuangpin_session(&fixture.paths, NineKeyShuangpinKind::Migala3);
8336 session.digits = "94264".into();
8337 let alternatives = session.grid.spelling_table().paths("94264");
8338 let candidates = session.candidates.clone();
8339 let boundaries = [2, 4];
8340 let (_, allocations) = crate::ime::personal_rerank::allocations::count(|| {
8341 session.expressive_readings(&alternatives, &boundaries, &candidates)
8342 });
8343 assert!(
8344 allocations <= 100,
8345 "discarded shuangpin readings still materialized boundaries: {allocations}"
8346 );
8347 }
8348
8349 #[test]
8350 fn ordinary_nine_key_readings_borrow_shared_boundaries() {
8351 let fixture = expressive_fixture();
8352 let mut session =
8353 open_expressive(&fixture, EnglishInputOptions::default(), BOTH_EXPRESSIVE);
8354 session.digits = "634486".into();
8355 let alternatives = spelling_table().paths("634486");
8356 let candidates = session.candidates.clone();
8357 let boundaries = [3, 6];
8358 let (_, allocations) = crate::ime::personal_rerank::allocations::count(|| {
8359 session.expressive_readings(&alternatives, &boundaries, &candidates)
8360 });
8361 assert!(
8362 allocations <= 84,
8363 "ordinary readings copied shared boundaries: {allocations}"
8364 );
8365 }
8366
8367 #[test]
8368 fn expressive_readings_do_not_allocate_a_separate_locked_prefix() {
8369 let fixture = expressive_fixture();
8370 let mut session =
8371 open_expressive(&fixture, EnglishInputOptions::default(), BOTH_EXPRESSIVE);
8372 type_digits(&mut session, "634486");
8373 assert!(
8374 session
8375 .choose_spelling(spelling_index(&session, "mei"))
8376 .handled
8377 );
8378 let alternatives = session.grid.spelling_table().paths("486");
8379 let candidates = session.candidates.clone();
8380 let (_, allocations) = crate::ime::personal_rerank::allocations::count(|| {
8381 session.expressive_readings(&alternatives, &[], &candidates)
8382 });
8383 assert!(
8384 allocations <= 15,
8385 "locked expressive readings allocated {allocations} buffers"
8386 );
8387 }
8388
8389 const SYMBOL_MAIN_FIXTURE: &str = "CREATE TABLE tbl_1_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_z VALUES('zheng','z','正',300),('zheng','z','政',200);CREATE TABLE tbl_1_f(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_f VALUES('fu','f','负',300),('fu','f','府',200);CREATE TABLE tbl_2_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_z VALUES('zheng''fu','zf','政府',1000),('zheng''fu','zf','正负',500);CREATE TABLE tbl_1_p(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_p VALUES('pai','p','派',900),('pai','p','排',800),('pai','p','牌',700),('pai','p','拍',600);";
8393
8394 const SYMBOLS_ONLY: MixedExpressiveOptions = MixedExpressiveOptions {
8395 emoji_candidates: false,
8396 kaomoji_candidates: false,
8397 symbol_candidates: true,
8398 };
8399
8400 fn has_symbol_row(session: &NineKeySession, symbol: &str) -> bool {
8401 sources(session)
8402 .iter()
8403 .zip(words(session))
8404 .any(|(source, word)| *source == CandidateSource::Emoji && word == symbol)
8405 }
8406
8407 #[test]
8408 fn digits_spelling_a_whole_symbol_name_add_its_symbol_after_the_name() {
8409 let fixture = fixture_with(SYMBOL_MAIN_FIXTURE);
8410 let mut session = open_expressive(&fixture, EnglishInputOptions::default(), SYMBOLS_ONLY);
8411 type_digits(&mut session, "9436438");
8413 let listed = words(&session);
8414 assert_eq!(listed[..3], ["政府", "正负", "±"]);
8415 assert_eq!(sources(&session)[2], CandidateSource::Emoji);
8416 assert_eq!(session.snapshot().candidates[2].pinyin, "9436438");
8417 session.command(Command::Cancel);
8418
8419 type_digits(&mut session, "943643");
8421 assert!(!has_symbol_row(&session, "±"));
8422 session.command(Command::Cancel);
8423
8424 type_digits(&mut session, "724");
8426 let listed = words(&session);
8427 assert_eq!(listed[0], "派");
8428 assert_eq!(listed[3], "π");
8429 session.command(Command::Cancel);
8430
8431 type_digits(&mut session, "9436438");
8433 assert!(session.set_filter(true, "").handled);
8434 assert!(!has_symbol_row(&session, "±"));
8435 session.command(Command::Cancel);
8436
8437 let mut off = open_expressive(
8439 &fixture,
8440 EnglishInputOptions::default(),
8441 MixedExpressiveOptions::default(),
8442 );
8443 let mut on = open_expressive(&fixture, EnglishInputOptions::default(), SYMBOLS_ONLY);
8444 type_digits(&mut off, "9436438");
8445 type_digits(&mut on, "9436438");
8446 assert!(!has_symbol_row(&off, "±"));
8447 let mut without_symbols = on.snapshot().candidates;
8448 without_symbols.retain(|item| item.source != CandidateSource::Emoji);
8449 assert_eq!(without_symbols, off.snapshot().candidates);
8450 }
8451
8452 #[test]
8453 fn choosing_a_symbol_row_commits_it_and_eats_every_digit() {
8454 let fixture = fixture_with(SYMBOL_MAIN_FIXTURE);
8455 let mut session = open_expressive(&fixture, EnglishInputOptions::default(), SYMBOLS_ONLY);
8456 type_digits(&mut session, "9436438");
8457 let chosen = session.select(index_of(&session, "±"));
8458 assert_eq!(chosen.commit.as_deref(), Some("±"));
8459 assert!(!session.active());
8460 type_digits(&mut session, "9436438");
8462 let index = index_of(&session, "±");
8463 assert!(!session.pin(index).handled);
8464 assert!(!session.remove(index).handled);
8465 assert!(!session.set_position(index, 2).handled);
8466 }
8467
8468 #[test]
8470 fn a_symbol_row_never_leads_after_its_name_is_fixed_further_down() {
8471 let fixture = fixture_with("CREATE TABLE tbl_1_d(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_d VALUES('da','d','大',300);CREATE TABLE tbl_1_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_y VALUES('yu','y','于',300);CREATE TABLE tbl_2_d(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_d VALUES('da''yu','dy','大于',900);");
8472 let mut session = open_expressive(&fixture, EnglishInputOptions::default(), SYMBOLS_ONLY);
8473 type_digits(&mut session, "3298");
8474 assert_eq!(words(&session)[..3], ["大于", ">", ">"]);
8475 assert!(session.set_position(0, 3).handled);
8476 let listed = words(&session);
8478 assert_ne!(sources(&session)[0], CandidateSource::Emoji, "{listed:?}");
8479 assert_eq!(listed[2], "大于", "{listed:?}");
8480 assert!(listed.contains(&">".to_owned()) && listed.contains(&">".to_owned()));
8481 }
8482
8483 #[test]
8484 fn symbols_only_does_not_allocate_an_unused_other_dictionary_path() {
8485 let fixture = fixture_with(SYMBOL_MAIN_FIXTURE);
8486 let mut session = open_expressive(&fixture, EnglishInputOptions::default(), SYMBOLS_ONLY);
8487 session.digits = "9436438".into();
8488 let alternatives = spelling_table().paths("9436438");
8489 assert!(!session
8490 .expressive_readings(&alternatives, &[], &[])
8491 .is_empty());
8492 let candidates = session.candidates.clone();
8493 let (_, allocations) = crate::ime::personal_rerank::allocations::count(|| {
8494 session.expressive_candidates(&alternatives, &candidates)
8495 });
8496 assert!(
8497 allocations <= 1150,
8498 "symbols-only expressive path allocated an unused resource path: {allocations}"
8499 );
8500 }
8501
8502 fn frozen_splits_into_syllables_at(key: &str, boundaries: &[usize]) -> bool {
8503 let mut start = 0;
8504 for &end in boundaries.iter().chain(std::iter::once(&key.len())) {
8505 let Some(piece) = key.get(start..end) else {
8506 return false;
8507 };
8508 if !piece.is_empty()
8509 && crate::pinyin::segment::cut_one_piece_min_segments(piece, true).is_empty()
8510 {
8511 return false;
8512 }
8513 start = end;
8514 }
8515 true
8516 }
8517
8518 #[test]
8519 fn expressive_boundary_check_matches_owned_cuts() {
8520 let mut inputs = vec![
8521 String::new(),
8522 "nihao".into(),
8523 "meiguo".into(),
8524 "xiangjiao".into(),
8525 "xi'an".into(),
8526 "ni''hao".into(),
8527 "'ni".into(),
8528 "ni'".into(),
8529 "你好".into(),
8530 "ni好".into(),
8531 "NIHAO".into(),
8532 "gamma".into(),
8533 "ni".repeat(32),
8534 "ni".repeat(33),
8535 "ni".repeat(33) + "h",
8536 ];
8537 for syllable in intact_pinyin_list() {
8538 for suffix in ["", "ni", "h", "'an", "好", "sahng", "zhonguo"] {
8539 inputs.push(format!("{syllable}{suffix}"));
8540 }
8541 }
8542 for key in inputs {
8543 let mut boundaries = vec![
8544 vec![],
8545 vec![0],
8546 vec![key.len()],
8547 vec![0, key.len()],
8548 vec![key.len(), key.len()],
8549 vec![key.len() + 1],
8550 vec![usize::MAX],
8551 vec![key.len(), 0],
8552 ];
8553 boundaries.extend((0..=key.len()).map(|at| vec![at, at]));
8554 for boundaries in boundaries {
8555 assert_eq!(
8556 splits_into_syllables_at(&key, &boundaries),
8557 frozen_splits_into_syllables_at(&key, &boundaries),
8558 "编码 {key:?},边界 {boundaries:?}"
8559 );
8560 }
8561 }
8562 }
8563
8564 #[test]
8565 fn expressive_boundary_check_does_not_materialize_syllables() {
8566 use crate::ime::personal_rerank::allocations::count;
8567 assert!(splits_into_syllables_at("nihao", &[]));
8568 for (key, boundaries, expected_allocations) in [
8569 ("nihao".to_owned(), vec![], 0),
8570 ("meiguo".to_owned(), vec![3], 0),
8571 ("xiangjiao".to_owned(), vec![2, 5], 0),
8572 ("ni".repeat(32), vec![], 0),
8573 ("ni".repeat(33), vec![], 1),
8574 ] {
8575 let (expected, old) = count(|| frozen_splits_into_syllables_at(&key, &boundaries));
8576 let (actual, new) = count(|| splits_into_syllables_at(&key, &boundaries));
8577 assert!(expected);
8578 assert_eq!(actual, expected);
8579 eprintln!("表情边界:{} 字节 {boundaries:?},{old} → {new}", key.len());
8580 assert_eq!(new, expected_allocations, "只应保留超长片段的 DP 表分配");
8581 assert!(new < old);
8582 }
8583 }
8584
8585 #[test]
8586 fn expressive_symbol_query_preserves_rows_without_cut_results() {
8587 use crate::ime::personal_rerank::allocations::count;
8588 let readings = vec!["zhengfu".to_owned()];
8589 let boundaries = [5];
8590 let current = |key: &str| splits_into_syllables_at(key, &boundaries);
8591 let frozen = |key: &str| frozen_splits_into_syllables_at(key, &boundaries);
8592 assert!(!query_symbol_readings(&readings, ¤t).is_empty());
8593 let (expected, old) = count(|| query_symbol_readings(&readings, &frozen));
8594 let (actual, new) = count(|| query_symbol_readings(&readings, ¤t));
8595 assert_eq!(actual, expected);
8596 eprintln!("符号查询:{old} → {new}");
8597 assert!(new < old, "符号查询仍然物化了边界切分结果");
8598 }
8599
8600 #[test]
8601 fn expressive_keys_must_split_at_the_fixed_boundaries() {
8602 assert!(splits_into_syllables_at("xiangjiao", &[]));
8603 assert!(splits_into_syllables_at("meiguo", &[3]));
8604 assert!(splits_into_syllables_at("meiguo", &[6]));
8605 assert!(splits_into_syllables_at("meiguobentuwaixiaodaoyu", &[3, 6]));
8606 assert!(splits_into_syllables_at("xiangjiao", &[2]));
8608 assert!(!splits_into_syllables_at("xiangjiao", &[4]));
8609 assert!(!splits_into_syllables_at("meiguo", &[2]));
8610 assert!(!splits_into_syllables_at("laugh", &[2]));
8611 assert!(!splits_into_syllables_at("mei", &[4]));
8612 assert!(letters_start_with("mei'guo", "meig"));
8613 assert!(!letters_start_with("mei", "meig"));
8614 }
8615
8616 #[test]
8618 fn real_dictionary_lists_exact_syllables_before_completions() {
8619 let Some(resources) = std::env::var_os("MSIME_EVAL_RESOURCES") else {
8620 eprintln!("skipped: MSIME_EVAL_RESOURCES is not set to a resource directory");
8621 return;
8622 };
8623 let user = tempfile::tempdir().expect("user directory");
8624 let resources = PathBuf::from(resources);
8625 let paths = RuntimePaths {
8626 resources: resources.clone(),
8627 user_data: user.path().to_path_buf(),
8628 cache: user.path().to_path_buf(),
8629 dictionaries: resources,
8630 };
8631 let mut session = open(&paths, false, mixed());
8632 type_digits(&mut session, "74");
8633 let spellings = session.snapshot().nine_key_spellings;
8634 let preferred = &spellings[0];
8635 let exact: Vec<&str> = spellings[1..]
8636 .iter()
8637 .map(String::as_str)
8638 .filter(|spelling| spelling != preferred)
8639 .take_while(|spelling| encode(spelling) == "74")
8640 .collect();
8641 let mut expected: Vec<&str> = ["pi", "qi", "ri", "si"]
8642 .into_iter()
8643 .filter(|spelling| spelling != preferred)
8644 .collect();
8645 let mut sorted = exact.clone();
8646 sorted.sort_unstable();
8647 expected.sort_unstable();
8648 assert_eq!(sorted, expected, "{spellings:?}");
8649 }
8650
8651 const GRIDS: [KeyGrid; 3] = [KeyGrid::NineKey, KeyGrid::FourteenKey, KeyGrid::EighteenKey];
8654
8655 const FOURTEEN_FIXTURE: &str = "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_n VALUES('ni','n','你',100);CREATE TABLE tbl_1_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_b VALUES('bu','b','不',90);CREATE TABLE tbl_1_h(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_h VALUES('hao','h','好',100);CREATE TABLE tbl_2_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_n VALUES('ni''hao','nh','你好',1000);CREATE TABLE tbl_2_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_b VALUES('bu''hao','bh','不好',500);CREATE TABLE tbl_1_x(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_x VALUES('xi','x','西',100),('xian','x','先',300);CREATE TABLE tbl_1_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_a VALUES('an','a','安',100);CREATE TABLE tbl_2_x(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_x VALUES('xi''an','xa','西安',800);CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_s VALUES('shi','s','是',1000),('shu','s','书',500);";
8657
8658 fn fourteen_key_session(paths: &RuntimePaths, english: EnglishInputOptions) -> NineKeySession {
8659 let mut session = open(paths, false, english);
8660 session.set_grid(KeyGrid::FourteenKey);
8661 session
8662 }
8663
8664 #[test]
8665 fn grid_codes_come_from_the_printed_keys() {
8666 for grid in GRIDS {
8667 let mut derived = [0u8; 26];
8669 let codes: Vec<u8> = match grid {
8670 KeyGrid::NineKey => (b'2'..=b'9').collect(),
8671 KeyGrid::FourteenKey => FOURTEEN_KEY_GROUPS
8672 .iter()
8673 .map(|group| group.as_bytes()[0])
8674 .collect(),
8675 KeyGrid::EighteenKey => EIGHTEEN_KEY_GROUPS
8676 .iter()
8677 .map(|group| group.as_bytes()[0])
8678 .collect(),
8679 KeyGrid::NineKeyShuangpin(_)
8680 | KeyGrid::FourteenKeyShuangpin(_)
8681 | KeyGrid::EighteenKeyShuangpin(_) => {
8682 unreachable!("GRIDS 只有按字母归组的键盘")
8683 }
8684 };
8685 for &code in &codes {
8686 assert!(grid.is_code(code));
8687 for letter in grid.letters(code).bytes() {
8688 let slot = &mut derived[usize::from(letter - b'a')];
8689 assert_eq!(*slot, 0, "{grid:?}: {} on two keys", char::from(letter));
8690 *slot = code;
8691 }
8692 }
8693 assert_eq!(Some(&derived), grid.keypad(), "{grid:?}");
8694 for letter in b'a'..=b'z' {
8695 let code = grid.code_of(letter).unwrap();
8696 assert!(grid.letters(code).as_bytes().contains(&letter));
8697 }
8698 assert_eq!(grid.code_of(b'A'), None);
8699 assert_eq!(grid.code_of(b'\''), None);
8700 assert!(!grid.is_code(b'\'') && !grid.is_code(b'A') && !grid.is_code(b'1'));
8701 }
8702 assert_eq!(FOURTEEN_KEYPAD, b"abcdedggujjlmbooqeatucqztz");
8703 assert_eq!(KeyGrid::FourteenKey.encode("ni'hao"), "bugao");
8704 assert_eq!(KeyGrid::FourteenKey.letters(b'l'), "l");
8705 assert_eq!(KeyGrid::FourteenKey.letters(b'w'), "", "w is not a code");
8706 assert!(KeyGrid::NineKey.offers_raw_key());
8707 assert!(!KeyGrid::FourteenKey.offers_raw_key());
8708 }
8709
8710 #[test]
8711 fn syllable_codes_per_grid() {
8712 for (grid, expected) in [(KeyGrid::NineKey, 218), (KeyGrid::FourteenKey, 314)] {
8713 let codes: HashSet<String> = intact_pinyin_list()
8714 .iter()
8715 .map(|syllable| grid.encode(syllable))
8716 .collect();
8717 assert_eq!(intact_pinyin_list().len(), 421);
8718 assert_eq!(codes.len(), expected, "{grid:?}");
8719 }
8720 }
8721
8722 #[test]
8723 fn fourteen_key_shares_codes_between_syllables() {
8724 let grid = KeyGrid::FourteenKey;
8725 for (code, syllables) in [
8726 ("bu", ["bi", "bu", "ni", "nu"]),
8727 ("zu", ["xi", "xu", "zi", "zu"]),
8728 ] {
8729 for syllable in syllables {
8730 assert_eq!(grid.encode(syllable), code, "{syllable}");
8731 }
8732 }
8733 assert_eq!(grid.encode("shi"), grid.encode("shu"));
8734 assert_eq!(grid.encode("sha"), grid.encode("a'ga"));
8736 let paths = grid
8737 .spelling_table()
8738 .split_paths("aga", &[], &SyllablePrior::default());
8739 assert!(paths.contains(&path(&["sha"])), "{paths:?}");
8740 assert!(paths.contains(&path(&["a", "ga"])), "{paths:?}");
8741 let table = grid.spelling_table();
8742 let spellings = table.spellings_for("agu", 0, None, &SyllablePrior::default());
8743 assert!(spellings.contains(&"shi".to_owned()) && spellings.contains(&"shu".to_owned()));
8744 }
8745
8746 #[test]
8747 fn grid_initials_expand_each_key_and_stop_at_keys_without_initials() {
8748 assert_eq!(
8749 super::initials_codes(KeyGrid::FourteenKey, "bg", 1024).unwrap(),
8750 ["bg", "bh", "ng", "nh"]
8751 );
8752 assert!(super::initials_codes(KeyGrid::FourteenKey, "bug", 1024).is_none());
8754 assert_eq!(
8756 super::initials_codes(KeyGrid::FourteenKey, "qqqqqqqqqq", 1024)
8757 .unwrap()
8758 .len(),
8759 1024
8760 );
8761 assert!(super::initials_codes(KeyGrid::FourteenKey, "qqqqqqqqqqq", 1024).is_none());
8762 assert!(super::initials_codes(KeyGrid::NineKey, "61", 1024).is_none());
8764 }
8765
8766 #[test]
8767 fn grid_letter_prefixes_and_english_matches() {
8768 assert_eq!(
8769 super::letter_prefixes(KeyGrid::FourteenKey, "oj", 64),
8770 ["oj", "ok", "pj", "pk"]
8771 );
8772 assert_eq!(super::letter_prefixes(KeyGrid::FourteenKey, "l", 64), ["l"]);
8773 assert!(super::word_matches_digits(KeyGrid::FourteenKey, "OK", "oj"));
8774 assert!(super::word_matches_digits(KeyGrid::FourteenKey, "ok", "o"));
8775 assert!(!super::word_matches_digits(
8776 KeyGrid::FourteenKey,
8777 "ok",
8778 "ob"
8779 ));
8780 assert!(!super::word_matches_digits(KeyGrid::NineKey, "ok", "oj"));
8781 assert!(super::keyword_spells_digits(
8782 KeyGrid::FourteenKey,
8783 "riqi",
8784 "euqu"
8785 ));
8786 assert!(super::letters_spell_code(KeyGrid::FourteenKey, "ni", "bu"));
8787 assert!(!super::letters_spell_code(KeyGrid::FourteenKey, "ni", "64"));
8788 }
8789
8790 #[test]
8791 fn fourteen_key_left_column_has_letters_but_no_raw_key() {
8792 let fixture = fixture_with(FOURTEEN_FIXTURE);
8793 let mut nine = open(&fixture.paths, false, EnglishInputOptions::default());
8794 let mut fourteen = fourteen_key_session(&fixture.paths, EnglishInputOptions::default());
8795 assert!(
8796 !fourteen.character(b'6').handled,
8797 "digits are not 14-key codes"
8798 );
8799 assert!(
8800 !fourteen.character(b'w').handled,
8801 "only the first letter is a code"
8802 );
8803 type_digits(&mut nine, "7");
8804 type_digits(&mut fourteen, "q");
8805 let spellings = fourteen.snapshot().nine_key_spellings;
8806 assert_eq!(spellings[spellings.len() - 2..], ["Q", "W"]);
8807 assert!(!spellings.contains(&"q".to_owned()), "{spellings:?}");
8808 assert_eq!(
8809 nine.snapshot()
8810 .nine_key_spellings
8811 .last()
8812 .map(String::as_str),
8813 Some("7")
8814 );
8815 fourteen.command(Command::Cancel);
8816 type_digits(&mut fourteen, "u");
8818 assert!(fourteen.snapshot().nine_key_spellings.is_empty());
8819 fourteen.command(Command::Cancel);
8820 type_digits(&mut fourteen, "agu");
8822 let spellings = fourteen.snapshot().nine_key_spellings;
8823 assert!(
8824 spellings.starts_with(&["shi".to_owned(), "shu".to_owned()]),
8825 "{spellings:?}"
8826 );
8827 assert_eq!(words(&fourteen)[..2], ["是", "书"]);
8828 assert_eq!(fourteen.snapshot().nine_key_reading, "shi");
8829 }
8830
8831 #[test]
8832 fn fourteen_key_types_words_sentences_and_splits() {
8833 let fixture = fixture_with(FOURTEEN_FIXTURE);
8834 let mut session = fourteen_key_session(&fixture.paths, EnglishInputOptions::default());
8835 type_digits(&mut session, "bugao");
8837 let view = session.snapshot();
8838 assert_eq!(view.editing_text, "bugao");
8839 assert_eq!(view.preedit, "bugao");
8840 assert_eq!(words(&session)[..2], ["你好", "不好"]);
8841 assert_eq!(view.nine_key_reading, "ni'hao");
8842 assert_eq!(view.candidates[0].pinyin, "bugao");
8843 let result = session.select(0);
8844 assert_eq!(result.commit.as_deref(), Some("你好"));
8845 assert!(!session.active());
8846
8847 type_digits(&mut session, "zu");
8849 assert!(session.character(b'\'').handled);
8850 type_digits(&mut session, "ab");
8851 assert_eq!(session.snapshot().preedit, "zu'ab");
8852 assert_eq!(words(&session).first().map(String::as_str), Some("西安"));
8853 assert_eq!(session.snapshot().nine_key_reading, "xi'an");
8854 assert!(!words(&session).contains(&"先".to_owned()));
8855 session.command(Command::Cancel);
8856 type_digits(&mut session, "zuab");
8857 assert!(words(&session).contains(&"先".to_owned()));
8858 }
8859
8860 #[test]
8861 fn fourteen_key_locks_undoes_and_backspaces() {
8862 let fixture = fixture_with(FOURTEEN_FIXTURE);
8863 let mut session = fourteen_key_session(&fixture.paths, EnglishInputOptions::default());
8864 type_digits(&mut session, "bu");
8865 let spellings = session.snapshot().nine_key_spellings;
8866 let ni = spellings
8867 .iter()
8868 .position(|spelling| spelling == "ni")
8869 .unwrap();
8870 assert!(session.choose_spelling(ni).handled);
8871 assert_eq!(session.snapshot().preedit, "ni");
8872 assert_eq!(words(&session).first().map(String::as_str), Some("你"));
8873 assert!(session.command(Command::Backspace).handled);
8875 assert_eq!(session.snapshot().preedit, "bu");
8876 assert!(session.locked.is_empty());
8877 assert!(session.command(Command::Backspace).handled);
8878 assert_eq!(session.snapshot().preedit, "b");
8879 type_digits(&mut session, "ugao");
8881 let spellings = session.snapshot().nine_key_spellings;
8882 let bu = spellings
8883 .iter()
8884 .position(|spelling| spelling == "bu")
8885 .unwrap();
8886 assert!(session.choose_spelling(bu).handled);
8887 assert_eq!(session.snapshot().preedit, "bu'gao");
8888 assert_eq!(words(&session).first().map(String::as_str), Some("不好"));
8889 assert!(session.command(Command::Backspace).handled);
8890 assert_eq!(session.snapshot().preedit, "bu'ga");
8891 assert_eq!(session.locked, ["bu"]);
8892 session.command(Command::Cancel);
8894 type_digits(&mut session, "q");
8895 let spellings = session.snapshot().nine_key_spellings;
8896 let w = spellings
8897 .iter()
8898 .position(|spelling| spelling == "W")
8899 .unwrap();
8900 assert!(session.choose_spelling(w).handled);
8901 assert_eq!(session.snapshot().preedit, "w");
8902 assert!(session.command(Command::Backspace).handled);
8903 assert_eq!(session.snapshot().preedit, "q");
8904 }
8905
8906 #[test]
8907 fn fourteen_key_mixes_english_words_and_reads_them() {
8908 let fixture = fixture_with(FOURTEEN_FIXTURE);
8909 let mut session = fourteen_key_session(&fixture.paths, mixed());
8910 type_digits(&mut session, "oj");
8912 assert_eq!(words(&session).first().map(String::as_str), Some("ok"));
8913 assert_eq!(session.snapshot().nine_key_reading, "ok");
8914 session.command(Command::Cancel);
8915 type_digits(&mut session, "ol");
8917 assert_eq!(words(&session).first().map(String::as_str), Some("old"));
8918 assert_eq!(session.snapshot().nine_key_reading, "ol");
8919 let result = session.select(0);
8920 assert_eq!(result.commit.as_deref(), Some("old"));
8921 assert!(!session.active());
8922 }
8923
8924 #[test]
8926 fn reading_is_judged_by_where_the_leading_row_came_from() {
8927 let mut nine = detached();
8928 nine.digits = "94".into();
8929 let mut fourteen = detached();
8930 fourteen.grid = KeyGrid::FourteenKey;
8931 fourteen.digits = "zu".into();
8932 let pinyin_sources = [
8933 CandidateSource::Database,
8934 CandidateSource::UserDatabase,
8935 CandidateSource::Generated,
8936 CandidateSource::Fallback,
8937 CandidateSource::NeuralKeyboard,
8938 ];
8939 for source in pinyin_sources {
8940 assert_eq!(
8941 nine.reading_for(Some(&spelled("西", "94", "xi", 1, source))),
8942 "xi"
8943 );
8944 assert_eq!(
8945 fourteen.reading_for(Some(&spelled("西", "zu", "xi", 1, source))),
8946 "xi"
8947 );
8948 assert_eq!(nine.reading_for(Some(&item("2026年", "94", 1, source))), "");
8950 }
8951 for source in [CandidateSource::Emoji, CandidateSource::Kaomoji] {
8952 assert_eq!(nine.reading_for(Some(&item("😀", "94", 1, source))), "");
8954 assert_eq!(fourteen.reading_for(Some(&item("😀", "zu", 1, source))), "");
8955 }
8956 let english = item("xi", "xi", 1, CandidateSource::EnglishDictionary);
8957 assert_eq!(nine.reading_for(Some(&english)), "");
8959 assert_eq!(fourteen.reading_for(Some(&english)), "xi");
8960 let longer = item("Xian", "xian", 1, CandidateSource::EnglishDictionary);
8961 assert_eq!(fourteen.reading_for(Some(&longer)), "xi");
8962 assert_eq!(nine.reading_for(None), "");
8963 }
8964
8965 #[test]
8966 fn commit_raw_moves_the_digit_buffer_into_the_result() {
8967 let fixture = fixture();
8968 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
8969 type_digits(&mut session, "64426");
8970
8971 let (result, allocations) =
8972 crate::ime::personal_rerank::allocations::count(|| session.command(Command::CommitRaw));
8973 assert_eq!(result.commit.as_deref(), Some("64426"));
8974 assert!(!session.active());
8975 assert_eq!(
8976 allocations, 0,
8977 "九键原样上屏复制了数字缓冲:{allocations} 次分配"
8978 );
8979 }
8980
8981 #[test]
8982 fn switching_grids_ends_the_composition() {
8983 let fixture = fixture_with(FOURTEEN_FIXTURE);
8984 let mut session = open(&fixture.paths, false, EnglishInputOptions::default());
8985 type_digits(&mut session, "64426");
8986 assert!(!session.sentence_seeds.is_empty());
8987 session.set_grid(KeyGrid::FourteenKey);
8988 assert!(!session.active());
8989 assert!(session.sentence_seeds.is_empty());
8990 assert!(!session.character(b'6').handled);
8991 type_digits(&mut session, "bugao");
8992 assert_eq!(words(&session).first().map(String::as_str), Some("你好"));
8993 }
8994
8995 #[test]
8997 fn fourteen_key_commit_raw_finishes_instead_of_writing_the_codes() {
8998 let fixture = fixture_with(FOURTEEN_FIXTURE);
8999 let mut nine = open(&fixture.paths, false, EnglishInputOptions::default());
9000 type_digits(&mut nine, "64426");
9001 assert_eq!(
9002 nine.command(Command::CommitRaw).commit.as_deref(),
9003 Some("64426")
9004 );
9005
9006 let mut session = fourteen_key_session(&fixture.paths, EnglishInputOptions::default());
9007 type_digits(&mut session, "bugao");
9008 assert_eq!(
9009 session.command(Command::CommitRaw).commit.as_deref(),
9010 Some("你好")
9011 );
9012 assert!(!session.active());
9013 type_digits(&mut session, "bugao");
9014 assert_eq!(session.finish(999).commit.as_deref(), Some("你好"));
9015 assert!(!session.active());
9016
9017 type_digits(&mut session, "bu");
9018 let bu = session
9019 .snapshot()
9020 .nine_key_spellings
9021 .iter()
9022 .position(|spelling| spelling == "bu")
9023 .unwrap();
9024 assert!(session.choose_spelling(bu).handled);
9025 type_digits(&mut session, "gao");
9026 assert_eq!(
9027 session.command(Command::CommitRaw).commit.as_deref(),
9028 Some("不好")
9029 );
9030
9031 type_digits(&mut session, "qqq");
9032 assert!(session.snapshot().candidates.is_empty());
9033 assert_eq!(
9034 session.command(Command::CommitRaw).commit.as_deref(),
9035 Some("qqq")
9036 );
9037 assert!(!session.active());
9038 }
9039
9040 const SHUANGPIN_FIXTURE: &str = "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_n VALUES('ni','n','你',100);CREATE TABLE tbl_1_h(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_h VALUES('hao','h','好',100);CREATE TABLE tbl_1_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_z VALUES('zhong','z','中',300);CREATE TABLE tbl_1_g(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_g VALUES('guo','g','国',200);CREATE TABLE tbl_1_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_a VALUES('an','a','安',150);CREATE TABLE tbl_1_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_b VALUES('ban','b','班',120);CREATE TABLE tbl_1_l(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_l VALUES('lu','l','路',90);CREATE TABLE tbl_2_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_n VALUES('ni''hao','nh','你好',1000);CREATE TABLE tbl_2_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_2_z VALUES('zhong''guo','zg','中国',900),('zou''guo','zg','走过',950);";
9042
9043 #[test]
9044 fn eighteen_key_shuangpin_locks_readings_and_finishes_real_pinyin() {
9045 let fixture = fixture_with(SHUANGPIN_FIXTURE);
9046 let mut session = open(&fixture.paths, false, mixed());
9047 let table = crate::nine_key_shuangpin::resolve_eighteen_key_shuangpin(
9048 crate::ShuangpinProfileKind::Xiaohe,
9049 None,
9050 )
9051 .unwrap();
9052 session.set_grid(KeyGrid::EighteenKeyShuangpin(table));
9053 type_digits(&mut session, "bi");
9054 let choices = session.snapshot().nine_key_spellings;
9055 assert!(choices.iter().any(|value| value == "ni"), "{choices:?}");
9056 assert!(choices.iter().any(|value| value == "bi"), "{choices:?}");
9057 assert!(
9058 session
9059 .choose_spelling(spelling_index(&session, "ni"))
9060 .handled
9061 );
9062 assert!(session.command(Command::Backspace).handled);
9063 assert_eq!(session.snapshot().editing_text, "bi");
9064 assert!(
9065 session
9066 .choose_spelling(spelling_index(&session, "ni"))
9067 .handled
9068 );
9069 type_digits(&mut session, "hx");
9070 assert!(words(&session).contains(&"你好".to_owned()));
9071 assert_eq!(session.snapshot().nine_key_reading, "ni'hao");
9072 assert_eq!(session.snapshot().scheme, SchemeType::Shuangpin);
9073 assert_eq!(
9074 session.command(Command::CommitRaw).commit.as_deref(),
9075 Some("你好")
9076 );
9077 assert!(!session.active());
9078 }
9079
9080 #[test]
9081 fn fourteen_key_shuangpin_locks_readings_and_finishes_real_pinyin() {
9082 let fixture = fixture_with(SHUANGPIN_FIXTURE);
9083 let mut session = open(&fixture.paths, false, mixed());
9084 let table = crate::nine_key_shuangpin::resolve_fourteen_key_shuangpin(
9085 crate::ShuangpinProfileKind::Xiaohe,
9086 None,
9087 )
9088 .unwrap();
9089 session.set_grid(KeyGrid::FourteenKeyShuangpin(table));
9090 type_digits(&mut session, "bu");
9091 let choices = session.snapshot().nine_key_spellings;
9092 assert!(choices.iter().any(|value| value == "ni"), "{choices:?}");
9093 assert!(choices.iter().any(|value| value == "bu"), "{choices:?}");
9094 let selected = spelling_index(&session, "ni");
9095 assert!(session.choose_spelling(selected).handled);
9096 assert!(session.command(Command::Backspace).handled);
9097 assert_eq!(session.snapshot().editing_text, "bu");
9098 assert!(
9099 session
9100 .choose_spelling(spelling_index(&session, "ni"))
9101 .handled
9102 );
9103 type_digits(&mut session, "gc");
9104 assert!(words(&session).contains(&"你好".to_owned()));
9105 assert_eq!(session.snapshot().nine_key_reading, "ni'hao");
9106 assert_eq!(
9107 session.command(Command::CommitRaw).commit.as_deref(),
9108 Some("你好")
9109 );
9110 assert!(!session.active());
9111 }
9112
9113 #[test]
9114 fn fourteen_key_shuangpin_queries_semicolon_and_alternate_codes() {
9115 let fixture = fixture_with("CREATE TABLE tbl_1_q(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_q VALUES('qing','q','青',100);CREATE TABLE tbl_1_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);INSERT INTO tbl_1_j VALUES('ju','j','居',100);");
9116 let mut session = open(&fixture.paths, false, mixed());
9117 let table = crate::nine_key_shuangpin::resolve_fourteen_key_shuangpin(
9118 crate::ShuangpinProfileKind::Microsoft,
9119 None,
9120 )
9121 .unwrap();
9122 session.set_grid(KeyGrid::FourteenKeyShuangpin(table));
9123 for (code, word) in [("ql", "青"), ("ju", "居"), ("jt", "居")] {
9124 type_digits(&mut session, code);
9125 assert!(
9126 words(&session).contains(&word.to_owned()),
9127 "{code}: {:?}",
9128 words(&session)
9129 );
9130 assert_eq!(
9131 session.command(Command::CommitRaw).commit.as_deref(),
9132 Some(word)
9133 );
9134 }
9135 }
9136
9137 fn shuangpin_session(paths: &RuntimePaths, kind: NineKeyShuangpinKind) -> NineKeySession {
9138 let mut session = open(paths, false, mixed());
9139 let table = crate::nine_key_shuangpin::resolve(kind, None).unwrap();
9140 session.set_grid(KeyGrid::NineKeyShuangpin(table));
9141 session
9142 }
9143
9144 #[test]
9145 fn nine_key_shuangpin_types_two_digits_per_syllable() {
9146 let fixture = fixture_with(SHUANGPIN_FIXTURE);
9147 let mut session = shuangpin_session(&fixture.paths, NineKeyShuangpinKind::Migala3);
9148 type_digits(&mut session, "6447");
9149 let view = session.snapshot();
9150 assert_eq!(view.editing_text, "6447");
9151 assert_eq!(words(&session).first().map(String::as_str), Some("你好"));
9152 assert_eq!(view.nine_key_reading, "ni'hao");
9153 assert_eq!(view.candidates[0].pinyin, "6447");
9154 assert!(view
9156 .candidate_sources
9157 .iter()
9158 .all(|source| *source != CandidateSource::EnglishDictionary));
9159 assert_eq!(session.select(0).commit.as_deref(), Some("你好"));
9160 assert!(!session.active());
9161
9162 type_digits(&mut session, "814");
9164 assert_eq!(words(&session).first().map(String::as_str), Some("中国"));
9165 assert_eq!(session.snapshot().nine_key_reading, "zhong'g");
9166 session.command(Command::Cancel);
9167
9168 assert!(session.character(b'5').handled);
9170 assert!(session.character(b'0').handled);
9171 assert_eq!(words(&session).first().map(String::as_str), Some("路"));
9172 session.command(Command::Cancel);
9173 let mut migala2 = shuangpin_session(&fixture.paths, NineKeyShuangpinKind::Migala2);
9175 assert!(!migala2.character(b'0').handled);
9176 assert!(!migala2.active());
9177 }
9178
9179 #[test]
9180 fn nine_key_shuangpin_shares_codes_and_locks_one_reading() {
9181 let fixture = fixture_with(SHUANGPIN_FIXTURE);
9182 let mut session = shuangpin_session(&fixture.paths, NineKeyShuangpinKind::Migala3);
9183 type_digits(&mut session, "25");
9185 let found = words(&session);
9186 assert!(found.contains(&"安".to_owned()), "{found:?}");
9187 assert!(found.contains(&"班".to_owned()), "{found:?}");
9188 let spellings = session.snapshot().nine_key_spellings;
9189 for spelling in ["an", "ban", "ang", "can"] {
9190 assert!(spellings.contains(&spelling.to_owned()), "{spellings:?}");
9191 }
9192 assert!(
9194 session
9195 .choose_spelling(spelling_index(&session, "ban"))
9196 .handled
9197 );
9198 assert_eq!(session.snapshot().preedit, "ban");
9199 assert_eq!(session.snapshot().editing_text, "25");
9200 assert_eq!(words(&session), ["班"]);
9201 assert!(session.command(Command::Backspace).handled);
9203 assert_eq!(session.snapshot().preedit, "25");
9204 assert!(session.locked.is_empty());
9205 assert!(session.command(Command::Backspace).handled);
9206 assert_eq!(session.snapshot().preedit, "2");
9207
9208 session.command(Command::Cancel);
9210 type_digits(&mut session, "8146");
9211 assert!(
9212 session
9213 .choose_spelling(spelling_index(&session, "zhong"))
9214 .handled
9215 );
9216 assert_eq!(session.snapshot().preedit, "zhong'46");
9217 assert_eq!(words(&session).first().map(String::as_str), Some("中国"));
9218 assert!(session.command(Command::Backspace).handled);
9219 assert_eq!(session.snapshot().preedit, "zhong'4");
9220 assert_eq!(session.locked, ["zhong"]);
9221 session.command(Command::Cancel);
9223 type_digits(&mut session, "8146");
9224 let zhong = index_of(&session, "中");
9225 assert_eq!(session.select(zhong).commit.as_deref(), Some("中"));
9226 assert_eq!(session.snapshot().editing_text, "46");
9227 assert_eq!(words(&session).first().map(String::as_str), Some("国"));
9228 }
9229
9230 #[test]
9231 fn nine_key_shuangpin_initials_check_the_key_of_each_initial() {
9232 let fixture = fixture_with(SHUANGPIN_FIXTURE);
9233 let mut session = shuangpin_session(&fixture.paths, NineKeyShuangpinKind::Migala3);
9234 type_digits(&mut session, "84");
9236 let found = words(&session);
9237 assert!(found.contains(&"中国".to_owned()), "{found:?}");
9238 assert!(!found.contains(&"走过".to_owned()), "{found:?}");
9239 assert_eq!(session.snapshot().nine_key_reading, "z'g");
9240 session.command(Command::Cancel);
9241 type_digits(&mut session, "94");
9242 let found = words(&session);
9243 assert!(found.contains(&"走过".to_owned()), "{found:?}");
9244 assert!(!found.contains(&"中国".to_owned()), "{found:?}");
9245 }
9246
9247 #[test]
9248 fn nine_key_shuangpin_splits_and_letters_follow_the_table() {
9249 let fixture = fixture_with(SHUANGPIN_FIXTURE);
9250 let mut session = shuangpin_session(&fixture.paths, NineKeyShuangpinKind::Migala3);
9251 assert!(session.character(b'8').handled);
9253 assert!(session.character(b'\'').handled);
9254 assert!(session.character(b'4').handled);
9255 assert_eq!(session.snapshot().preedit, "8'4");
9256 assert_eq!(words(&session).first().map(String::as_str), Some("中国"));
9257 session.command(Command::Cancel);
9258 type_digits(&mut session, "2");
9260 let spellings = session.snapshot().nine_key_spellings;
9261 for letter in ["A", "B", "C", "E", "O", "2"] {
9262 assert!(spellings.contains(&letter.to_owned()), "{spellings:?}");
9263 }
9264 assert!(
9265 session
9266 .choose_spelling(spelling_index(&session, "A"))
9267 .handled
9268 );
9269 assert_eq!(session.snapshot().preedit, "a");
9270 assert_eq!(words(&session), ["安"]);
9271 assert_eq!(
9273 session.command(Command::CommitRaw).commit.as_deref(),
9274 Some("2")
9275 );
9276 }
9277
9278 #[test]
9279 fn nine_key_shuangpin_does_not_change_the_nine_key_grid() {
9280 let fixture = fixture_with(SHUANGPIN_FIXTURE);
9281 let mut session = open(&fixture.paths, false, mixed());
9282 type_digits(&mut session, "64426");
9283 assert_eq!(words(&session).first().map(String::as_str), Some("你好"));
9284 let mut shuangpin = shuangpin_session(&fixture.paths, NineKeyShuangpinKind::Migala2);
9286 type_digits(&mut shuangpin, "6442");
9287 assert_eq!(words(&shuangpin).first().map(String::as_str), Some("你好"));
9288 assert!(KeyGrid::NineKey.spelling_table().longest_code > 2);
9289 assert_ne!(KeyGrid::NineKey, shuangpin.grid);
9290 }
9291
9292 #[test]
9293 fn nine_key_shuangpin_words_are_found_by_their_initial_keys() {
9294 let table =
9295 crate::nine_key_shuangpin::resolve(NineKeyShuangpinKind::Migala3, None).unwrap();
9296 let grid = KeyGrid::NineKeyShuangpin(table);
9297 assert_eq!(shuangpin_word_codes(grid, "8146"), None);
9299 let codes = shuangpin_word_codes(grid, "814643").unwrap();
9300 assert!(codes.contains(&"zgr".to_owned()));
9301 assert!(codes.contains(&"tgh".to_owned()));
9302 assert!(!codes.iter().any(|code| code.starts_with('x')));
9303 assert!(shuangpin_word_spells(table, "zhong'guo'ren", "814643"));
9305 assert!(!shuangpin_word_spells(table, "zhang'guo'rong", "814643"));
9306 assert!(shuangpin_word_spells(table, "zhong'guo'ren", "81464"));
9308 assert!(!shuangpin_word_spells(table, "zhong'guo'ren'min", "81464"));
9309 assert!(!shuangpin_word_spells(table, "zhong'guo", "81464"));
9310 }
9311}
9312
9313#[cfg(test)]
9314#[path = "nine_key/english_page_append_tests.rs"]
9315mod english_page_append_tests;