use std::collections::HashSet;
use std::fmt;
use std::sync::{Mutex, MutexGuard, OnceLock, PoisonError};
use super::profile::{accepted_syllables, default_profile, profile};
use super::ShuangpinProfile;
use crate::diagnostics;
use crate::error::{EngineError, Result};
use crate::types::{ShuangpinCustomTable, ShuangpinProfileKind};
pub const MULTI_LETTER_INITIALS: [&str; 3] = ["zh", "ch", "sh"];
const SINGLE_LETTER_INITIALS: &[u8] = b"bpmfdtnlgkhjqxrzcsyw";
pub const FINALS: [&str; 33] = [
"a", "o", "e", "i", "u", "v", "ai", "ei", "ui", "ao", "ou", "iu", "ie", "ve", "ue", "an", "en",
"in", "un", "ang", "eng", "ing", "ong", "ia", "ua", "uo", "uai", "ian", "uan", "iao", "iang",
"uang", "iong",
];
pub const ZERO_INITIALS: [&str; 12] = [
"a", "ai", "an", "ang", "ao", "e", "ei", "en", "eng", "er", "o", "ou",
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CustomTablePart {
Initials,
Finals,
ZeroInitials,
}
impl CustomTablePart {
pub fn name(self) -> &'static str {
match self {
Self::Initials => "initials",
Self::Finals => "finals",
Self::ZeroInitials => "zero_initials",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CustomProfileError {
UnknownUnit { part: CustomTablePart, unit: String },
DuplicateUnit { part: CustomTablePart, unit: String },
MissingUnit {
part: CustomTablePart,
unit: &'static str,
},
InvalidKey {
part: CustomTablePart,
unit: String,
key: String,
},
InitialKeyTaken { unit: &'static str, key: String },
AmbiguousCode {
code: String,
first: String,
second: String,
},
}
impl fmt::Display for CustomProfileError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnknownUnit { part, unit } => {
write!(formatter, "unknown {} unit {unit:?}", part.name())
}
Self::DuplicateUnit { part, unit } => {
write!(formatter, "{} unit {unit:?} appears twice", part.name())
}
Self::MissingUnit { part, unit } => {
write!(formatter, "{} unit {unit:?} has no key", part.name())
}
Self::InvalidKey { part, unit, key } => {
write!(
formatter,
"{} unit {unit:?} has an invalid key {key:?}",
part.name()
)
}
Self::InitialKeyTaken { unit, key } => {
write!(formatter, "initial {unit:?} cannot take key {key:?}")
}
Self::AmbiguousCode {
code,
first,
second,
} => write!(
formatter,
"code {code:?} reads as both {first:?} and {second:?}"
),
}
}
}
impl std::error::Error for CustomProfileError {}
struct Canonical {
initials: Vec<(&'static str, String)>,
finals: Vec<(&'static str, String)>,
zero_initials: Vec<(&'static str, String)>,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum InitialText {
Single(u8),
Multi(&'static str),
}
#[derive(Clone, Copy, PartialEq, Eq)]
struct DecodedSyllable {
initial: InitialText,
final_unit: &'static str,
}
impl DecodedSyllable {
fn matches(self, text: &str) -> bool {
match self.initial {
InitialText::Single(key) => {
text.as_bytes().first() == Some(&key) && text.get(1..) == Some(self.final_unit)
}
InitialText::Multi(initial) => text
.strip_prefix(initial)
.is_some_and(|rest| rest == self.final_unit),
}
}
fn write_to(self, target: &mut String) {
target.clear();
match self.initial {
InitialText::Single(key) => target.push(char::from(key)),
InitialText::Multi(initial) => target.push_str(initial),
}
target.push_str(self.final_unit);
}
}
pub fn custom_profile(
table: &ShuangpinCustomTable,
) -> Result<&'static ShuangpinProfile, CustomProfileError> {
let canonical = canonicalize(table)?;
let mut interned = interned();
if let Some(found) = interned.iter().find(|profile| canonical.matches(profile)) {
return Ok(found);
}
check_codes(&canonical)?;
let profile: &'static ShuangpinProfile = Box::leak(Box::new(ShuangpinProfile {
kind: ShuangpinProfileKind::Custom,
initials: leak_pairs(canonical.initials),
zero_initials: leak_pairs(canonical.zero_initials),
finals: leak_pairs(canonical.finals),
}));
interned.push(profile);
Ok(profile)
}
pub fn session_profile(
kind: ShuangpinProfileKind,
table: Option<&ShuangpinCustomTable>,
shuangpin_enabled: bool,
) -> Result<&'static ShuangpinProfile> {
if let Some(builtin) = profile(kind) {
return Ok(builtin);
}
let resolved = match table {
Some(table) => custom_profile(table).map_err(|error| error.to_string()),
None => Err("no table".to_owned()),
};
match resolved {
Ok(custom) => Ok(custom),
Err(_) if !shuangpin_enabled => Ok(default_profile()),
Err(reason) => Err(EngineError::invalid(format!(
"{}: {reason}",
diagnostics::INVALID_CUSTOM_SHUANGPIN_PROFILE
))),
}
}
pub fn validate_custom_profile(table: &ShuangpinCustomTable) -> Result<(), CustomProfileError> {
custom_profile(table).map(|_| ())
}
pub fn check_custom_profile(table: &ShuangpinCustomTable) -> Result<(), CustomProfileError> {
check_codes(&canonicalize(table)?)
}
pub fn builtin_table(kind: ShuangpinProfileKind) -> Option<ShuangpinCustomTable> {
fn owned(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
pairs
.iter()
.map(|(unit, key)| ((*unit).to_owned(), (*key).to_owned()))
.collect()
}
let source = profile(kind)?;
Some(ShuangpinCustomTable {
initials: owned(source.initials),
finals: owned(source.finals),
zero_initials: owned(source.zero_initials),
})
}
fn interned() -> MutexGuard<'static, Vec<&'static ShuangpinProfile>> {
static INTERNED: OnceLock<Mutex<Vec<&'static ShuangpinProfile>>> = OnceLock::new();
INTERNED
.get_or_init(Mutex::default)
.lock()
.unwrap_or_else(PoisonError::into_inner)
}
fn leak_pairs(pairs: Vec<(&'static str, String)>) -> &'static [(&'static str, &'static str)] {
let leaked: Vec<(&'static str, &'static str)> = pairs
.into_iter()
.map(|(unit, key)| (unit, &*Box::leak(key.into_boxed_str())))
.collect();
Box::leak(leaked.into_boxed_slice())
}
impl Canonical {
fn matches(&self, profile: &ShuangpinProfile) -> bool {
fn same(left: &[(&'static str, String)], right: &[(&str, &str)]) -> bool {
left.len() == right.len()
&& left.iter().all(|(unit, key)| {
right
.iter()
.any(|(other_unit, other_key)| unit == other_unit && key == other_key)
})
}
same(&self.initials, profile.initials)
&& same(&self.finals, profile.finals)
&& same(&self.zero_initials, profile.zero_initials)
}
fn initial_for_key(&self, key: u8) -> InitialText {
self.initials
.iter()
.find(|(_, mapped)| mapped.as_bytes() == [key])
.map_or(InitialText::Single(key), |(unit, _)| {
InitialText::Multi(unit)
})
}
fn decode_pair(
&self,
first: u8,
second: u8,
buffer: &mut String,
) -> Result<Option<DecodedSyllable>, CustomProfileError> {
let accepted = accepted_syllables();
let mut found: Option<DecodedSyllable> = None;
let initial = self.initial_for_key(first);
for (unit, _) in self
.finals
.iter()
.filter(|(_, key)| key.as_bytes() == [second])
{
buffer.clear();
match initial {
InitialText::Single(key) => buffer.push(char::from(key)),
InitialText::Multi(unit) => buffer.push_str(unit),
}
let normalized = if *unit == "v"
&& matches!(initial, InitialText::Single(b'j' | b'q' | b'x' | b'y'))
{
"u"
} else {
unit
};
buffer.push_str(normalized);
if !accepted.contains(buffer.as_str()) {
continue;
}
let decoded = DecodedSyllable {
initial,
final_unit: normalized,
};
match &found {
Some(previous) if *previous != decoded => {
let mut previous_text = String::with_capacity(8);
previous.write_to(&mut previous_text);
return Err(CustomProfileError::AmbiguousCode {
code: code_text(first, second),
first: previous_text,
second: buffer.clone(),
});
}
Some(_) => {}
None => found = Some(decoded),
}
}
Ok(found)
}
}
fn code_text(first: u8, second: u8) -> String {
[char::from(first), char::from(second)].iter().collect()
}
fn canonicalize(table: &ShuangpinCustomTable) -> Result<Canonical, CustomProfileError> {
Ok(Canonical {
initials: canonical_part(
CustomTablePart::Initials,
&table.initials,
&MULTI_LETTER_INITIALS,
)?,
finals: canonical_part(CustomTablePart::Finals, &table.finals, &FINALS)?,
zero_initials: canonical_part(
CustomTablePart::ZeroInitials,
&table.zero_initials,
&ZERO_INITIALS,
)?,
})
}
fn canonical_part(
part: CustomTablePart,
entries: &[(String, String)],
units: &[&'static str],
) -> Result<Vec<(&'static str, String)>, CustomProfileError> {
let mut keys: Vec<Option<String>> = vec![None; units.len()];
for (unit, key) in entries {
let Some(index) = units.iter().position(|known| known == unit) else {
return Err(CustomProfileError::UnknownUnit {
part,
unit: unit.clone(),
});
};
if keys[index].is_some() {
return Err(CustomProfileError::DuplicateUnit {
part,
unit: unit.clone(),
});
}
let key = key.to_ascii_lowercase();
if !key_is_valid(part, key.as_bytes()) {
return Err(CustomProfileError::InvalidKey {
part,
unit: unit.clone(),
key,
});
}
keys[index] = Some(key);
}
units
.iter()
.zip(keys)
.map(|(unit, key)| {
key.map(|key| (*unit, key))
.ok_or(CustomProfileError::MissingUnit { part, unit })
})
.collect()
}
fn key_is_valid(part: CustomTablePart, key: &[u8]) -> bool {
match (part, key) {
(CustomTablePart::Initials, [key]) => key.is_ascii_lowercase(),
(CustomTablePart::Finals, [key]) => key.is_ascii_lowercase() || *key == b';',
(CustomTablePart::ZeroInitials, [first, second]) => {
first.is_ascii_lowercase() && (second.is_ascii_lowercase() || *second == b';')
}
_ => false,
}
}
fn check_codes(canonical: &Canonical) -> Result<(), CustomProfileError> {
let inherited_shoudao = canonical.matches(profile(ShuangpinProfileKind::Shoudao).unwrap());
let mut initial_keys: HashSet<u8> = HashSet::new();
for (unit, key) in &canonical.initials {
let byte = key.as_bytes()[0];
if SINGLE_LETTER_INITIALS.contains(&byte) || !initial_keys.insert(byte) {
return Err(CustomProfileError::InitialKeyTaken {
unit,
key: key.clone(),
});
}
}
let mut final_keys: Vec<u8> = canonical
.finals
.iter()
.map(|(_, key)| key.as_bytes()[0])
.collect();
final_keys.sort_unstable();
final_keys.dedup();
let mut buffer = String::with_capacity(8);
for first in b'a'..=b'z' {
for &second in &final_keys {
match canonical.decode_pair(first, second, &mut buffer) {
Err(CustomProfileError::AmbiguousCode {
code,
first,
second,
}) if inherited_shoudao && code == "dk" && first == "den" && second == "dia" => {}
result => {
result?;
}
}
}
}
let mut zero_codes: Vec<(&str, &str)> = Vec::with_capacity(canonical.zero_initials.len());
for (syllable, code) in &canonical.zero_initials {
if let Some((other, _)) = zero_codes.iter().find(|(_, used)| used == code) {
return Err(CustomProfileError::AmbiguousCode {
code: code.clone(),
first: (*other).to_owned(),
second: (*syllable).to_owned(),
});
}
let bytes = code.as_bytes();
if let Some(decoded) = canonical.decode_pair(bytes[0], bytes[1], &mut buffer)? {
if !decoded.matches(syllable) {
return Err(CustomProfileError::AmbiguousCode {
code: code.clone(),
first: buffer.clone(),
second: (*syllable).to_owned(),
});
}
}
zero_codes.push((syllable, code));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::shuangpin::profile::profile;
use crate::shuangpin::scheme::ShuangpinScheme;
use crate::shuangpin::utils::cvt_single_sp_to_pinyin;
use crate::types::SchemeKey;
const BUILTIN: [ShuangpinProfileKind; 5] = [
ShuangpinProfileKind::Xiaohe,
ShuangpinProfileKind::Ziranma,
ShuangpinProfileKind::Shoudao,
ShuangpinProfileKind::Microsoft,
ShuangpinProfileKind::SmartAbc,
];
fn owned(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
pairs
.iter()
.map(|(unit, key)| ((*unit).to_owned(), (*key).to_owned()))
.collect()
}
fn table_of(kind: ShuangpinProfileKind) -> ShuangpinCustomTable {
builtin_table(kind).unwrap()
}
fn set(units: impl IntoIterator<Item = &'static str>) -> Vec<&'static str> {
let mut units: Vec<&'static str> = units.into_iter().collect();
units.sort_unstable();
units
}
fn replace(entries: &mut [(String, String)], unit: &str, key: &str) {
entries
.iter_mut()
.find(|(known, _)| known == unit)
.unwrap()
.1 = key.to_owned();
}
#[test]
fn builtin_profiles_use_the_same_units() {
for kind in BUILTIN {
let source = profile(kind).unwrap();
assert_eq!(
set(source.initials.iter().map(|(unit, _)| *unit)),
set(MULTI_LETTER_INITIALS),
"{kind:?}"
);
assert_eq!(
set(source.finals.iter().map(|(unit, _)| *unit)),
set(FINALS),
"{kind:?}"
);
assert_eq!(
set(source.zero_initials.iter().map(|(unit, _)| *unit)),
set(ZERO_INITIALS),
"{kind:?}"
);
}
}
#[test]
fn builtin_tables_pass_and_decode_as_the_builtin_profile() {
for kind in BUILTIN {
let builtin = profile(kind).unwrap();
let custom = custom_profile(&table_of(kind)).unwrap();
assert_eq!(custom.kind, ShuangpinProfileKind::Custom);
assert_eq!(custom.uses_semicolon_key(), builtin.uses_semicolon_key());
for first in b'a'..=b'z' {
for second in (b'a'..=b'z').chain(*b";") {
let code = code_text(first, second);
assert_eq!(
cvt_single_sp_to_pinyin(&code, custom),
cvt_single_sp_to_pinyin(&code, builtin),
"{kind:?} {code}"
);
}
}
}
}
#[test]
fn shoudao_copy_only_inherits_its_known_collision_while_unchanged() {
let mut changed = table_of(ShuangpinProfileKind::Shoudao);
replace(&mut changed.zero_initials, "a", "oa");
assert!(matches!(
check_custom_profile(&changed),
Err(CustomProfileError::AmbiguousCode { code, .. }) if code == "dk"
));
}
#[test]
fn the_same_table_is_interned_once() {
let first = custom_profile(&table_of(ShuangpinProfileKind::Ziranma)).unwrap();
let mut shuffled = table_of(ShuangpinProfileKind::Ziranma);
shuffled.finals.reverse();
for (_, key) in &mut shuffled.zero_initials {
*key = key.to_ascii_uppercase();
}
let second = custom_profile(&shuffled).unwrap();
assert!(std::ptr::eq(first, second));
let other = custom_profile(&table_of(ShuangpinProfileKind::Xiaohe)).unwrap();
assert!(!std::ptr::eq(first, other));
}
#[test]
fn checking_a_custom_profile_reuses_the_initial_text_buffer() {
let table = table_of(ShuangpinProfileKind::Xiaohe);
let canonical = canonicalize(&table).unwrap();
let (_, allocations) = crate::ime::personal_rerank::allocations::count(|| {
check_codes(&canonical).unwrap();
});
assert!(
allocations <= 10,
"custom profile code validation allocated {allocations} buffers"
);
}
#[test]
fn a_custom_profile_moves_finals_and_initials() {
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
replace(&mut table.initials, "zh", "a");
table.zero_initials = owned(&[
("a", "oa"),
("ai", "od"),
("an", "oj"),
("ang", "oh"),
("ao", "oc"),
("e", "oe"),
("ei", "ow"),
("en", "of"),
("eng", "og"),
("er", "or"),
("o", "oo"),
("ou", "oz"),
]);
replace(&mut table.finals, "ing", ";");
let custom = custom_profile(&table).unwrap();
assert!(custom.uses_semicolon_key());
assert_eq!(cvt_single_sp_to_pinyin("ah", custom), "zhang");
assert_eq!(cvt_single_sp_to_pinyin("oa", custom), "a");
assert_eq!(cvt_single_sp_to_pinyin("x;", custom), "xing");
assert_eq!(cvt_single_sp_to_pinyin("xk", custom), "");
assert_eq!(cvt_single_sp_to_pinyin("vh", custom), "");
}
#[test]
fn a_zero_initial_can_use_semicolon_without_a_final_on_semicolon() {
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
replace(&mut table.zero_initials, "o", "o;");
let custom = custom_profile(&table).unwrap();
assert_eq!(cvt_single_sp_to_pinyin("o;", custom), "o");
assert!(custom.uses_semicolon_key());
let mut scheme = ShuangpinScheme::new(custom);
assert!(scheme.handle_key(SchemeKey::Letter(b'o')));
assert!(scheme.handle_key(SchemeKey::Semicolon));
assert_eq!(scheme.preedit(), "o;");
}
#[test]
fn incomplete_or_unknown_units_are_refused() {
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
table.finals.retain(|(unit, _)| unit != "iong");
assert_eq!(
validate_custom_profile(&table),
Err(CustomProfileError::MissingUnit {
part: CustomTablePart::Finals,
unit: "iong",
})
);
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
table.initials.push(("b".to_owned(), "b".to_owned()));
assert_eq!(
validate_custom_profile(&table),
Err(CustomProfileError::UnknownUnit {
part: CustomTablePart::Initials,
unit: "b".to_owned(),
})
);
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
table.finals.push(("iu".to_owned(), "q".to_owned()));
assert_eq!(
validate_custom_profile(&table),
Err(CustomProfileError::DuplicateUnit {
part: CustomTablePart::Finals,
unit: "iu".to_owned(),
})
);
assert!(matches!(
validate_custom_profile(&ShuangpinCustomTable::default()),
Err(CustomProfileError::MissingUnit { .. })
));
}
#[test]
fn keys_must_fit_their_place() {
for (part, unit, key) in [
(CustomTablePart::Finals, "iu", "qq"),
(CustomTablePart::Finals, "iu", "1"),
(CustomTablePart::Finals, "iu", ""),
(CustomTablePart::Initials, "zh", ";"),
(CustomTablePart::ZeroInitials, "a", "a"),
(CustomTablePart::ZeroInitials, "a", ";a"),
(CustomTablePart::ZeroInitials, "a", "aé"),
] {
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
let entries = match part {
CustomTablePart::Initials => &mut table.initials,
CustomTablePart::Finals => &mut table.finals,
CustomTablePart::ZeroInitials => &mut table.zero_initials,
};
replace(entries, unit, key);
assert_eq!(
validate_custom_profile(&table),
Err(CustomProfileError::InvalidKey {
part,
unit: unit.to_owned(),
key: key.to_ascii_lowercase(),
}),
"{unit} {key:?}"
);
}
}
#[test]
fn initials_cannot_take_a_single_letter_initial_or_share_a_key() {
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
replace(&mut table.initials, "sh", "s");
assert_eq!(
validate_custom_profile(&table),
Err(CustomProfileError::InitialKeyTaken {
unit: "sh",
key: "s".to_owned(),
})
);
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
replace(&mut table.initials, "ch", "u");
assert!(matches!(
validate_custom_profile(&table),
Err(CustomProfileError::InitialKeyTaken { key, .. }) if key == "u"
));
}
#[test]
fn colliding_codes_are_refused() {
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
replace(&mut table.finals, "ang", "k");
assert!(matches!(
validate_custom_profile(&table),
Err(CustomProfileError::AmbiguousCode { .. })
));
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
replace(&mut table.zero_initials, "a", "ua");
assert_eq!(
validate_custom_profile(&table),
Err(CustomProfileError::AmbiguousCode {
code: "ua".to_owned(),
first: "sha".to_owned(),
second: "a".to_owned(),
})
);
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
replace(&mut table.zero_initials, "o", "aa");
assert_eq!(
validate_custom_profile(&table),
Err(CustomProfileError::AmbiguousCode {
code: "aa".to_owned(),
first: "a".to_owned(),
second: "o".to_owned(),
})
);
}
#[test]
fn custom_has_no_builtin_table() {
assert_eq!(builtin_table(ShuangpinProfileKind::Custom), None);
}
#[test]
fn check_agrees_with_validate() {
let mut tables: Vec<ShuangpinCustomTable> = BUILTIN.into_iter().map(table_of).collect();
tables.push(ShuangpinCustomTable::default());
for (part, unit, key) in [
(CustomTablePart::Finals, "ang", "k"),
(CustomTablePart::Finals, "ing", ";"),
(CustomTablePart::Finals, "iu", "1"),
(CustomTablePart::Initials, "sh", "s"),
(CustomTablePart::Initials, "zh", "a"),
(CustomTablePart::ZeroInitials, "a", "ua"),
(CustomTablePart::ZeroInitials, "o", "aa"),
(CustomTablePart::ZeroInitials, "o", "o;"),
] {
let mut table = table_of(ShuangpinProfileKind::Xiaohe);
let entries = match part {
CustomTablePart::Initials => &mut table.initials,
CustomTablePart::Finals => &mut table.finals,
CustomTablePart::ZeroInitials => &mut table.zero_initials,
};
replace(entries, unit, key);
tables.push(table);
}
let mut missing = table_of(ShuangpinProfileKind::Microsoft);
missing.finals.retain(|(unit, _)| unit != "iong");
tables.push(missing);
for table in &tables {
assert_eq!(check_custom_profile(table), validate_custom_profile(table));
}
assert!(tables
.iter()
.any(|table| check_custom_profile(table).is_ok()));
assert!(tables
.iter()
.any(|table| check_custom_profile(table).is_err()));
}
#[test]
fn check_leaves_nothing_behind() {
let mut table = table_of(ShuangpinProfileKind::Ziranma);
replace(&mut table.zero_initials, "o", "o;");
let canonical = canonicalize(&table).unwrap();
let interned_now = || interned().iter().any(|profile| canonical.matches(profile));
assert_eq!(check_custom_profile(&table), Ok(()));
assert_eq!(check_custom_profile(&table), Ok(()));
assert!(!interned_now());
validate_custom_profile(&table).unwrap();
assert!(interned_now());
}
#[test]
fn an_invalid_table_is_not_interned() {
let mut table = table_of(ShuangpinProfileKind::Shoudao);
replace(&mut table.finals, "ang", "k");
assert!(custom_profile(&table).is_err());
assert!(custom_profile(&table).is_err());
}
}