use crate::modules::core::unknown_handler::{TransliterationMetadata, TransliterationResult};
use crate::modules::hub::{HubError, HubInput};
use crate::modules::registry::SchemaRegistryTrait;
use rustc_hash::FxHashMap;
use thiserror::Error;
#[derive(Error, Debug, Clone)]
pub enum ConverterError {
#[error("Invalid input for script {script}: {message}")]
InvalidInput { script: String, message: String },
#[error("Mapping not found for script {script}: {token}")]
MappingNotFound { script: String, token: String },
#[error("Conversion failed for script {script}: {reason}")]
ConversionFailed { script: String, reason: String },
#[error("Hub error: {0}")]
HubError(#[from] HubError),
}
#[derive(Debug, Clone)]
pub struct ConverterStats {
pub total_converters: usize,
pub total_scripts: usize,
pub bidirectional_scripts: usize,
pub implicit_a_scripts: usize,
}
pub trait ScriptConverter: Send + Sync {
fn to_hub(&self, script: &str, input: &str) -> Result<HubInput, ConverterError>;
#[allow(clippy::wrong_self_convention)]
fn from_hub(&self, script: &str, _hub_input: &HubInput) -> Result<String, ConverterError> {
Err(ConverterError::ConversionFailed {
script: script.to_string(),
reason: "Reverse conversion not supported by this converter".to_string(),
})
}
fn to_hub_with_metadata(
&self,
script: &str,
input: &str,
) -> Result<(HubInput, TransliterationMetadata), ConverterError> {
let hub_input = self.to_hub(script, input)?;
let metadata = TransliterationMetadata::new(script, "hub");
Ok((hub_input, metadata))
}
#[allow(clippy::wrong_self_convention)]
fn from_hub_with_metadata(
&self,
script: &str,
hub_input: &HubInput,
) -> Result<TransliterationResult, ConverterError> {
let output = self.from_hub(script, hub_input)?;
Ok(TransliterationResult::simple(output))
}
fn supported_scripts(&self) -> Vec<&'static str>;
fn supports_script(&self, script: &str) -> bool {
self.supported_scripts().contains(&script)
}
fn script_has_implicit_a(&self, script: &str) -> bool;
fn supports_reverse_conversion(&self, script: &str) -> bool {
use crate::modules::hub::HubFormat;
let dummy_input = HubFormat::AlphabetTokens(vec![]);
self.from_hub(script, &dummy_input).is_ok()
}
}
pub trait TokenConverter: Send + Sync {
fn string_to_tokens(&self, input: &str) -> HubTokenSequence;
fn tokens_to_string(&self, tokens: &HubTokenSequence) -> String;
fn script_name(&self) -> &'static str;
fn is_alphabet(&self) -> bool;
}
#[derive(Default)]
pub struct TokenConverterRegistry {
converters: Vec<Box<dyn TokenConverter>>,
script_to_converter: FxHashMap<String, usize>,
}
impl TokenConverterRegistry {
pub fn new() -> Self {
Self {
converters: Vec::new(),
script_to_converter: FxHashMap::default(),
}
}
pub fn register_converter(&mut self, converter: Box<dyn TokenConverter>) {
let converter_index = self.converters.len();
let script_name = converter.script_name().to_string();
self.script_to_converter
.insert(script_name, converter_index);
self.converters.push(converter);
}
pub fn register_converter_with_aliases(
&mut self,
converter: Box<dyn TokenConverter>,
aliases: &[&str],
) {
let converter_index = self.converters.len();
let script_name = converter.script_name().to_string();
self.script_to_converter
.insert(script_name, converter_index);
for alias in aliases {
self.script_to_converter
.insert(alias.to_string(), converter_index);
}
self.converters.push(converter);
}
pub fn convert_to_tokens(
&self,
script: &str,
input: &str,
) -> Result<HubTokenSequence, ConverterError> {
if let Some(&converter_index) = self.script_to_converter.get(script) {
let tokens = self.converters[converter_index].string_to_tokens(input);
return Ok(tokens);
}
Err(ConverterError::ConversionFailed {
script: script.to_string(),
reason: format!("No token converter found for script: {}", script),
})
}
pub fn convert_from_tokens(
&self,
script: &str,
tokens: &HubTokenSequence,
) -> Result<String, ConverterError> {
if let Some(&converter_index) = self.script_to_converter.get(script) {
let output = self.converters[converter_index].tokens_to_string(tokens);
return Ok(output);
}
Err(ConverterError::ConversionFailed {
script: script.to_string(),
reason: format!("No token converter found for script: {}", script),
})
}
pub fn supports_script(&self, script: &str) -> bool {
self.script_to_converter.contains_key(script)
}
pub fn list_supported_scripts(&self) -> Vec<String> {
self.script_to_converter.keys().cloned().collect()
}
pub fn is_alphabet_script(&self, script: &str) -> bool {
self.script_to_converter
.get(script)
.map(|&idx| self.converters[idx].is_alphabet())
.unwrap_or(false)
}
}
pub struct ScriptConverterRegistry {
converters: Vec<Box<dyn ScriptConverter>>,
script_to_converter: FxHashMap<String, usize>,
token_converters: TokenConverterRegistry,
}
impl ScriptConverterRegistry {
pub fn new() -> Self {
Self {
converters: Vec::new(),
script_to_converter: FxHashMap::default(),
token_converters: TokenConverterRegistry::new(),
}
}
pub fn register_converter(&mut self, converter: Box<dyn ScriptConverter>) {
let converter_index = self.converters.len();
for script in converter.supported_scripts() {
self.script_to_converter
.insert(script.to_string(), converter_index);
}
self.converters.push(converter);
}
pub fn to_hub(&self, script: &str, input: &str) -> Result<HubInput, ConverterError> {
self.to_hub_with_schema_registry(script, input, None)
}
pub fn to_hub_with_schema_registry(
&self,
script: &str,
input: &str,
schema_registry: Option<&crate::modules::registry::SchemaRegistry>,
) -> Result<HubInput, ConverterError> {
let resolved_script = if let Some(registry) = schema_registry {
if let Some(schema) = registry.find_schema_by_alias(script) {
&schema.name
} else {
script
}
} else {
script
};
if self.token_converters.supports_script(resolved_script) {
let tokens = self
.token_converters
.convert_to_tokens(resolved_script, input)?;
let hub_format = if self.token_converters.is_alphabet_script(resolved_script) {
HubFormat::AlphabetTokens(tokens)
} else {
HubFormat::AbugidaTokens(tokens)
};
return Ok(hub_format);
}
let canonical_script = self.resolve_script_alias_with_registry(script, schema_registry);
if let Some(&converter_index) = self.script_to_converter.get(&canonical_script) {
return self.converters[converter_index].to_hub(&canonical_script, input);
}
if let Some(registry) = schema_registry {
let lookup_name = if canonical_script != script {
canonical_script.as_str()
} else {
resolved_script
};
if let Some(schema) = registry.get_schema(lookup_name) {
return self.to_hub_from_runtime_schema(input, schema);
}
if let Some(schema) = registry.get_schema(script) {
return self.to_hub_from_runtime_schema(input, schema);
}
}
Err(ConverterError::ConversionFailed {
script: script.to_string(),
reason: "No converter found for script".to_string(),
})
}
fn to_hub_from_runtime_schema(
&self,
input: &str,
schema: &crate::modules::registry::Schema,
) -> Result<HubInput, ConverterError> {
use std::str::FromStr;
let mut reverse: rustc_hash::FxHashMap<&str, &str> = rustc_hash::FxHashMap::default();
for (token_name, char_value) in &schema.mappings {
reverse.insert(char_value.as_str(), token_name.as_str());
}
let mut candidates: Vec<(&str, &str)> = reverse.iter().map(|(&k, &v)| (k, v)).collect();
candidates.sort_by_key(|b| std::cmp::Reverse(b.0.len()));
let is_alphabet = schema.metadata.script_type == "roman"
|| schema.target == "alphabet_tokens"
|| schema.target == "iso15919";
let mut tokens: HubTokenSequence = Vec::new();
let bytes = input.as_bytes();
let len = input.len();
let mut pos = 0usize;
while pos < len {
let mut matched = false;
for &(pat, token_name) in &candidates {
let pat_len = pat.len();
if pos + pat_len <= len && &bytes[pos..pos + pat_len] == pat.as_bytes() {
let hub_token = if is_alphabet {
match AlphabetToken::from_str(token_name) {
Ok(t) => HubToken::Alphabet(t),
Err(_) => {
HubToken::Alphabet(AlphabetToken::Unknown(pat.to_string()))
}
}
} else {
match AbugidaToken::from_str(token_name) {
Ok(t) => HubToken::Abugida(t),
Err(_) => HubToken::Abugida(AbugidaToken::Unknown(pat.to_string())),
}
};
tokens.push(hub_token);
pos += pat_len;
matched = true;
break;
}
}
if !matched {
let rest = &input[pos..];
let ch = rest.chars().next().unwrap();
let unknown_str = ch.to_string();
if is_alphabet {
tokens.push(HubToken::Alphabet(AlphabetToken::Unknown(unknown_str)));
} else {
tokens.push(HubToken::Abugida(AbugidaToken::Unknown(unknown_str)));
}
pos += ch.len_utf8();
}
}
if is_alphabet {
Ok(HubFormat::AlphabetTokens(tokens))
} else {
Ok(HubFormat::AbugidaTokens(tokens))
}
}
pub fn from_hub(&self, script: &str, hub_input: &HubInput) -> Result<String, ConverterError> {
self.from_hub_with_schema_registry(script, hub_input, None)
}
pub fn from_hub_with_schema_registry(
&self,
script: &str,
hub_input: &HubInput,
schema_registry: Option<&crate::modules::registry::SchemaRegistry>,
) -> Result<String, ConverterError> {
let resolved_script = if let Some(registry) = schema_registry {
if let Some(schema) = registry.find_schema_by_alias(script) {
&schema.name
} else {
script
}
} else {
script
};
if self.token_converters.supports_script(resolved_script) {
let tokens = match hub_input {
HubFormat::AlphabetTokens(tokens) => tokens,
HubFormat::AbugidaTokens(tokens) => tokens,
};
let result = self
.token_converters
.convert_from_tokens(resolved_script, tokens)?;
return Ok(result);
}
let canonical_script = self.resolve_script_alias_with_registry(script, schema_registry);
if let Some(&converter_index) = self.script_to_converter.get(&canonical_script) {
return self.converters[converter_index].from_hub(&canonical_script, hub_input);
}
Err(ConverterError::ConversionFailed {
script: script.to_string(),
reason: "No converter found for script".to_string(),
})
}
pub fn to_hub_with_metadata(
&self,
script: &str,
input: &str,
) -> Result<(HubInput, TransliterationMetadata), ConverterError> {
if self.token_converters.supports_script(script) {
let tokens = self.token_converters.convert_to_tokens(script, input)?;
let hub_format = if self.token_converters.is_alphabet_script(script) {
HubFormat::AlphabetTokens(tokens)
} else {
HubFormat::AbugidaTokens(tokens)
};
let metadata = TransliterationMetadata::new(script, script);
return Ok((hub_format, metadata));
}
if let Some(&converter_index) = self.script_to_converter.get(script) {
return self.converters[converter_index].to_hub_with_metadata(script, input);
}
Err(ConverterError::ConversionFailed {
script: script.to_string(),
reason: "No converter found for script".to_string(),
})
}
pub fn from_hub_with_metadata(
&self,
script: &str,
hub_input: &HubInput,
) -> Result<TransliterationResult, ConverterError> {
if self.token_converters.supports_script(script) {
let tokens = match hub_input {
HubFormat::AlphabetTokens(tokens) => tokens,
HubFormat::AbugidaTokens(tokens) => tokens,
};
let result = self.token_converters.convert_from_tokens(script, tokens)?;
let metadata = TransliterationMetadata::new(script, script);
return Ok(TransliterationResult {
output: result,
metadata: Some(metadata),
});
}
let canonical_script = self.resolve_script_alias(script);
if let Some(&converter_index) = self.script_to_converter.get(canonical_script) {
return self.converters[converter_index]
.from_hub_with_metadata(canonical_script, hub_input);
}
Err(ConverterError::ConversionFailed {
script: script.to_string(),
reason: "No converter found for script".to_string(),
})
}
pub fn supports_script(&self, script: &str) -> bool {
self.supports_script_with_registry(script, None)
}
pub fn supports_script_with_registry(
&self,
script: &str,
schema_registry: Option<&crate::modules::registry::SchemaRegistry>,
) -> bool {
if script.to_lowercase() == "devanagari" || script.to_lowercase() == "deva" {
return true;
}
if self.script_to_converter.contains_key(script)
|| self.token_converters.supports_script(script)
{
return true;
}
let resolved_script = if let Some(registry) = schema_registry {
if let Some(schema) = registry.find_schema_by_alias(script) {
&schema.name
} else {
self.resolve_script_alias(script)
}
} else {
self.resolve_script_alias(script)
};
if resolved_script != script {
self.script_to_converter.contains_key(resolved_script)
|| self.token_converters.supports_script(resolved_script)
} else {
false
}
}
fn resolve_script_alias<'a>(&self, script: &'a str) -> &'a str {
match script {
"hk" => "harvard_kyoto",
"bn" => "bengali",
"ta" => "tamil",
"te" => "telugu",
"gu" => "gujarati",
"kn" => "kannada",
"ml" => "malayalam",
"or" => "odia",
"pa" => "gurmukhi",
"si" => "sinhala",
"deva" => "devanagari",
"iso" => "iso15919",
_ => script,
}
}
fn resolve_script_alias_with_registry(
&self,
script: &str,
schema_registry: Option<&crate::modules::registry::SchemaRegistry>,
) -> String {
let resolved = self.resolve_script_alias(script);
if resolved != script {
return resolved.to_string();
}
if let Some(registry) = schema_registry {
if let Some(schema) = registry.find_schema_by_alias(script) {
return schema.name.clone();
}
}
script.to_string()
}
pub fn list_supported_scripts(&self) -> Vec<String> {
let mut scripts: Vec<String> = self.script_to_converter.keys().cloned().collect();
let token_scripts = self.token_converters.list_supported_scripts();
scripts.extend(token_scripts);
scripts.sort();
scripts.dedup();
scripts
}
pub fn supports_reverse_conversion(&self, script: &str) -> bool {
if script.to_lowercase() == "devanagari" || script.to_lowercase() == "deva" {
return true;
}
let canonical_script = self.resolve_script_alias(script);
if let Some(&converter_index) = self.script_to_converter.get(canonical_script) {
return self.converters[converter_index].supports_reverse_conversion(canonical_script);
}
false
}
pub fn script_has_implicit_a(&self, script: &str) -> bool {
if script.to_lowercase() == "devanagari" || script.to_lowercase() == "deva" {
return true;
}
let canonical_script = self.resolve_script_alias(script);
if let Some(&converter_index) = self.script_to_converter.get(canonical_script) {
return self.converters[converter_index].script_has_implicit_a(canonical_script);
}
false
}
pub fn get_stats(&self) -> ConverterStats {
let total_converters = self.converters.len();
let total_scripts = self.list_supported_scripts().len();
let bidirectional_scripts = self
.list_supported_scripts()
.iter()
.filter(|script| self.supports_reverse_conversion(script))
.count();
let implicit_a_scripts = self
.list_supported_scripts()
.iter()
.filter(|script| self.script_has_implicit_a(script))
.count();
ConverterStats {
total_converters,
total_scripts,
bidirectional_scripts,
implicit_a_scripts,
}
}
}
impl Default for ScriptConverterRegistry {
fn default() -> Self {
Self::new_with_all_converters()
}
}
impl ScriptConverterRegistry {
pub fn new_with_all_converters() -> Self {
let mut registry = Self::new();
register_schema_generated_converters(&mut registry);
for (converter, aliases) in register_token_converters_with_aliases() {
if aliases.is_empty() {
registry.token_converters.register_converter(converter);
} else {
let alias_refs: Vec<&str> = aliases.iter().map(|s| s.as_str()).collect();
registry
.token_converters
.register_converter_with_aliases(converter, &alias_refs);
}
}
registry
}
pub fn convert_to_hub(&self, script: &str, input: &str) -> Result<HubInput, ConverterError> {
self.to_hub(script, input)
}
pub fn script_has_implicit_vowels(&self, script: &str) -> Result<bool, ConverterError> {
if script.to_lowercase() == "devanagari" || script.to_lowercase() == "deva" {
return Ok(true);
}
let canonical_script = self.resolve_script_alias(script);
if let Some(&converter_index) = self.script_to_converter.get(canonical_script) {
return Ok(self.converters[converter_index].script_has_implicit_a(canonical_script));
}
Err(ConverterError::ConversionFailed {
script: script.to_string(),
reason: "Script not supported".to_string(),
})
}
}
pub mod processors;
include!(concat!(env!("OUT_DIR"), "/schema_generated.rs"));
pub use ScriptConverterRegistry as ConverterRegistry;
#[cfg(test)]
mod send_sync_tests {
use super::*;
use std::sync::Arc;
use std::thread;
#[test]
fn test_script_converter_send_sync() {
fn assert_send<T: Send>() {}
fn assert_sync<T: Sync>() {}
fn assert_send_sync<T: Send + Sync>() {}
assert_send::<Box<dyn ScriptConverter>>();
assert_sync::<Box<dyn ScriptConverter>>();
assert_send_sync::<Box<dyn ScriptConverter>>();
assert_send::<Arc<dyn ScriptConverter>>();
assert_sync::<Arc<dyn ScriptConverter>>();
assert_send_sync::<Arc<dyn ScriptConverter>>();
}
#[test]
fn test_script_converter_registry_send_sync() {
fn assert_send<T: Send>() {}
fn assert_sync<T: Sync>() {}
fn assert_send_sync<T: Send + Sync>() {}
assert_send::<ScriptConverterRegistry>();
assert_sync::<ScriptConverterRegistry>();
assert_send_sync::<ScriptConverterRegistry>();
assert_send::<Arc<ScriptConverterRegistry>>();
assert_sync::<Arc<ScriptConverterRegistry>>();
assert_send_sync::<Arc<ScriptConverterRegistry>>();
}
#[test]
fn test_registry_thread_safety() {
let registry = Arc::new(ScriptConverterRegistry::default());
let registry_clone = Arc::clone(®istry);
let handle = thread::spawn(move || {
let scripts = registry_clone.list_supported_scripts();
assert!(scripts.contains(&"devanagari".to_string()));
});
handle.join().unwrap();
}
}