html_translation_lib/

utils.rs

//! 工具函数模块
//!
//! 提供常用的工具函数和辅助功能

/// HTML工具函数
pub mod html {
    use crate::error::{TranslationError, TranslationResult};
    
    /// 验证HTML格式
    pub fn validate_html(html: &str) -> TranslationResult<()> {
        // 基本的HTML验证
        if html.trim().is_empty() {
            return Err(TranslationError::ParseError("HTML内容为空".to_string()));
        }
        
        // 检查基本的HTML结构
        if !html.contains('<') || !html.contains('>') {
            return Err(TranslationError::ParseError("无效的HTML格式".to_string()));
        }
        
        Ok(())
    }
    
    /// 清理HTML中的空白字符
    pub fn clean_whitespace(html: &str) -> String {
        // 移除多余的空白字符，但保持基本格式
        html.lines()
            .map(|line| line.trim())
            .filter(|line| !line.is_empty())
            .collect::<Vec<_>>()
            .join("\n")
    }
    
    /// 提取HTML中的纯文本内容
    pub fn extract_text_content(html: &str) -> TranslationResult<String> {
        use regex::Regex;
        
        // 移除HTML标签
        let tag_regex = Regex::new(r"<[^>]*>").unwrap();
        let text = tag_regex.replace_all(html, " ");
        
        // 清理空白字符
        let whitespace_regex = Regex::new(r"\s+").unwrap();
        let cleaned = whitespace_regex.replace_all(&text, " ");
        
        Ok(cleaned.trim().to_string())
    }
}

/// 文本工具函数
pub mod text {
    /// 计算文本相似度
    pub fn similarity(text1: &str, text2: &str) -> f32 {
        if text1 == text2 {
            return 1.0;
        }
        
        if text1.is_empty() || text2.is_empty() {
            return 0.0;
        }
        
        // 简单的编辑距离算法
        let len1 = text1.chars().count();
        let len2 = text2.chars().count();
        
        if len1 == 0 {
            return 0.0;
        }
        if len2 == 0 {
            return 0.0;
        }
        
        let mut matrix = vec![vec![0usize; len2 + 1]; len1 + 1];
        
        for (i, row) in matrix.iter_mut().enumerate().take(len1 + 1) {
            row[0] = i;
        }
        for j in 0..=len2 {
            matrix[0][j] = j;
        }
        
        let chars1: Vec<char> = text1.chars().collect();
        let chars2: Vec<char> = text2.chars().collect();
        
        for i in 1..=len1 {
            for j in 1..=len2 {
                let cost = if chars1[i - 1] == chars2[j - 1] { 0 } else { 1 };
                matrix[i][j] = std::cmp::min(
                    std::cmp::min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1),
                    matrix[i - 1][j - 1] + cost,
                );
            }
        }
        
        let distance = matrix[len1][len2];
        let max_len = std::cmp::max(len1, len2);
        
        1.0 - (distance as f32 / max_len as f32)
    }
    
    /// 检查文本是否主要由ASCII字符组成
    pub fn is_mostly_ascii(text: &str) -> bool {
        if text.is_empty() {
            return true;
        }
        
        let ascii_count = text.chars().filter(|c| c.is_ascii()).count();
        let total_count = text.chars().count();
        
        ascii_count as f32 / total_count as f32 > 0.8
    }
    
    /// 检查文本是否包含CJK字符
    pub fn contains_cjk(text: &str) -> bool {
        text.chars().any(|c| {
            // 中文字符范围
            ('\u{4E00}'..='\u{9FFF}').contains(&c) ||
            // 日文平假名
            ('\u{3040}'..='\u{309F}').contains(&c) ||
            // 日文片假名
            ('\u{30A0}'..='\u{30FF}').contains(&c) ||
            // 韩文字符
            ('\u{AC00}'..='\u{D7AF}').contains(&c)
        })
    }
    
    /// 统计文本中的单词数
    pub fn word_count(text: &str) -> usize {
        if contains_cjk(text) {
            // 对于CJK文本，字符数就是"单词数"
            text.chars().filter(|c| !c.is_whitespace()).count()
        } else {
            // 对于其他语言，按空白字符分割
            text.split_whitespace().count()
        }
    }
    
    /// 截断文本到指定长度，保持单词完整性
    pub fn truncate_words(text: &str, max_length: usize) -> String {
        if text.len() <= max_length {
            return text.to_string();
        }
        
        if contains_cjk(text) {
            // 对于CJK文本，直接截断
            text.chars().take(max_length).collect()
        } else {
            // 对于其他语言，保持单词完整性
            let mut result = String::new();
            for word in text.split_whitespace() {
                if result.len() + word.len() + 1 > max_length {
                    break;
                }
                if !result.is_empty() {
                    result.push(' ');
                }
                result.push_str(word);
            }
            result
        }
    }
}

/// 语言检测工具
pub mod language {
    /// 简单的语言检测
    pub fn detect_language(text: &str) -> Language {
        let chars: Vec<char> = text.chars().collect();
        let total_chars = chars.len();
        
        if total_chars == 0 {
            return Language::Unknown;
        }
        
        let mut chinese_count = 0;
        let mut japanese_count = 0;
        let mut korean_count = 0;
        let mut cyrillic_count = 0;
        let mut latin_count = 0;
        
        for &ch in &chars {
            if ('\u{4E00}'..='\u{9FFF}').contains(&ch) {
                chinese_count += 1;
            } else if ('\u{3040}'..='\u{309F}').contains(&ch) || ('\u{30A0}'..='\u{30FF}').contains(&ch) {
                japanese_count += 1;
            } else if ('\u{AC00}'..='\u{D7AF}').contains(&ch) {
                korean_count += 1;
            } else if ('\u{0400}'..='\u{04FF}').contains(&ch) {
                cyrillic_count += 1;
            } else if ch.is_ascii_alphabetic() {
                latin_count += 1;
            }
        }
        
        let threshold = total_chars / 10; // 10%的阈值
        
        if chinese_count > threshold {
            Language::Chinese
        } else if japanese_count > threshold {
            Language::Japanese
        } else if korean_count > threshold {
            Language::Korean
        } else if cyrillic_count > threshold {
            Language::Russian
        } else if latin_count > threshold {
            Language::English
        } else {
            Language::Unknown
        }
    }
    
    /// 语言枚举
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    pub enum Language {
        /// 未知语言
        Unknown,
        /// 英语
        English,
        /// 中文
        Chinese,
        /// 日语
        Japanese,
        /// 韩语
        Korean,
        /// 俄语
        Russian,
    }
    
    impl Language {
        /// 获取语言代码
        pub fn code(&self) -> &'static str {
            match self {
                Language::Unknown => "auto",
                Language::English => "en",
                Language::Chinese => "zh",
                Language::Japanese => "ja",
                Language::Korean => "ko",
                Language::Russian => "ru",
            }
        }
        
        /// 获取语言名称
        pub fn name(&self) -> &'static str {
            match self {
                Language::Unknown => "Unknown",
                Language::English => "English",
                Language::Chinese => "Chinese",
                Language::Japanese => "Japanese", 
                Language::Korean => "Korean",
                Language::Russian => "Russian",
            }
        }
    }
}

/// 性能监控工具
pub mod performance {
    use std::time::{Duration, Instant};
    
    /// 性能计时器
    pub struct Timer {
        start: Instant,
        label: String,
    }
    
    impl Timer {
        /// 开始计时
        pub fn start(label: &str) -> Self {
            Self {
                start: Instant::now(),
                label: label.to_string(),
            }
        }
        
        /// 结束计时并返回持续时间
        pub fn stop(self) -> Duration {
            let duration = self.start.elapsed();
            tracing::debug!("Timer '{}': {:?}", self.label, duration);
            duration
        }
        
        /// 获取中间时间（不停止计时器）
        pub fn lap(&self) -> Duration {
            self.start.elapsed()
        }
    }
    
    /// 简单的计时宏
    #[macro_export]
    macro_rules! time_it {
        ($label:expr, $code:block) => {{
            let timer = $crate::utils::performance::Timer::start($label);
            let result = $code;
            timer.stop();
            result
        }};
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    
    #[test]
    fn test_text_similarity() {
        assert_eq!(text::similarity("hello", "hello"), 1.0);
        assert!(text::similarity("hello", "world") < 0.3); // 这两个单词的相似度确实不是0
        assert!(text::similarity("hello", "hallo") > 0.5);
    }
    
    #[test]
    fn test_cjk_detection() {
        assert!(text::contains_cjk("你好世界"));
        assert!(text::contains_cjk("こんにちは"));
        assert!(text::contains_cjk("안녕하세요"));
        assert!(!text::contains_cjk("Hello World"));
    }
    
    #[test]
    fn test_language_detection() {
        assert_eq!(language::detect_language("Hello World"), language::Language::English);
        assert_eq!(language::detect_language("你好世界"), language::Language::Chinese);
        assert_eq!(language::detect_language("こんにちは"), language::Language::Japanese);
    }
}