use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection};
use encoding_rs::Encoding;
use log::{debug, warn};
use crate::i18n::{tr_log, tr_log_args};
use lru::LruCache;
use parking_lot::Mutex;
use std::sync::Arc;
use thiserror::Error;
#[derive(Error, Debug, Clone)]
pub enum TextEncodingError {
#[error("Encoding detection failed: {0}")]
DetectionFailed(String),
#[error("Encoding conversion failed: {0}")]
ConversionFailed(String),
#[error("Unicode conversion failed: {0}")]
UnicodeConversionFailed(String),
#[error("Invalid encoding format: {0}")]
InvalidEncoding(String),
#[error("Text processing timeout")]
ProcessingTimeout,
#[error("Cache lock acquisition failed: {0}")]
CacheLockError(String),
}
#[derive(Debug, Clone)]
pub struct EncodingDetection {
pub encoding: String,
pub confidence: f32,
pub is_utf8: bool,
}
#[derive(Clone)]
pub struct TextEncodingProcessor {
encoding_cache: Arc<Mutex<LruCache<String, EncodingDetection>>>,
short_text_threshold: usize,
}
impl Default for TextEncodingProcessor {
fn default() -> Self {
Self::new()
}
}
impl TextEncodingProcessor {
pub fn new() -> Self {
Self {
encoding_cache: Arc::new(Mutex::new(LruCache::new(
std::num::NonZeroUsize::new(1000).expect("Cache size must be non-zero"),
))),
short_text_threshold: 1000,
}
}
pub fn with_config(cache_size: usize, short_text_threshold: usize) -> Self {
Self {
encoding_cache: Arc::new(Mutex::new(LruCache::new(
std::num::NonZeroUsize::new(cache_size).expect("Cache size must be non-zero"),
))),
short_text_threshold,
}
}
pub fn process_text(&self, input: &[u8]) -> Result<String, TextEncodingError> {
let result = if input.len() < self.short_text_threshold {
self.process_short_text(input)?
} else {
self.process_long_text(input)?
};
Ok(self.trim_newlines(&result))
}
fn process_short_text(&self, input: &[u8]) -> Result<String, TextEncodingError> {
debug!(
"{}",
tr_log_args("text-short-input", &[("length", input.len().to_string())],)
);
if let Ok(utf8_str) = std::str::from_utf8(input) {
if self.contains_unicode_escapes(utf8_str) {
return self.normalize_unicode(utf8_str);
}
return Ok(utf8_str.to_string());
}
let (result, _detection) = self.detect_and_convert_encoding(input);
result
}
fn process_long_text(&self, input: &[u8]) -> Result<String, TextEncodingError> {
debug!(
"{}",
tr_log_args("text-long-input", &[("length", input.len().to_string())],)
);
let cache_key = self.generate_cache_key(input);
if let Some(cached_result) = self.get_cached_detection(&cache_key) {
debug!(
"{}",
tr_log_args(
"text-cached-detection",
&[("detection", format!("{:?}", cached_result))],
)
);
return self.apply_cached_conversion(input, &cached_result);
}
let (result, detection) = self.detect_and_convert_encoding(input);
if result.is_ok() {
self.cache_detection_result(&cache_key, &detection);
}
result
}
fn contains_unicode_escapes(&self, text: &str) -> bool {
text.contains("\\u") || text.contains("\\U") || text.contains("\\x")
}
fn parse_unicode_escapes(&self, text: &str) -> String {
let mut result = String::new();
let mut chars = text.chars().peekable();
while let Some(c) = chars.next() {
if c == '\\' {
if chars.peek() == Some(&'u') {
chars.next();
if let Some(ch) = Self::decode_hex_escape(&mut chars, 4) {
result.push(ch);
continue;
}
} else if chars.peek() == Some(&'U') {
chars.next();
if let Some(ch) = Self::decode_hex_escape(&mut chars, 8) {
result.push(ch);
continue;
}
} else if chars.peek() == Some(&'x') {
chars.next();
if let Some(ch) = Self::decode_hex_escape(&mut chars, 2) {
result.push(ch);
continue;
}
}
}
result.push(c);
}
result
}
fn decode_hex_escape(
chars: &mut std::iter::Peekable<std::str::Chars<'_>>,
n: usize,
) -> Option<char> {
const MAX_HEX_CHARS: usize = 8;
debug_assert!(n <= MAX_HEX_CHARS);
let mut buf = [0u8; MAX_HEX_CHARS * 4];
let mut len = 0usize;
for _ in 0..n {
if let Some(h) = chars.next() {
len += h.encode_utf8(&mut buf[len..]).len();
}
}
let hex_str = std::str::from_utf8(&buf[..len]).ok()?;
let code_point = u32::from_str_radix(hex_str, 16).ok()?;
char::from_u32(code_point)
}
fn normalize_unicode(&self, text: &str) -> Result<String, TextEncodingError> {
debug!("{}", tr_log("text-unicode-escapes-detected"));
let parsed = self.parse_unicode_escapes(text);
debug!(
"{}",
tr_log_args(
"text-unicode-escapes-parsed",
&[("result", parsed.clone().to_string())],
)
);
Ok(parsed)
}
fn detect_and_convert_encoding(
&self,
input: &[u8],
) -> (Result<String, TextEncodingError>, EncodingDetection) {
debug!("{}", tr_log("text-detection-started"));
let mut detector = EncodingDetector::new(Iso2022JpDetection::Deny);
detector.feed(input, true);
let encoding = detector.guess(None, Utf8Detection::Allow);
let encoding_name = encoding.name();
let is_utf8 = encoding_name == "utf-8";
let confidence = if is_utf8 { 1.0 } else { 0.9 };
let detection = EncodingDetection {
encoding: encoding_name.to_string(),
confidence,
is_utf8,
};
debug!(
"{}",
tr_log_args(
"text-encoding-detected",
&[
("encoding", encoding_name.to_string()),
("confidence", format!("{:.2}", confidence)),
],
)
);
if is_utf8 {
let result = std::str::from_utf8(input)
.map(|s| s.to_string())
.map_err(|e| TextEncodingError::ConversionFailed(e.to_string()));
return (result, detection);
}
let result = self.convert_encoding(input, encoding, confidence);
(result, detection)
}
pub fn trim_newlines(&self, text: &str) -> String {
let mut result = text.to_string();
while result.starts_with('\n') {
result = result[1..].to_string();
}
while result.ends_with('\n') && !result.is_empty() {
result.pop();
}
result
}
fn convert_encoding(
&self,
input: &[u8],
encoding: &'static Encoding,
confidence: f32,
) -> Result<String, TextEncodingError> {
if confidence < 0.3 {
warn!(
"{}",
tr_log_args(
"text-detection-low-confidence",
&[("confidence", format!("{:.2}", confidence))],
)
);
}
let (decoded, _, had_errors) = encoding.decode(input);
if had_errors && confidence < 0.5 {
return Err(TextEncodingError::ConversionFailed(format!(
"Encoding conversion error, detected encoding: {}, confidence: {:.2}",
encoding.name(),
confidence
)));
}
let result = decoded.into_owned();
if self.contains_unicode_escapes(&result) {
return self.normalize_unicode(&result);
}
Ok(result)
}
fn generate_cache_key(&self, input: &[u8]) -> String {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
input.hash(&mut hasher);
format!("{:x}", hasher.finish())
}
fn get_cached_detection(&self, key: &str) -> Option<EncodingDetection> {
let mut cache = self.encoding_cache.lock();
cache.get(key).cloned()
}
fn cache_detection_result(&self, key: &str, detection: &EncodingDetection) {
let mut cache = self.encoding_cache.lock();
cache.put(key.to_string(), detection.clone());
}
fn apply_cached_conversion(
&self,
input: &[u8],
detection: &EncodingDetection,
) -> Result<String, TextEncodingError> {
if detection.is_utf8 {
std::str::from_utf8(input)
.map(|s| s.to_string())
.map_err(|e| TextEncodingError::ConversionFailed(e.to_string()))
} else {
let encoding = Encoding::for_label(detection.encoding.as_bytes())
.ok_or_else(|| TextEncodingError::InvalidEncoding(detection.encoding.clone()))?;
self.convert_encoding(input, encoding, detection.confidence)
}
}
pub fn process_batch(&self, inputs: Vec<&[u8]>) -> Vec<Result<String, TextEncodingError>> {
inputs
.into_iter()
.map(|input| self.process_text(input))
.collect()
}
pub fn get_stats(&self) -> TextProcessorStats {
let cache_size = self.encoding_cache.lock().len();
TextProcessorStats {
cache_size,
cache_capacity: 1000,
short_text_threshold: self.short_text_threshold,
}
}
}
#[derive(Debug, Clone)]
pub struct TextProcessorStats {
pub cache_size: usize,
pub cache_capacity: usize,
pub short_text_threshold: usize,
}
pub fn process_with_processor(
processor: &TextEncodingProcessor,
input: &[u8],
) -> Result<String, TextEncodingError> {
processor.process_text(input)
}
pub fn process_batch_with_processor(
processor: &TextEncodingProcessor,
inputs: Vec<&[u8]>,
) -> Vec<Result<String, TextEncodingError>> {
processor.process_batch(inputs)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_utf8_text_processing() {
let processor = TextEncodingProcessor::new();
let input = "Hello, 世界! This is a test.";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, input);
}
#[test]
fn test_unicode_escape_processing() {
let processor = TextEncodingProcessor::new();
let input = "Hello 世界!";
let result = processor.process_text(input.as_bytes()).unwrap();
assert!(result.contains("世界"));
}
#[test]
fn test_unicode_escape_sequence_conversion() {
let processor = TextEncodingProcessor::new();
let input = "\u{4e00}\u{7ad9}\u{5f0f}\u{7f51}\u{7ad9}\u{5efa}\u{8bbe}";
let result = processor
.process_text(input.as_bytes())
.expect("Unicode conversion should succeed");
assert!(!result.is_empty());
}
#[test]
fn test_short_text_optimization() {
let processor = TextEncodingProcessor::new();
let short_text = "Short text";
let result = processor.process_text(short_text.as_bytes()).unwrap();
assert_eq!(result, short_text);
}
#[test]
fn test_batch_processing() {
let processor = TextEncodingProcessor::new();
let inputs = vec![
"Text 1".as_bytes(),
"Text 2".as_bytes(),
"Text 3".as_bytes(),
];
let results = processor.process_batch(inputs);
assert_eq!(results.len(), 3);
assert!(results.iter().all(|r| r.is_ok()));
}
#[test]
fn test_trim_newlines_basic() {
let processor = TextEncodingProcessor::new();
assert_eq!(processor.trim_newlines("hello"), "hello");
assert_eq!(processor.trim_newlines("\nhello"), "hello");
assert_eq!(processor.trim_newlines("hello\n"), "hello");
assert_eq!(processor.trim_newlines("\nhello\n"), "hello");
assert_eq!(processor.trim_newlines("\n\nhello\n\n"), "hello");
}
#[test]
fn test_trim_newlines_middle_preserved() {
let processor = TextEncodingProcessor::new();
assert_eq!(processor.trim_newlines("hello\nworld"), "hello\nworld");
assert_eq!(processor.trim_newlines("\nhello\nworld\n"), "hello\nworld");
}
#[test]
fn test_trim_newlines_empty() {
let processor = TextEncodingProcessor::new();
assert_eq!(processor.trim_newlines(""), "");
assert_eq!(processor.trim_newlines("\n"), "");
assert_eq!(processor.trim_newlines("\n\n\n"), "");
}
#[test]
fn test_with_config_custom_values() {
let processor = TextEncodingProcessor::with_config(50, 200);
let stats = processor.get_stats();
assert_eq!(stats.short_text_threshold, 200);
}
#[test]
fn test_default_impl() {
let processor = TextEncodingProcessor::default();
let stats = processor.get_stats();
assert_eq!(stats.short_text_threshold, 1000);
}
#[test]
fn test_long_text_processing_utf8() {
let processor = TextEncodingProcessor::new();
let long_text = "a".repeat(1500);
let result = processor.process_text(long_text.as_bytes()).unwrap();
assert_eq!(result, long_text);
}
#[test]
fn test_long_text_caching_increases_cache_size() {
let processor = TextEncodingProcessor::new();
let long_text = "x".repeat(1500);
assert_eq!(processor.get_stats().cache_size, 0);
processor.process_text(long_text.as_bytes()).unwrap();
assert_eq!(processor.get_stats().cache_size, 1);
processor.process_text(long_text.as_bytes()).unwrap();
assert_eq!(processor.get_stats().cache_size, 1);
}
#[test]
fn test_get_stats_empty_cache() {
let processor = TextEncodingProcessor::new();
let stats = processor.get_stats();
assert_eq!(stats.cache_size, 0);
assert_eq!(stats.cache_capacity, 1000);
assert_eq!(stats.short_text_threshold, 1000);
}
#[test]
fn test_get_stats_after_processing() {
let processor = TextEncodingProcessor::new();
let long_text = "c".repeat(1500);
processor.process_text(long_text.as_bytes()).unwrap();
let stats = processor.get_stats();
assert_eq!(stats.cache_size, 1);
}
#[test]
fn test_unicode_lowercase_u_escape_sequence() {
let processor = TextEncodingProcessor::new();
let input = r"Hello \u0041 World";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "Hello A World");
}
#[test]
fn test_unicode_uppercase_u_escape_sequence() {
let processor = TextEncodingProcessor::new();
let input = r"Hello \U00000041 World";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "Hello A World");
}
#[test]
fn test_unicode_x_escape_sequence() {
let processor = TextEncodingProcessor::new();
let input = r"Hello \x41 World";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "Hello A World");
}
#[test]
fn test_unicode_escape_chinese() {
let processor = TextEncodingProcessor::new();
let input = r"Hello \u4e16界";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "Hello 世界");
}
#[test]
fn test_invalid_unicode_escape_fallback() {
let processor = TextEncodingProcessor::new();
let input = r"Hello \uGGGG World";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "Hello \\ World");
}
#[test]
fn test_backslash_without_escape() {
let processor = TextEncodingProcessor::new();
let input = r"Hello \n World";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "Hello \\n World");
}
#[test]
fn test_process_with_processor_free_function() {
let processor = TextEncodingProcessor::new();
let input = "test text";
let result = process_with_processor(&processor, input.as_bytes()).unwrap();
assert_eq!(result, input);
}
#[test]
fn test_process_batch_with_processor_free_function() {
let processor = TextEncodingProcessor::new();
let inputs = vec!["a".as_bytes(), "b".as_bytes()];
let results = process_batch_with_processor(&processor, inputs);
assert_eq!(results.len(), 2);
assert!(results.iter().all(|r| r.is_ok()));
}
#[test]
fn test_text_encoding_error_display() {
let err = TextEncodingError::DetectionFailed("test reason".to_string());
assert_eq!(err.to_string(), "Encoding detection failed: test reason");
let err = TextEncodingError::ConversionFailed("conv error".to_string());
assert_eq!(err.to_string(), "Encoding conversion failed: conv error");
let err = TextEncodingError::UnicodeConversionFailed("uni error".to_string());
assert_eq!(err.to_string(), "Unicode conversion failed: uni error");
let err = TextEncodingError::InvalidEncoding("bad enc".to_string());
assert_eq!(err.to_string(), "Invalid encoding format: bad enc");
let err = TextEncodingError::ProcessingTimeout;
assert_eq!(err.to_string(), "Text processing timeout");
let err = TextEncodingError::CacheLockError("lock fail".to_string());
assert_eq!(err.to_string(), "Cache lock acquisition failed: lock fail");
}
#[test]
fn test_encoding_detection_struct() {
let detection = EncodingDetection {
encoding: "utf-8".to_string(),
confidence: 1.0,
is_utf8: true,
};
assert_eq!(detection.encoding, "utf-8");
assert_eq!(detection.confidence, 1.0);
assert!(detection.is_utf8);
}
#[test]
fn test_text_processor_stats_struct() {
let stats = TextProcessorStats {
cache_size: 10,
cache_capacity: 100,
short_text_threshold: 500,
};
assert_eq!(stats.cache_size, 10);
assert_eq!(stats.cache_capacity, 100);
assert_eq!(stats.short_text_threshold, 500);
}
#[test]
fn test_batch_empty_input() {
let processor = TextEncodingProcessor::new();
let results = processor.process_batch(vec![]);
assert_eq!(results.len(), 0);
}
#[test]
fn test_long_text_with_unicode_escapes_is_normalized() {
let processor = TextEncodingProcessor::new();
let prefix = "a".repeat(1000);
let suffix = r"\u4e16\u754c";
let long_text = format!("{}{}", prefix, suffix);
let result = processor
.process_text(long_text.as_bytes())
.expect("long text with unicode escapes should process");
assert!(
result.contains("世界"),
"unicode escapes should be normalized: got {}",
result
);
}
#[test]
fn test_long_text_caching_returns_same_result_on_second_call() {
let processor = TextEncodingProcessor::new();
let long_text = "b".repeat(1500);
let first = processor.process_text(long_text.as_bytes()).unwrap();
let second = processor.process_text(long_text.as_bytes()).unwrap();
assert_eq!(first, second, "cached result must match first result");
assert_eq!(processor.get_stats().cache_size, 1);
}
#[test]
fn test_with_config_custom_cache_size() {
let processor = TextEncodingProcessor::with_config(50, 200);
let stats = processor.get_stats();
assert_eq!(stats.short_text_threshold, 200);
assert_eq!(stats.cache_capacity, 1000);
}
#[test]
fn test_with_config_small_threshold_uses_long_text_path() {
let processor = TextEncodingProcessor::with_config(50, 5);
assert_eq!(processor.get_stats().cache_size, 0);
let result = processor.process_text(b"hello world").unwrap();
assert_eq!(result, "hello world");
assert_eq!(
processor.get_stats().cache_size,
1,
"content longer than threshold should be cached"
);
}
#[test]
fn test_long_text_unicode_escape_uppercase_u_in_long_content() {
let processor = TextEncodingProcessor::new();
let prefix = "x".repeat(1000);
let suffix = r"\U00000041"; let long_text = format!("{}{}", prefix, suffix);
let result = processor.process_text(long_text.as_bytes()).unwrap();
assert!(
result.contains('A'),
"\\U escape should be normalized to 'A': got {}",
result
);
}
#[test]
fn test_long_text_unicode_escape_x_in_long_content() {
let processor = TextEncodingProcessor::new();
let prefix = "y".repeat(1000);
let suffix = r"\x42"; let long_text = format!("{}{}", prefix, suffix);
let result = processor.process_text(long_text.as_bytes()).unwrap();
assert!(
result.contains('B'),
"\\x escape should be normalized to 'B': got {}",
result
);
}
#[test]
fn test_cache_key_is_deterministic_for_same_input() {
let processor = TextEncodingProcessor::with_config(100, 10);
let content = "z".repeat(20);
processor.process_text(content.as_bytes()).unwrap();
assert_eq!(processor.get_stats().cache_size, 1);
let content2 = "w".repeat(20);
processor.process_text(content2.as_bytes()).unwrap();
assert_eq!(processor.get_stats().cache_size, 2);
}
#[test]
fn test_trim_newlines_only_newlines_at_start_and_end() {
let processor = TextEncodingProcessor::new();
assert_eq!(processor.trim_newlines("\n\n\n"), "");
assert_eq!(processor.trim_newlines("content"), "content");
assert_eq!(
processor.trim_newlines("\n\ncontent\nmore\n"),
"content\nmore"
);
}
#[test]
fn test_process_text_with_empty_input() {
let processor = TextEncodingProcessor::new();
let result = processor
.process_text(b"")
.expect("empty input should succeed");
assert_eq!(result, "");
}
#[test]
fn test_process_batch_with_mixed_utf8_and_empty() {
let processor = TextEncodingProcessor::new();
let inputs: Vec<&[u8]> = vec![b"", b"hello", b""];
let results = processor.process_batch(inputs);
assert_eq!(results.len(), 3);
assert!(results.iter().all(|r| r.is_ok()));
assert_eq!(results[0].as_ref().unwrap(), "");
assert_eq!(results[1].as_ref().unwrap(), "hello");
assert_eq!(results[2].as_ref().unwrap(), "");
}
#[test]
fn test_non_utf8_content_is_decoded() {
let processor = TextEncodingProcessor::new();
let gbk_bytes: [u8; 4] = [0xD6, 0xD0, 0xCE, 0xC4];
let result = processor.process_text(&gbk_bytes);
match result {
Ok(text) => {
assert!(!text.is_empty(), "decoded text should be non-empty");
}
Err(_) => {
}
}
}
#[test]
fn test_non_utf8_long_content_caching() {
let processor = TextEncodingProcessor::new();
let mut non_utf8 = Vec::new();
for _ in 0..300 {
non_utf8.extend_from_slice(&[0xD6, 0xD0, 0xCE, 0xC4]);
}
let result = processor.process_text(&non_utf8);
if let Ok(text) = result {
assert!(!text.is_empty());
}
}
#[test]
fn test_text_processor_clone_preserves_config() {
let processor = TextEncodingProcessor::with_config(50, 500);
let cloned = processor.clone();
assert_eq!(
processor.get_stats().short_text_threshold,
cloned.get_stats().short_text_threshold
);
}
#[test]
fn test_invalid_utf8_short_text_does_not_panic() {
let processor = TextEncodingProcessor::new();
let invalid_utf8: [u8; 4] = [0xFF, 0xFE, 0xFD, 0xFC];
let result = processor.process_text(&invalid_utf8);
let _ = result;
}
#[test]
fn test_get_stats_cache_capacity_is_constant() {
let processor = TextEncodingProcessor::with_config(50, 200);
let stats = processor.get_stats();
assert_eq!(stats.cache_capacity, 1000);
}
#[test]
fn test_unicode_escape_surrogate_code_point_falls_back_to_backslash() {
let processor = TextEncodingProcessor::new();
let input = r"\uD800";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "\\");
}
#[test]
fn test_unicode_uppercase_u_escape_beyond_max_code_point() {
let processor = TextEncodingProcessor::new();
let input = r"\U00110000";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "\\");
}
#[test]
fn test_unicode_escape_incomplete_at_end_of_string() {
let processor = TextEncodingProcessor::new();
let input = r"\u41";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "A");
}
#[test]
fn test_unicode_x_escape_with_non_hex_chars() {
let processor = TextEncodingProcessor::new();
let input = r"\xZZ";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "\\");
}
#[test]
fn test_unicode_x_escape_single_hex_digit() {
let processor = TextEncodingProcessor::new();
let input = r"\xA";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "");
}
#[test]
fn test_multiple_unicode_escapes_in_one_string() {
let processor = TextEncodingProcessor::new();
let input = r"\u0041\U00000042\x43";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "ABC");
}
#[test]
fn test_backslash_u_at_end_of_string_with_no_hex() {
let processor = TextEncodingProcessor::new();
let input = r"\u";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "\\");
}
#[test]
fn test_backslash_followed_by_non_escape_char() {
let processor = TextEncodingProcessor::new();
let input = r"\n";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "\\n");
}
#[test]
fn test_unicode_escape_max_valid_code_point() {
let processor = TextEncodingProcessor::new();
let input = r"\U0010FFFF";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "\u{10FFFF}");
}
#[test]
fn test_unicode_escape_null_code_point() {
let processor = TextEncodingProcessor::new();
let input = r"\u0000";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "\0");
}
#[test]
fn test_apply_cached_conversion_non_utf8_path() {
let processor = TextEncodingProcessor::with_config(100, 2);
let gbk_bytes: [u8; 4] = [0xD6, 0xD0, 0xCE, 0xC4];
let first = processor.process_text(&gbk_bytes);
let second = processor.process_text(&gbk_bytes);
match (first, second) {
(Ok(a), Ok(b)) => assert_eq!(a, b, "cached non-utf8 result should match first"),
(Err(_), Err(_)) => {}
(a, b) => panic!("both calls should agree: first={:?}, second={:?}", a, b),
}
assert_eq!(processor.get_stats().cache_size, 1);
}
#[test]
fn test_convert_encoding_with_unicode_escapes_in_non_utf8() {
let processor = TextEncodingProcessor::with_config(100, 2);
let mut non_utf8 = Vec::new();
for _ in 0..100 {
non_utf8.extend_from_slice(&[0xD6, 0xD0, 0xCE, 0xC4]);
}
let result = processor.process_text(&non_utf8);
let _ = result;
}
#[test]
fn test_long_text_processing_caches_detection_only_on_success() {
let processor = TextEncodingProcessor::with_config(100, 2);
let weird_bytes: Vec<u8> = vec![0xFF; 10];
let _ = processor.process_text(&weird_bytes);
let stats = processor.get_stats();
assert!(stats.cache_size <= 1);
}
#[test]
fn test_process_text_preserves_content_with_special_chars() {
let processor = TextEncodingProcessor::new();
let input = "Hello\tWorld\nNew Line\r\nCarriage";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, input);
}
#[test]
fn test_trim_newlines_preserves_carriage_returns() {
let processor = TextEncodingProcessor::new();
assert_eq!(processor.trim_newlines("\rhello\r"), "\rhello\r");
assert_eq!(processor.trim_newlines("\r\nhello\r\n"), "\r\nhello\r");
}
#[test]
fn test_unicode_escape_mixed_with_regular_backslashes() {
let processor = TextEncodingProcessor::new();
let input = r"\\u0041\n\x42";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "\\A\\nB");
}
#[test]
fn test_detect_and_convert_encoding_utf8_branch_with_chinese_long_text() {
let processor = TextEncodingProcessor::new();
let prefix = "a".repeat(900);
let chinese = "你好世界测试文本".repeat(20);
let long_text = format!("{}{}", prefix, chinese);
assert!(
long_text.len() >= 1000,
"long_text must be >= 1000 bytes for long-text path"
);
let result = processor
.process_text(long_text.as_bytes())
.expect("long UTF-8 text with Chinese should process");
assert!(
result.contains("你好世界测试文本"),
"result should contain Chinese text"
);
}
#[test]
fn test_apply_cached_conversion_utf8_branch_on_second_call() {
let processor = TextEncodingProcessor::new();
let prefix = "b".repeat(900);
let chinese = "你好世界".repeat(30);
let long_text = format!("{}{}", prefix, chinese);
assert!(long_text.len() >= 1000);
let first = processor
.process_text(long_text.as_bytes())
.expect("first call should succeed");
assert_eq!(processor.get_stats().cache_size, 1);
let second = processor
.process_text(long_text.as_bytes())
.expect("second call should use cache");
assert_eq!(first, second, "cached result should match first result");
}
#[test]
fn test_detect_and_convert_encoding_utf8_branch_with_mixed_long_text() {
let processor = TextEncodingProcessor::new();
let prefix = "x".repeat(800);
let mixed = "你好世界こんにちは안녕하세요".repeat(20);
let long_text = format!("{}{}", prefix, mixed);
assert!(long_text.len() >= 1000);
let result = processor
.process_text(long_text.as_bytes())
.expect("mixed long UTF-8 text should process");
assert!(!result.is_empty());
}
}