use crawlrs::utils::text_processing::encoding::{
process_batch_with_processor, process_with_processor, EncodingDetection, TextEncodingError,
TextEncodingProcessor, TextProcessorStats,
};
#[test]
fn tc_short_gbk_text_triggers_encoding_detection() {
let processor = TextEncodingProcessor::new();
let (gbk_bytes, _, _) = encoding_rs::GBK.encode("你好");
let result = processor.process_text(&gbk_bytes);
match result {
Ok(text) => assert!(!text.is_empty(), "decoded text should not be empty"),
Err(e) => assert!(
matches!(e, TextEncodingError::ConversionFailed(_)),
"expected ConversionFailed, got {:?}",
e
),
}
}
#[test]
fn tc_short_big5_text_triggers_encoding_detection() {
let processor = TextEncodingProcessor::new();
let (big5_bytes, _, _) = encoding_rs::BIG5.encode("這是測試");
let result = processor.process_text(&big5_bytes);
match result {
Ok(text) => assert!(!text.is_empty()),
Err(e) => assert!(
matches!(e, TextEncodingError::ConversionFailed(_)),
"expected ConversionFailed, got {:?}",
e
),
}
}
#[test]
fn tc_short_iso_8859_1_text_triggers_detection() {
let processor = TextEncodingProcessor::new();
let iso_bytes: &[u8] = &[0xE0, 0xE9, 0xFC, 0xF6, 0xDF];
let result = processor.process_text(iso_bytes);
match result {
Ok(text) => assert!(!text.is_empty()),
Err(e) => assert!(
matches!(e, TextEncodingError::ConversionFailed(_)),
"expected ConversionFailed, got {:?}",
e
),
}
}
#[test]
fn tc_short_windows_1252_text_triggers_detection() {
let processor = TextEncodingProcessor::new();
let (bytes, _, _) = encoding_rs::WINDOWS_1252.encode("café — résumé");
let result = processor.process_text(&bytes);
match result {
Ok(text) => assert!(!text.is_empty()),
Err(e) => assert!(
matches!(e, TextEncodingError::ConversionFailed(_)),
"expected ConversionFailed, got {:?}",
e
),
}
}
#[test]
fn tc_long_gbk_text_caches_detection_result() {
let processor = TextEncodingProcessor::new();
let chinese_text = "你好世界,这是一个测试".repeat(100);
let (gbk_bytes, _, _) = encoding_rs::GBK.encode(&chinese_text);
assert!(
gbk_bytes.len() > 1000,
"GBK text should exceed short text threshold"
);
let stats_before = processor.get_stats();
let result1 = processor.process_text(&gbk_bytes);
let stats_after = processor.get_stats();
assert_eq!(
stats_after.cache_size,
stats_before.cache_size + 1,
"cache should grow by 1 after processing long non-UTF-8 text"
);
if let Ok(text) = result1 {
assert!(!text.is_empty());
}
}
#[test]
fn tc_long_gbk_text_uses_cached_detection_on_second_call() {
let processor = TextEncodingProcessor::new();
let chinese_text = "你好世界,这是一个测试".repeat(100);
let (gbk_bytes, _, _) = encoding_rs::GBK.encode(&chinese_text);
let result1 = processor.process_text(&gbk_bytes);
let stats_after_first = processor.get_stats();
assert_eq!(stats_after_first.cache_size, 1);
let result2 = processor.process_text(&gbk_bytes);
let stats_after_second = processor.get_stats();
assert_eq!(
stats_after_second.cache_size, 1,
"cache should not grow on second call (cache hit)"
);
assert_eq!(
result1.is_ok(),
result2.is_ok(),
"both calls should produce consistent success/failure"
);
}
#[test]
fn tc_long_big5_text_exercises_apply_cached_conversion_non_utf8() {
let processor = TextEncodingProcessor::new();
let text = "這是一個繁體中文測試".repeat(150);
let (big5_bytes, _, _) = encoding_rs::BIG5.encode(&text);
assert!(big5_bytes.len() > 1000);
let result1 = processor.process_text(&big5_bytes);
let result2 = processor.process_text(&big5_bytes);
assert_eq!(
result1.is_ok(),
result2.is_ok(),
"cached and non-cached results should be consistent"
);
}
#[test]
fn tc_long_shift_jis_text_caches_and_reuses() {
let processor = TextEncodingProcessor::new();
let japanese_text = "これは日本語のテストです".repeat(100);
let (sjis_bytes, _, _) = encoding_rs::SHIFT_JIS.encode(&japanese_text);
assert!(sjis_bytes.len() > 1000);
let result1 = processor.process_text(&sjis_bytes);
let stats1 = processor.get_stats();
assert_eq!(stats1.cache_size, 1);
let result2 = processor.process_text(&sjis_bytes);
let stats2 = processor.get_stats();
assert_eq!(stats2.cache_size, 1);
assert_eq!(result1.is_ok(), result2.is_ok());
}
#[test]
fn tc_long_euc_kr_text_caches_and_reuses() {
let processor = TextEncodingProcessor::new();
let korean_text = "이것은 한국어 테스트입니다".repeat(100);
let (euckr_bytes, _, _) = encoding_rs::EUC_KR.encode(&korean_text);
assert!(euckr_bytes.len() > 1000);
let result1 = processor.process_text(&euckr_bytes);
let result2 = processor.process_text(&euckr_bytes);
assert_eq!(result1.is_ok(), result2.is_ok());
assert_eq!(processor.get_stats().cache_size, 1);
}
#[test]
fn tc_with_config_small_threshold_routes_to_long_text_path() {
let processor = TextEncodingProcessor::with_config(100, 10);
let input = "a".repeat(20); let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, input);
let stats = processor.get_stats();
assert_eq!(stats.cache_size, 1);
assert_eq!(stats.short_text_threshold, 10);
}
#[test]
fn tc_process_with_processor_handles_non_utf8_gbk() {
let processor = TextEncodingProcessor::new();
let (gbk_bytes, _, _) = encoding_rs::GBK.encode("测试");
let result = process_with_processor(&processor, &gbk_bytes);
if let Ok(text) = result {
assert!(!text.is_empty());
}
}
#[test]
fn tc_process_batch_with_processor_mixed_encodings() {
let processor = TextEncodingProcessor::new();
let utf8_text = "Hello".as_bytes();
let (gbk_bytes, _, _) = encoding_rs::GBK.encode("你好");
let ascii_text = b"plain ascii";
let inputs: Vec<&[u8]> = vec![utf8_text, &gbk_bytes, ascii_text];
let results = process_batch_with_processor(&processor, inputs);
assert_eq!(results.len(), 3);
assert!(results[0].is_ok(), "UTF-8 should succeed");
assert!(results[2].is_ok(), "ASCII should succeed");
}
#[test]
fn tc_process_batch_with_processor_all_non_utf8() {
let processor = TextEncodingProcessor::new();
let (gbk, _, _) = encoding_rs::GBK.encode("你好");
let (big5, _, _) = encoding_rs::BIG5.encode("測試");
let (sjis, _, _) = encoding_rs::SHIFT_JIS.encode("テスト");
let inputs: Vec<&[u8]> = vec![&gbk, &big5, &sjis];
let results = process_batch_with_processor(&processor, inputs);
assert_eq!(results.len(), 3);
for r in &results {
match r {
Ok(_) => {}
Err(e) => assert!(matches!(e, TextEncodingError::ConversionFailed(_))),
}
}
}
#[test]
fn tc_empty_input_returns_empty_string() {
let processor = TextEncodingProcessor::new();
let result = processor.process_text(b"").unwrap();
assert_eq!(result, "");
}
#[test]
fn tc_single_byte_non_utf8_input() {
let processor = TextEncodingProcessor::new();
let result = processor.process_text(&[0xFE]);
match result {
Ok(text) => assert!(!text.is_empty()),
Err(e) => assert!(
matches!(e, TextEncodingError::ConversionFailed(_)),
"expected ConversionFailed, got {:?}",
e
),
}
}
#[test]
fn tc_two_byte_non_utf8_input() {
let processor = TextEncodingProcessor::new();
let result = processor
.process_text(&[0xFF, 0xFE])
.expect("UTF-16LE BOM-only input should decode");
assert!(
result.is_empty(),
"BOM-only input should decode to empty string, got {:?}",
result
);
}
#[test]
fn tc_trim_newlines_preserves_internal_whitespace() {
let processor = TextEncodingProcessor::new();
assert_eq!(processor.trim_newlines("\n\n text \n\n"), " text ");
}
#[test]
fn tc_trim_newlines_only_newlines_returns_empty() {
let processor = TextEncodingProcessor::new();
assert_eq!(processor.trim_newlines("\n\n\n\n"), "");
}
#[test]
fn tc_trim_newlines_single_leading_and_trailing() {
let processor = TextEncodingProcessor::new();
assert_eq!(processor.trim_newlines("\nhello\n"), "hello");
}
#[test]
fn tc_encoding_detection_clone_preserves_all_fields() {
let detection = EncodingDetection {
encoding: "gbk".to_string(),
confidence: 0.9,
is_utf8: false,
};
let cloned = detection.clone();
assert_eq!(detection.encoding, cloned.encoding);
assert_eq!(detection.confidence, cloned.confidence);
assert_eq!(detection.is_utf8, cloned.is_utf8);
}
#[test]
fn tc_encoding_detection_debug_format() {
let detection = EncodingDetection {
encoding: "utf-8".to_string(),
confidence: 1.0,
is_utf8: true,
};
let debug = format!("{:?}", detection);
assert!(debug.contains("utf-8"));
assert!(debug.contains("EncodingDetection"));
}
#[test]
fn tc_text_encoding_error_clone_preserves_message() {
let err = TextEncodingError::ConversionFailed("test error".to_string());
let cloned = err.clone();
assert_eq!(err.to_string(), cloned.to_string());
}
#[test]
fn tc_text_processor_stats_clone_preserves_values() {
let stats = TextProcessorStats {
cache_size: 42,
cache_capacity: 1000,
short_text_threshold: 500,
};
let cloned = stats.clone();
assert_eq!(stats.cache_size, cloned.cache_size);
assert_eq!(stats.cache_capacity, cloned.cache_capacity);
assert_eq!(stats.short_text_threshold, cloned.short_text_threshold);
}
#[test]
fn tc_text_processor_stats_debug_format() {
let stats = TextProcessorStats {
cache_size: 10,
cache_capacity: 100,
short_text_threshold: 50,
};
let debug = format!("{:?}", stats);
assert!(debug.contains("TextProcessorStats"));
assert!(debug.contains("cache_size"));
}
#[test]
fn tc_process_text_with_unicode_escape_in_short_text() {
let processor = TextEncodingProcessor::new();
let input = r"\u0048\u0065\u006C\u006C\u006F";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "Hello");
}
#[test]
fn tc_process_text_with_mixed_escape_sequences() {
let processor = TextEncodingProcessor::new();
let input = r"\u0041\x42\U00000043";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "ABC");
}
#[test]
#[ignore = "encoding behavior changed; test expectation outdated"]
fn tc_process_text_with_truncated_unicode_escape() {
let processor = TextEncodingProcessor::new();
let input = r"\u41";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "A");
}
#[test]
fn tc_process_text_with_backslash_no_escape() {
let processor = TextEncodingProcessor::new();
let input = r"hello \z world";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "hello \\z world");
}
#[test]
fn tc_process_text_with_chinese_unicode_escape() {
let processor = TextEncodingProcessor::new();
let input = r"\u4e16\u754c";
let result = processor.process_text(input.as_bytes()).unwrap();
assert_eq!(result, "世界");
}
#[test]
fn tc_text_encoding_error_detection_failed_display() {
let err = TextEncodingError::DetectionFailed("detect fail".to_string());
assert_eq!(err.to_string(), "Encoding detection failed: detect fail");
}
#[test]
fn tc_text_encoding_error_conversion_failed_display() {
let err = TextEncodingError::ConversionFailed("conv fail".to_string());
assert_eq!(err.to_string(), "Encoding conversion failed: conv fail");
}
#[test]
fn tc_text_encoding_error_unicode_conversion_failed_display() {
let err = TextEncodingError::UnicodeConversionFailed("uni fail".to_string());
assert_eq!(err.to_string(), "Unicode conversion failed: uni fail");
}
#[test]
fn tc_text_encoding_error_invalid_encoding_display() {
let err = TextEncodingError::InvalidEncoding("bad enc".to_string());
assert_eq!(err.to_string(), "Invalid encoding format: bad enc");
}
#[test]
fn tc_text_encoding_error_processing_timeout_display() {
let err = TextEncodingError::ProcessingTimeout;
assert_eq!(err.to_string(), "Text processing timeout");
}
#[test]
fn tc_text_encoding_error_cache_lock_error_display() {
let err = TextEncodingError::CacheLockError("lock fail".to_string());
assert_eq!(err.to_string(), "Cache lock acquisition failed: lock fail");
}