pub mod encoding;
pub mod processor;
pub use encoding::{
process_batch_with_processor, process_with_processor, EncodingDetection, TextEncodingError,
TextEncodingProcessor, TextProcessorStats,
};
pub use processor::{
detect_html_structure, init_encoding_patterns, process_crawled_batch_with_processor,
process_crawled_content_with_processor, process_web_content_with_processor, ContentQuality,
CrawlProcessingError, CrawlProcessorConfig, CrawlTextProcessor, ProcessedCrawlContent,
ProcessedWebContent, WebContentError, WebContentProcessor,
};
pub fn health_check() -> Result<(), String> {
let test_content = "health check sample content";
match TextEncodingProcessor::new().process_text(test_content.as_bytes()) {
Ok(_) => Ok(()),
Err(e) => Err(format!("Text processing module failure: {}", e)),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_health_check_returns_ok_with_valid_content() {
let result = health_check();
assert!(
result.is_ok(),
"health_check should return Ok for valid UTF-8 content"
);
}
#[test]
fn test_health_check_ok_value_is_unit() {
let result = health_check();
assert!(result.is_ok());
result.expect("health_check should succeed");
}
#[test]
fn test_text_encoding_processor_new_does_not_panic() {
let _processor = TextEncodingProcessor::new();
}
#[test]
fn test_text_encoding_processor_processes_ascii() {
let processor = TextEncodingProcessor::new();
let input = b"Hello, World!";
let result = processor.process_text(input);
assert!(result.is_ok(), "process_text should succeed on ASCII input");
let processed = result.expect("ASCII processing should succeed");
assert!(
!processed.is_empty(),
"processed ASCII text should be non-empty"
);
}
#[test]
fn test_text_encoding_processor_processes_chinese() {
let processor = TextEncodingProcessor::new();
let input = "你好,世界".as_bytes();
let result = processor.process_text(input);
assert!(
result.is_ok(),
"process_text should succeed on Chinese input"
);
}
#[test]
fn test_text_encoding_processor_processes_empty_input() {
let processor = TextEncodingProcessor::new();
let result = processor.process_text(b"");
assert!(result.is_ok(), "process_text should succeed on empty input");
}
#[test]
fn test_process_with_processor_is_callable() {
let processor = TextEncodingProcessor::new();
let input = b"re-export test";
let result = process_with_processor(&processor, input);
assert!(
result.is_ok(),
"process_with_processor should succeed on ASCII"
);
}
#[test]
fn test_process_batch_with_processor_empty_batch() {
let processor = TextEncodingProcessor::new();
let batch: Vec<&[u8]> = vec![];
let results = process_batch_with_processor(&processor, batch);
assert!(
results.is_empty(),
"empty batch should produce empty results"
);
}
}