use thiserror::Error;
#[derive(Debug, Clone)]
pub struct ProcessorConfig {
pub chunk_size: usize,
pub parallel_threshold: usize,
pub max_threads: Option<usize>,
pub overlap_size: usize,
}
impl Default for ProcessorConfig {
fn default() -> Self {
Self {
chunk_size: 256 * 1024, parallel_threshold: 1024 * 1024, max_threads: None, overlap_size: 256, }
}
}
impl ProcessorConfig {
pub fn builder() -> ProcessorConfigBuilder {
ProcessorConfigBuilder::new()
}
pub fn small_text() -> Self {
Self {
chunk_size: 8 * 1024, parallel_threshold: usize::MAX, overlap_size: 64, ..Default::default()
}
}
pub fn large_text() -> Self {
Self {
chunk_size: 512 * 1024, parallel_threshold: 512 * 1024, max_threads: None, overlap_size: 512, }
}
pub fn streaming() -> Self {
Self {
chunk_size: 32 * 1024, parallel_threshold: 256 * 1024, max_threads: Some(2), overlap_size: 128, }
}
pub fn validate(&self) -> Result<(), ProcessingError> {
if self.chunk_size == 0 {
return Err(ProcessingError::InvalidConfig {
reason: "Chunk size must be greater than 0".to_string(),
});
}
if self.overlap_size >= self.chunk_size {
return Err(ProcessingError::InvalidConfig {
reason: "Overlap size must be less than chunk size".to_string(),
});
}
if let Some(threads) = self.max_threads {
if threads == 0 {
return Err(ProcessingError::InvalidConfig {
reason: "Max threads must be greater than 0".to_string(),
});
}
}
Ok(())
}
}
#[derive(Debug, Error)]
pub enum ProcessingError {
#[error("Text too large for processing: {size} bytes (max: {max} bytes)")]
TextTooLarge { size: usize, max: usize },
#[error("Invalid configuration: {reason}")]
InvalidConfig { reason: String },
#[error("Parallel processing failed")]
ParallelError {
#[source]
source: Box<dyn std::error::Error + Send + Sync>,
},
#[error("Invalid UTF-8 in text at position {position}")]
Utf8Error { position: usize },
#[error("Failed to calculate chunk boundaries: {reason}")]
ChunkingError { reason: String },
#[error("Failed to find UTF-8 boundary at position {position}")]
Utf8BoundaryError { position: usize },
#[error("Failed to find word boundary near position {position}")]
WordBoundaryError { position: usize },
#[error("Invalid chunk boundaries: start={start}, end={end}, next={next}")]
InvalidChunkBoundaries {
start: usize,
end: usize,
next: usize,
},
#[error("Memory allocation failed: {reason}")]
AllocationError { reason: String },
#[error("I/O operation failed")]
IoError {
#[from]
source: std::io::Error,
},
#[error("Language rules processing failed: {reason}")]
LanguageRulesError { reason: String },
#[error("Other error: {0}")]
Other(String),
}
pub type ProcessingResult<T> = Result<T, ProcessingError>;
#[derive(Debug, Clone)]
pub struct ProcessorConfigBuilder {
config: ProcessorConfig,
}
impl ProcessorConfigBuilder {
pub fn new() -> Self {
Self {
config: ProcessorConfig::default(),
}
}
pub fn chunk_size(mut self, size: usize) -> Self {
self.config.chunk_size = size;
self
}
pub fn parallel_threshold(mut self, threshold: usize) -> Self {
self.config.parallel_threshold = threshold;
self
}
pub fn max_threads(mut self, threads: Option<usize>) -> Self {
self.config.max_threads = threads;
self
}
pub fn overlap_size(mut self, size: usize) -> Self {
self.config.overlap_size = size;
self
}
pub fn build(self) -> ProcessingResult<ProcessorConfig> {
self.config.validate()?;
Ok(self.config)
}
pub fn build_unchecked(self) -> ProcessorConfig {
self.config
}
}
impl Default for ProcessorConfigBuilder {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Default)]
pub struct ProcessingMetrics {
pub total_time_us: u64,
pub chunking_time_us: u64,
pub parallel_time_us: u64,
pub merge_time_us: u64,
pub chunk_count: usize,
pub thread_count: usize,
pub bytes_processed: usize,
pub boundaries_found: usize,
}
impl ProcessingMetrics {
pub fn throughput_mbps(&self) -> f64 {
if self.total_time_us == 0 {
return 0.0;
}
let mb = self.bytes_processed as f64 / (1024.0 * 1024.0);
let seconds = self.total_time_us as f64 / 1_000_000.0;
mb / seconds
}
pub fn parallel_efficiency(&self) -> f64 {
if self.thread_count <= 1 || self.parallel_time_us == 0 {
return 1.0;
}
let ideal_time = self.total_time_us as f64 / self.thread_count as f64;
let actual_time = self.parallel_time_us as f64;
(ideal_time / actual_time).min(1.0)
}
}
#[cfg(feature = "parallel")]
#[derive(Debug, Clone)]
pub struct ThreadPoolConfig {
pub num_threads: usize,
pub stack_size: Option<usize>,
pub thread_name_prefix: String,
}
#[cfg(feature = "parallel")]
impl Default for ThreadPoolConfig {
fn default() -> Self {
Self {
num_threads: num_cpus::get(),
stack_size: None,
thread_name_prefix: "sakurs-worker".to_string(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_default_config() {
let config = ProcessorConfig::default();
assert_eq!(config.chunk_size, 256 * 1024);
assert_eq!(config.parallel_threshold, 1024 * 1024);
assert!(config.validate().is_ok());
}
#[test]
fn test_config_validation() {
let config = ProcessorConfig {
chunk_size: 0,
..Default::default()
};
assert!(config.validate().is_err());
let config = ProcessorConfig {
chunk_size: 1024,
overlap_size: 2048,
..Default::default()
};
assert!(config.validate().is_err());
let config = ProcessorConfig {
max_threads: Some(0),
..Default::default()
};
assert!(config.validate().is_err());
}
#[test]
fn test_preset_configs() {
let small = ProcessorConfig::small_text();
assert_eq!(small.chunk_size, 8 * 1024);
assert_eq!(small.parallel_threshold, usize::MAX);
let large = ProcessorConfig::large_text();
assert_eq!(large.chunk_size, 512 * 1024);
let streaming = ProcessorConfig::streaming();
assert_eq!(streaming.max_threads, Some(2));
}
#[test]
fn test_processing_metrics() {
let metrics = ProcessingMetrics {
bytes_processed: 10 * 1024 * 1024, total_time_us: 1_000_000, ..Default::default()
};
assert_eq!(metrics.throughput_mbps(), 10.0);
let metrics_parallel = ProcessingMetrics {
bytes_processed: 10 * 1024 * 1024,
total_time_us: 1_000_000,
thread_count: 4,
parallel_time_us: 300_000, ..Default::default()
};
assert!(metrics_parallel.parallel_efficiency() > 0.8);
}
}