html_translation_lib/pipeline/

batch.rs

//! 批次管理模块
//!
//! 管理翻译文本的批次处理，优化API调用效率

use crate::config::TranslationConfig;
use crate::pipeline::collector::TextItem;

/// 批次管理器
pub struct BatchManager {
    /// 批次大小
    batch_size: usize,
    
    /// 最大字符数限制
    max_chars_per_batch: usize,
    
    /// 是否启用智能分批
    smart_batching: bool,
}

impl BatchManager {
    /// 创建新的批次管理器
    pub fn new(config: &TranslationConfig) -> Self {
        Self {
            batch_size: config.batch_size,
            max_chars_per_batch: 8000, // API通常有字符限制
            smart_batching: true,
        }
    }
    
    /// 创建翻译批次
    pub fn create_batches(&self, items: Vec<TextItem>) -> Vec<Batch> {
        if items.is_empty() {
            return Vec::new();
        }
        
        if self.smart_batching {
            self.create_smart_batches(items)
        } else {
            self.create_simple_batches(items)
        }
    }
    
    /// 创建智能批次（考虑字符数限制）
    fn create_smart_batches(&self, items: Vec<TextItem>) -> Vec<Batch> {
        let mut batches = Vec::new();
        let mut current_batch = Vec::new();
        let mut current_chars = 0;
        
        for item in items {
            let item_chars = item.text.len();
            
            // 检查是否需要开始新批次
            if !current_batch.is_empty() && 
               (current_batch.len() >= self.batch_size || 
                current_chars + item_chars > self.max_chars_per_batch) {
                
                batches.push(Batch::new(
                    std::mem::take(&mut current_batch),
                    current_chars,
                    BatchType::Smart,
                    BatchPriority::Normal,
                ));
                current_chars = 0;
            }
            
            current_batch.push(item);
            current_chars += item_chars;
        }
        
        // 添加最后一个批次
        if !current_batch.is_empty() {
            batches.push(Batch::new(
                current_batch,
                current_chars,
                BatchType::Smart,
                BatchPriority::Normal,
            ));
        }
        
        // 优化批次优先级
        self.optimize_batch_priorities(&mut batches);
        
        batches
    }
    
    /// 创建简单批次（仅按数量分割）
    fn create_simple_batches(&self, items: Vec<TextItem>) -> Vec<Batch> {
        items
            .chunks(self.batch_size)
            .map(|chunk| {
                let chars = chunk.iter().map(|item| item.text.len()).sum();
                Batch::new(
                    chunk.to_vec(),
                    chars,
                    BatchType::Simple,
                    BatchPriority::Normal,
                )
            })
            .collect()
    }
    
    /// 优化批次优先级
    fn optimize_batch_priorities(&self, batches: &mut [Batch]) {
        for batch in batches.iter_mut() {
            // 基于内容类型设置优先级
            let has_title = batch.items.iter().any(|item| {
                matches!(item.text_type, crate::pipeline::collector::TextType::Title)
            });
            
            let has_important_attrs = batch.items.iter().any(|item| {
                matches!(item.text_type, 
                    crate::pipeline::collector::TextType::ImageAlt |
                    crate::pipeline::collector::TextType::FormLabel
                )
            });
            
            if has_title {
                batch.priority = BatchPriority::High;
            } else if has_important_attrs {
                batch.priority = BatchPriority::Medium;
            }
            
            // 小批次优先处理
            if batch.items.len() <= 5 {
                batch.priority = match batch.priority {
                    BatchPriority::Low => BatchPriority::Medium,
                    BatchPriority::Normal => BatchPriority::High,
                    other => other,
                };
            }
        }
        
        // 按优先级排序
        batches.sort_by(|a, b| b.priority.cmp(&a.priority));
    }
    
    /// 估算批次处理时间
    pub fn estimate_processing_time(&self, batches: &[Batch]) -> std::time::Duration {
        let base_time_per_batch = std::time::Duration::from_millis(500); // 基础API延迟
        let time_per_char = std::time::Duration::from_nanos(100); // 每字符处理时间
        
        let total_time: std::time::Duration = batches
            .iter()
            .map(|batch| {
                base_time_per_batch + time_per_char * batch.estimated_chars as u32
            })
            .sum();
        
        total_time
    }
}

/// 翻译批次
#[derive(Debug, Clone)]
pub struct Batch {
    /// 批次中的文本项
    pub items: Vec<TextItem>,
    
    /// 估计的字符数
    pub estimated_chars: usize,
    
    /// 批次类型
    pub batch_type: BatchType,
    
    /// 批次优先级
    pub priority: BatchPriority,
    
    /// 创建时间
    pub created_at: std::time::Instant,
}

impl Batch {
    /// 创建新批次
    pub fn new(
        items: Vec<TextItem>,
        estimated_chars: usize,
        batch_type: BatchType,
        priority: BatchPriority,
    ) -> Self {
        Self {
            items,
            estimated_chars,
            batch_type,
            priority,
            created_at: std::time::Instant::now(),
        }
    }
    
    /// 获取批次大小
    pub fn size(&self) -> usize {
        self.items.len()
    }
    
    /// 是否为空批次
    pub fn is_empty(&self) -> bool {
        self.items.is_empty()
    }
    
    /// 获取批次的平均文本长度
    pub fn average_text_length(&self) -> f32 {
        if self.items.is_empty() {
            0.0
        } else {
            self.estimated_chars as f32 / self.items.len() as f32
        }
    }
    
    /// 拆分批次（如果太大）
    pub fn split_if_needed(&self, max_size: usize, max_chars: usize) -> Vec<Batch> {
        if self.items.len() <= max_size && self.estimated_chars <= max_chars {
            return vec![self.clone()];
        }
        
        let mut result = Vec::new();
        let mut current_items = Vec::new();
        let mut current_chars = 0;
        
        for item in &self.items {
            let item_chars = item.text.len();
            
            if (current_items.len() >= max_size || current_chars + item_chars > max_chars)
                && !current_items.is_empty() {
                    result.push(Batch::new(
                        std::mem::take(&mut current_items),
                        current_chars,
                        self.batch_type,
                        self.priority,
                    ));
                    current_chars = 0;
                }
            
            current_items.push(item.clone());
            current_chars += item_chars;
        }
        
        if !current_items.is_empty() {
            result.push(Batch::new(
                current_items,
                current_chars,
                self.batch_type,
                self.priority,
            ));
        }
        
        result
    }
}

/// 批次类型
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BatchType {
    /// 简单批次（仅按数量分割）
    Simple,
    
    /// 智能批次（考虑字符限制）
    Smart,
    
    /// 优化批次（考虑内容相关性）
    Optimized,
}

/// 批次优先级
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum BatchPriority {
    /// 低优先级
    Low,
    
    /// 普通优先级
    Normal,
    
    /// 中等优先级
    Medium,
    
    /// 高优先级
    High,
    
    /// 紧急优先级
    Urgent,
}

impl Default for BatchManager {
    fn default() -> Self {
        Self {
            batch_size: 20,
            max_chars_per_batch: 8000,
            smart_batching: true,
        }
    }
}