Skip to main content

trustformers_models/
batch_inference.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! # Batch Inference Utilities for Trustformers Models
4//!
5//! This module provides efficient batch processing capabilities for transformer models,
6//! enabling high-throughput inference with automatic optimization and memory management.
7//!
8//! ## Features
9//!
10//! - **Dynamic batching**: Automatically group requests into optimal batch sizes
11//! - **Smart padding**: Efficient padding strategies to minimize computation waste
12//! - **Memory management**: Automatic batch size adjustment based on available memory
13//! - **Parallel processing**: Leverage multi-core CPUs for batch preprocessing
14//! - **Adaptive batching**: Adjust batch size based on sequence lengths
15//! - **Batch queue management**: Priority-based request scheduling
16//!
17//! ## Usage Example
18//!
19//! ```rust
20//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
21//! use trustformers_models::batch_inference::{BatchProcessor, BatchConfig, PaddingStrategy};
22//!
23//! // Configure batch processing
24//! let config = BatchConfig {
25//!     max_batch_size: 32,
26//!     timeout_ms: 100,
27//!     padding_strategy: PaddingStrategy::Longest,
28//!     ..Default::default()
29//! };
30//!
31//! let processor = BatchProcessor::new(config)?;
32//!
33//! // Process requests in batches
34//! // let results = processor.process_batch(requests)?;
35//! # let _ = processor;
36//! # Ok(())
37//! # }
38//! ```
39
40use scirs2_core::ndarray::Array2;
41use scirs2_core::parallel_ops::*;
42use serde::{Deserialize, Serialize};
43use std::collections::VecDeque;
44use std::sync::{Arc, Mutex};
45use std::time::{Duration, Instant};
46use trustformers_core::errors::{Result, TrustformersError};
47use trustformers_core::tensor::Tensor;
48
49/// Padding strategy for batch processing
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
51pub enum PaddingStrategy {
52    /// Pad all sequences to the longest sequence in the batch
53    Longest,
54
55    /// Pad all sequences to a fixed length
56    Fixed(usize),
57
58    /// Pad to multiples of a specific value (e.g., 8 for better GPU utilization)
59    Multiple(usize),
60
61    /// No padding (requires all sequences to be same length)
62    None,
63}
64
65impl Default for PaddingStrategy {
66    fn default() -> Self {
67        Self::Longest
68    }
69}
70
71/// Truncation strategy for sequences that exceed max length
72#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
73pub enum TruncationStrategy {
74    /// Truncate from the end (keep beginning)
75    End,
76
77    /// Truncate from the beginning (keep end)
78    Beginning,
79
80    /// Error if sequence exceeds max length
81    Error,
82
83    /// No truncation
84    None,
85}
86
87impl Default for TruncationStrategy {
88    fn default() -> Self {
89        Self::End
90    }
91}
92
93/// Batch processing configuration
94#[derive(Debug, Clone, Serialize, Deserialize)]
95pub struct BatchConfig {
96    /// Maximum batch size
97    pub max_batch_size: usize,
98
99    /// Minimum batch size before processing (for efficiency)
100    pub min_batch_size: usize,
101
102    /// Maximum wait time before processing incomplete batch (milliseconds)
103    pub timeout_ms: u64,
104
105    /// Padding strategy
106    pub padding_strategy: PaddingStrategy,
107
108    /// Truncation strategy
109    pub truncation_strategy: TruncationStrategy,
110
111    /// Maximum sequence length (for truncation/padding)
112    pub max_sequence_length: Option<usize>,
113
114    /// Padding token ID
115    pub pad_token_id: u32,
116
117    /// Enable dynamic batch size adjustment based on sequence lengths
118    pub dynamic_batching: bool,
119
120    /// Target memory usage in MB (for automatic batch size adjustment)
121    pub target_memory_mb: Option<usize>,
122
123    /// Enable parallel preprocessing
124    pub parallel_preprocessing: bool,
125
126    /// Sort sequences by length for more efficient batching
127    pub sort_by_length: bool,
128}
129
130impl Default for BatchConfig {
131    fn default() -> Self {
132        Self {
133            max_batch_size: 32,
134            min_batch_size: 1,
135            timeout_ms: 100,
136            padding_strategy: PaddingStrategy::default(),
137            truncation_strategy: TruncationStrategy::default(),
138            max_sequence_length: None,
139            pad_token_id: 0,
140            dynamic_batching: true,
141            target_memory_mb: None,
142            parallel_preprocessing: true,
143            sort_by_length: true,
144        }
145    }
146}
147
148impl BatchConfig {
149    /// Create a configuration optimized for low latency
150    pub fn low_latency() -> Self {
151        Self {
152            max_batch_size: 8,
153            min_batch_size: 1,
154            timeout_ms: 10,
155            dynamic_batching: false,
156            ..Default::default()
157        }
158    }
159
160    /// Create a configuration optimized for high throughput
161    pub fn high_throughput() -> Self {
162        Self {
163            max_batch_size: 128,
164            min_batch_size: 32,
165            timeout_ms: 500,
166            dynamic_batching: true,
167            sort_by_length: true,
168            ..Default::default()
169        }
170    }
171
172    /// Validate configuration
173    pub fn validate(&self) -> Result<()> {
174        if self.max_batch_size == 0 {
175            return Err(TrustformersError::invalid_config(
176                "max_batch_size must be positive".to_string(),
177            ));
178        }
179
180        if self.min_batch_size > self.max_batch_size {
181            return Err(TrustformersError::invalid_config(
182                "min_batch_size cannot exceed max_batch_size".to_string(),
183            ));
184        }
185
186        if let PaddingStrategy::Fixed(len) = self.padding_strategy {
187            if len == 0 {
188                return Err(TrustformersError::invalid_config(
189                    "fixed padding length must be positive".to_string(),
190                ));
191            }
192        }
193
194        if let PaddingStrategy::Multiple(multiple) = self.padding_strategy {
195            if multiple == 0 {
196                return Err(TrustformersError::invalid_config(
197                    "padding multiple must be positive".to_string(),
198                ));
199            }
200        }
201
202        Ok(())
203    }
204}
205
206/// A batch of input sequences ready for model inference
207#[derive(Debug, Clone)]
208pub struct BatchedInput {
209    /// Input IDs tensor (batch_size, max_seq_len)
210    pub input_ids: Tensor,
211
212    /// Attention mask (batch_size, max_seq_len) - 1 for real tokens, 0 for padding
213    pub attention_mask: Tensor,
214
215    /// Original sequence lengths before padding
216    pub sequence_lengths: Vec<usize>,
217
218    /// Batch size
219    pub batch_size: usize,
220
221    /// Maximum sequence length in this batch
222    pub max_seq_length: usize,
223}
224
225/// A single inference request
226#[derive(Debug, Clone)]
227pub struct InferenceRequest {
228    /// Unique request ID
229    pub id: String,
230
231    /// Input token IDs
232    pub input_ids: Vec<u32>,
233
234    /// Optional priority (higher = more urgent)
235    pub priority: i32,
236
237    /// Request timestamp
238    pub timestamp: Instant,
239}
240
241impl InferenceRequest {
242    /// Create a new inference request
243    pub fn new(id: String, input_ids: Vec<u32>) -> Self {
244        Self {
245            id,
246            input_ids,
247            priority: 0,
248            timestamp: Instant::now(),
249        }
250    }
251
252    /// Create a request with priority
253    pub fn with_priority(id: String, input_ids: Vec<u32>, priority: i32) -> Self {
254        Self {
255            id,
256            input_ids,
257            priority,
258            timestamp: Instant::now(),
259        }
260    }
261}
262
263/// Batch processor for efficient inference
264pub struct BatchProcessor {
265    config: BatchConfig,
266    request_queue: Arc<Mutex<VecDeque<InferenceRequest>>>,
267    batch_start_time: Arc<Mutex<Option<Instant>>>,
268}
269
270impl BatchProcessor {
271    /// Create a new batch processor
272    pub fn new(config: BatchConfig) -> Result<Self> {
273        config.validate()?;
274
275        Ok(Self {
276            config,
277            request_queue: Arc::new(Mutex::new(VecDeque::new())),
278            batch_start_time: Arc::new(Mutex::new(None)),
279        })
280    }
281
282    /// Add a request to the queue
283    pub fn enqueue(&self, request: InferenceRequest) -> Result<()> {
284        let mut queue = self.request_queue.lock().map_err(|e| {
285            TrustformersError::lock_error(format!("Failed to lock request queue: {}", e))
286        })?;
287
288        // Insert based on priority (higher priority first)
289        let insert_pos =
290            queue.iter().position(|r| r.priority < request.priority).unwrap_or(queue.len());
291
292        queue.insert(insert_pos, request);
293
294        // Set batch start time if this is the first request
295        let mut start_time = self.batch_start_time.lock().map_err(|e| {
296            TrustformersError::lock_error(format!("Failed to lock batch start time: {}", e))
297        })?;
298
299        if start_time.is_none() {
300            *start_time = Some(Instant::now());
301        }
302
303        Ok(())
304    }
305
306    /// Check if batch should be processed
307    pub fn should_process_batch(&self) -> Result<bool> {
308        let queue = self.request_queue.lock().map_err(|e| {
309            TrustformersError::lock_error(format!("Failed to lock request queue: {}", e))
310        })?;
311
312        if queue.is_empty() {
313            return Ok(false);
314        }
315
316        // Process if we have enough requests
317        if queue.len() >= self.config.min_batch_size {
318            return Ok(true);
319        }
320
321        // Process if timeout exceeded
322        let start_time = self.batch_start_time.lock().map_err(|e| {
323            TrustformersError::lock_error(format!("Failed to lock batch start time: {}", e))
324        })?;
325
326        if let Some(start) = *start_time {
327            let elapsed = start.elapsed();
328            if elapsed >= Duration::from_millis(self.config.timeout_ms) {
329                return Ok(true);
330            }
331        }
332
333        Ok(false)
334    }
335
336    /// Get next batch of requests from queue
337    pub fn get_next_batch(&self) -> Result<Vec<InferenceRequest>> {
338        let mut queue = self.request_queue.lock().map_err(|e| {
339            TrustformersError::lock_error(format!("Failed to lock request queue: {}", e))
340        })?;
341
342        let batch_size = self.config.max_batch_size.min(queue.len());
343        let mut batch: Vec<InferenceRequest> = Vec::with_capacity(batch_size);
344
345        for _ in 0..batch_size {
346            if let Some(request) = queue.pop_front() {
347                batch.push(request);
348            }
349        }
350
351        // Reset batch start time if queue is empty
352        if queue.is_empty() {
353            let mut start_time = self.batch_start_time.lock().map_err(|e| {
354                TrustformersError::lock_error(format!("Failed to lock batch start time: {}", e))
355            })?;
356            *start_time = None;
357        }
358
359        Ok(batch)
360    }
361
362    /// Prepare a batch for inference
363    pub fn prepare_batch(&self, requests: Vec<InferenceRequest>) -> Result<BatchedInput> {
364        if requests.is_empty() {
365            return Err(TrustformersError::invalid_input(
366                "Cannot prepare batch from empty requests".to_string(),
367            ));
368        }
369
370        let mut sequences: Vec<Vec<u32>> = requests.into_iter().map(|r| r.input_ids).collect();
371
372        // Sort by length if enabled (for more efficient batching)
373        if self.config.sort_by_length {
374            sequences.sort_by_key(|s| std::cmp::Reverse(s.len()));
375        }
376
377        // Apply truncation
378        if let Some(max_len) = self.config.max_sequence_length {
379            sequences = self.apply_truncation(sequences, max_len)?;
380        }
381
382        // Determine target length for padding
383        let max_len = sequences.iter().map(|s| s.len()).max().unwrap_or(0);
384        let target_len = self.compute_target_length(max_len)?;
385
386        // Apply padding
387        let (padded_ids, attention_masks, seq_lengths) =
388            self.apply_padding(&sequences, target_len)?;
389
390        let batch_size = sequences.len();
391
392        Ok(BatchedInput {
393            input_ids: padded_ids,
394            attention_mask: attention_masks,
395            sequence_lengths: seq_lengths,
396            batch_size,
397            max_seq_length: target_len,
398        })
399    }
400
401    /// Apply truncation to sequences
402    fn apply_truncation(&self, sequences: Vec<Vec<u32>>, max_len: usize) -> Result<Vec<Vec<u32>>> {
403        let process_seq = |seq: Vec<u32>| -> Result<Vec<u32>> {
404            if seq.len() <= max_len {
405                return Ok(seq);
406            }
407
408            match self.config.truncation_strategy {
409                TruncationStrategy::End => Ok(seq[..max_len].to_vec()),
410                TruncationStrategy::Beginning => Ok(seq[seq.len() - max_len..].to_vec()),
411                TruncationStrategy::Error => Err(TrustformersError::invalid_input(format!(
412                    "Sequence length {} exceeds max length {}",
413                    seq.len(),
414                    max_len
415                ))),
416                TruncationStrategy::None => Ok(seq),
417            }
418        };
419
420        if self.config.parallel_preprocessing && sequences.len() > 4 {
421            sequences.into_par_iter().map(process_seq).collect::<Result<Vec<_>>>()
422        } else {
423            sequences.into_iter().map(process_seq).collect::<Result<Vec<_>>>()
424        }
425    }
426
427    /// Compute target length for padding
428    fn compute_target_length(&self, max_len: usize) -> Result<usize> {
429        match self.config.padding_strategy {
430            PaddingStrategy::Longest => Ok(max_len),
431            PaddingStrategy::Fixed(len) => Ok(len),
432            PaddingStrategy::Multiple(multiple) => {
433                // Round up to nearest multiple
434                Ok(max_len.div_ceil(multiple) * multiple)
435            },
436            PaddingStrategy::None => Ok(max_len),
437        }
438    }
439
440    /// Apply padding to sequences
441    fn apply_padding(
442        &self,
443        sequences: &[Vec<u32>],
444        target_len: usize,
445    ) -> Result<(Tensor, Tensor, Vec<usize>)> {
446        let batch_size = sequences.len();
447        let pad_token = self.config.pad_token_id;
448
449        // Create padded input IDs and attention masks
450        let mut input_ids = Array2::<u32>::from_elem((batch_size, target_len), pad_token);
451        let mut attention_mask = Array2::<i32>::zeros((batch_size, target_len));
452        let mut seq_lengths = Vec::with_capacity(batch_size);
453
454        for (i, seq) in sequences.iter().enumerate() {
455            let seq_len = seq.len();
456            seq_lengths.push(seq_len);
457
458            // Copy sequence
459            for (j, &token) in seq.iter().enumerate() {
460                input_ids[[i, j]] = token;
461                attention_mask[[i, j]] = 1;
462            }
463        }
464
465        let input_ids_tensor = Tensor::F32(input_ids.mapv(|x| x as f32).into_dyn());
466
467        let attention_mask_tensor = Tensor::F32(attention_mask.mapv(|x| x as f32).into_dyn());
468
469        Ok((input_ids_tensor, attention_mask_tensor, seq_lengths))
470    }
471
472    /// Estimate memory usage for a batch
473    pub fn estimate_memory_mb(&self, batch_size: usize, seq_length: usize) -> usize {
474        // Rough estimate: 4 bytes per token (float32)
475        // Account for input_ids, attention_mask, and intermediate tensors
476        let tokens = batch_size * seq_length;
477        let bytes_per_token = 4;
478        let overhead_multiplier = 10; // Account for intermediate activations
479
480        let memory_bytes = tokens * bytes_per_token * overhead_multiplier;
481        let memory_mb = memory_bytes / (1024 * 1024);
482
483        // Ensure at least 1 MB for small batches
484        memory_mb.max(1)
485    }
486
487    /// Adjust batch size based on memory constraints
488    pub fn adjust_batch_size(&self, sequences: &[Vec<u32>]) -> usize {
489        if let Some(target_mb) = self.config.target_memory_mb {
490            let max_seq_len = sequences.iter().map(|s| s.len()).max().unwrap_or(0);
491
492            // Binary search for optimal batch size
493            let mut left = 1;
494            let mut right = self.config.max_batch_size;
495            let mut best_size = 1;
496
497            while left <= right {
498                let mid = (left + right) / 2;
499                let estimated_mb = self.estimate_memory_mb(mid, max_seq_len);
500
501                if estimated_mb <= target_mb {
502                    best_size = mid;
503                    left = mid + 1;
504                } else {
505                    right = mid - 1;
506                }
507            }
508
509            best_size.min(sequences.len())
510        } else {
511            self.config.max_batch_size.min(sequences.len())
512        }
513    }
514
515    /// Get current queue size
516    pub fn queue_size(&self) -> Result<usize> {
517        let queue = self.request_queue.lock().map_err(|e| {
518            TrustformersError::lock_error(format!("Failed to lock request queue: {}", e))
519        })?;
520        Ok(queue.len())
521    }
522}
523
524/// Batch statistics for monitoring and optimization
525#[derive(Debug, Clone)]
526pub struct BatchStatistics {
527    /// Average batch size
528    pub avg_batch_size: f32,
529
530    /// Average sequence length
531    pub avg_sequence_length: f32,
532
533    /// Average wait time in milliseconds
534    pub avg_wait_time_ms: f32,
535
536    /// Total number of batches processed
537    pub total_batches: usize,
538
539    /// Total number of requests processed
540    pub total_requests: usize,
541
542    /// Average padding ratio (padded tokens / total tokens)
543    pub avg_padding_ratio: f32,
544}
545
546impl BatchStatistics {
547    /// Create new statistics tracker
548    pub fn new() -> Self {
549        Self {
550            avg_batch_size: 0.0,
551            avg_sequence_length: 0.0,
552            avg_wait_time_ms: 0.0,
553            total_batches: 0,
554            total_requests: 0,
555            avg_padding_ratio: 0.0,
556        }
557    }
558
559    /// Update statistics with new batch
560    pub fn update(&mut self, batch: &BatchedInput, wait_time_ms: f32) {
561        self.total_batches += 1;
562        self.total_requests += batch.batch_size;
563
564        // Update running averages
565        let n = self.total_batches as f32;
566        self.avg_batch_size = (self.avg_batch_size * (n - 1.0) + batch.batch_size as f32) / n;
567
568        let actual_tokens: usize = batch.sequence_lengths.iter().sum();
569        let total_tokens = batch.batch_size * batch.max_seq_length;
570        let padding_ratio = 1.0 - (actual_tokens as f32 / total_tokens as f32);
571
572        self.avg_padding_ratio = (self.avg_padding_ratio * (n - 1.0) + padding_ratio) / n;
573        self.avg_wait_time_ms = (self.avg_wait_time_ms * (n - 1.0) + wait_time_ms) / n;
574
575        let avg_seq_len: f32 =
576            batch.sequence_lengths.iter().sum::<usize>() as f32 / batch.batch_size as f32;
577        self.avg_sequence_length = (self.avg_sequence_length * (n - 1.0) + avg_seq_len) / n;
578    }
579}
580
581impl Default for BatchStatistics {
582    fn default() -> Self {
583        Self::new()
584    }
585}
586
587#[cfg(test)]
588mod tests {
589    use super::*;
590
591    #[test]
592    fn test_batch_config_validation() {
593        let valid_config = BatchConfig::default();
594        assert!(valid_config.validate().is_ok());
595
596        let invalid_config = BatchConfig {
597            max_batch_size: 0,
598            ..BatchConfig::default()
599        };
600        assert!(invalid_config.validate().is_err());
601
602        let invalid_config2 = BatchConfig {
603            min_batch_size: 100,
604            max_batch_size: 10,
605            ..BatchConfig::default()
606        };
607        assert!(invalid_config2.validate().is_err());
608    }
609
610    #[test]
611    fn test_batch_config_presets() {
612        let low_latency = BatchConfig::low_latency();
613        assert_eq!(low_latency.max_batch_size, 8);
614        assert_eq!(low_latency.timeout_ms, 10);
615
616        let high_throughput = BatchConfig::high_throughput();
617        assert_eq!(high_throughput.max_batch_size, 128);
618        assert!(high_throughput.sort_by_length);
619    }
620
621    #[test]
622    fn test_padding_strategy() {
623        let longest = PaddingStrategy::Longest;
624        let fixed = PaddingStrategy::Fixed(128);
625        let multiple = PaddingStrategy::Multiple(8);
626
627        assert!(matches!(longest, PaddingStrategy::Longest));
628        assert!(matches!(fixed, PaddingStrategy::Fixed(128)));
629        assert!(matches!(multiple, PaddingStrategy::Multiple(8)));
630    }
631
632    #[test]
633    fn test_batch_processor_creation() -> Result<()> {
634        let config = BatchConfig::default();
635        let processor = BatchProcessor::new(config)?;
636        assert_eq!(processor.queue_size()?, 0);
637        Ok(())
638    }
639
640    #[test]
641    fn test_enqueue_requests() -> Result<()> {
642        let config = BatchConfig::default();
643        let processor = BatchProcessor::new(config)?;
644
645        let request1 = InferenceRequest::new("req1".to_string(), vec![1, 2, 3]);
646        let request2 = InferenceRequest::new("req2".to_string(), vec![4, 5, 6, 7]);
647
648        processor.enqueue(request1)?;
649        processor.enqueue(request2)?;
650
651        assert_eq!(processor.queue_size()?, 2);
652        Ok(())
653    }
654
655    #[test]
656    fn test_priority_ordering() -> Result<()> {
657        let config = BatchConfig::default();
658        let processor = BatchProcessor::new(config)?;
659
660        let low_priority = InferenceRequest::with_priority("low".to_string(), vec![1, 2], 1);
661        let high_priority = InferenceRequest::with_priority("high".to_string(), vec![3, 4], 10);
662
663        processor.enqueue(low_priority)?;
664        processor.enqueue(high_priority)?;
665
666        let batch = processor.get_next_batch()?;
667        assert_eq!(batch[0].id, "high"); // High priority should be first
668        Ok(())
669    }
670
671    #[test]
672    fn test_truncation() -> Result<()> {
673        let config = BatchConfig {
674            max_sequence_length: Some(5),
675            truncation_strategy: TruncationStrategy::End,
676            ..BatchConfig::default()
677        };
678
679        let processor = BatchProcessor::new(config)?;
680
681        let sequences = vec![vec![1, 2, 3, 4, 5, 6, 7, 8], vec![9, 10, 11]];
682
683        let truncated = processor.apply_truncation(sequences, 5)?;
684        assert_eq!(truncated[0].len(), 5);
685        assert_eq!(truncated[0], vec![1, 2, 3, 4, 5]);
686        assert_eq!(truncated[1], vec![9, 10, 11]);
687
688        Ok(())
689    }
690
691    #[test]
692    fn test_padding_computation() -> Result<()> {
693        let config = BatchConfig::default();
694        let processor = BatchProcessor::new(config)?;
695
696        // Test longest padding
697        let target = processor.compute_target_length(10)?;
698        assert_eq!(target, 10);
699
700        // Test multiple padding
701        let config2 = BatchConfig {
702            padding_strategy: PaddingStrategy::Multiple(8),
703            ..BatchConfig::default()
704        };
705        let processor2 = BatchProcessor::new(config2)?;
706
707        let target2 = processor2.compute_target_length(10)?;
708        assert_eq!(target2, 16); // Rounded up to multiple of 8
709
710        Ok(())
711    }
712
713    #[test]
714    fn test_memory_estimation() {
715        let config = BatchConfig::default();
716        let processor = BatchProcessor::new(config).expect("operation failed");
717
718        // Test with larger sequences to see actual memory differences
719        let mem_mb_small = processor.estimate_memory_mb(32, 2048);
720        assert!(mem_mb_small > 1);
721
722        // Larger batch should use more memory
723        let mem_mb_large = processor.estimate_memory_mb(64, 2048);
724        assert!(mem_mb_large > mem_mb_small);
725
726        // Very small batch should still return at least 1 MB
727        let mem_mb_tiny = processor.estimate_memory_mb(1, 10);
728        assert_eq!(mem_mb_tiny, 1);
729    }
730
731    #[test]
732    fn test_batch_statistics() {
733        let mut stats = BatchStatistics::new();
734        assert_eq!(stats.total_batches, 0);
735
736        let batch = BatchedInput {
737            input_ids: Tensor::F32(Array2::zeros((2, 10)).into_dyn()),
738            attention_mask: Tensor::F32(Array2::zeros((2, 10)).into_dyn()),
739            sequence_lengths: vec![8, 6],
740            batch_size: 2,
741            max_seq_length: 10,
742        };
743
744        stats.update(&batch, 50.0);
745        assert_eq!(stats.total_batches, 1);
746        assert_eq!(stats.total_requests, 2);
747        assert_eq!(stats.avg_batch_size, 2.0);
748    }
749
750    #[test]
751    fn test_prepare_batch() -> Result<()> {
752        let config = BatchConfig::default();
753        let processor = BatchProcessor::new(config)?;
754
755        let requests = vec![
756            InferenceRequest::new("1".to_string(), vec![1, 2, 3]),
757            InferenceRequest::new("2".to_string(), vec![4, 5]),
758        ];
759
760        let batched = processor.prepare_batch(requests)?;
761        assert_eq!(batched.batch_size, 2);
762        assert_eq!(batched.max_seq_length, 3);
763        assert_eq!(batched.sequence_lengths, vec![3, 2]);
764
765        Ok(())
766    }
767}