1use scirs2_core::ndarray::Array2;
41use scirs2_core::parallel_ops::*;
42use serde::{Deserialize, Serialize};
43use std::collections::VecDeque;
44use std::sync::{Arc, Mutex};
45use std::time::{Duration, Instant};
46use trustformers_core::errors::{Result, TrustformersError};
47use trustformers_core::tensor::Tensor;
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
51pub enum PaddingStrategy {
52 Longest,
54
55 Fixed(usize),
57
58 Multiple(usize),
60
61 None,
63}
64
65impl Default for PaddingStrategy {
66 fn default() -> Self {
67 Self::Longest
68 }
69}
70
71#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
73pub enum TruncationStrategy {
74 End,
76
77 Beginning,
79
80 Error,
82
83 None,
85}
86
87impl Default for TruncationStrategy {
88 fn default() -> Self {
89 Self::End
90 }
91}
92
93#[derive(Debug, Clone, Serialize, Deserialize)]
95pub struct BatchConfig {
96 pub max_batch_size: usize,
98
99 pub min_batch_size: usize,
101
102 pub timeout_ms: u64,
104
105 pub padding_strategy: PaddingStrategy,
107
108 pub truncation_strategy: TruncationStrategy,
110
111 pub max_sequence_length: Option<usize>,
113
114 pub pad_token_id: u32,
116
117 pub dynamic_batching: bool,
119
120 pub target_memory_mb: Option<usize>,
122
123 pub parallel_preprocessing: bool,
125
126 pub sort_by_length: bool,
128}
129
130impl Default for BatchConfig {
131 fn default() -> Self {
132 Self {
133 max_batch_size: 32,
134 min_batch_size: 1,
135 timeout_ms: 100,
136 padding_strategy: PaddingStrategy::default(),
137 truncation_strategy: TruncationStrategy::default(),
138 max_sequence_length: None,
139 pad_token_id: 0,
140 dynamic_batching: true,
141 target_memory_mb: None,
142 parallel_preprocessing: true,
143 sort_by_length: true,
144 }
145 }
146}
147
148impl BatchConfig {
149 pub fn low_latency() -> Self {
151 Self {
152 max_batch_size: 8,
153 min_batch_size: 1,
154 timeout_ms: 10,
155 dynamic_batching: false,
156 ..Default::default()
157 }
158 }
159
160 pub fn high_throughput() -> Self {
162 Self {
163 max_batch_size: 128,
164 min_batch_size: 32,
165 timeout_ms: 500,
166 dynamic_batching: true,
167 sort_by_length: true,
168 ..Default::default()
169 }
170 }
171
172 pub fn validate(&self) -> Result<()> {
174 if self.max_batch_size == 0 {
175 return Err(TrustformersError::invalid_config(
176 "max_batch_size must be positive".to_string(),
177 ));
178 }
179
180 if self.min_batch_size > self.max_batch_size {
181 return Err(TrustformersError::invalid_config(
182 "min_batch_size cannot exceed max_batch_size".to_string(),
183 ));
184 }
185
186 if let PaddingStrategy::Fixed(len) = self.padding_strategy {
187 if len == 0 {
188 return Err(TrustformersError::invalid_config(
189 "fixed padding length must be positive".to_string(),
190 ));
191 }
192 }
193
194 if let PaddingStrategy::Multiple(multiple) = self.padding_strategy {
195 if multiple == 0 {
196 return Err(TrustformersError::invalid_config(
197 "padding multiple must be positive".to_string(),
198 ));
199 }
200 }
201
202 Ok(())
203 }
204}
205
206#[derive(Debug, Clone)]
208pub struct BatchedInput {
209 pub input_ids: Tensor,
211
212 pub attention_mask: Tensor,
214
215 pub sequence_lengths: Vec<usize>,
217
218 pub batch_size: usize,
220
221 pub max_seq_length: usize,
223}
224
225#[derive(Debug, Clone)]
227pub struct InferenceRequest {
228 pub id: String,
230
231 pub input_ids: Vec<u32>,
233
234 pub priority: i32,
236
237 pub timestamp: Instant,
239}
240
241impl InferenceRequest {
242 pub fn new(id: String, input_ids: Vec<u32>) -> Self {
244 Self {
245 id,
246 input_ids,
247 priority: 0,
248 timestamp: Instant::now(),
249 }
250 }
251
252 pub fn with_priority(id: String, input_ids: Vec<u32>, priority: i32) -> Self {
254 Self {
255 id,
256 input_ids,
257 priority,
258 timestamp: Instant::now(),
259 }
260 }
261}
262
263pub struct BatchProcessor {
265 config: BatchConfig,
266 request_queue: Arc<Mutex<VecDeque<InferenceRequest>>>,
267 batch_start_time: Arc<Mutex<Option<Instant>>>,
268}
269
270impl BatchProcessor {
271 pub fn new(config: BatchConfig) -> Result<Self> {
273 config.validate()?;
274
275 Ok(Self {
276 config,
277 request_queue: Arc::new(Mutex::new(VecDeque::new())),
278 batch_start_time: Arc::new(Mutex::new(None)),
279 })
280 }
281
282 pub fn enqueue(&self, request: InferenceRequest) -> Result<()> {
284 let mut queue = self.request_queue.lock().map_err(|e| {
285 TrustformersError::lock_error(format!("Failed to lock request queue: {}", e))
286 })?;
287
288 let insert_pos =
290 queue.iter().position(|r| r.priority < request.priority).unwrap_or(queue.len());
291
292 queue.insert(insert_pos, request);
293
294 let mut start_time = self.batch_start_time.lock().map_err(|e| {
296 TrustformersError::lock_error(format!("Failed to lock batch start time: {}", e))
297 })?;
298
299 if start_time.is_none() {
300 *start_time = Some(Instant::now());
301 }
302
303 Ok(())
304 }
305
306 pub fn should_process_batch(&self) -> Result<bool> {
308 let queue = self.request_queue.lock().map_err(|e| {
309 TrustformersError::lock_error(format!("Failed to lock request queue: {}", e))
310 })?;
311
312 if queue.is_empty() {
313 return Ok(false);
314 }
315
316 if queue.len() >= self.config.min_batch_size {
318 return Ok(true);
319 }
320
321 let start_time = self.batch_start_time.lock().map_err(|e| {
323 TrustformersError::lock_error(format!("Failed to lock batch start time: {}", e))
324 })?;
325
326 if let Some(start) = *start_time {
327 let elapsed = start.elapsed();
328 if elapsed >= Duration::from_millis(self.config.timeout_ms) {
329 return Ok(true);
330 }
331 }
332
333 Ok(false)
334 }
335
336 pub fn get_next_batch(&self) -> Result<Vec<InferenceRequest>> {
338 let mut queue = self.request_queue.lock().map_err(|e| {
339 TrustformersError::lock_error(format!("Failed to lock request queue: {}", e))
340 })?;
341
342 let batch_size = self.config.max_batch_size.min(queue.len());
343 let mut batch: Vec<InferenceRequest> = Vec::with_capacity(batch_size);
344
345 for _ in 0..batch_size {
346 if let Some(request) = queue.pop_front() {
347 batch.push(request);
348 }
349 }
350
351 if queue.is_empty() {
353 let mut start_time = self.batch_start_time.lock().map_err(|e| {
354 TrustformersError::lock_error(format!("Failed to lock batch start time: {}", e))
355 })?;
356 *start_time = None;
357 }
358
359 Ok(batch)
360 }
361
362 pub fn prepare_batch(&self, requests: Vec<InferenceRequest>) -> Result<BatchedInput> {
364 if requests.is_empty() {
365 return Err(TrustformersError::invalid_input(
366 "Cannot prepare batch from empty requests".to_string(),
367 ));
368 }
369
370 let mut sequences: Vec<Vec<u32>> = requests.into_iter().map(|r| r.input_ids).collect();
371
372 if self.config.sort_by_length {
374 sequences.sort_by_key(|s| std::cmp::Reverse(s.len()));
375 }
376
377 if let Some(max_len) = self.config.max_sequence_length {
379 sequences = self.apply_truncation(sequences, max_len)?;
380 }
381
382 let max_len = sequences.iter().map(|s| s.len()).max().unwrap_or(0);
384 let target_len = self.compute_target_length(max_len)?;
385
386 let (padded_ids, attention_masks, seq_lengths) =
388 self.apply_padding(&sequences, target_len)?;
389
390 let batch_size = sequences.len();
391
392 Ok(BatchedInput {
393 input_ids: padded_ids,
394 attention_mask: attention_masks,
395 sequence_lengths: seq_lengths,
396 batch_size,
397 max_seq_length: target_len,
398 })
399 }
400
401 fn apply_truncation(&self, sequences: Vec<Vec<u32>>, max_len: usize) -> Result<Vec<Vec<u32>>> {
403 let process_seq = |seq: Vec<u32>| -> Result<Vec<u32>> {
404 if seq.len() <= max_len {
405 return Ok(seq);
406 }
407
408 match self.config.truncation_strategy {
409 TruncationStrategy::End => Ok(seq[..max_len].to_vec()),
410 TruncationStrategy::Beginning => Ok(seq[seq.len() - max_len..].to_vec()),
411 TruncationStrategy::Error => Err(TrustformersError::invalid_input(format!(
412 "Sequence length {} exceeds max length {}",
413 seq.len(),
414 max_len
415 ))),
416 TruncationStrategy::None => Ok(seq),
417 }
418 };
419
420 if self.config.parallel_preprocessing && sequences.len() > 4 {
421 sequences.into_par_iter().map(process_seq).collect::<Result<Vec<_>>>()
422 } else {
423 sequences.into_iter().map(process_seq).collect::<Result<Vec<_>>>()
424 }
425 }
426
427 fn compute_target_length(&self, max_len: usize) -> Result<usize> {
429 match self.config.padding_strategy {
430 PaddingStrategy::Longest => Ok(max_len),
431 PaddingStrategy::Fixed(len) => Ok(len),
432 PaddingStrategy::Multiple(multiple) => {
433 Ok(max_len.div_ceil(multiple) * multiple)
435 },
436 PaddingStrategy::None => Ok(max_len),
437 }
438 }
439
440 fn apply_padding(
442 &self,
443 sequences: &[Vec<u32>],
444 target_len: usize,
445 ) -> Result<(Tensor, Tensor, Vec<usize>)> {
446 let batch_size = sequences.len();
447 let pad_token = self.config.pad_token_id;
448
449 let mut input_ids = Array2::<u32>::from_elem((batch_size, target_len), pad_token);
451 let mut attention_mask = Array2::<i32>::zeros((batch_size, target_len));
452 let mut seq_lengths = Vec::with_capacity(batch_size);
453
454 for (i, seq) in sequences.iter().enumerate() {
455 let seq_len = seq.len();
456 seq_lengths.push(seq_len);
457
458 for (j, &token) in seq.iter().enumerate() {
460 input_ids[[i, j]] = token;
461 attention_mask[[i, j]] = 1;
462 }
463 }
464
465 let input_ids_tensor = Tensor::F32(input_ids.mapv(|x| x as f32).into_dyn());
466
467 let attention_mask_tensor = Tensor::F32(attention_mask.mapv(|x| x as f32).into_dyn());
468
469 Ok((input_ids_tensor, attention_mask_tensor, seq_lengths))
470 }
471
472 pub fn estimate_memory_mb(&self, batch_size: usize, seq_length: usize) -> usize {
474 let tokens = batch_size * seq_length;
477 let bytes_per_token = 4;
478 let overhead_multiplier = 10; let memory_bytes = tokens * bytes_per_token * overhead_multiplier;
481 let memory_mb = memory_bytes / (1024 * 1024);
482
483 memory_mb.max(1)
485 }
486
487 pub fn adjust_batch_size(&self, sequences: &[Vec<u32>]) -> usize {
489 if let Some(target_mb) = self.config.target_memory_mb {
490 let max_seq_len = sequences.iter().map(|s| s.len()).max().unwrap_or(0);
491
492 let mut left = 1;
494 let mut right = self.config.max_batch_size;
495 let mut best_size = 1;
496
497 while left <= right {
498 let mid = (left + right) / 2;
499 let estimated_mb = self.estimate_memory_mb(mid, max_seq_len);
500
501 if estimated_mb <= target_mb {
502 best_size = mid;
503 left = mid + 1;
504 } else {
505 right = mid - 1;
506 }
507 }
508
509 best_size.min(sequences.len())
510 } else {
511 self.config.max_batch_size.min(sequences.len())
512 }
513 }
514
515 pub fn queue_size(&self) -> Result<usize> {
517 let queue = self.request_queue.lock().map_err(|e| {
518 TrustformersError::lock_error(format!("Failed to lock request queue: {}", e))
519 })?;
520 Ok(queue.len())
521 }
522}
523
524#[derive(Debug, Clone)]
526pub struct BatchStatistics {
527 pub avg_batch_size: f32,
529
530 pub avg_sequence_length: f32,
532
533 pub avg_wait_time_ms: f32,
535
536 pub total_batches: usize,
538
539 pub total_requests: usize,
541
542 pub avg_padding_ratio: f32,
544}
545
546impl BatchStatistics {
547 pub fn new() -> Self {
549 Self {
550 avg_batch_size: 0.0,
551 avg_sequence_length: 0.0,
552 avg_wait_time_ms: 0.0,
553 total_batches: 0,
554 total_requests: 0,
555 avg_padding_ratio: 0.0,
556 }
557 }
558
559 pub fn update(&mut self, batch: &BatchedInput, wait_time_ms: f32) {
561 self.total_batches += 1;
562 self.total_requests += batch.batch_size;
563
564 let n = self.total_batches as f32;
566 self.avg_batch_size = (self.avg_batch_size * (n - 1.0) + batch.batch_size as f32) / n;
567
568 let actual_tokens: usize = batch.sequence_lengths.iter().sum();
569 let total_tokens = batch.batch_size * batch.max_seq_length;
570 let padding_ratio = 1.0 - (actual_tokens as f32 / total_tokens as f32);
571
572 self.avg_padding_ratio = (self.avg_padding_ratio * (n - 1.0) + padding_ratio) / n;
573 self.avg_wait_time_ms = (self.avg_wait_time_ms * (n - 1.0) + wait_time_ms) / n;
574
575 let avg_seq_len: f32 =
576 batch.sequence_lengths.iter().sum::<usize>() as f32 / batch.batch_size as f32;
577 self.avg_sequence_length = (self.avg_sequence_length * (n - 1.0) + avg_seq_len) / n;
578 }
579}
580
581impl Default for BatchStatistics {
582 fn default() -> Self {
583 Self::new()
584 }
585}
586
587#[cfg(test)]
588mod tests {
589 use super::*;
590
591 #[test]
592 fn test_batch_config_validation() {
593 let valid_config = BatchConfig::default();
594 assert!(valid_config.validate().is_ok());
595
596 let invalid_config = BatchConfig {
597 max_batch_size: 0,
598 ..BatchConfig::default()
599 };
600 assert!(invalid_config.validate().is_err());
601
602 let invalid_config2 = BatchConfig {
603 min_batch_size: 100,
604 max_batch_size: 10,
605 ..BatchConfig::default()
606 };
607 assert!(invalid_config2.validate().is_err());
608 }
609
610 #[test]
611 fn test_batch_config_presets() {
612 let low_latency = BatchConfig::low_latency();
613 assert_eq!(low_latency.max_batch_size, 8);
614 assert_eq!(low_latency.timeout_ms, 10);
615
616 let high_throughput = BatchConfig::high_throughput();
617 assert_eq!(high_throughput.max_batch_size, 128);
618 assert!(high_throughput.sort_by_length);
619 }
620
621 #[test]
622 fn test_padding_strategy() {
623 let longest = PaddingStrategy::Longest;
624 let fixed = PaddingStrategy::Fixed(128);
625 let multiple = PaddingStrategy::Multiple(8);
626
627 assert!(matches!(longest, PaddingStrategy::Longest));
628 assert!(matches!(fixed, PaddingStrategy::Fixed(128)));
629 assert!(matches!(multiple, PaddingStrategy::Multiple(8)));
630 }
631
632 #[test]
633 fn test_batch_processor_creation() -> Result<()> {
634 let config = BatchConfig::default();
635 let processor = BatchProcessor::new(config)?;
636 assert_eq!(processor.queue_size()?, 0);
637 Ok(())
638 }
639
640 #[test]
641 fn test_enqueue_requests() -> Result<()> {
642 let config = BatchConfig::default();
643 let processor = BatchProcessor::new(config)?;
644
645 let request1 = InferenceRequest::new("req1".to_string(), vec![1, 2, 3]);
646 let request2 = InferenceRequest::new("req2".to_string(), vec![4, 5, 6, 7]);
647
648 processor.enqueue(request1)?;
649 processor.enqueue(request2)?;
650
651 assert_eq!(processor.queue_size()?, 2);
652 Ok(())
653 }
654
655 #[test]
656 fn test_priority_ordering() -> Result<()> {
657 let config = BatchConfig::default();
658 let processor = BatchProcessor::new(config)?;
659
660 let low_priority = InferenceRequest::with_priority("low".to_string(), vec![1, 2], 1);
661 let high_priority = InferenceRequest::with_priority("high".to_string(), vec![3, 4], 10);
662
663 processor.enqueue(low_priority)?;
664 processor.enqueue(high_priority)?;
665
666 let batch = processor.get_next_batch()?;
667 assert_eq!(batch[0].id, "high"); Ok(())
669 }
670
671 #[test]
672 fn test_truncation() -> Result<()> {
673 let config = BatchConfig {
674 max_sequence_length: Some(5),
675 truncation_strategy: TruncationStrategy::End,
676 ..BatchConfig::default()
677 };
678
679 let processor = BatchProcessor::new(config)?;
680
681 let sequences = vec![vec![1, 2, 3, 4, 5, 6, 7, 8], vec![9, 10, 11]];
682
683 let truncated = processor.apply_truncation(sequences, 5)?;
684 assert_eq!(truncated[0].len(), 5);
685 assert_eq!(truncated[0], vec![1, 2, 3, 4, 5]);
686 assert_eq!(truncated[1], vec![9, 10, 11]);
687
688 Ok(())
689 }
690
691 #[test]
692 fn test_padding_computation() -> Result<()> {
693 let config = BatchConfig::default();
694 let processor = BatchProcessor::new(config)?;
695
696 let target = processor.compute_target_length(10)?;
698 assert_eq!(target, 10);
699
700 let config2 = BatchConfig {
702 padding_strategy: PaddingStrategy::Multiple(8),
703 ..BatchConfig::default()
704 };
705 let processor2 = BatchProcessor::new(config2)?;
706
707 let target2 = processor2.compute_target_length(10)?;
708 assert_eq!(target2, 16); Ok(())
711 }
712
713 #[test]
714 fn test_memory_estimation() {
715 let config = BatchConfig::default();
716 let processor = BatchProcessor::new(config).expect("operation failed");
717
718 let mem_mb_small = processor.estimate_memory_mb(32, 2048);
720 assert!(mem_mb_small > 1);
721
722 let mem_mb_large = processor.estimate_memory_mb(64, 2048);
724 assert!(mem_mb_large > mem_mb_small);
725
726 let mem_mb_tiny = processor.estimate_memory_mb(1, 10);
728 assert_eq!(mem_mb_tiny, 1);
729 }
730
731 #[test]
732 fn test_batch_statistics() {
733 let mut stats = BatchStatistics::new();
734 assert_eq!(stats.total_batches, 0);
735
736 let batch = BatchedInput {
737 input_ids: Tensor::F32(Array2::zeros((2, 10)).into_dyn()),
738 attention_mask: Tensor::F32(Array2::zeros((2, 10)).into_dyn()),
739 sequence_lengths: vec![8, 6],
740 batch_size: 2,
741 max_seq_length: 10,
742 };
743
744 stats.update(&batch, 50.0);
745 assert_eq!(stats.total_batches, 1);
746 assert_eq!(stats.total_requests, 2);
747 assert_eq!(stats.avg_batch_size, 2.0);
748 }
749
750 #[test]
751 fn test_prepare_batch() -> Result<()> {
752 let config = BatchConfig::default();
753 let processor = BatchProcessor::new(config)?;
754
755 let requests = vec![
756 InferenceRequest::new("1".to_string(), vec![1, 2, 3]),
757 InferenceRequest::new("2".to_string(), vec![4, 5]),
758 ];
759
760 let batched = processor.prepare_batch(requests)?;
761 assert_eq!(batched.batch_size, 2);
762 assert_eq!(batched.max_seq_length, 3);
763 assert_eq!(batched.sequence_lengths, vec![3, 2]);
764
765 Ok(())
766 }
767}