Struct BatchProcessor 

pub struct BatchProcessor { /* private fields */ }

Batch processor for efficient inference

Implementations

impl BatchProcessor

pub fn new(config: PerformanceConfig) -> Self

Create a new batch processor

pub fn add_to_batch(&mut self, tensor: Tensor) -> Result<Option<Vec<Tensor>>>

Add a tensor to the current batch

pub fn flush_batch(&mut self) -> Result<Vec<Tensor>>

Flush the current batch and return it

pub fn cache_tensor(&mut self, key: String, tensor: Tensor) -> Result<()>

Cache a tensor with a given key

pub fn cache_stats(&self) -> CacheStatistics

Cache statistics

pub fn get_cached_tensor(&mut self, key: &str) -> Option<&Tensor>

Retrieve a cached tensor

pub fn clear_cache(&mut self)

Clear the cache

pub fn current_batch_size(&self) -> usize

Get current batch size

Trait Implementations

impl Debug for BatchProcessor

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.