pub struct DynamicBatcher<T> { /* private fields */ }Expand description
Dynamic batcher for inference requests.
Implementations§
Source§impl<T> DynamicBatcher<T>
impl<T> DynamicBatcher<T>
Sourcepub fn new(config: DynamicBatchConfig) -> Self
pub fn new(config: DynamicBatchConfig) -> Self
Create a new dynamic batcher.
Sourcepub fn submit(&mut self, request: BatchRequest<T>) -> Result<(), BatchingError>
pub fn submit(&mut self, request: BatchRequest<T>) -> Result<(), BatchingError>
Submit a request for batching.
Sourcepub fn try_form_batch(&mut self) -> Option<Vec<BatchRequest<T>>>
pub fn try_form_batch(&mut self) -> Option<Vec<BatchRequest<T>>>
Try to form a batch if criteria are met.
Sourcepub fn stats(&self) -> &BatchingStats
pub fn stats(&self) -> &BatchingStats
Get statistics.
Sourcepub fn record_batch(
&mut self,
batch_size: usize,
wait_time: Duration,
latency: Duration,
)
pub fn record_batch( &mut self, batch_size: usize, wait_time: Duration, latency: Duration, )
Record batch execution results.
Sourcepub fn queue_depth(&self) -> usize
pub fn queue_depth(&self) -> usize
Get current queue depth.
Auto Trait Implementations§
impl<T> Freeze for DynamicBatcher<T>
impl<T> RefUnwindSafe for DynamicBatcher<T>where
T: RefUnwindSafe,
impl<T> Send for DynamicBatcher<T>where
T: Send,
impl<T> Sync for DynamicBatcher<T>where
T: Sync,
impl<T> Unpin for DynamicBatcher<T>where
T: Unpin,
impl<T> UnwindSafe for DynamicBatcher<T>where
T: UnwindSafe,
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more