pub struct BatchedEngine { /* private fields */ }Expand description
Batched inference engine using continuous batching
Implementations§
Source§impl BatchedEngine
impl BatchedEngine
Sourcepub fn new(
model: Arc<dyn Model>,
tokenizer: Arc<Tokenizer>,
_model_config: ModelConfig,
backend: Arc<dyn Backend>,
config: BatchedEngineConfig,
) -> Self
pub fn new( model: Arc<dyn Model>, tokenizer: Arc<Tokenizer>, _model_config: ModelConfig, backend: Arc<dyn Backend>, config: BatchedEngineConfig, ) -> Self
Create a new batched engine and spawn the background processing loop.
Auto Trait Implementations§
impl !RefUnwindSafe for BatchedEngine
impl !UnwindSafe for BatchedEngine
impl Freeze for BatchedEngine
impl Send for BatchedEngine
impl Sync for BatchedEngine
impl Unpin for BatchedEngine
impl UnsafeUnpin for BatchedEngine
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
impl<A, B, T> HttpServerConnExec<A, B> for Twhere
B: Body,
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
Source§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more