pub trait ModelEngine: Send + Sync {
// Required methods
fn forward_batch(
&mut self,
tokens: &[u32],
) -> impl Future<Output = Result<Vec<Logits>, EngineError>> + Send;
fn forward_token(
&mut self,
token: u32,
) -> impl Future<Output = Result<Logits, EngineError>> + Send;
}Expand description
Abstract inference engine interface for draft and target models. Compatible with Candle, Burn, ONNX Runtime, or custom model runtimes.
Required Methods§
Sourcefn forward_batch(
&mut self,
tokens: &[u32],
) -> impl Future<Output = Result<Vec<Logits>, EngineError>> + Send
fn forward_batch( &mut self, tokens: &[u32], ) -> impl Future<Output = Result<Vec<Logits>, EngineError>> + Send
Computes logits for a sequence of tokens in a single parallel forward pass.
Sourcefn forward_token(
&mut self,
token: u32,
) -> impl Future<Output = Result<Logits, EngineError>> + Send
fn forward_token( &mut self, token: u32, ) -> impl Future<Output = Result<Logits, EngineError>> + Send
Computes logits for a single token step.
Dyn Compatibility§
This trait is not dyn compatible.
In older versions of Rust, dyn compatibility was called "object safety".