Enum ForwardEngine 

pub enum ForwardEngine {
    Llama(LlamaForward),
    Phi(PhiForward),
}

Architecture-specific inference engine with KV-cache support.

Variants

Llama(LlamaForward)

Phi(PhiForward)

Implementations

impl ForwardEngine

pub fn from_pretrained( info: ModelInfo, weight_paths: &[PathBuf], ) -> Result<Self>

pub fn from_weight_paths( info: ModelInfo, weight_paths: &[PathBuf], ) -> Result<Self>

pub fn from_weight_paths_with_backend( info: ModelInfo, weight_paths: &[PathBuf], backend: LlmBackendKind, ) -> Result<Self>

pub fn from_gguf(info: ModelInfo, path: &Path) -> Result<Self>

pub fn from_gguf_with_backend( info: ModelInfo, path: &Path, backend: LlmBackendKind, ) -> Result<Self>

pub fn reset_cache(&mut self)

pub fn forward_logits( &mut self, input_ids: &[u32], use_cache: bool, ) -> Result<Vec<f32>>

pub fn embed(&mut self, input_ids: &[u32]) -> Result<Vec<f32>>