Struct SparseInferenceBackend 

pub struct SparseInferenceBackend { /* private fields */ }

Sparse inference backend for RuvLLM integration

Implementations

impl SparseInferenceBackend

pub fn new( num_layers: usize, hidden_dim: usize, intermediate_dim: usize, vocab_size: usize, sparsity_ratio: f32, ) -> Result<Self>

Create a new sparse inference backend

pub fn from_gguf(path: &Path) -> Result<Self>

Create from a GGUF model file

pub fn from_gguf_bytes(data: &[u8]) -> Result<Self>

Create from GGUF model bytes

pub fn next_token( &mut self, input_ids: &[u32], kv_cache: &mut KVCache, ) -> Result<u32>

Generate next token

pub fn generate( &mut self, input_ids: &[u32], config: &GenerationConfig, ) -> Result<Vec<u32>>

Generate multiple tokens

pub fn metadata(&self) -> &ModelMetadata

Get model metadata

pub fn generation_stats(&self) -> &GenerationStats

Get generation statistics

pub fn set_sparsity(&mut self, threshold: f32)

Set sparsity threshold

pub fn calibrate(&mut self, samples: &[Vec<f32>]) -> Result<()>

Calibrate predictors with sample data

pub fn reset(&mut self)

Reset KV cache (for new conversation)

Trait Implementations

impl InferenceBackend for SparseInferenceBackend

fn forward(&mut self, input_ids: &[u32]) -> Result<Vec<f32>>

Generate next token probabilities

fn generate( &mut self, input_ids: &[u32], max_new_tokens: usize, ) -> Result<Vec<u32>>

Generate tokens

fn vocab_size(&self) -> usize

Get vocabulary size

fn name(&self) -> &str

Backend name