Struct UniLLMBackend 

pub struct UniLLMBackend { /* private fields */ }

UniLLM inference backend

Implementations

impl UniLLMBackend

pub fn new() -> Self

Create new UniLLM backend

Trait Implementations

impl Default for UniLLMBackend

fn default() -> Self

Returns the “default value” for a type.

impl InferenceBackend for UniLLMBackend

fn name(&self) -> &str

Backend identifier name

fn load_model(&mut self, path: &Path) -> Result<f64>

Load model from path, return load time in milliseconds

fn generate( &mut self, prompt: &str, max_tokens: usize, ) -> Result<GenerationResult>

Generate text from prompt, return generation result

fn memory_usage(&self) -> u64

Get current memory usage in bytes

fn unload(&mut self)

Unload model and free resources