Struct GGMLRuntime 

pub struct GGMLRuntime { /* private fields */ }

GGML runtime - reuses GGUF runtime implementation since llama-server supports both

Implementations

impl GGMLRuntime

pub fn new() -> Self

Trait Implementations

impl Default for GGMLRuntime

fn default() -> Self

Returns the “default value” for a type.

impl ModelRuntime for GGMLRuntime

fn supported_format(&self) -> ModelFormat

Get the format this runtime supports

fn initialize<'life0, 'async_trait>( &'life0 mut self, config: RuntimeConfig, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Initialize the runtime (start server process, load model, etc.)

fn is_ready<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = bool> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Check if runtime is ready for inference

fn health_check<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Get health status

fn base_url(&self) -> String

Get the base URL for inference API (e.g., “http://127.0.0.1:8001”)

fn generate<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<InferenceResponse>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Perform inference (non-streaming)

fn generate_stream<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<Box<dyn Stream<Item = Result<String, Error>> + Send + Unpin>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Perform streaming inference

fn shutdown<'life0, 'async_trait>( &'life0 mut self, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Shutdown the runtime (stop server, cleanup resources)

fn metadata(&self) -> RuntimeMetadata

Get runtime metadata

fn completions_url(&self) -> String

Get the OpenAI-compatible chat completions endpoint