Struct LLMWorker 

pub struct LLMWorker { /* private fields */ }

Implementations

impl LLMWorker

pub fn new(shared_state: Arc<SharedState>) -> Self

Create with shared state (legacy constructor)

pub fn new_with_backend(backend_url: String) -> Self

Create with explicit backend URL

pub fn set_runtime_manager(&self, runtime_manager: Arc<RuntimeManager>)

Set the runtime manager

pub async fn is_runtime_ready(&self) -> bool

Check if runtime is ready for inference

pub async fn generate_response( &self, _session_id: String, context: Vec<Message>, ) -> Result<String>

Generate a complete (non-streaming) response from the LLM.

pub async fn stream_response( &self, messages: Vec<Message>, max_tokens: u32, temperature: f32, ) -> Result<impl Stream<Item = Result<String, Error>>>

Stream response tokens from the LLM as Server-Sent Events. Returns a stream of SSE-formatted strings ready to send to the client.

pub async fn batch_process( &self, prompts: Vec<(String, Vec<Message>)>, ) -> Result<Vec<String>>

Batch process multiple prompts (non-streaming)

pub async fn initialize_model(&self, model_path: &str) -> Result<()>

Initialize LLM model (no-op for HTTP proxy mode)

pub async fn generate_embeddings( &self, texts: Vec<String>, ) -> Result<Vec<Vec<f32>>>

Generate embeddings for one or more text inputs via llama-server’s /v1/embeddings endpoint. This reuses the vectors llama.cpp already computes during inference — no separate model needed. Returns a Vec of embedding vectors (one per input string).

pub async fn generate_title( &self, prompt: &str, max_tokens: u32, ) -> Result<String>

Generate title for a chat using the LLM