Struct VLLMClient 

pub struct VLLMClient { /* private fields */ }

vLLM/OpenAI-compatible client

Implementations

impl VLLMClient

pub fn new(config: LLMServiceConfig) -> Result<Self>

Create new vLLM client from configuration

pub fn from_env() -> Result<Self>

Create from environment variables

pub fn metrics(&self) -> MetricsSnapshot

Get current API metrics

pub async fn embed_batch_optimized<F>( &self, texts: &[String], batch_size: usize, progress_callback: Option<F>, ) -> Result<Vec<Vec<f32>>>

where F: Fn(usize, usize) + Send + Sync,

Embed texts with optimized batching

Splits large batches into optimal chunks for better throughput and parallel processing. Returns progress updates via callback.

pub async fn embed_batch_parallel( &self, texts: &[String], batch_size: usize, max_concurrent: usize, ) -> Result<Vec<Vec<f32>>>

Embed texts in parallel with multiple concurrent batches

Uses tokio to process multiple batches concurrently for maximum throughput. Useful for embedding large document collections.

Trait Implementations

impl LLMClient for VLLMClient

fn chat_completion<'life0, 'async_trait>( &'life0 self, messages: Vec<ChatMessage>, ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Generate chat completion

fn embed<'life0, 'life1, 'async_trait>( &'life0 self, text: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Generate embeddings for text

fn embed_batch<'life0, 'life1, 'async_trait>( &'life0 self, texts: &'life1 [String], ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Generate embeddings for multiple texts

fn embedding_dimensions(&self) -> usize

Get embedding dimensions

fn model_name(&self) -> &str

Get model name