Struct CachedProvider 

pub struct CachedProvider<P> { /* private fields */ }

A cached LLM provider that wraps another provider with caching.

Implementations

impl<P> CachedProvider<P>

pub fn new(inner: P, cache: Arc<LLMCache>) -> Self

Create a new cached provider.

pub fn with_default_cache(inner: P) -> Self

Create with default cache config.

pub async fn cache_stats(&self) -> CacheStats

Get cache statistics.

pub async fn clear_cache(&self)

Clear the cache.

Trait Implementations

impl<P: EmbeddingProvider> EmbeddingProvider for CachedProvider<P>

fn name(&self) -> &str

Get the name of this provider.

fn model(&self) -> &str

Get the embedding model.

fn dimension(&self) -> usize

Get the dimension of the embeddings.

fn max_tokens(&self) -> usize

Get the maximum number of tokens per input.

fn embed<'life0, 'life1, 'async_trait>( &'life0 self, texts: &'life1 [String], ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Generate embeddings for a batch of texts.

fn embed_one<'life0, 'life1, 'async_trait>( &'life0 self, text: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Generate embedding for a single text.

impl<P: LLMProvider> LLMProvider for CachedProvider<P>

fn name(&self) -> &str

Get the name of this provider.

fn model(&self) -> &str

Get the current model.

fn max_context_length(&self) -> usize

Get the maximum context length for the model.

fn complete<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<LLMResponse>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Generate a completion for the given prompt.

fn complete_with_options<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, prompt: &'life1 str, options: &'life2 CompletionOptions, ) -> Pin<Box<dyn Future<Output = Result<LLMResponse>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Generate a completion with custom options.

fn chat<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, messages: &'life1 [ChatMessage], options: Option<&'life2 CompletionOptions>, ) -> Pin<Box<dyn Future<Output = Result<LLMResponse>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Generate a chat completion with messages.

fn chat_with_tools<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, messages: &'life1 [ChatMessage], tools: &'life2 [ToolDefinition], tool_choice: Option<ToolChoice>, options: Option<&'life3 CompletionOptions>, ) -> Pin<Box<dyn Future<Output = Result<LLMResponse>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Generate a chat completion with tool/function calling support.

fn stream<'life0, 'life1, 'async_trait>( &'life0 self, _prompt: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<BoxStream<'static, Result<String>>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Generate a streaming completion.

fn chat_with_tools_stream<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, _messages: &'life1 [ChatMessage], _tools: &'life2 [ToolDefinition], _tool_choice: Option<ToolChoice>, _options: Option<&'life3 CompletionOptions>, ) -> Pin<Box<dyn Future<Output = Result<BoxStream<'static, Result<StreamChunk>>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Stream chat completion with tool calling support. Returns a stream of events containing content chunks, tool call deltas, and finish reasons.

fn supports_streaming(&self) -> bool

Check if the model supports streaming.

fn supports_tool_streaming(&self) -> bool

Check if the provider supports streaming with tool calls.

fn supports_json_mode(&self) -> bool

Check if the model supports JSON mode.

fn supports_function_calling(&self) -> bool

Check if the model supports function/tool calling.

fn model_name(&self) -> Option<String>

Get the model name as an Option<String>.