synaptic_cache/
cached_model.rs

1use std::sync::Arc;
2
3use async_trait::async_trait;
4use synaptic_core::{ChatModel, ChatRequest, ChatResponse, ChatStream, SynapticError};
5
6use crate::LlmCache;
7
8/// A ChatModel wrapper that caches responses using an [`LlmCache`].
9///
10/// Cache keys are generated by serializing the [`ChatRequest`] to JSON.
11/// Streaming (`stream_chat`) delegates directly to the inner model without caching.
12pub struct CachedChatModel {
13    inner: Arc<dyn ChatModel>,
14    cache: Arc<dyn LlmCache>,
15}
16
17impl CachedChatModel {
18    /// Create a new cached model wrapping the given model and cache.
19    pub fn new(inner: Arc<dyn ChatModel>, cache: Arc<dyn LlmCache>) -> Self {
20        Self { inner, cache }
21    }
22}
23
24#[async_trait]
25impl ChatModel for CachedChatModel {
26    async fn chat(&self, request: ChatRequest) -> Result<ChatResponse, SynapticError> {
27        let key = serde_json::to_string(&request)
28            .map_err(|e| SynapticError::Cache(format!("failed to serialize request: {e}")))?;
29
30        // Check cache first
31        if let Some(cached) = self.cache.get(&key).await? {
32            return Ok(cached);
33        }
34
35        // Cache miss: call inner model
36        let response = self.inner.chat(request).await?;
37
38        // Store in cache
39        self.cache.put(&key, &response).await?;
40
41        Ok(response)
42    }
43
44    fn stream_chat(&self, request: ChatRequest) -> ChatStream<'_> {
45        // Streaming bypasses the cache
46        self.inner.stream_chat(request)
47    }
48}
synaptic_cache/cached_model.rs

synaptic_cache/
cached_model.rs
