synaptic-cache 0.4.0

LLM response caching: InMemory, Semantic, CachedChatModel
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

use std::time::Duration;

use synaptic_cache::{InMemoryCache, LlmCache};
use synaptic_core::{ChatResponse, Message};

fn make_response(text: &str) -> ChatResponse {
    ChatResponse {
        message: Message::ai(text),
        usage: None,
    }
}

#[tokio::test]
async fn cache_stores_and_retrieves() {
    let cache = InMemoryCache::new();
    let response = make_response("hello");

    cache.put("key1", &response).await.unwrap();
    let result = cache.get("key1").await.unwrap();

    assert!(result.is_some());
    assert_eq!(result.unwrap().message.content(), "hello");
}

#[tokio::test]
async fn cache_returns_none_for_miss() {
    let cache = InMemoryCache::new();
    let result = cache.get("nonexistent").await.unwrap();
    assert!(result.is_none());
}

#[tokio::test]
async fn cache_ttl_expires() {
    let cache = InMemoryCache::with_ttl(Duration::from_millis(50));
    let response = make_response("ephemeral");

    cache.put("key1", &response).await.unwrap();

    // Should be present immediately
    assert!(cache.get("key1").await.unwrap().is_some());

    // Wait for TTL to expire
    tokio::time::sleep(Duration::from_millis(100)).await;

    // Should be expired now
    assert!(cache.get("key1").await.unwrap().is_none());
}

#[tokio::test]
async fn cache_no_ttl_persists() {
    let cache = InMemoryCache::new();
    let response = make_response("persistent");

    cache.put("key1", &response).await.unwrap();

    // Wait a bit
    tokio::time::sleep(Duration::from_millis(50)).await;

    // Should still be present
    let result = cache.get("key1").await.unwrap();
    assert!(result.is_some());
    assert_eq!(result.unwrap().message.content(), "persistent");
}

#[tokio::test]
async fn cache_clear_removes_all_entries() {
    let cache = InMemoryCache::new();

    cache.put("key1", &make_response("a")).await.unwrap();
    cache.put("key2", &make_response("b")).await.unwrap();

    // Both should be present
    assert!(cache.get("key1").await.unwrap().is_some());
    assert!(cache.get("key2").await.unwrap().is_some());

    // Clear the cache
    cache.clear().await.unwrap();

    // Both should be gone
    assert!(cache.get("key1").await.unwrap().is_none());
    assert!(cache.get("key2").await.unwrap().is_none());
}