ai_client 0.1.0

A Rust crate for interacting with AI language model APIs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

use std::num::NonZeroUsize;
use super::super::entities::ChatCompletionResponse;
use lru::LruCache;
use tokio::sync::RwLock;
use std::sync::Arc;

/// Cache for storing API responses
#[derive(Debug)]
pub struct ResponseCache {
    cache: Arc<RwLock<LruCache<String, ChatCompletionResponse>>>,
}

impl ResponseCache {
    pub fn new(capacity: usize) -> Self {
        ResponseCache {
            cache: Arc::new(RwLock::new(LruCache::new(NonZeroUsize::try_from(capacity).unwrap()))),
        }
    }

    pub async fn get(&self, key: &str) -> Option<ChatCompletionResponse> {
        let mut cache = self.cache.write().await;
        cache.get(key).cloned()
    }

    pub async fn put(&self, key: String, value: ChatCompletionResponse) -> Result<(), super::super::error::LlmClientError> {
        let mut cache = self.cache.write().await;
        cache.put(key, value);
        Ok(())
    }
}