litellm_rs/core/cache/cloud/
mod.rs

1//! Cloud storage cache backends
2//!
3//! This module provides cache implementations backed by cloud object storage services:
4//! - AWS S3
5//! - Google Cloud Storage (GCS)
6//! - Azure Blob Storage
7//!
8//! These backends are suitable for persistent caching across distributed systems
9//! where durability and cross-region access are important.
10
11pub mod azure_blob;
12pub mod gcs;
13pub mod s3;
14
15#[cfg(feature = "s3")]
16pub use s3::{S3Cache, S3CacheConfig, S3StorageClass};
17
18#[cfg(feature = "s3")]
19pub use gcs::{GcsCache, GcsCacheConfig};
20
21#[cfg(feature = "s3")]
22pub use azure_blob::{AzureBlobCache, AzureBlobCacheConfig};
23
24use async_trait::async_trait;
25use serde::{Serialize, de::DeserializeOwned};
26use std::time::Duration;
27
28use super::types::CacheKey;
29use crate::utils::error::gateway_error::Result;
30
31/// Trait for cloud storage cache backends
32#[async_trait]
33pub trait CloudCache: Send + Sync {
34    /// Get a value from the cache
35    async fn get<T: DeserializeOwned + Send>(&self, key: &CacheKey) -> Result<Option<T>>;
36
37    /// Set a value in the cache with TTL
38    async fn set<T: Serialize + Send + Sync>(
39        &self,
40        key: &CacheKey,
41        value: &T,
42        ttl: Duration,
43    ) -> Result<()>;
44
45    /// Delete a value from the cache
46    async fn delete(&self, key: &CacheKey) -> Result<bool>;
47
48    /// Check if a key exists
49    async fn exists(&self, key: &CacheKey) -> Result<bool>;
50
51    /// List keys with a prefix
52    async fn list_keys(&self, prefix: &str) -> Result<Vec<String>>;
53
54    /// Clear all cache entries (use with caution)
55    async fn clear(&self) -> Result<()>;
56
57    /// Get the backend name
58    fn name(&self) -> &'static str;
59}
60
61/// Cloud cache configuration
62#[derive(Debug, Clone)]
63pub struct CloudCacheConfig {
64    /// Bucket/container name
65    pub bucket: String,
66    /// Key prefix for all cache entries
67    pub prefix: String,
68    /// Default TTL for cache entries
69    pub default_ttl: Duration,
70    /// Enable compression for large values
71    pub enable_compression: bool,
72    /// Compression threshold in bytes
73    pub compression_threshold: usize,
74}
75
76impl Default for CloudCacheConfig {
77    fn default() -> Self {
78        Self {
79            bucket: String::new(),
80            prefix: "litellm-cache/".to_string(),
81            default_ttl: Duration::from_secs(3600),
82            enable_compression: true,
83            compression_threshold: 1024,
84        }
85    }
86}
87
88impl CloudCacheConfig {
89    /// Create a new configuration with the specified bucket
90    pub fn new(bucket: impl Into<String>) -> Self {
91        Self {
92            bucket: bucket.into(),
93            ..Default::default()
94        }
95    }
96
97    /// Set the key prefix
98    pub fn prefix(mut self, prefix: impl Into<String>) -> Self {
99        self.prefix = prefix.into();
100        self
101    }
102
103    /// Set the default TTL
104    pub fn default_ttl(mut self, ttl: Duration) -> Self {
105        self.default_ttl = ttl;
106        self
107    }
108
109    /// Enable or disable compression
110    pub fn compression(mut self, enabled: bool) -> Self {
111        self.enable_compression = enabled;
112        self
113    }
114
115    /// Set the compression threshold
116    pub fn compression_threshold(mut self, threshold: usize) -> Self {
117        self.compression_threshold = threshold;
118        self
119    }
120}
121
122/// Cache entry metadata stored alongside the value
123#[derive(Debug, Clone, Serialize, serde::Deserialize)]
124pub struct CacheMetadata {
125    /// Unix timestamp when the entry expires
126    pub expires_at: u64,
127    /// Original size before compression
128    pub original_size: usize,
129    /// Whether the value is compressed
130    pub compressed: bool,
131    /// Content type
132    pub content_type: String,
133}
134
135impl CacheMetadata {
136    /// Create new metadata
137    pub fn new(ttl: Duration, original_size: usize, compressed: bool) -> Self {
138        let expires_at = std::time::SystemTime::now()
139            .duration_since(std::time::UNIX_EPOCH)
140            .unwrap_or_default()
141            .as_secs()
142            + ttl.as_secs();
143
144        Self {
145            expires_at,
146            original_size,
147            compressed,
148            content_type: "application/json".to_string(),
149        }
150    }
151
152    /// Check if the entry has expired
153    pub fn is_expired(&self) -> bool {
154        let now = std::time::SystemTime::now()
155            .duration_since(std::time::UNIX_EPOCH)
156            .unwrap_or_default()
157            .as_secs();
158        now >= self.expires_at
159    }
160}
161
162#[cfg(test)]
163mod tests {
164    use super::*;
165
166    #[test]
167    fn test_cloud_cache_config_default() {
168        let config = CloudCacheConfig::default();
169        assert!(config.bucket.is_empty());
170        assert_eq!(config.prefix, "litellm-cache/");
171        assert_eq!(config.default_ttl, Duration::from_secs(3600));
172        assert!(config.enable_compression);
173    }
174
175    #[test]
176    fn test_cloud_cache_config_builder() {
177        let config = CloudCacheConfig::new("my-bucket")
178            .prefix("cache/")
179            .default_ttl(Duration::from_secs(7200))
180            .compression(false);
181
182        assert_eq!(config.bucket, "my-bucket");
183        assert_eq!(config.prefix, "cache/");
184        assert_eq!(config.default_ttl, Duration::from_secs(7200));
185        assert!(!config.enable_compression);
186    }
187
188    #[test]
189    fn test_cache_metadata_not_expired() {
190        let metadata = CacheMetadata::new(Duration::from_secs(3600), 100, false);
191        assert!(!metadata.is_expired());
192    }
193
194    #[test]
195    fn test_cache_metadata_expired() {
196        let mut metadata = CacheMetadata::new(Duration::from_secs(0), 100, false);
197        metadata.expires_at = 0; // Force expiration
198        assert!(metadata.is_expired());
199    }
200}
litellm_rs/core/cache/cloud/mod.rs

litellm_rs/core/cache/cloud/
mod.rs
