#![ allow( clippy::missing_inline_in_public_items, clippy::unused_async, clippy::explicit_iter_loop ) ]
mod private
{
use std::
{
collections ::{ HashMap, VecDeque },
sync ::Arc,
time ::Instant,
};
use core::
{
hash ::{ Hash, Hasher },
sync ::atomic::{ AtomicU32, AtomicU64, AtomicBool, Ordering },
time ::Duration,
};
use tokio::sync::{ RwLock, Mutex };
use serde::{ Serialize, Deserialize };
use std::collections::hash_map::DefaultHasher;
#[ derive( Debug, Clone, Serialize, Deserialize ) ]
#[ allow( clippy::struct_excessive_bools ) ]
pub struct EnhancedCacheConfig
{
pub max_size : usize,
pub default_ttl : Duration,
pub min_ttl : Duration,
pub max_ttl : Duration,
pub adaptive_ttl : bool,
pub predictive_caching : bool,
pub cache_warming : bool,
pub cleanup_interval : Duration,
pub detailed_metrics : bool,
pub eviction_policy : EvictionPolicy,
pub compression_threshold : usize,
pub persistence : bool,
pub max_memory_usage : usize,
}
impl Default for EnhancedCacheConfig
{
fn default() -> Self
{
Self
{
max_size : 5000,
default_ttl : Duration::from_secs( 300 ), min_ttl : Duration::from_secs( 30 ), max_ttl : Duration::from_secs( 3600 ), adaptive_ttl : true,
predictive_caching : true,
cache_warming : false,
cleanup_interval : Duration::from_secs( 60 ),
detailed_metrics : true,
eviction_policy : EvictionPolicy::AdaptiveLRU,
compression_threshold : 1024, persistence : false,
max_memory_usage : 100 * 1024 * 1024, }
}
}
#[ derive( Debug, Clone, Copy, Serialize, Deserialize, PartialEq ) ]
pub enum EvictionPolicy
{
LRU,
LFU,
AdaptiveLRU,
TimeWeighted,
SizeAware,
}
#[ derive( Debug ) ]
pub struct EnhancedCacheStatistics
{
pub hits : AtomicU64,
pub misses : AtomicU64,
pub insertions : AtomicU64,
pub evictions : AtomicU64,
pub entries : AtomicU32,
pub memory_usage : AtomicU64,
pub avg_access_time : AtomicU64,
pub warming_hits : AtomicU64,
pub predictive_hits : AtomicU64,
pub compression_saves : AtomicU64,
pub ttl_adjustments : AtomicU64,
}
impl EnhancedCacheStatistics
{
#[ must_use ]
pub fn new() -> Self
{
Self
{
hits : AtomicU64::new( 0 ),
misses : AtomicU64::new( 0 ),
insertions : AtomicU64::new( 0 ),
evictions : AtomicU64::new( 0 ),
entries : AtomicU32::new( 0 ),
memory_usage : AtomicU64::new( 0 ),
avg_access_time : AtomicU64::new( 0 ),
warming_hits : AtomicU64::new( 0 ),
predictive_hits : AtomicU64::new( 0 ),
compression_saves : AtomicU64::new( 0 ),
ttl_adjustments : AtomicU64::new( 0 ),
}
}
pub fn hit_ratio( &self ) -> f64
{
let hits = self.hits.load( Ordering::Relaxed );
let misses = self.misses.load( Ordering::Relaxed );
let total = hits + misses;
if total == 0 { 0.0 } else { hits as f64 / total as f64 }
}
pub fn memory_usage_mb( &self ) -> f64
{
self.memory_usage.load( Ordering::Relaxed ) as f64 / ( 1024.0 * 1024.0 )
}
}
impl Default for EnhancedCacheStatistics
{
fn default() -> Self
{
Self::new()
}
}
#[ derive( Debug, Clone ) ]
pub struct EnhancedCacheEntry< V >
{
pub value : V,
pub created_at : Instant,
pub expires_at : Instant,
pub last_accessed : Arc< Mutex< Instant > >,
pub access_count : Arc< AtomicU32 >,
pub size_bytes : usize,
pub compressed : bool,
pub priority : CachePriority,
pub access_pattern : Arc< Mutex< AccessPattern > >,
}
#[ derive( Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord ) ]
pub enum CachePriority
{
Low = 1,
Normal = 2,
High = 3,
Critical = 4,
}
#[ derive( Debug, Clone ) ]
pub struct AccessPattern
{
pub recent_accesses : VecDeque< Instant >,
pub avg_interval : Option< Duration >,
pub predicted_next_access : Option< Instant >,
pub in_burst : bool,
}
impl< V > EnhancedCacheEntry< V >
where
V : Clone + Send + Sync,
{
pub fn new( value : V, ttl : Duration, size_bytes : usize, priority : CachePriority ) -> Self
{
let now = Instant::now();
Self
{
value,
created_at : now,
expires_at : now + ttl,
last_accessed : Arc::new( Mutex::new( now ) ),
access_count : Arc::new( AtomicU32::new( 1 ) ),
size_bytes,
compressed : false,
priority,
access_pattern : Arc::new( Mutex::new( AccessPattern
{
recent_accesses : VecDeque::new(),
avg_interval : None,
predicted_next_access : None,
in_burst : false,
} ) ),
}
}
pub fn is_expired( &self ) -> bool
{
Instant::now() >= self.expires_at
}
pub async fn mark_accessed( &self )
{
let now = Instant::now();
self.access_count.fetch_add( 1, Ordering::Relaxed );
{
let mut last_accessed = self.last_accessed.lock().await;
*last_accessed = now;
}
{
let mut pattern = self.access_pattern.lock().await;
pattern.recent_accesses.push_back( now );
while pattern.recent_accesses.len() > 10
{
pattern.recent_accesses.pop_front();
}
if pattern.recent_accesses.len() >= 2
{
let intervals : Vec< Duration > = pattern.recent_accesses
.iter()
.zip( pattern.recent_accesses.iter().skip( 1 ) )
.map( |( a, b )| b.duration_since( *a ) )
.collect();
if !intervals.is_empty()
{
let total_nanos : u64 = intervals.iter().map( |d| u64::try_from( d.as_nanos() ).unwrap_or( u64::MAX ) ).sum();
pattern.avg_interval = Some( Duration::from_nanos( total_nanos / intervals.len() as u64 ) );
if let Some( avg_interval ) = pattern.avg_interval
{
pattern.predicted_next_access = Some( now + avg_interval );
}
}
}
let recent_threshold = now.checked_sub( Duration::from_secs( 10 ) ).unwrap_or( now );
let recent_count = pattern.recent_accesses.iter()
.filter( |&&access_time| access_time >= recent_threshold )
.count();
pattern.in_burst = recent_count > 3;
}
}
pub async fn calculate_adaptive_ttl( &self, base_ttl : Duration, min_ttl : Duration, max_ttl : Duration ) -> Duration
{
let access_count = self.access_count.load( Ordering::Relaxed );
{
let pattern = self.access_pattern.lock().await;
let mut multiplier = 1.0;
if access_count > 10
{
multiplier *= 1.5;
}
else if access_count > 5
{
multiplier *= 1.2;
}
if pattern.in_burst
{
multiplier *= 2.0;
}
if pattern.avg_interval.is_some()
{
multiplier *= 1.3;
}
let adjusted_ttl = base_ttl.mul_f64( multiplier );
adjusted_ttl.max( min_ttl ).min( max_ttl )
}
}
}
#[ derive( Debug ) ]
pub struct EnhancedRequestCache< K, V >
where
K : Hash + Eq + Clone + Send + Sync + 'static,
V : Clone + Send + Sync + 'static,
{
entries : Arc< RwLock< HashMap< K, EnhancedCacheEntry< V > > > >,
config : EnhancedCacheConfig,
stats : EnhancedCacheStatistics,
access_frequencies : Arc< RwLock< HashMap< K, u32 > > >,
predictive_candidates : Arc< RwLock< HashMap< K, Instant > > >,
#[ allow( dead_code ) ]
warming_active : AtomicBool,
last_cleanup : Arc< Mutex< Instant > >,
}
impl< K, V > EnhancedRequestCache< K, V >
where
K : Hash + Eq + Clone + Send + Sync + 'static,
V : Clone + Send + Sync + 'static,
{
#[ must_use ]
pub fn new( config : EnhancedCacheConfig ) -> Self
{
Self
{
entries : Arc::new( RwLock::new( HashMap::with_capacity( config.max_size ) ) ),
config,
stats : EnhancedCacheStatistics::new(),
access_frequencies : Arc::new( RwLock::new( HashMap::new() ) ),
predictive_candidates : Arc::new( RwLock::new( HashMap::new() ) ),
warming_active : AtomicBool::new( false ),
last_cleanup : Arc::new( Mutex::new( Instant::now() ) ),
}
}
pub async fn insert_enhanced( &self, key : K, value : V, priority : CachePriority ) -> bool
{
let size_estimate = Self::estimate_size( &value );
let ttl = if self.config.adaptive_ttl
{
self.calculate_intelligent_ttl( &key, size_estimate ).await
}
else
{
self.config.default_ttl
};
let entry = EnhancedCacheEntry::new( value, ttl, size_estimate, priority );
let mut entries = self.entries.write().await;
while entries.len() >= self.config.max_size
{
self.evict_by_policy( &mut entries ).await;
}
let was_replaced = entries.insert( key.clone(), entry ).is_some();
if !was_replaced
{
self.stats.entries.fetch_add( 1, Ordering::Relaxed );
self.stats.memory_usage.fetch_add( size_estimate as u64, Ordering::Relaxed );
}
self.stats.insertions.fetch_add( 1, Ordering::Relaxed );
{
let mut frequencies = self.access_frequencies.write().await;
*frequencies.entry( key.clone() ).or_insert( 0 ) += 1;
}
if self.config.predictive_caching
{
self.update_predictive_candidates( &key ).await;
}
true
}
pub async fn get_enhanced( &self, key : &K ) -> Option< V >
{
let start_time = Instant::now();
let entry = {
let entries = self.entries.read().await;
entries.get( key ).cloned()
};
match entry
{
Some( entry ) if !entry.is_expired() =>
{
entry.mark_accessed().await;
{
let mut entries = self.entries.write().await;
entries.insert( key.clone(), entry.clone() );
}
self.stats.hits.fetch_add( 1, Ordering::Relaxed );
let access_time = u64::try_from( start_time.elapsed().as_nanos() ).unwrap_or( u64::MAX );
self.update_avg_access_time( access_time );
{
let mut frequencies = self.access_frequencies.write().await;
*frequencies.entry( key.clone() ).or_insert( 0 ) += 1;
}
Some( entry.value )
}
Some( _expired_entry ) =>
{
self.remove_entry( key ).await;
self.stats.misses.fetch_add( 1, Ordering::Relaxed );
None
}
None =>
{
self.stats.misses.fetch_add( 1, Ordering::Relaxed );
if self.config.predictive_caching
{
Self::consider_predictive_load( key );
}
None
}
}
}
async fn calculate_intelligent_ttl( &self, key : &K, size_bytes : usize ) -> Duration
{
let mut ttl = self.config.default_ttl;
{
let frequencies = self.access_frequencies.read().await;
if let Some( &frequency ) = frequencies.get( key )
{
if frequency > 10
{
ttl = ttl.mul_f64( 2.0 );
}
else if frequency > 5
{
ttl = ttl.mul_f64( 1.5 );
}
}
}
if size_bytes > self.config.compression_threshold * 10
{
ttl = ttl.mul_f64( 0.7 );
}
ttl.max( self.config.min_ttl ).min( self.config.max_ttl )
}
async fn evict_by_policy( &self, entries : &mut HashMap< K, EnhancedCacheEntry< V > > )
{
if entries.is_empty()
{
return;
}
let key_to_remove = match self.config.eviction_policy
{
EvictionPolicy::LRU => self.find_lru_key( entries ).await,
EvictionPolicy::LFU => self.find_lfu_key( entries ).await,
EvictionPolicy::AdaptiveLRU => self.find_adaptive_lru_key( entries ).await,
EvictionPolicy::TimeWeighted => self.find_time_weighted_key( entries ).await,
EvictionPolicy::SizeAware => self.find_size_aware_key( entries ).await,
};
if let Some( key ) = key_to_remove
{
if let Some( entry ) = entries.remove( &key )
{
self.stats.entries.fetch_sub( 1, Ordering::Relaxed );
self.stats.evictions.fetch_add( 1, Ordering::Relaxed );
self.stats.memory_usage.fetch_sub( entry.size_bytes as u64, Ordering::Relaxed );
}
}
}
async fn find_lru_key( &self, entries : &HashMap< K, EnhancedCacheEntry< V > > ) -> Option< K >
{
let mut oldest_key = None;
let mut oldest_time = Instant::now();
for ( key, entry ) in entries.iter()
{
{
let last_accessed = entry.last_accessed.lock().await;
if oldest_key.is_none() || *last_accessed < oldest_time
{
oldest_time = *last_accessed;
oldest_key = Some( key.clone() );
}
}
}
oldest_key
}
async fn find_lfu_key( &self, entries : &HashMap< K, EnhancedCacheEntry< V > > ) -> Option< K >
{
let mut min_count = u32::MAX;
let mut min_key = None;
for ( key, entry ) in entries.iter()
{
let count = entry.access_count.load( Ordering::Relaxed );
if count < min_count
{
min_count = count;
min_key = Some( key.clone() );
}
}
min_key
}
async fn find_adaptive_lru_key( &self, entries : &HashMap< K, EnhancedCacheEntry< V > > ) -> Option< K >
{
let mut best_score = f64::MIN;
let mut best_key = None;
let now = Instant::now();
for ( key, entry ) in entries.iter()
{
{
let last_accessed = entry.last_accessed.lock().await;
let time_since_access = now.duration_since( *last_accessed ).as_secs_f64();
let access_count = f64::from(entry.access_count.load( Ordering::Relaxed ));
let priority_weight = f64::from(entry.priority as u8);
let score = time_since_access / ( access_count + 1.0 ) / priority_weight;
if score > best_score
{
best_score = score;
best_key = Some( key.clone() );
}
}
}
best_key
}
async fn find_time_weighted_key( &self, entries : &HashMap< K, EnhancedCacheEntry< V > > ) -> Option< K >
{
let mut best_score = f64::MIN;
let mut best_key = None;
let now = Instant::now();
for ( key, entry ) in entries.iter()
{
let age = now.duration_since( entry.created_at ).as_secs_f64();
let access_count = f64::from(entry.access_count.load( Ordering::Relaxed ));
let score = age / ( access_count + 1.0 ) / f64::from(entry.priority as u8);
if score > best_score
{
best_score = score;
best_key = Some( key.clone() );
}
}
best_key
}
async fn find_size_aware_key( &self, entries : &HashMap< K, EnhancedCacheEntry< V > > ) -> Option< K >
{
let mut best_score = 0.0;
let mut best_key = None;
for ( key, entry ) in entries.iter()
{
let size_score = entry.size_bytes as f64;
let access_count = f64::from(entry.access_count.load( Ordering::Relaxed ));
let priority_penalty = f64::from(entry.priority as u8);
let score = size_score / ( access_count + 1.0 ) / priority_penalty;
if score > best_score
{
best_score = score;
best_key = Some( key.clone() );
}
}
best_key
}
async fn remove_entry( &self, key : &K ) -> bool
{
let mut entries = self.entries.write().await;
if let Some( entry ) = entries.remove( key )
{
self.stats.entries.fetch_sub( 1, Ordering::Relaxed );
self.stats.memory_usage.fetch_sub( entry.size_bytes as u64, Ordering::Relaxed );
true
}
else
{
false
}
}
async fn update_predictive_candidates( &self, key : &K )
{
let mut candidates = self.predictive_candidates.write().await;
candidates.insert( key.clone(), Instant::now() );
let cutoff = Instant::now().checked_sub( Duration::from_secs( 300 ) ).unwrap(); candidates.retain( |_, &mut time| time >= cutoff );
}
fn consider_predictive_load( _key : &K )
{
}
fn estimate_size( _value : &V ) -> usize
{
core ::mem::size_of::< V >().max( 256 )
}
fn update_avg_access_time( &self, new_time : u64 )
{
let current_avg = self.stats.avg_access_time.load( Ordering::Relaxed );
let new_avg = if current_avg == 0
{
new_time
}
else
{
( current_avg * 9 + new_time ) / 10 };
self.stats.avg_access_time.store( new_avg, Ordering::Relaxed );
}
pub fn stats( &self ) -> &EnhancedCacheStatistics
{
&self.stats
}
pub async fn cleanup_and_maintain( &self ) -> usize
{
let mut removed = 0;
let now = Instant::now();
{
let mut last_cleanup = self.last_cleanup.lock().await;
if now.duration_since( *last_cleanup ) < self.config.cleanup_interval
{
return 0;
}
*last_cleanup = now;
}
{
let mut entries = self.entries.write().await;
let mut keys_to_remove = Vec::new();
for ( key, entry ) in &*entries
{
if entry.is_expired()
{
keys_to_remove.push( key.clone() );
}
}
for key in keys_to_remove
{
if let Some( entry ) = entries.remove( &key )
{
self.stats.memory_usage.fetch_sub( entry.size_bytes as u64, Ordering::Relaxed );
removed += 1;
}
}
self.stats.entries.fetch_sub( u32::try_from(removed).unwrap_or(u32::MAX), Ordering::Relaxed );
self.stats.evictions.fetch_add( removed as u64, Ordering::Relaxed );
}
{
let entries = self.entries.read().await;
let mut frequencies = self.access_frequencies.write().await;
frequencies.retain( |k, _| entries.contains_key( k ) );
}
removed
}
pub async fn clear( &self )
{
let mut entries = self.entries.write().await;
let count = entries.len();
entries.clear();
self.stats.entries.store( 0, Ordering::Relaxed );
self.stats.memory_usage.store( 0, Ordering::Relaxed );
self.stats.evictions.fetch_add( count as u64, Ordering::Relaxed );
{
let mut frequencies = self.access_frequencies.write().await;
frequencies.clear();
}
{
let mut candidates = self.predictive_candidates.write().await;
candidates.clear();
}
}
}
pub type EnhancedApiRequestCache = EnhancedRequestCache< RequestCacheKey, serde_json::Value >;
#[ derive( Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize ) ]
pub struct RequestCacheKey
{
pub endpoint : String,
pub method : String,
pub body_hash : u64,
pub headers_hash : u64,
}
impl RequestCacheKey
{
pub fn new< T : Serialize >(
endpoint : &str,
method : &str,
request_body : Option< &T >,
headers : &HashMap< String, String >,
) -> crate::error::Result< Self >
{
let body_hash = if let Some( body ) = request_body
{
let body_json = serde_json::to_string( body ).map_err( |e|
crate ::error::OpenAIError::Internal( format!( "Failed to serialize request body for cache key : {e}" ) )
)?;
Self::hash_string( &body_json )
}
else
{
0
};
let relevant_headers : HashMap< String, String > = headers.iter()
.filter( |( key, _ )| Self::is_relevant_header( key ) )
.map( |( k, v )| ( k.clone(), v.clone() ) )
.collect();
let headers_json = serde_json::to_string( &relevant_headers ).map_err( |e|
crate ::error::OpenAIError::Internal( format!( "Failed to serialize headers for cache key : {e}" ) )
)?;
Ok( Self
{
endpoint : endpoint.to_string(),
method : method.to_string(),
body_hash,
headers_hash : Self::hash_string( &headers_json ),
} )
}
fn is_relevant_header( key : &str ) -> bool
{
matches!( key.to_lowercase().as_str(),
"content-type" | "accept" | "openai-organization" | "openai-project" | "authorization"
)
}
fn hash_string( s : &str ) -> u64
{
let mut hasher = DefaultHasher::new();
s.hash( &mut hasher );
hasher.finish()
}
}
}
crate ::mod_interface!
{
orphan use EnhancedCacheConfig;
orphan use EvictionPolicy;
orphan use EnhancedCacheStatistics;
orphan use EnhancedCacheEntry;
orphan use CachePriority;
orphan use AccessPattern;
orphan use EnhancedRequestCache;
orphan use EnhancedApiRequestCache;
orphan use RequestCacheKey;
}