Struct CacheLayer 

pub struct CacheLayer { /* private fields */ }

Main cache layer with three specialized caches

Implementations

impl CacheLayer

pub fn new() -> Self

Create a new cache layer with default configuration

pub fn with_config(config: CacheConfig) -> Self

Create a new cache layer with custom configuration

pub fn generate_cache_key( url: &str, headers: Option<&[(String, String)]>, ) -> String

Generate a cache key from URL and optional headers

Uses BLAKE3 hashing for fast, collision-resistant cache keys.

Example

let key = CacheLayer::generate_cache_key("https://example.com", None);

pub async fn get_or_fetch_content<F, Fut>( &self, url: &str, headers: Option<&[(String, String)]>, fetch_fn: F, ) -> Result<CachedContent, ScrapeError>

where F: FnOnce() -> Fut, Fut: Future<Output = Result<CachedContent, ScrapeError>>,

Get cached content or fetch it using the provided async function

This implements a cache-first strategy where the cache is checked first, and only if there’s a miss does it call the fetch function.

Example

let content = cache.get_or_fetch_content(
    "https://example.com",
    None,
    || async {
        // Fetch logic here
        Ok(CachedContent { ... })
    }
).await?;

pub async fn get_or_fetch_robots<F, Fut>( &self, domain: &str, fetch_fn: F, ) -> Result<CachedRobots, ScrapeError>

where F: FnOnce() -> Fut, Fut: Future<Output = Result<CachedRobots, ScrapeError>>,

Get cached robots.txt or fetch it

pub async fn get_or_fetch_redirect<F, Fut>( &self, url: &str, fetch_fn: F, ) -> Result<Option<CachedRedirect>, ScrapeError>

where F: FnOnce() -> Fut, Fut: Future<Output = Result<Option<CachedRedirect>, ScrapeError>>,

Get cached redirect or fetch it

pub async fn get_metrics(&self) -> CacheMetrics

Get current cache metrics

pub async fn get_stats(&self) -> CacheStats

Get cache statistics

pub async fn clear_all(&self)

Clear all caches

pub async fn clear_content(&self)

Clear content cache only

pub async fn clear_robots(&self)

Clear robots cache only

pub async fn clear_redirect(&self)

Clear redirect cache only

Trait Implementations

impl Default for CacheLayer

fn default() -> Self

Returns the “default value” for a type.