Struct wasmtime_cache::CacheConfig

pub struct CacheConfig { /* private fields */ }

Global configuration for how the cache is managed

Implementations

impl CacheConfig

pub fn worker_event_queue_size(&self) -> u64

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 60)

    pub(super) fn start_new(
        cache_config: &CacheConfig,
        init_file_per_thread_logger: Option<&'static str>,
    ) -> Self {
        let queue_size = match cache_config.worker_event_queue_size() {
            num if num <= usize::max_value() as u64 => num as usize,
            _ => usize::max_value(),
        };
        let (tx, rx) = sync_channel(queue_size);

        #[cfg(test)]
        let stats = Arc::new((Mutex::new(WorkerStats::default()), Condvar::new()));

        let worker_thread = WorkerThread {
            receiver: rx,
            cache_config: cache_config.clone(),
            #[cfg(test)]
            stats: stats.clone(),
        };

        // when self is dropped, sender will be dropped, what will cause the channel
        // to hang, and the worker thread to exit -- it happens in the tests
        // non-tests binary has only a static worker, so Rust doesn't drop it
        thread::spawn(move || worker_thread.run(init_file_per_thread_logger));

        Self {
            sender: tx,
            #[cfg(test)]
            stats,
        }
    }

pub fn baseline_compression_level(&self) -> i32

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 165)

    fn default(cache_config: &CacheConfig) -> Self {
        Self {
            usages: 0,
            compression_level: cache_config.baseline_compression_level(),
        }
    }

src/lib.rs (line 167)

    fn update_data(&self, hash: &str, serialized_data: &[u8]) -> Option<()> {
        let mod_cache_path = self.root_path.join(hash);
        trace!("update_data() for path: {}", mod_cache_path.display());
        let compressed_data = zstd::encode_all(
            &serialized_data[..],
            self.cache_config.baseline_compression_level(),
        )
        .map_err(|err| warn!("Failed to compress cached code: {}", err))
        .ok()?;

        // Optimize syscalls: first, try writing to disk. It should succeed in most cases.
        // Otherwise, try creating the cache directory and retry writing to the file.
        if fs_write_atomic(&mod_cache_path, "mod", &compressed_data) {
            return Some(());
        }

        debug!(
            "Attempting to create the cache directory, because \
             failed to write cached code to disk, path: {}",
            mod_cache_path.display(),
        );

        let cache_dir = mod_cache_path.parent().unwrap();
        fs::create_dir_all(cache_dir)
            .map_err(|err| {
                warn!(
                    "Failed to create cache directory, path: {}, message: {}",
                    cache_dir.display(),
                    err
                )
            })
            .ok()?;

        if fs_write_atomic(&mod_cache_path, "mod", &compressed_data) {
            Some(())
        } else {
            None
        }
    }

pub fn optimized_compression_level(&self) -> i32

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 290)

    fn handle_on_cache_get(&self, path: PathBuf) {
        trace!("handle_on_cache_get() for path: {}", path.display());

        // construct .stats file path
        let filename = path.file_name().unwrap().to_str().unwrap();
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // load .stats file (default if none or error)
        let mut stats = read_stats_file(stats_path.as_ref())
            .unwrap_or_else(|| ModuleCacheStatistics::default(&self.cache_config));

        // step 1: update the usage counter & write to the disk
        //         it's racy, but it's fine (the counter will be just smaller,
        //         sometimes will retrigger recompression)
        stats.usages += 1;
        if !write_stats_file(stats_path.as_ref(), &stats) {
            return;
        }

        // step 2: recompress if there's a need
        let opt_compr_lvl = self.cache_config.optimized_compression_level();
        if stats.compression_level >= opt_compr_lvl
            || stats.usages
                < self
                    .cache_config
                    .optimized_compression_usage_counter_threshold()
        {
            return;
        }

        let lock_path = if let Some(p) = acquire_task_fs_lock(
            path.as_ref(),
            self.cache_config.optimizing_compression_task_timeout(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        ) {
            p
        } else {
            return;
        };

        trace!("Trying to recompress file: {}", path.display());

        // recompress, write to other file, rename (it's atomic file content exchange)
        // and update the stats file
        let compressed_cache_bytes = unwrap_or_warn!(
            fs::read(&path),
            return,
            "Failed to read old cache file",
            path
        );

        let cache_bytes = unwrap_or_warn!(
            zstd::decode_all(&compressed_cache_bytes[..]),
            return,
            "Failed to decompress cached code",
            path
        );

        let recompressed_cache_bytes = unwrap_or_warn!(
            zstd::encode_all(&cache_bytes[..], opt_compr_lvl),
            return,
            "Failed to compress cached code",
            path
        );

        unwrap_or_warn!(
            fs::write(&lock_path, &recompressed_cache_bytes),
            return,
            "Failed to write recompressed cache",
            lock_path
        );

        unwrap_or_warn!(
            fs::rename(&lock_path, &path),
            {
                if let Err(error) = fs::remove_file(&lock_path) {
                    warn!(
                        "Failed to clean up (remove) recompressed cache, path {}, err: {}",
                        lock_path.display(),
                        error
                    );
                }

                return;
            },
            "Failed to rename recompressed cache",
            lock_path
        );

        // update stats file (reload it! recompression can take some time)
        if let Some(mut new_stats) = read_stats_file(stats_path.as_ref()) {
            if new_stats.compression_level >= opt_compr_lvl {
                // Rare race:
                //    two instances with different opt_compr_lvl: we don't know in which order they updated
                //    the cache file and the stats file (they are not updated together atomically)
                // Possible solution is to use directories per cache entry, but it complicates the system
                // and is not worth it.
                debug!(
                    "DETECTED task did more than once (or race with new file): \
                     recompression of {}. Note: if optimized compression level setting \
                     has changed in the meantine, the stats file might contain \
                     inconsistent compression level due to race.",
                    path.display()
                );
            } else {
                new_stats.compression_level = opt_compr_lvl;
                let _ = write_stats_file(stats_path.as_ref(), &new_stats);
            }

            if new_stats.usages < stats.usages {
                debug!(
                    "DETECTED lower usage count (new file or race with counter \
                     increasing): file {}",
                    path.display()
                );
            }
        } else {
            debug!(
                "Can't read stats file again to update compression level (it might got \
                 cleaned up): file {}",
                stats_path.display()
            );
        }

        trace!("Task finished: recompress file: {}", path.display());
    }

pub fn optimized_compression_usage_counter_threshold(&self) -> u64

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 295)

    fn handle_on_cache_get(&self, path: PathBuf) {
        trace!("handle_on_cache_get() for path: {}", path.display());

        // construct .stats file path
        let filename = path.file_name().unwrap().to_str().unwrap();
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // load .stats file (default if none or error)
        let mut stats = read_stats_file(stats_path.as_ref())
            .unwrap_or_else(|| ModuleCacheStatistics::default(&self.cache_config));

        // step 1: update the usage counter & write to the disk
        //         it's racy, but it's fine (the counter will be just smaller,
        //         sometimes will retrigger recompression)
        stats.usages += 1;
        if !write_stats_file(stats_path.as_ref(), &stats) {
            return;
        }

        // step 2: recompress if there's a need
        let opt_compr_lvl = self.cache_config.optimized_compression_level();
        if stats.compression_level >= opt_compr_lvl
            || stats.usages
                < self
                    .cache_config
                    .optimized_compression_usage_counter_threshold()
        {
            return;
        }

        let lock_path = if let Some(p) = acquire_task_fs_lock(
            path.as_ref(),
            self.cache_config.optimizing_compression_task_timeout(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        ) {
            p
        } else {
            return;
        };

        trace!("Trying to recompress file: {}", path.display());

        // recompress, write to other file, rename (it's atomic file content exchange)
        // and update the stats file
        let compressed_cache_bytes = unwrap_or_warn!(
            fs::read(&path),
            return,
            "Failed to read old cache file",
            path
        );

        let cache_bytes = unwrap_or_warn!(
            zstd::decode_all(&compressed_cache_bytes[..]),
            return,
            "Failed to decompress cached code",
            path
        );

        let recompressed_cache_bytes = unwrap_or_warn!(
            zstd::encode_all(&cache_bytes[..], opt_compr_lvl),
            return,
            "Failed to compress cached code",
            path
        );

        unwrap_or_warn!(
            fs::write(&lock_path, &recompressed_cache_bytes),
            return,
            "Failed to write recompressed cache",
            lock_path
        );

        unwrap_or_warn!(
            fs::rename(&lock_path, &path),
            {
                if let Err(error) = fs::remove_file(&lock_path) {
                    warn!(
                        "Failed to clean up (remove) recompressed cache, path {}, err: {}",
                        lock_path.display(),
                        error
                    );
                }

                return;
            },
            "Failed to rename recompressed cache",
            lock_path
        );

        // update stats file (reload it! recompression can take some time)
        if let Some(mut new_stats) = read_stats_file(stats_path.as_ref()) {
            if new_stats.compression_level >= opt_compr_lvl {
                // Rare race:
                //    two instances with different opt_compr_lvl: we don't know in which order they updated
                //    the cache file and the stats file (they are not updated together atomically)
                // Possible solution is to use directories per cache entry, but it complicates the system
                // and is not worth it.
                debug!(
                    "DETECTED task did more than once (or race with new file): \
                     recompression of {}. Note: if optimized compression level setting \
                     has changed in the meantine, the stats file might contain \
                     inconsistent compression level due to race.",
                    path.display()
                );
            } else {
                new_stats.compression_level = opt_compr_lvl;
                let _ = write_stats_file(stats_path.as_ref(), &new_stats);
            }

            if new_stats.usages < stats.usages {
                debug!(
                    "DETECTED lower usage count (new file or race with counter \
                     increasing): file {}",
                    path.display()
                );
            }
        } else {
            debug!(
                "Can't read stats file again to update compression level (it might got \
                 cleaned up): file {}",
                stats_path.display()
            );
        }

        trace!("Task finished: recompress file: {}", path.display());
    }

pub fn cleanup_interval(&self) -> Duration

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 424)

    fn handle_on_cache_update(&self, path: PathBuf) {
        trace!("handle_on_cache_update() for path: {}", path.display());

        // ---------------------- step 1: create .stats file

        // construct .stats file path
        let filename = path
            .file_name()
            .expect("Expected valid cache file name")
            .to_str()
            .expect("Expected valid cache file name");
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // create and write stats file
        let mut stats = ModuleCacheStatistics::default(&self.cache_config);
        stats.usages += 1;
        write_stats_file(&stats_path, &stats);

        // ---------------------- step 2: perform cleanup task if needed

        // acquire lock for cleanup task
        // Lock is a proof of recent cleanup task, so we don't want to delete them.
        // Expired locks will be deleted by the cleanup task.
        let cleanup_file = self.cache_config.directory().join(".cleanup"); // some non existing marker file
        if acquire_task_fs_lock(
            &cleanup_file,
            self.cache_config.cleanup_interval(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        )
        .is_none()
        {
            return;
        }

        trace!("Trying to clean up cache");

        let mut cache_index = self.list_cache_contents();
        let future_tolerance = SystemTime::now()
            .checked_add(
                self.cache_config
                    .allowed_clock_drift_for_files_from_future(),
            )
            .expect("Brace your cache, the next Big Bang is coming (time overflow)");
        cache_index.sort_unstable_by(|lhs, rhs| {
            // sort by age
            use CacheEntry::*;
            match (lhs, rhs) {
                (Recognized { mtime: lhs_mt, .. }, Recognized { mtime: rhs_mt, .. }) => {
                    match (*lhs_mt > future_tolerance, *rhs_mt > future_tolerance) {
                        // later == younger
                        (false, false) => rhs_mt.cmp(lhs_mt),
                        // files from far future are treated as oldest recognized files
                        // we want to delete them, so the cache keeps track of recent files
                        // however, we don't delete them uncodintionally,
                        // because .stats file can be overwritten with a meaningful mtime
                        (true, false) => cmp::Ordering::Greater,
                        (false, true) => cmp::Ordering::Less,
                        (true, true) => cmp::Ordering::Equal,
                    }
                }
                // unrecognized is kind of infinity
                (Recognized { .. }, Unrecognized { .. }) => cmp::Ordering::Less,
                (Unrecognized { .. }, Recognized { .. }) => cmp::Ordering::Greater,
                (Unrecognized { .. }, Unrecognized { .. }) => cmp::Ordering::Equal,
            }
        });

        // find "cut" boundary:
        // - remove unrecognized files anyway,
        // - remove some cache files if some quota has been exceeded
        let mut total_size = 0u64;
        let mut start_delete_idx = None;
        let mut start_delete_idx_if_deleting_recognized_items: Option<usize> = None;

        let total_size_limit = self.cache_config.files_total_size_soft_limit();
        let file_count_limit = self.cache_config.file_count_soft_limit();
        let tsl_if_deleting = total_size_limit
            .checked_mul(
                self.cache_config
                    .files_total_size_limit_percent_if_deleting() as u64,
            )
            .unwrap()
            / 100;
        let fcl_if_deleting = file_count_limit
            .checked_mul(self.cache_config.file_count_limit_percent_if_deleting() as u64)
            .unwrap()
            / 100;

        for (idx, item) in cache_index.iter().enumerate() {
            let size = if let CacheEntry::Recognized { size, .. } = item {
                size
            } else {
                start_delete_idx = Some(idx);
                break;
            };

            total_size += size;
            if start_delete_idx_if_deleting_recognized_items.is_none()
                && (total_size > tsl_if_deleting || (idx + 1) as u64 > fcl_if_deleting)
            {
                start_delete_idx_if_deleting_recognized_items = Some(idx);
            }

            if total_size > total_size_limit || (idx + 1) as u64 > file_count_limit {
                start_delete_idx = start_delete_idx_if_deleting_recognized_items;
                break;
            }
        }

        if let Some(idx) = start_delete_idx {
            for item in &cache_index[idx..] {
                let (result, path, entity) = match item {
                    CacheEntry::Recognized { path, .. }
                    | CacheEntry::Unrecognized {
                        path,
                        is_dir: false,
                    } => (fs::remove_file(path), path, "file"),
                    CacheEntry::Unrecognized { path, is_dir: true } => {
                        (fs::remove_dir_all(path), path, "directory")
                    }
                };
                if let Err(err) = result {
                    warn!(
                        "Failed to remove {} during cleanup, path: {}, err: {}",
                        entity,
                        path.display(),
                        err
                    );
                }
            }
        }

        trace!("Task finished: clean up cache");
    }

    // Be fault tolerant: list as much as you can, and ignore the rest
    fn list_cache_contents(&self) -> Vec<CacheEntry> {
        fn enter_dir(
            vec: &mut Vec<CacheEntry>,
            dir_path: &Path,
            level: u8,
            cache_config: &CacheConfig,
        ) {
            macro_rules! add_unrecognized {
                (file: $path:expr) => {
                    add_unrecognized!(false, $path)
                };
                (dir: $path:expr) => {
                    add_unrecognized!(true, $path)
                };
                ($is_dir:expr, $path:expr) => {
                    vec.push(CacheEntry::Unrecognized {
                        path: $path.to_path_buf(),
                        is_dir: $is_dir,
                    })
                };
            }
            macro_rules! add_unrecognized_and {
                ([ $( $ty:ident: $path:expr ),* ], $cont:stmt) => {{
                    $( add_unrecognized!($ty: $path); )*
                        $cont
                }};
            }

            macro_rules! unwrap_or {
                ($result:expr, $cont:stmt, $err_msg:expr) => {
                    unwrap_or!($result, $cont, $err_msg, dir_path)
                };
                ($result:expr, $cont:stmt, $err_msg:expr, $path:expr) => {
                    unwrap_or_warn!(
                        $result,
                        $cont,
                        format!("{}, level: {}", $err_msg, level),
                        $path
                    )
                };
            }

            // If we fail to list a directory, something bad is happening anyway
            // (something touches our cache or we have disk failure)
            // Try to delete it, so we can stay within soft limits of the cache size.
            // This comment applies later in this function, too.
            let it = unwrap_or!(
                fs::read_dir(dir_path),
                add_unrecognized_and!([dir: dir_path], return),
                "Failed to list cache directory, deleting it"
            );

            let mut cache_files = HashMap::new();
            for entry in it {
                // read_dir() returns an iterator over results - in case some of them are errors
                // we don't know their names, so we can't delete them. We don't want to delete
                // the whole directory with good entries too, so we just ignore the erroneous entries.
                let entry = unwrap_or!(
                    entry,
                    continue,
                    "Failed to read a cache dir entry (NOT deleting it, it still occupies space)"
                );
                let path = entry.path();
                match (level, path.is_dir()) {
                    (0..=1, true) => enter_dir(vec, &path, level + 1, cache_config),
                    (0..=1, false) => {
                        if level == 0
                            && path.file_stem() == Some(OsStr::new(".cleanup"))
                                && path.extension().is_some()
                                // assume it's cleanup lock
                                && !is_fs_lock_expired(
                                    Some(&entry),
                                    &path,
                                    cache_config.cleanup_interval(),
                                    cache_config.allowed_clock_drift_for_files_from_future(),
                                )
                        {
                            continue; // skip active lock
                        }
                        add_unrecognized!(file: path);
                    }
                    (2, false) => {
                        match path.extension().and_then(OsStr::to_str) {
                            // mod or stats file
                            None | Some("stats") => {
                                cache_files.insert(path, entry);
                            }

                            Some(ext) => {
                                // check if valid lock
                                let recognized = ext.starts_with("wip-")
                                    && !is_fs_lock_expired(
                                        Some(&entry),
                                        &path,
                                        cache_config.optimizing_compression_task_timeout(),
                                        cache_config.allowed_clock_drift_for_files_from_future(),
                                    );

                                if !recognized {
                                    add_unrecognized!(file: path);
                                }
                            }
                        }
                    }
                    (_, is_dir) => add_unrecognized!(is_dir, path),
                }
            }

            // associate module with its stats & handle them
            // assumption: just mods and stats
            for (path, entry) in cache_files.iter() {
                let path_buf: PathBuf;
                let (mod_, stats_, is_mod) = match path.extension() {
                    Some(_) => {
                        path_buf = path.with_extension("");
                        (
                            cache_files.get(&path_buf).map(|v| (&path_buf, v)),
                            Some((path, entry)),
                            false,
                        )
                    }
                    None => {
                        path_buf = path.with_extension("stats");
                        (
                            Some((path, entry)),
                            cache_files.get(&path_buf).map(|v| (&path_buf, v)),
                            true,
                        )
                    }
                };

                // construct a cache entry
                match (mod_, stats_, is_mod) {
                    (Some((mod_path, mod_entry)), Some((stats_path, stats_entry)), true) => {
                        let mod_metadata = unwrap_or!(
                            mod_entry.metadata(),
                            add_unrecognized_and!([file: stats_path, file: mod_path], continue),
                            "Failed to get metadata, deleting BOTH module cache and stats files",
                            mod_path
                        );
                        let stats_mtime = unwrap_or!(
                            stats_entry.metadata().and_then(|m| m.modified()),
                            add_unrecognized_and!(
                                [file: stats_path],
                                unwrap_or!(
                                    mod_metadata.modified(),
                                    add_unrecognized_and!(
                                        [file: stats_path, file: mod_path],
                                        continue
                                    ),
                                    "Failed to get mtime, deleting BOTH module cache and stats \
                                     files",
                                    mod_path
                                )
                            ),
                            "Failed to get metadata/mtime, deleting the file",
                            stats_path
                        );
                        // .into() called for the SystemTimeStub if cfg(test)
                        #[allow(clippy::identity_conversion)]
                        vec.push(CacheEntry::Recognized {
                            path: mod_path.to_path_buf(),
                            mtime: stats_mtime.into(),
                            size: mod_metadata.len(),
                        })
                    }
                    (Some(_), Some(_), false) => (), // was or will be handled by previous branch
                    (Some((mod_path, mod_entry)), None, _) => {
                        let (mod_metadata, mod_mtime) = unwrap_or!(
                            mod_entry
                                .metadata()
                                .and_then(|md| md.modified().map(|mt| (md, mt))),
                            add_unrecognized_and!([file: mod_path], continue),
                            "Failed to get metadata/mtime, deleting the file",
                            mod_path
                        );
                        // .into() called for the SystemTimeStub if cfg(test)
                        #[allow(clippy::identity_conversion)]
                        vec.push(CacheEntry::Recognized {
                            path: mod_path.to_path_buf(),
                            mtime: mod_mtime.into(),
                            size: mod_metadata.len(),
                        })
                    }
                    (None, Some((stats_path, _stats_entry)), _) => {
                        debug!("Found orphaned stats file: {}", stats_path.display());
                        add_unrecognized!(file: stats_path);
                    }
                    _ => unreachable!(),
                }
            }
        }

pub fn optimizing_compression_task_timeout(&self) -> Duration

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 302)

    fn handle_on_cache_get(&self, path: PathBuf) {
        trace!("handle_on_cache_get() for path: {}", path.display());

        // construct .stats file path
        let filename = path.file_name().unwrap().to_str().unwrap();
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // load .stats file (default if none or error)
        let mut stats = read_stats_file(stats_path.as_ref())
            .unwrap_or_else(|| ModuleCacheStatistics::default(&self.cache_config));

        // step 1: update the usage counter & write to the disk
        //         it's racy, but it's fine (the counter will be just smaller,
        //         sometimes will retrigger recompression)
        stats.usages += 1;
        if !write_stats_file(stats_path.as_ref(), &stats) {
            return;
        }

        // step 2: recompress if there's a need
        let opt_compr_lvl = self.cache_config.optimized_compression_level();
        if stats.compression_level >= opt_compr_lvl
            || stats.usages
                < self
                    .cache_config
                    .optimized_compression_usage_counter_threshold()
        {
            return;
        }

        let lock_path = if let Some(p) = acquire_task_fs_lock(
            path.as_ref(),
            self.cache_config.optimizing_compression_task_timeout(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        ) {
            p
        } else {
            return;
        };

        trace!("Trying to recompress file: {}", path.display());

        // recompress, write to other file, rename (it's atomic file content exchange)
        // and update the stats file
        let compressed_cache_bytes = unwrap_or_warn!(
            fs::read(&path),
            return,
            "Failed to read old cache file",
            path
        );

        let cache_bytes = unwrap_or_warn!(
            zstd::decode_all(&compressed_cache_bytes[..]),
            return,
            "Failed to decompress cached code",
            path
        );

        let recompressed_cache_bytes = unwrap_or_warn!(
            zstd::encode_all(&cache_bytes[..], opt_compr_lvl),
            return,
            "Failed to compress cached code",
            path
        );

        unwrap_or_warn!(
            fs::write(&lock_path, &recompressed_cache_bytes),
            return,
            "Failed to write recompressed cache",
            lock_path
        );

        unwrap_or_warn!(
            fs::rename(&lock_path, &path),
            {
                if let Err(error) = fs::remove_file(&lock_path) {
                    warn!(
                        "Failed to clean up (remove) recompressed cache, path {}, err: {}",
                        lock_path.display(),
                        error
                    );
                }

                return;
            },
            "Failed to rename recompressed cache",
            lock_path
        );

        // update stats file (reload it! recompression can take some time)
        if let Some(mut new_stats) = read_stats_file(stats_path.as_ref()) {
            if new_stats.compression_level >= opt_compr_lvl {
                // Rare race:
                //    two instances with different opt_compr_lvl: we don't know in which order they updated
                //    the cache file and the stats file (they are not updated together atomically)
                // Possible solution is to use directories per cache entry, but it complicates the system
                // and is not worth it.
                debug!(
                    "DETECTED task did more than once (or race with new file): \
                     recompression of {}. Note: if optimized compression level setting \
                     has changed in the meantine, the stats file might contain \
                     inconsistent compression level due to race.",
                    path.display()
                );
            } else {
                new_stats.compression_level = opt_compr_lvl;
                let _ = write_stats_file(stats_path.as_ref(), &new_stats);
            }

            if new_stats.usages < stats.usages {
                debug!(
                    "DETECTED lower usage count (new file or race with counter \
                     increasing): file {}",
                    path.display()
                );
            }
        } else {
            debug!(
                "Can't read stats file again to update compression level (it might got \
                 cleaned up): file {}",
                stats_path.display()
            );
        }

        trace!("Task finished: recompress file: {}", path.display());
    }

    fn handle_on_cache_update(&self, path: PathBuf) {
        trace!("handle_on_cache_update() for path: {}", path.display());

        // ---------------------- step 1: create .stats file

        // construct .stats file path
        let filename = path
            .file_name()
            .expect("Expected valid cache file name")
            .to_str()
            .expect("Expected valid cache file name");
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // create and write stats file
        let mut stats = ModuleCacheStatistics::default(&self.cache_config);
        stats.usages += 1;
        write_stats_file(&stats_path, &stats);

        // ---------------------- step 2: perform cleanup task if needed

        // acquire lock for cleanup task
        // Lock is a proof of recent cleanup task, so we don't want to delete them.
        // Expired locks will be deleted by the cleanup task.
        let cleanup_file = self.cache_config.directory().join(".cleanup"); // some non existing marker file
        if acquire_task_fs_lock(
            &cleanup_file,
            self.cache_config.cleanup_interval(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        )
        .is_none()
        {
            return;
        }

        trace!("Trying to clean up cache");

        let mut cache_index = self.list_cache_contents();
        let future_tolerance = SystemTime::now()
            .checked_add(
                self.cache_config
                    .allowed_clock_drift_for_files_from_future(),
            )
            .expect("Brace your cache, the next Big Bang is coming (time overflow)");
        cache_index.sort_unstable_by(|lhs, rhs| {
            // sort by age
            use CacheEntry::*;
            match (lhs, rhs) {
                (Recognized { mtime: lhs_mt, .. }, Recognized { mtime: rhs_mt, .. }) => {
                    match (*lhs_mt > future_tolerance, *rhs_mt > future_tolerance) {
                        // later == younger
                        (false, false) => rhs_mt.cmp(lhs_mt),
                        // files from far future are treated as oldest recognized files
                        // we want to delete them, so the cache keeps track of recent files
                        // however, we don't delete them uncodintionally,
                        // because .stats file can be overwritten with a meaningful mtime
                        (true, false) => cmp::Ordering::Greater,
                        (false, true) => cmp::Ordering::Less,
                        (true, true) => cmp::Ordering::Equal,
                    }
                }
                // unrecognized is kind of infinity
                (Recognized { .. }, Unrecognized { .. }) => cmp::Ordering::Less,
                (Unrecognized { .. }, Recognized { .. }) => cmp::Ordering::Greater,
                (Unrecognized { .. }, Unrecognized { .. }) => cmp::Ordering::Equal,
            }
        });

        // find "cut" boundary:
        // - remove unrecognized files anyway,
        // - remove some cache files if some quota has been exceeded
        let mut total_size = 0u64;
        let mut start_delete_idx = None;
        let mut start_delete_idx_if_deleting_recognized_items: Option<usize> = None;

        let total_size_limit = self.cache_config.files_total_size_soft_limit();
        let file_count_limit = self.cache_config.file_count_soft_limit();
        let tsl_if_deleting = total_size_limit
            .checked_mul(
                self.cache_config
                    .files_total_size_limit_percent_if_deleting() as u64,
            )
            .unwrap()
            / 100;
        let fcl_if_deleting = file_count_limit
            .checked_mul(self.cache_config.file_count_limit_percent_if_deleting() as u64)
            .unwrap()
            / 100;

        for (idx, item) in cache_index.iter().enumerate() {
            let size = if let CacheEntry::Recognized { size, .. } = item {
                size
            } else {
                start_delete_idx = Some(idx);
                break;
            };

            total_size += size;
            if start_delete_idx_if_deleting_recognized_items.is_none()
                && (total_size > tsl_if_deleting || (idx + 1) as u64 > fcl_if_deleting)
            {
                start_delete_idx_if_deleting_recognized_items = Some(idx);
            }

            if total_size > total_size_limit || (idx + 1) as u64 > file_count_limit {
                start_delete_idx = start_delete_idx_if_deleting_recognized_items;
                break;
            }
        }

        if let Some(idx) = start_delete_idx {
            for item in &cache_index[idx..] {
                let (result, path, entity) = match item {
                    CacheEntry::Recognized { path, .. }
                    | CacheEntry::Unrecognized {
                        path,
                        is_dir: false,
                    } => (fs::remove_file(path), path, "file"),
                    CacheEntry::Unrecognized { path, is_dir: true } => {
                        (fs::remove_dir_all(path), path, "directory")
                    }
                };
                if let Err(err) = result {
                    warn!(
                        "Failed to remove {} during cleanup, path: {}, err: {}",
                        entity,
                        path.display(),
                        err
                    );
                }
            }
        }

        trace!("Task finished: clean up cache");
    }

    // Be fault tolerant: list as much as you can, and ignore the rest
    fn list_cache_contents(&self) -> Vec<CacheEntry> {
        fn enter_dir(
            vec: &mut Vec<CacheEntry>,
            dir_path: &Path,
            level: u8,
            cache_config: &CacheConfig,
        ) {
            macro_rules! add_unrecognized {
                (file: $path:expr) => {
                    add_unrecognized!(false, $path)
                };
                (dir: $path:expr) => {
                    add_unrecognized!(true, $path)
                };
                ($is_dir:expr, $path:expr) => {
                    vec.push(CacheEntry::Unrecognized {
                        path: $path.to_path_buf(),
                        is_dir: $is_dir,
                    })
                };
            }
            macro_rules! add_unrecognized_and {
                ([ $( $ty:ident: $path:expr ),* ], $cont:stmt) => {{
                    $( add_unrecognized!($ty: $path); )*
                        $cont
                }};
            }

            macro_rules! unwrap_or {
                ($result:expr, $cont:stmt, $err_msg:expr) => {
                    unwrap_or!($result, $cont, $err_msg, dir_path)
                };
                ($result:expr, $cont:stmt, $err_msg:expr, $path:expr) => {
                    unwrap_or_warn!(
                        $result,
                        $cont,
                        format!("{}, level: {}", $err_msg, level),
                        $path
                    )
                };
            }

            // If we fail to list a directory, something bad is happening anyway
            // (something touches our cache or we have disk failure)
            // Try to delete it, so we can stay within soft limits of the cache size.
            // This comment applies later in this function, too.
            let it = unwrap_or!(
                fs::read_dir(dir_path),
                add_unrecognized_and!([dir: dir_path], return),
                "Failed to list cache directory, deleting it"
            );

            let mut cache_files = HashMap::new();
            for entry in it {
                // read_dir() returns an iterator over results - in case some of them are errors
                // we don't know their names, so we can't delete them. We don't want to delete
                // the whole directory with good entries too, so we just ignore the erroneous entries.
                let entry = unwrap_or!(
                    entry,
                    continue,
                    "Failed to read a cache dir entry (NOT deleting it, it still occupies space)"
                );
                let path = entry.path();
                match (level, path.is_dir()) {
                    (0..=1, true) => enter_dir(vec, &path, level + 1, cache_config),
                    (0..=1, false) => {
                        if level == 0
                            && path.file_stem() == Some(OsStr::new(".cleanup"))
                                && path.extension().is_some()
                                // assume it's cleanup lock
                                && !is_fs_lock_expired(
                                    Some(&entry),
                                    &path,
                                    cache_config.cleanup_interval(),
                                    cache_config.allowed_clock_drift_for_files_from_future(),
                                )
                        {
                            continue; // skip active lock
                        }
                        add_unrecognized!(file: path);
                    }
                    (2, false) => {
                        match path.extension().and_then(OsStr::to_str) {
                            // mod or stats file
                            None | Some("stats") => {
                                cache_files.insert(path, entry);
                            }

                            Some(ext) => {
                                // check if valid lock
                                let recognized = ext.starts_with("wip-")
                                    && !is_fs_lock_expired(
                                        Some(&entry),
                                        &path,
                                        cache_config.optimizing_compression_task_timeout(),
                                        cache_config.allowed_clock_drift_for_files_from_future(),
                                    );

                                if !recognized {
                                    add_unrecognized!(file: path);
                                }
                            }
                        }
                    }
                    (_, is_dir) => add_unrecognized!(is_dir, path),
                }
            }

            // associate module with its stats & handle them
            // assumption: just mods and stats
            for (path, entry) in cache_files.iter() {
                let path_buf: PathBuf;
                let (mod_, stats_, is_mod) = match path.extension() {
                    Some(_) => {
                        path_buf = path.with_extension("");
                        (
                            cache_files.get(&path_buf).map(|v| (&path_buf, v)),
                            Some((path, entry)),
                            false,
                        )
                    }
                    None => {
                        path_buf = path.with_extension("stats");
                        (
                            Some((path, entry)),
                            cache_files.get(&path_buf).map(|v| (&path_buf, v)),
                            true,
                        )
                    }
                };

                // construct a cache entry
                match (mod_, stats_, is_mod) {
                    (Some((mod_path, mod_entry)), Some((stats_path, stats_entry)), true) => {
                        let mod_metadata = unwrap_or!(
                            mod_entry.metadata(),
                            add_unrecognized_and!([file: stats_path, file: mod_path], continue),
                            "Failed to get metadata, deleting BOTH module cache and stats files",
                            mod_path
                        );
                        let stats_mtime = unwrap_or!(
                            stats_entry.metadata().and_then(|m| m.modified()),
                            add_unrecognized_and!(
                                [file: stats_path],
                                unwrap_or!(
                                    mod_metadata.modified(),
                                    add_unrecognized_and!(
                                        [file: stats_path, file: mod_path],
                                        continue
                                    ),
                                    "Failed to get mtime, deleting BOTH module cache and stats \
                                     files",
                                    mod_path
                                )
                            ),
                            "Failed to get metadata/mtime, deleting the file",
                            stats_path
                        );
                        // .into() called for the SystemTimeStub if cfg(test)
                        #[allow(clippy::identity_conversion)]
                        vec.push(CacheEntry::Recognized {
                            path: mod_path.to_path_buf(),
                            mtime: stats_mtime.into(),
                            size: mod_metadata.len(),
                        })
                    }
                    (Some(_), Some(_), false) => (), // was or will be handled by previous branch
                    (Some((mod_path, mod_entry)), None, _) => {
                        let (mod_metadata, mod_mtime) = unwrap_or!(
                            mod_entry
                                .metadata()
                                .and_then(|md| md.modified().map(|mt| (md, mt))),
                            add_unrecognized_and!([file: mod_path], continue),
                            "Failed to get metadata/mtime, deleting the file",
                            mod_path
                        );
                        // .into() called for the SystemTimeStub if cfg(test)
                        #[allow(clippy::identity_conversion)]
                        vec.push(CacheEntry::Recognized {
                            path: mod_path.to_path_buf(),
                            mtime: mod_mtime.into(),
                            size: mod_metadata.len(),
                        })
                    }
                    (None, Some((stats_path, _stats_entry)), _) => {
                        debug!("Found orphaned stats file: {}", stats_path.display());
                        add_unrecognized!(file: stats_path);
                    }
                    _ => unreachable!(),
                }
            }
        }

pub fn allowed_clock_drift_for_files_from_future(&self) -> Duration

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 304)

    fn handle_on_cache_get(&self, path: PathBuf) {
        trace!("handle_on_cache_get() for path: {}", path.display());

        // construct .stats file path
        let filename = path.file_name().unwrap().to_str().unwrap();
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // load .stats file (default if none or error)
        let mut stats = read_stats_file(stats_path.as_ref())
            .unwrap_or_else(|| ModuleCacheStatistics::default(&self.cache_config));

        // step 1: update the usage counter & write to the disk
        //         it's racy, but it's fine (the counter will be just smaller,
        //         sometimes will retrigger recompression)
        stats.usages += 1;
        if !write_stats_file(stats_path.as_ref(), &stats) {
            return;
        }

        // step 2: recompress if there's a need
        let opt_compr_lvl = self.cache_config.optimized_compression_level();
        if stats.compression_level >= opt_compr_lvl
            || stats.usages
                < self
                    .cache_config
                    .optimized_compression_usage_counter_threshold()
        {
            return;
        }

        let lock_path = if let Some(p) = acquire_task_fs_lock(
            path.as_ref(),
            self.cache_config.optimizing_compression_task_timeout(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        ) {
            p
        } else {
            return;
        };

        trace!("Trying to recompress file: {}", path.display());

        // recompress, write to other file, rename (it's atomic file content exchange)
        // and update the stats file
        let compressed_cache_bytes = unwrap_or_warn!(
            fs::read(&path),
            return,
            "Failed to read old cache file",
            path
        );

        let cache_bytes = unwrap_or_warn!(
            zstd::decode_all(&compressed_cache_bytes[..]),
            return,
            "Failed to decompress cached code",
            path
        );

        let recompressed_cache_bytes = unwrap_or_warn!(
            zstd::encode_all(&cache_bytes[..], opt_compr_lvl),
            return,
            "Failed to compress cached code",
            path
        );

        unwrap_or_warn!(
            fs::write(&lock_path, &recompressed_cache_bytes),
            return,
            "Failed to write recompressed cache",
            lock_path
        );

        unwrap_or_warn!(
            fs::rename(&lock_path, &path),
            {
                if let Err(error) = fs::remove_file(&lock_path) {
                    warn!(
                        "Failed to clean up (remove) recompressed cache, path {}, err: {}",
                        lock_path.display(),
                        error
                    );
                }

                return;
            },
            "Failed to rename recompressed cache",
            lock_path
        );

        // update stats file (reload it! recompression can take some time)
        if let Some(mut new_stats) = read_stats_file(stats_path.as_ref()) {
            if new_stats.compression_level >= opt_compr_lvl {
                // Rare race:
                //    two instances with different opt_compr_lvl: we don't know in which order they updated
                //    the cache file and the stats file (they are not updated together atomically)
                // Possible solution is to use directories per cache entry, but it complicates the system
                // and is not worth it.
                debug!(
                    "DETECTED task did more than once (or race with new file): \
                     recompression of {}. Note: if optimized compression level setting \
                     has changed in the meantine, the stats file might contain \
                     inconsistent compression level due to race.",
                    path.display()
                );
            } else {
                new_stats.compression_level = opt_compr_lvl;
                let _ = write_stats_file(stats_path.as_ref(), &new_stats);
            }

            if new_stats.usages < stats.usages {
                debug!(
                    "DETECTED lower usage count (new file or race with counter \
                     increasing): file {}",
                    path.display()
                );
            }
        } else {
            debug!(
                "Can't read stats file again to update compression level (it might got \
                 cleaned up): file {}",
                stats_path.display()
            );
        }

        trace!("Task finished: recompress file: {}", path.display());
    }

    fn handle_on_cache_update(&self, path: PathBuf) {
        trace!("handle_on_cache_update() for path: {}", path.display());

        // ---------------------- step 1: create .stats file

        // construct .stats file path
        let filename = path
            .file_name()
            .expect("Expected valid cache file name")
            .to_str()
            .expect("Expected valid cache file name");
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // create and write stats file
        let mut stats = ModuleCacheStatistics::default(&self.cache_config);
        stats.usages += 1;
        write_stats_file(&stats_path, &stats);

        // ---------------------- step 2: perform cleanup task if needed

        // acquire lock for cleanup task
        // Lock is a proof of recent cleanup task, so we don't want to delete them.
        // Expired locks will be deleted by the cleanup task.
        let cleanup_file = self.cache_config.directory().join(".cleanup"); // some non existing marker file
        if acquire_task_fs_lock(
            &cleanup_file,
            self.cache_config.cleanup_interval(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        )
        .is_none()
        {
            return;
        }

        trace!("Trying to clean up cache");

        let mut cache_index = self.list_cache_contents();
        let future_tolerance = SystemTime::now()
            .checked_add(
                self.cache_config
                    .allowed_clock_drift_for_files_from_future(),
            )
            .expect("Brace your cache, the next Big Bang is coming (time overflow)");
        cache_index.sort_unstable_by(|lhs, rhs| {
            // sort by age
            use CacheEntry::*;
            match (lhs, rhs) {
                (Recognized { mtime: lhs_mt, .. }, Recognized { mtime: rhs_mt, .. }) => {
                    match (*lhs_mt > future_tolerance, *rhs_mt > future_tolerance) {
                        // later == younger
                        (false, false) => rhs_mt.cmp(lhs_mt),
                        // files from far future are treated as oldest recognized files
                        // we want to delete them, so the cache keeps track of recent files
                        // however, we don't delete them uncodintionally,
                        // because .stats file can be overwritten with a meaningful mtime
                        (true, false) => cmp::Ordering::Greater,
                        (false, true) => cmp::Ordering::Less,
                        (true, true) => cmp::Ordering::Equal,
                    }
                }
                // unrecognized is kind of infinity
                (Recognized { .. }, Unrecognized { .. }) => cmp::Ordering::Less,
                (Unrecognized { .. }, Recognized { .. }) => cmp::Ordering::Greater,
                (Unrecognized { .. }, Unrecognized { .. }) => cmp::Ordering::Equal,
            }
        });

        // find "cut" boundary:
        // - remove unrecognized files anyway,
        // - remove some cache files if some quota has been exceeded
        let mut total_size = 0u64;
        let mut start_delete_idx = None;
        let mut start_delete_idx_if_deleting_recognized_items: Option<usize> = None;

        let total_size_limit = self.cache_config.files_total_size_soft_limit();
        let file_count_limit = self.cache_config.file_count_soft_limit();
        let tsl_if_deleting = total_size_limit
            .checked_mul(
                self.cache_config
                    .files_total_size_limit_percent_if_deleting() as u64,
            )
            .unwrap()
            / 100;
        let fcl_if_deleting = file_count_limit
            .checked_mul(self.cache_config.file_count_limit_percent_if_deleting() as u64)
            .unwrap()
            / 100;

        for (idx, item) in cache_index.iter().enumerate() {
            let size = if let CacheEntry::Recognized { size, .. } = item {
                size
            } else {
                start_delete_idx = Some(idx);
                break;
            };

            total_size += size;
            if start_delete_idx_if_deleting_recognized_items.is_none()
                && (total_size > tsl_if_deleting || (idx + 1) as u64 > fcl_if_deleting)
            {
                start_delete_idx_if_deleting_recognized_items = Some(idx);
            }

            if total_size > total_size_limit || (idx + 1) as u64 > file_count_limit {
                start_delete_idx = start_delete_idx_if_deleting_recognized_items;
                break;
            }
        }

        if let Some(idx) = start_delete_idx {
            for item in &cache_index[idx..] {
                let (result, path, entity) = match item {
                    CacheEntry::Recognized { path, .. }
                    | CacheEntry::Unrecognized {
                        path,
                        is_dir: false,
                    } => (fs::remove_file(path), path, "file"),
                    CacheEntry::Unrecognized { path, is_dir: true } => {
                        (fs::remove_dir_all(path), path, "directory")
                    }
                };
                if let Err(err) = result {
                    warn!(
                        "Failed to remove {} during cleanup, path: {}, err: {}",
                        entity,
                        path.display(),
                        err
                    );
                }
            }
        }

        trace!("Task finished: clean up cache");
    }

    // Be fault tolerant: list as much as you can, and ignore the rest
    fn list_cache_contents(&self) -> Vec<CacheEntry> {
        fn enter_dir(
            vec: &mut Vec<CacheEntry>,
            dir_path: &Path,
            level: u8,
            cache_config: &CacheConfig,
        ) {
            macro_rules! add_unrecognized {
                (file: $path:expr) => {
                    add_unrecognized!(false, $path)
                };
                (dir: $path:expr) => {
                    add_unrecognized!(true, $path)
                };
                ($is_dir:expr, $path:expr) => {
                    vec.push(CacheEntry::Unrecognized {
                        path: $path.to_path_buf(),
                        is_dir: $is_dir,
                    })
                };
            }
            macro_rules! add_unrecognized_and {
                ([ $( $ty:ident: $path:expr ),* ], $cont:stmt) => {{
                    $( add_unrecognized!($ty: $path); )*
                        $cont
                }};
            }

            macro_rules! unwrap_or {
                ($result:expr, $cont:stmt, $err_msg:expr) => {
                    unwrap_or!($result, $cont, $err_msg, dir_path)
                };
                ($result:expr, $cont:stmt, $err_msg:expr, $path:expr) => {
                    unwrap_or_warn!(
                        $result,
                        $cont,
                        format!("{}, level: {}", $err_msg, level),
                        $path
                    )
                };
            }

            // If we fail to list a directory, something bad is happening anyway
            // (something touches our cache or we have disk failure)
            // Try to delete it, so we can stay within soft limits of the cache size.
            // This comment applies later in this function, too.
            let it = unwrap_or!(
                fs::read_dir(dir_path),
                add_unrecognized_and!([dir: dir_path], return),
                "Failed to list cache directory, deleting it"
            );

            let mut cache_files = HashMap::new();
            for entry in it {
                // read_dir() returns an iterator over results - in case some of them are errors
                // we don't know their names, so we can't delete them. We don't want to delete
                // the whole directory with good entries too, so we just ignore the erroneous entries.
                let entry = unwrap_or!(
                    entry,
                    continue,
                    "Failed to read a cache dir entry (NOT deleting it, it still occupies space)"
                );
                let path = entry.path();
                match (level, path.is_dir()) {
                    (0..=1, true) => enter_dir(vec, &path, level + 1, cache_config),
                    (0..=1, false) => {
                        if level == 0
                            && path.file_stem() == Some(OsStr::new(".cleanup"))
                                && path.extension().is_some()
                                // assume it's cleanup lock
                                && !is_fs_lock_expired(
                                    Some(&entry),
                                    &path,
                                    cache_config.cleanup_interval(),
                                    cache_config.allowed_clock_drift_for_files_from_future(),
                                )
                        {
                            continue; // skip active lock
                        }
                        add_unrecognized!(file: path);
                    }
                    (2, false) => {
                        match path.extension().and_then(OsStr::to_str) {
                            // mod or stats file
                            None | Some("stats") => {
                                cache_files.insert(path, entry);
                            }

                            Some(ext) => {
                                // check if valid lock
                                let recognized = ext.starts_with("wip-")
                                    && !is_fs_lock_expired(
                                        Some(&entry),
                                        &path,
                                        cache_config.optimizing_compression_task_timeout(),
                                        cache_config.allowed_clock_drift_for_files_from_future(),
                                    );

                                if !recognized {
                                    add_unrecognized!(file: path);
                                }
                            }
                        }
                    }
                    (_, is_dir) => add_unrecognized!(is_dir, path),
                }
            }

            // associate module with its stats & handle them
            // assumption: just mods and stats
            for (path, entry) in cache_files.iter() {
                let path_buf: PathBuf;
                let (mod_, stats_, is_mod) = match path.extension() {
                    Some(_) => {
                        path_buf = path.with_extension("");
                        (
                            cache_files.get(&path_buf).map(|v| (&path_buf, v)),
                            Some((path, entry)),
                            false,
                        )
                    }
                    None => {
                        path_buf = path.with_extension("stats");
                        (
                            Some((path, entry)),
                            cache_files.get(&path_buf).map(|v| (&path_buf, v)),
                            true,
                        )
                    }
                };

                // construct a cache entry
                match (mod_, stats_, is_mod) {
                    (Some((mod_path, mod_entry)), Some((stats_path, stats_entry)), true) => {
                        let mod_metadata = unwrap_or!(
                            mod_entry.metadata(),
                            add_unrecognized_and!([file: stats_path, file: mod_path], continue),
                            "Failed to get metadata, deleting BOTH module cache and stats files",
                            mod_path
                        );
                        let stats_mtime = unwrap_or!(
                            stats_entry.metadata().and_then(|m| m.modified()),
                            add_unrecognized_and!(
                                [file: stats_path],
                                unwrap_or!(
                                    mod_metadata.modified(),
                                    add_unrecognized_and!(
                                        [file: stats_path, file: mod_path],
                                        continue
                                    ),
                                    "Failed to get mtime, deleting BOTH module cache and stats \
                                     files",
                                    mod_path
                                )
                            ),
                            "Failed to get metadata/mtime, deleting the file",
                            stats_path
                        );
                        // .into() called for the SystemTimeStub if cfg(test)
                        #[allow(clippy::identity_conversion)]
                        vec.push(CacheEntry::Recognized {
                            path: mod_path.to_path_buf(),
                            mtime: stats_mtime.into(),
                            size: mod_metadata.len(),
                        })
                    }
                    (Some(_), Some(_), false) => (), // was or will be handled by previous branch
                    (Some((mod_path, mod_entry)), None, _) => {
                        let (mod_metadata, mod_mtime) = unwrap_or!(
                            mod_entry
                                .metadata()
                                .and_then(|md| md.modified().map(|mt| (md, mt))),
                            add_unrecognized_and!([file: mod_path], continue),
                            "Failed to get metadata/mtime, deleting the file",
                            mod_path
                        );
                        // .into() called for the SystemTimeStub if cfg(test)
                        #[allow(clippy::identity_conversion)]
                        vec.push(CacheEntry::Recognized {
                            path: mod_path.to_path_buf(),
                            mtime: mod_mtime.into(),
                            size: mod_metadata.len(),
                        })
                    }
                    (None, Some((stats_path, _stats_entry)), _) => {
                        debug!("Found orphaned stats file: {}", stats_path.display());
                        add_unrecognized!(file: stats_path);
                    }
                    _ => unreachable!(),
                }
            }
        }

pub fn file_count_soft_limit(&self) -> u64

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 474)

    fn handle_on_cache_update(&self, path: PathBuf) {
        trace!("handle_on_cache_update() for path: {}", path.display());

        // ---------------------- step 1: create .stats file

        // construct .stats file path
        let filename = path
            .file_name()
            .expect("Expected valid cache file name")
            .to_str()
            .expect("Expected valid cache file name");
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // create and write stats file
        let mut stats = ModuleCacheStatistics::default(&self.cache_config);
        stats.usages += 1;
        write_stats_file(&stats_path, &stats);

        // ---------------------- step 2: perform cleanup task if needed

        // acquire lock for cleanup task
        // Lock is a proof of recent cleanup task, so we don't want to delete them.
        // Expired locks will be deleted by the cleanup task.
        let cleanup_file = self.cache_config.directory().join(".cleanup"); // some non existing marker file
        if acquire_task_fs_lock(
            &cleanup_file,
            self.cache_config.cleanup_interval(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        )
        .is_none()
        {
            return;
        }

        trace!("Trying to clean up cache");

        let mut cache_index = self.list_cache_contents();
        let future_tolerance = SystemTime::now()
            .checked_add(
                self.cache_config
                    .allowed_clock_drift_for_files_from_future(),
            )
            .expect("Brace your cache, the next Big Bang is coming (time overflow)");
        cache_index.sort_unstable_by(|lhs, rhs| {
            // sort by age
            use CacheEntry::*;
            match (lhs, rhs) {
                (Recognized { mtime: lhs_mt, .. }, Recognized { mtime: rhs_mt, .. }) => {
                    match (*lhs_mt > future_tolerance, *rhs_mt > future_tolerance) {
                        // later == younger
                        (false, false) => rhs_mt.cmp(lhs_mt),
                        // files from far future are treated as oldest recognized files
                        // we want to delete them, so the cache keeps track of recent files
                        // however, we don't delete them uncodintionally,
                        // because .stats file can be overwritten with a meaningful mtime
                        (true, false) => cmp::Ordering::Greater,
                        (false, true) => cmp::Ordering::Less,
                        (true, true) => cmp::Ordering::Equal,
                    }
                }
                // unrecognized is kind of infinity
                (Recognized { .. }, Unrecognized { .. }) => cmp::Ordering::Less,
                (Unrecognized { .. }, Recognized { .. }) => cmp::Ordering::Greater,
                (Unrecognized { .. }, Unrecognized { .. }) => cmp::Ordering::Equal,
            }
        });

        // find "cut" boundary:
        // - remove unrecognized files anyway,
        // - remove some cache files if some quota has been exceeded
        let mut total_size = 0u64;
        let mut start_delete_idx = None;
        let mut start_delete_idx_if_deleting_recognized_items: Option<usize> = None;

        let total_size_limit = self.cache_config.files_total_size_soft_limit();
        let file_count_limit = self.cache_config.file_count_soft_limit();
        let tsl_if_deleting = total_size_limit
            .checked_mul(
                self.cache_config
                    .files_total_size_limit_percent_if_deleting() as u64,
            )
            .unwrap()
            / 100;
        let fcl_if_deleting = file_count_limit
            .checked_mul(self.cache_config.file_count_limit_percent_if_deleting() as u64)
            .unwrap()
            / 100;

        for (idx, item) in cache_index.iter().enumerate() {
            let size = if let CacheEntry::Recognized { size, .. } = item {
                size
            } else {
                start_delete_idx = Some(idx);
                break;
            };

            total_size += size;
            if start_delete_idx_if_deleting_recognized_items.is_none()
                && (total_size > tsl_if_deleting || (idx + 1) as u64 > fcl_if_deleting)
            {
                start_delete_idx_if_deleting_recognized_items = Some(idx);
            }

            if total_size > total_size_limit || (idx + 1) as u64 > file_count_limit {
                start_delete_idx = start_delete_idx_if_deleting_recognized_items;
                break;
            }
        }

        if let Some(idx) = start_delete_idx {
            for item in &cache_index[idx..] {
                let (result, path, entity) = match item {
                    CacheEntry::Recognized { path, .. }
                    | CacheEntry::Unrecognized {
                        path,
                        is_dir: false,
                    } => (fs::remove_file(path), path, "file"),
                    CacheEntry::Unrecognized { path, is_dir: true } => {
                        (fs::remove_dir_all(path), path, "directory")
                    }
                };
                if let Err(err) = result {
                    warn!(
                        "Failed to remove {} during cleanup, path: {}, err: {}",
                        entity,
                        path.display(),
                        err
                    );
                }
            }
        }

        trace!("Task finished: clean up cache");
    }

pub fn files_total_size_soft_limit(&self) -> u64

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 473)

    fn handle_on_cache_update(&self, path: PathBuf) {
        trace!("handle_on_cache_update() for path: {}", path.display());

        // ---------------------- step 1: create .stats file

        // construct .stats file path
        let filename = path
            .file_name()
            .expect("Expected valid cache file name")
            .to_str()
            .expect("Expected valid cache file name");
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // create and write stats file
        let mut stats = ModuleCacheStatistics::default(&self.cache_config);
        stats.usages += 1;
        write_stats_file(&stats_path, &stats);

        // ---------------------- step 2: perform cleanup task if needed

        // acquire lock for cleanup task
        // Lock is a proof of recent cleanup task, so we don't want to delete them.
        // Expired locks will be deleted by the cleanup task.
        let cleanup_file = self.cache_config.directory().join(".cleanup"); // some non existing marker file
        if acquire_task_fs_lock(
            &cleanup_file,
            self.cache_config.cleanup_interval(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        )
        .is_none()
        {
            return;
        }

        trace!("Trying to clean up cache");

        let mut cache_index = self.list_cache_contents();
        let future_tolerance = SystemTime::now()
            .checked_add(
                self.cache_config
                    .allowed_clock_drift_for_files_from_future(),
            )
            .expect("Brace your cache, the next Big Bang is coming (time overflow)");
        cache_index.sort_unstable_by(|lhs, rhs| {
            // sort by age
            use CacheEntry::*;
            match (lhs, rhs) {
                (Recognized { mtime: lhs_mt, .. }, Recognized { mtime: rhs_mt, .. }) => {
                    match (*lhs_mt > future_tolerance, *rhs_mt > future_tolerance) {
                        // later == younger
                        (false, false) => rhs_mt.cmp(lhs_mt),
                        // files from far future are treated as oldest recognized files
                        // we want to delete them, so the cache keeps track of recent files
                        // however, we don't delete them uncodintionally,
                        // because .stats file can be overwritten with a meaningful mtime
                        (true, false) => cmp::Ordering::Greater,
                        (false, true) => cmp::Ordering::Less,
                        (true, true) => cmp::Ordering::Equal,
                    }
                }
                // unrecognized is kind of infinity
                (Recognized { .. }, Unrecognized { .. }) => cmp::Ordering::Less,
                (Unrecognized { .. }, Recognized { .. }) => cmp::Ordering::Greater,
                (Unrecognized { .. }, Unrecognized { .. }) => cmp::Ordering::Equal,
            }
        });

        // find "cut" boundary:
        // - remove unrecognized files anyway,
        // - remove some cache files if some quota has been exceeded
        let mut total_size = 0u64;
        let mut start_delete_idx = None;
        let mut start_delete_idx_if_deleting_recognized_items: Option<usize> = None;

        let total_size_limit = self.cache_config.files_total_size_soft_limit();
        let file_count_limit = self.cache_config.file_count_soft_limit();
        let tsl_if_deleting = total_size_limit
            .checked_mul(
                self.cache_config
                    .files_total_size_limit_percent_if_deleting() as u64,
            )
            .unwrap()
            / 100;
        let fcl_if_deleting = file_count_limit
            .checked_mul(self.cache_config.file_count_limit_percent_if_deleting() as u64)
            .unwrap()
            / 100;

        for (idx, item) in cache_index.iter().enumerate() {
            let size = if let CacheEntry::Recognized { size, .. } = item {
                size
            } else {
                start_delete_idx = Some(idx);
                break;
            };

            total_size += size;
            if start_delete_idx_if_deleting_recognized_items.is_none()
                && (total_size > tsl_if_deleting || (idx + 1) as u64 > fcl_if_deleting)
            {
                start_delete_idx_if_deleting_recognized_items = Some(idx);
            }

            if total_size > total_size_limit || (idx + 1) as u64 > file_count_limit {
                start_delete_idx = start_delete_idx_if_deleting_recognized_items;
                break;
            }
        }

        if let Some(idx) = start_delete_idx {
            for item in &cache_index[idx..] {
                let (result, path, entity) = match item {
                    CacheEntry::Recognized { path, .. }
                    | CacheEntry::Unrecognized {
                        path,
                        is_dir: false,
                    } => (fs::remove_file(path), path, "file"),
                    CacheEntry::Unrecognized { path, is_dir: true } => {
                        (fs::remove_dir_all(path), path, "directory")
                    }
                };
                if let Err(err) = result {
                    warn!(
                        "Failed to remove {} during cleanup, path: {}, err: {}",
                        entity,
                        path.display(),
                        err
                    );
                }
            }
        }

        trace!("Task finished: clean up cache");
    }

pub fn file_count_limit_percent_if_deleting(&self) -> u8

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 483)

    fn handle_on_cache_update(&self, path: PathBuf) {
        trace!("handle_on_cache_update() for path: {}", path.display());

        // ---------------------- step 1: create .stats file

        // construct .stats file path
        let filename = path
            .file_name()
            .expect("Expected valid cache file name")
            .to_str()
            .expect("Expected valid cache file name");
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // create and write stats file
        let mut stats = ModuleCacheStatistics::default(&self.cache_config);
        stats.usages += 1;
        write_stats_file(&stats_path, &stats);

        // ---------------------- step 2: perform cleanup task if needed

        // acquire lock for cleanup task
        // Lock is a proof of recent cleanup task, so we don't want to delete them.
        // Expired locks will be deleted by the cleanup task.
        let cleanup_file = self.cache_config.directory().join(".cleanup"); // some non existing marker file
        if acquire_task_fs_lock(
            &cleanup_file,
            self.cache_config.cleanup_interval(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        )
        .is_none()
        {
            return;
        }

        trace!("Trying to clean up cache");

        let mut cache_index = self.list_cache_contents();
        let future_tolerance = SystemTime::now()
            .checked_add(
                self.cache_config
                    .allowed_clock_drift_for_files_from_future(),
            )
            .expect("Brace your cache, the next Big Bang is coming (time overflow)");
        cache_index.sort_unstable_by(|lhs, rhs| {
            // sort by age
            use CacheEntry::*;
            match (lhs, rhs) {
                (Recognized { mtime: lhs_mt, .. }, Recognized { mtime: rhs_mt, .. }) => {
                    match (*lhs_mt > future_tolerance, *rhs_mt > future_tolerance) {
                        // later == younger
                        (false, false) => rhs_mt.cmp(lhs_mt),
                        // files from far future are treated as oldest recognized files
                        // we want to delete them, so the cache keeps track of recent files
                        // however, we don't delete them uncodintionally,
                        // because .stats file can be overwritten with a meaningful mtime
                        (true, false) => cmp::Ordering::Greater,
                        (false, true) => cmp::Ordering::Less,
                        (true, true) => cmp::Ordering::Equal,
                    }
                }
                // unrecognized is kind of infinity
                (Recognized { .. }, Unrecognized { .. }) => cmp::Ordering::Less,
                (Unrecognized { .. }, Recognized { .. }) => cmp::Ordering::Greater,
                (Unrecognized { .. }, Unrecognized { .. }) => cmp::Ordering::Equal,
            }
        });

        // find "cut" boundary:
        // - remove unrecognized files anyway,
        // - remove some cache files if some quota has been exceeded
        let mut total_size = 0u64;
        let mut start_delete_idx = None;
        let mut start_delete_idx_if_deleting_recognized_items: Option<usize> = None;

        let total_size_limit = self.cache_config.files_total_size_soft_limit();
        let file_count_limit = self.cache_config.file_count_soft_limit();
        let tsl_if_deleting = total_size_limit
            .checked_mul(
                self.cache_config
                    .files_total_size_limit_percent_if_deleting() as u64,
            )
            .unwrap()
            / 100;
        let fcl_if_deleting = file_count_limit
            .checked_mul(self.cache_config.file_count_limit_percent_if_deleting() as u64)
            .unwrap()
            / 100;

        for (idx, item) in cache_index.iter().enumerate() {
            let size = if let CacheEntry::Recognized { size, .. } = item {
                size
            } else {
                start_delete_idx = Some(idx);
                break;
            };

            total_size += size;
            if start_delete_idx_if_deleting_recognized_items.is_none()
                && (total_size > tsl_if_deleting || (idx + 1) as u64 > fcl_if_deleting)
            {
                start_delete_idx_if_deleting_recognized_items = Some(idx);
            }

            if total_size > total_size_limit || (idx + 1) as u64 > file_count_limit {
                start_delete_idx = start_delete_idx_if_deleting_recognized_items;
                break;
            }
        }

        if let Some(idx) = start_delete_idx {
            for item in &cache_index[idx..] {
                let (result, path, entity) = match item {
                    CacheEntry::Recognized { path, .. }
                    | CacheEntry::Unrecognized {
                        path,
                        is_dir: false,
                    } => (fs::remove_file(path), path, "file"),
                    CacheEntry::Unrecognized { path, is_dir: true } => {
                        (fs::remove_dir_all(path), path, "directory")
                    }
                };
                if let Err(err) = result {
                    warn!(
                        "Failed to remove {} during cleanup, path: {}, err: {}",
                        entity,
                        path.display(),
                        err
                    );
                }
            }
        }

        trace!("Task finished: clean up cache");
    }

pub fn files_total_size_limit_percent_if_deleting(&self) -> u8

Returns $setting.

Panics if the cache is disabled.

Examples found in repository

src/worker.rs (line 478)

    fn handle_on_cache_update(&self, path: PathBuf) {
        trace!("handle_on_cache_update() for path: {}", path.display());

        // ---------------------- step 1: create .stats file

        // construct .stats file path
        let filename = path
            .file_name()
            .expect("Expected valid cache file name")
            .to_str()
            .expect("Expected valid cache file name");
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // create and write stats file
        let mut stats = ModuleCacheStatistics::default(&self.cache_config);
        stats.usages += 1;
        write_stats_file(&stats_path, &stats);

        // ---------------------- step 2: perform cleanup task if needed

        // acquire lock for cleanup task
        // Lock is a proof of recent cleanup task, so we don't want to delete them.
        // Expired locks will be deleted by the cleanup task.
        let cleanup_file = self.cache_config.directory().join(".cleanup"); // some non existing marker file
        if acquire_task_fs_lock(
            &cleanup_file,
            self.cache_config.cleanup_interval(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        )
        .is_none()
        {
            return;
        }

        trace!("Trying to clean up cache");

        let mut cache_index = self.list_cache_contents();
        let future_tolerance = SystemTime::now()
            .checked_add(
                self.cache_config
                    .allowed_clock_drift_for_files_from_future(),
            )
            .expect("Brace your cache, the next Big Bang is coming (time overflow)");
        cache_index.sort_unstable_by(|lhs, rhs| {
            // sort by age
            use CacheEntry::*;
            match (lhs, rhs) {
                (Recognized { mtime: lhs_mt, .. }, Recognized { mtime: rhs_mt, .. }) => {
                    match (*lhs_mt > future_tolerance, *rhs_mt > future_tolerance) {
                        // later == younger
                        (false, false) => rhs_mt.cmp(lhs_mt),
                        // files from far future are treated as oldest recognized files
                        // we want to delete them, so the cache keeps track of recent files
                        // however, we don't delete them uncodintionally,
                        // because .stats file can be overwritten with a meaningful mtime
                        (true, false) => cmp::Ordering::Greater,
                        (false, true) => cmp::Ordering::Less,
                        (true, true) => cmp::Ordering::Equal,
                    }
                }
                // unrecognized is kind of infinity
                (Recognized { .. }, Unrecognized { .. }) => cmp::Ordering::Less,
                (Unrecognized { .. }, Recognized { .. }) => cmp::Ordering::Greater,
                (Unrecognized { .. }, Unrecognized { .. }) => cmp::Ordering::Equal,
            }
        });

        // find "cut" boundary:
        // - remove unrecognized files anyway,
        // - remove some cache files if some quota has been exceeded
        let mut total_size = 0u64;
        let mut start_delete_idx = None;
        let mut start_delete_idx_if_deleting_recognized_items: Option<usize> = None;

        let total_size_limit = self.cache_config.files_total_size_soft_limit();
        let file_count_limit = self.cache_config.file_count_soft_limit();
        let tsl_if_deleting = total_size_limit
            .checked_mul(
                self.cache_config
                    .files_total_size_limit_percent_if_deleting() as u64,
            )
            .unwrap()
            / 100;
        let fcl_if_deleting = file_count_limit
            .checked_mul(self.cache_config.file_count_limit_percent_if_deleting() as u64)
            .unwrap()
            / 100;

        for (idx, item) in cache_index.iter().enumerate() {
            let size = if let CacheEntry::Recognized { size, .. } = item {
                size
            } else {
                start_delete_idx = Some(idx);
                break;
            };

            total_size += size;
            if start_delete_idx_if_deleting_recognized_items.is_none()
                && (total_size > tsl_if_deleting || (idx + 1) as u64 > fcl_if_deleting)
            {
                start_delete_idx_if_deleting_recognized_items = Some(idx);
            }

            if total_size > total_size_limit || (idx + 1) as u64 > file_count_limit {
                start_delete_idx = start_delete_idx_if_deleting_recognized_items;
                break;
            }
        }

        if let Some(idx) = start_delete_idx {
            for item in &cache_index[idx..] {
                let (result, path, entity) = match item {
                    CacheEntry::Recognized { path, .. }
                    | CacheEntry::Unrecognized {
                        path,
                        is_dir: false,
                    } => (fs::remove_file(path), path, "file"),
                    CacheEntry::Unrecognized { path, is_dir: true } => {
                        (fs::remove_dir_all(path), path, "directory")
                    }
                };
                if let Err(err) = result {
                    warn!(
                        "Failed to remove {} during cleanup, path: {}, err: {}",
                        entity,
                        path.display(),
                        err
                    );
                }
            }
        }

        trace!("Task finished: clean up cache");
    }

pub fn enabled(&self) -> bool

Returns true if and only if the cache is enabled.

Examples found in repository

src/lib.rs (line 30)

    pub fn new<'data>(compiler_name: &str, cache_config: &'config CacheConfig) -> Self {
        if cache_config.enabled() {
            Self(Some(ModuleCacheEntryInner::new(
                compiler_name,
                cache_config,
            )))
        } else {
            Self(None)
        }
    }

pub fn directory(&self) -> &PathBuf

Returns path to the cache directory.

Panics if the cache is disabled.

Examples found in repository

src/lib.rs (line 144)

    fn new<'data>(compiler_name: &str, cache_config: &'config CacheConfig) -> Self {
        // If debug assertions are enabled then assume that we're some sort of
        // local build. We don't want local builds to stomp over caches between
        // builds, so just use a separate cache directory based on the mtime of
        // our executable, which should roughly correlate with "you changed the
        // source code so you get a different directory".
        //
        // Otherwise if this is a release build we use the `GIT_REV` env var
        // which is either the git rev if installed from git or the crate
        // version if installed from crates.io.
        let compiler_dir = if cfg!(debug_assertions) {
            fn self_mtime() -> Option<String> {
                let path = std::env::current_exe().ok()?;
                let metadata = path.metadata().ok()?;
                let mtime = metadata.modified().ok()?;
                Some(match mtime.duration_since(std::time::UNIX_EPOCH) {
                    Ok(dur) => format!("{}", dur.as_millis()),
                    Err(err) => format!("m{}", err.duration().as_millis()),
                })
            }
            let self_mtime = self_mtime().unwrap_or("no-mtime".to_string());
            format!(
                "{comp_name}-{comp_ver}-{comp_mtime}",
                comp_name = compiler_name,
                comp_ver = env!("GIT_REV"),
                comp_mtime = self_mtime,
            )
        } else {
            format!(
                "{comp_name}-{comp_ver}",
                comp_name = compiler_name,
                comp_ver = env!("GIT_REV"),
            )
        };
        let root_path = cache_config.directory().join("modules").join(compiler_dir);

        Self {
            root_path,
            cache_config,
        }
    }

src/worker.rs (line 421)

    fn handle_on_cache_update(&self, path: PathBuf) {
        trace!("handle_on_cache_update() for path: {}", path.display());

        // ---------------------- step 1: create .stats file

        // construct .stats file path
        let filename = path
            .file_name()
            .expect("Expected valid cache file name")
            .to_str()
            .expect("Expected valid cache file name");
        let stats_path = path.with_file_name(format!("{}.stats", filename));

        // create and write stats file
        let mut stats = ModuleCacheStatistics::default(&self.cache_config);
        stats.usages += 1;
        write_stats_file(&stats_path, &stats);

        // ---------------------- step 2: perform cleanup task if needed

        // acquire lock for cleanup task
        // Lock is a proof of recent cleanup task, so we don't want to delete them.
        // Expired locks will be deleted by the cleanup task.
        let cleanup_file = self.cache_config.directory().join(".cleanup"); // some non existing marker file
        if acquire_task_fs_lock(
            &cleanup_file,
            self.cache_config.cleanup_interval(),
            self.cache_config
                .allowed_clock_drift_for_files_from_future(),
        )
        .is_none()
        {
            return;
        }

        trace!("Trying to clean up cache");

        let mut cache_index = self.list_cache_contents();
        let future_tolerance = SystemTime::now()
            .checked_add(
                self.cache_config
                    .allowed_clock_drift_for_files_from_future(),
            )
            .expect("Brace your cache, the next Big Bang is coming (time overflow)");
        cache_index.sort_unstable_by(|lhs, rhs| {
            // sort by age
            use CacheEntry::*;
            match (lhs, rhs) {
                (Recognized { mtime: lhs_mt, .. }, Recognized { mtime: rhs_mt, .. }) => {
                    match (*lhs_mt > future_tolerance, *rhs_mt > future_tolerance) {
                        // later == younger
                        (false, false) => rhs_mt.cmp(lhs_mt),
                        // files from far future are treated as oldest recognized files
                        // we want to delete them, so the cache keeps track of recent files
                        // however, we don't delete them uncodintionally,
                        // because .stats file can be overwritten with a meaningful mtime
                        (true, false) => cmp::Ordering::Greater,
                        (false, true) => cmp::Ordering::Less,
                        (true, true) => cmp::Ordering::Equal,
                    }
                }
                // unrecognized is kind of infinity
                (Recognized { .. }, Unrecognized { .. }) => cmp::Ordering::Less,
                (Unrecognized { .. }, Recognized { .. }) => cmp::Ordering::Greater,
                (Unrecognized { .. }, Unrecognized { .. }) => cmp::Ordering::Equal,
            }
        });

        // find "cut" boundary:
        // - remove unrecognized files anyway,
        // - remove some cache files if some quota has been exceeded
        let mut total_size = 0u64;
        let mut start_delete_idx = None;
        let mut start_delete_idx_if_deleting_recognized_items: Option<usize> = None;

        let total_size_limit = self.cache_config.files_total_size_soft_limit();
        let file_count_limit = self.cache_config.file_count_soft_limit();
        let tsl_if_deleting = total_size_limit
            .checked_mul(
                self.cache_config
                    .files_total_size_limit_percent_if_deleting() as u64,
            )
            .unwrap()
            / 100;
        let fcl_if_deleting = file_count_limit
            .checked_mul(self.cache_config.file_count_limit_percent_if_deleting() as u64)
            .unwrap()
            / 100;

        for (idx, item) in cache_index.iter().enumerate() {
            let size = if let CacheEntry::Recognized { size, .. } = item {
                size
            } else {
                start_delete_idx = Some(idx);
                break;
            };

            total_size += size;
            if start_delete_idx_if_deleting_recognized_items.is_none()
                && (total_size > tsl_if_deleting || (idx + 1) as u64 > fcl_if_deleting)
            {
                start_delete_idx_if_deleting_recognized_items = Some(idx);
            }

            if total_size > total_size_limit || (idx + 1) as u64 > file_count_limit {
                start_delete_idx = start_delete_idx_if_deleting_recognized_items;
                break;
            }
        }

        if let Some(idx) = start_delete_idx {
            for item in &cache_index[idx..] {
                let (result, path, entity) = match item {
                    CacheEntry::Recognized { path, .. }
                    | CacheEntry::Unrecognized {
                        path,
                        is_dir: false,
                    } => (fs::remove_file(path), path, "file"),
                    CacheEntry::Unrecognized { path, is_dir: true } => {
                        (fs::remove_dir_all(path), path, "directory")
                    }
                };
                if let Err(err) = result {
                    warn!(
                        "Failed to remove {} during cleanup, path: {}, err: {}",
                        entity,
                        path.display(),
                        err
                    );
                }
            }
        }

        trace!("Task finished: clean up cache");
    }

    // Be fault tolerant: list as much as you can, and ignore the rest
    fn list_cache_contents(&self) -> Vec<CacheEntry> {
        fn enter_dir(
            vec: &mut Vec<CacheEntry>,
            dir_path: &Path,
            level: u8,
            cache_config: &CacheConfig,
        ) {
            macro_rules! add_unrecognized {
                (file: $path:expr) => {
                    add_unrecognized!(false, $path)
                };
                (dir: $path:expr) => {
                    add_unrecognized!(true, $path)
                };
                ($is_dir:expr, $path:expr) => {
                    vec.push(CacheEntry::Unrecognized {
                        path: $path.to_path_buf(),
                        is_dir: $is_dir,
                    })
                };
            }
            macro_rules! add_unrecognized_and {
                ([ $( $ty:ident: $path:expr ),* ], $cont:stmt) => {{
                    $( add_unrecognized!($ty: $path); )*
                        $cont
                }};
            }

            macro_rules! unwrap_or {
                ($result:expr, $cont:stmt, $err_msg:expr) => {
                    unwrap_or!($result, $cont, $err_msg, dir_path)
                };
                ($result:expr, $cont:stmt, $err_msg:expr, $path:expr) => {
                    unwrap_or_warn!(
                        $result,
                        $cont,
                        format!("{}, level: {}", $err_msg, level),
                        $path
                    )
                };
            }

            // If we fail to list a directory, something bad is happening anyway
            // (something touches our cache or we have disk failure)
            // Try to delete it, so we can stay within soft limits of the cache size.
            // This comment applies later in this function, too.
            let it = unwrap_or!(
                fs::read_dir(dir_path),
                add_unrecognized_and!([dir: dir_path], return),
                "Failed to list cache directory, deleting it"
            );

            let mut cache_files = HashMap::new();
            for entry in it {
                // read_dir() returns an iterator over results - in case some of them are errors
                // we don't know their names, so we can't delete them. We don't want to delete
                // the whole directory with good entries too, so we just ignore the erroneous entries.
                let entry = unwrap_or!(
                    entry,
                    continue,
                    "Failed to read a cache dir entry (NOT deleting it, it still occupies space)"
                );
                let path = entry.path();
                match (level, path.is_dir()) {
                    (0..=1, true) => enter_dir(vec, &path, level + 1, cache_config),
                    (0..=1, false) => {
                        if level == 0
                            && path.file_stem() == Some(OsStr::new(".cleanup"))
                                && path.extension().is_some()
                                // assume it's cleanup lock
                                && !is_fs_lock_expired(
                                    Some(&entry),
                                    &path,
                                    cache_config.cleanup_interval(),
                                    cache_config.allowed_clock_drift_for_files_from_future(),
                                )
                        {
                            continue; // skip active lock
                        }
                        add_unrecognized!(file: path);
                    }
                    (2, false) => {
                        match path.extension().and_then(OsStr::to_str) {
                            // mod or stats file
                            None | Some("stats") => {
                                cache_files.insert(path, entry);
                            }

                            Some(ext) => {
                                // check if valid lock
                                let recognized = ext.starts_with("wip-")
                                    && !is_fs_lock_expired(
                                        Some(&entry),
                                        &path,
                                        cache_config.optimizing_compression_task_timeout(),
                                        cache_config.allowed_clock_drift_for_files_from_future(),
                                    );

                                if !recognized {
                                    add_unrecognized!(file: path);
                                }
                            }
                        }
                    }
                    (_, is_dir) => add_unrecognized!(is_dir, path),
                }
            }

            // associate module with its stats & handle them
            // assumption: just mods and stats
            for (path, entry) in cache_files.iter() {
                let path_buf: PathBuf;
                let (mod_, stats_, is_mod) = match path.extension() {
                    Some(_) => {
                        path_buf = path.with_extension("");
                        (
                            cache_files.get(&path_buf).map(|v| (&path_buf, v)),
                            Some((path, entry)),
                            false,
                        )
                    }
                    None => {
                        path_buf = path.with_extension("stats");
                        (
                            Some((path, entry)),
                            cache_files.get(&path_buf).map(|v| (&path_buf, v)),
                            true,
                        )
                    }
                };

                // construct a cache entry
                match (mod_, stats_, is_mod) {
                    (Some((mod_path, mod_entry)), Some((stats_path, stats_entry)), true) => {
                        let mod_metadata = unwrap_or!(
                            mod_entry.metadata(),
                            add_unrecognized_and!([file: stats_path, file: mod_path], continue),
                            "Failed to get metadata, deleting BOTH module cache and stats files",
                            mod_path
                        );
                        let stats_mtime = unwrap_or!(
                            stats_entry.metadata().and_then(|m| m.modified()),
                            add_unrecognized_and!(
                                [file: stats_path],
                                unwrap_or!(
                                    mod_metadata.modified(),
                                    add_unrecognized_and!(
                                        [file: stats_path, file: mod_path],
                                        continue
                                    ),
                                    "Failed to get mtime, deleting BOTH module cache and stats \
                                     files",
                                    mod_path
                                )
                            ),
                            "Failed to get metadata/mtime, deleting the file",
                            stats_path
                        );
                        // .into() called for the SystemTimeStub if cfg(test)
                        #[allow(clippy::identity_conversion)]
                        vec.push(CacheEntry::Recognized {
                            path: mod_path.to_path_buf(),
                            mtime: stats_mtime.into(),
                            size: mod_metadata.len(),
                        })
                    }
                    (Some(_), Some(_), false) => (), // was or will be handled by previous branch
                    (Some((mod_path, mod_entry)), None, _) => {
                        let (mod_metadata, mod_mtime) = unwrap_or!(
                            mod_entry
                                .metadata()
                                .and_then(|md| md.modified().map(|mt| (md, mt))),
                            add_unrecognized_and!([file: mod_path], continue),
                            "Failed to get metadata/mtime, deleting the file",
                            mod_path
                        );
                        // .into() called for the SystemTimeStub if cfg(test)
                        #[allow(clippy::identity_conversion)]
                        vec.push(CacheEntry::Recognized {
                            path: mod_path.to_path_buf(),
                            mtime: mod_mtime.into(),
                            size: mod_metadata.len(),
                        })
                    }
                    (None, Some((stats_path, _stats_entry)), _) => {
                        debug!("Found orphaned stats file: {}", stats_path.display());
                        add_unrecognized!(file: stats_path);
                    }
                    _ => unreachable!(),
                }
            }
        }

        let mut vec = Vec::new();
        enter_dir(
            &mut vec,
            self.cache_config.directory(),
            0,
            &self.cache_config,
        );
        vec
    }

pub fn new_cache_disabled() -> Self

Creates a new set of configuration which represents a disabled cache

Examples found in repository

src/config.rs (line 340)

    fn new_cache_enabled_template() -> Self {
        let mut conf = Self::new_cache_disabled();
        conf.enabled = true;
        conf
    }

pub fn from_file(config_file: Option<&Path>) -> Result<Self>

Parses cache configuration from the file specified

pub fn cache_hits(&self) -> usize

Returns the number of cache hits seen so far

pub fn cache_misses(&self) -> usize

Returns the number of cache misses seen so far

Trait Implementations

impl Clone for CacheConfig

fn clone(&self) -> CacheConfig

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CacheConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for CacheConfig

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
    __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.