use anyhow::Context;
use std::{
collections::HashMap,
fs,
path::{Path, PathBuf},
sync::{Arc, Mutex},
time::{Duration, Instant, SystemTime},
};
#[derive(Debug)]
pub(super) struct FsCache {
ttl: Duration,
entries: Mutex<HashMap<PathBuf, CacheEntry>>,
}
#[derive(Debug)]
struct DirectoryState {
dir: PathBuf,
modified: SystemTime,
}
#[derive(Debug, Clone)]
struct CacheEntry {
modified: SystemTime,
inserted: Instant,
listing: Arc<[DirectoryListingEntry]>,
}
#[derive(Debug, Clone)]
pub(super) struct DirectoryListingEntry {
pub(super) name: String,
pub(super) file_type: fs::FileType,
}
#[derive(Debug, Clone)]
pub(super) struct DirectoryListing {
pub(super) entries: Arc<[DirectoryListingEntry]>,
#[cfg(test)]
pub(super) cache_hit: bool,
}
impl Default for FsCache {
fn default() -> Self {
Self::with_ttl(Duration::from_secs(2))
}
}
impl FsCache {
pub(super) fn with_ttl(ttl: Duration) -> Self {
Self {
ttl,
entries: Mutex::new(HashMap::new()),
}
}
pub(super) fn read_dir_listing(&self, dir: &Path) -> anyhow::Result<DirectoryListing> {
let state = self.state_for_dir(dir)?;
if let Some(entries) = self.get_valid(&state)? {
return Ok(DirectoryListing {
entries,
#[cfg(test)]
cache_hit: true,
});
}
let entries = Arc::from(read_directory_entries(&state.dir)?);
self.store(state, Arc::clone(&entries))?;
Ok(DirectoryListing {
entries,
#[cfg(test)]
cache_hit: false,
})
}
pub(super) fn len(&self) -> anyhow::Result<usize> {
self.entries
.lock()
.map(|entries| entries.len())
.map_err(|_| anyhow::anyhow!("fs cache lock poisoned"))
}
fn state_for_dir(&self, dir: &Path) -> anyhow::Result<DirectoryState> {
let canonical = dir
.canonicalize()
.with_context(|| format!("failed to canonicalize directory '{}'", dir.display()))?;
let modified = fs::metadata(&canonical)
.with_context(|| format!("failed to read metadata for '{}'", canonical.display()))?
.modified()
.with_context(|| {
format!(
"failed to read modified time for directory '{}'",
canonical.display()
)
})?;
Ok(DirectoryState {
dir: canonical,
modified,
})
}
fn get_valid(
&self,
state: &DirectoryState,
) -> anyhow::Result<Option<Arc<[DirectoryListingEntry]>>> {
let mut entries = self
.entries
.lock()
.map_err(|_| anyhow::anyhow!("fs cache lock poisoned"))?;
entries.retain(|_, entry| entry.inserted.elapsed() <= self.ttl);
Ok(entries.get(&state.dir).and_then(|entry| {
(entry.modified == state.modified).then(|| Arc::clone(&entry.listing))
}))
}
fn store(
&self,
state: DirectoryState,
listing: Arc<[DirectoryListingEntry]>,
) -> anyhow::Result<()> {
let mut entries = self
.entries
.lock()
.map_err(|_| anyhow::anyhow!("fs cache lock poisoned"))?;
entries.retain(|_, entry| entry.inserted.elapsed() <= self.ttl);
entries.insert(
state.dir,
CacheEntry {
modified: state.modified,
inserted: Instant::now(),
listing,
},
);
Ok(())
}
}
fn read_directory_entries(dir: &Path) -> anyhow::Result<Vec<DirectoryListingEntry>> {
let mut entries = Vec::new();
for entry in fs::read_dir(dir)
.with_context(|| format!("failed to read directory '{}'", dir.display()))?
{
let entry = entry
.with_context(|| format!("failed to read entry in directory '{}'", dir.display()))?;
let path = entry.path();
let file_type = entry
.file_type()
.with_context(|| format!("failed to read file type for '{}'", path.display()))?;
entries.push(DirectoryListingEntry {
name: entry.file_name().to_string_lossy().into_owned(),
file_type,
});
}
Ok(entries)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fs_cache_repeated_read_hits_cache() {
let temp = tempfile::TempDir::new().unwrap();
fs::write(temp.path().join("visible.txt"), "content").unwrap();
let cache = FsCache::default();
let first = cache.read_dir_listing(temp.path()).unwrap();
let second = cache.read_dir_listing(temp.path()).unwrap();
assert!(!first.cache_hit);
assert!(second.cache_hit);
assert_eq!(second.entries.len(), 1);
assert_eq!(second.entries[0].name, "visible.txt");
assert!(second.entries[0].file_type.is_file());
}
#[test]
fn fs_cache_directory_mutation_replaces_stale_listing() {
let temp = tempfile::TempDir::new().unwrap();
fs::write(temp.path().join("before.txt"), "x").unwrap();
let cache = FsCache::default();
let first = cache.read_dir_listing(temp.path()).unwrap();
let initial_modified = cache.state_for_dir(temp.path()).unwrap().modified;
let mut changed = false;
for index in 0..100 {
fs::write(temp.path().join(format!("after-{index}.txt")), "x").unwrap();
if cache.state_for_dir(temp.path()).unwrap().modified != initial_modified {
changed = true;
break;
}
std::thread::sleep(Duration::from_millis(10));
}
assert!(
changed,
"test filesystem did not expose a directory mtime change"
);
let second = cache.read_dir_listing(temp.path()).unwrap();
assert!(!first.cache_hit);
assert!(!second.cache_hit);
assert!(second.entries.len() > first.entries.len());
assert_eq!(cache.len().unwrap(), 1);
}
#[test]
fn fs_cache_concurrent_hits_share_one_listing() {
let temp = tempfile::TempDir::new().unwrap();
fs::write(temp.path().join("shared.txt"), "x").unwrap();
let cache = Arc::new(FsCache::default());
let warm = cache.read_dir_listing(temp.path()).unwrap().entries;
let path = temp.path().to_path_buf();
let handles = (0..16)
.map(|_| {
let cache = Arc::clone(&cache);
let path = path.clone();
std::thread::spawn(move || cache.read_dir_listing(&path).unwrap())
})
.collect::<Vec<_>>();
for handle in handles {
let listing = handle.join().unwrap();
assert!(listing.cache_hit);
assert!(Arc::ptr_eq(&warm, &listing.entries));
}
assert_eq!(cache.len().unwrap(), 1);
}
#[test]
fn fs_cache_supersedes_modified_directory_without_growing() {
let temp = tempfile::TempDir::new().unwrap();
let cache = FsCache::default();
let state = cache.state_for_dir(temp.path()).unwrap();
cache
.store(
DirectoryState {
dir: state.dir.clone(),
modified: SystemTime::UNIX_EPOCH,
},
Arc::from(Vec::new()),
)
.unwrap();
cache
.store(
DirectoryState {
dir: state.dir,
modified: SystemTime::UNIX_EPOCH + Duration::from_secs(1),
},
Arc::from(Vec::new()),
)
.unwrap();
assert_eq!(cache.len().unwrap(), 1);
}
#[test]
fn fs_cache_prunes_expired_entries_during_normal_reads() {
let first = tempfile::TempDir::new().unwrap();
let second = tempfile::TempDir::new().unwrap();
let cache = FsCache::with_ttl(Duration::from_millis(1));
cache.read_dir_listing(first.path()).unwrap();
std::thread::sleep(Duration::from_millis(10));
cache.read_dir_listing(second.path()).unwrap();
assert_eq!(cache.len().unwrap(), 1);
}
#[test]
fn fs_cache_hits_share_large_listings_without_entry_clones() {
let temp = tempfile::TempDir::new().unwrap();
for index in 0..2_000 {
fs::write(temp.path().join(format!("entry-{index:04}.txt")), "x").unwrap();
}
let cache = FsCache::default();
let first = cache.read_dir_listing(temp.path()).unwrap();
let second = cache.read_dir_listing(temp.path()).unwrap();
assert!(second.cache_hit);
assert!(Arc::ptr_eq(&first.entries, &second.entries));
assert_eq!(second.entries.len(), 2_000);
}
#[test]
#[ignore = "release-mode filesystem cache measurement; run with --release --ignored --nocapture"]
fn filesystem_cache_large_directory_measurement() {
let temp = tempfile::TempDir::new().unwrap();
for index in 0..10_000 {
fs::write(temp.path().join(format!("entry-{index:05}.txt")), "x").unwrap();
}
let cache = FsCache::default();
let first = cache.read_dir_listing(temp.path()).unwrap();
let started = Instant::now();
let second = cache.read_dir_listing(temp.path()).unwrap();
let elapsed = started.elapsed();
let retained_name_bytes = second
.entries
.iter()
.map(|entry| entry.name.len())
.sum::<usize>();
eprintln!(
"fs_cache entries={} retained_paths={} retained_name_bytes={} hit_us={} shared_listing={} cloned_entries=0",
second.entries.len(),
cache.len().unwrap(),
retained_name_bytes,
elapsed.as_micros(),
Arc::ptr_eq(&first.entries, &second.entries),
);
}
}