use std::time::SystemTime;
use crate::query::{
Delivery, Report, ReportProvenance, ReportSource, Request, SizeMetric, SortKey, SummaryRow,
TreeStatus, ViewSpec, report, report_summary,
};
use crate::{CachePolicy, EntryKind, Error, OpenPath, PendingSave, Progress, Result, execute};
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(crate) enum RetainedState {
Summary,
Tree(TreeRetention),
FullIndex,
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(crate) struct TreeRetention {
pub(crate) largest_files: u32,
pub(crate) size: SizeMetric,
}
impl TreeRetention {
pub(crate) const MAX_FILES: u32 = 65_536;
fn for_request(request: &Request) -> Option<Self> {
let query = &request.query;
let tree_only = !query.views.is_empty()
&& query.views.iter().all(|view| {
matches!(view, ViewSpec::List | ViewSpec::Tree) && query.tree_for(*view)
});
if !tree_only
|| !query.selection.is_unfiltered()
|| matches!(query.selection.sort, Some(SortKey::Metric(_)))
|| request.basis.scope.population != crate::query::IgnoredEntries::Include
{
return None;
}
let largest_files = query
.min_share_for()
.admitted_parts_bound()
.and_then(|bound| u32::try_from(bound).ok())
.filter(|bound| *bound <= Self::MAX_FILES)?;
Some(Self { largest_files, size: query.selection.size })
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Route {
OneShot,
Retained,
Refresh,
Watch,
Opened,
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Load {
None,
Snapshot,
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Verify {
None,
Filesystem,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum OutcomeClass {
Success,
Partial,
}
#[derive(Clone, Debug)]
pub struct Plan {
pub(crate) basis: crate::query::Basis,
pub(crate) route: Route,
pub(crate) retained: RetainedState,
pub(crate) load: Load,
pub(crate) verify: Verify,
pub(crate) persist: bool,
pub(crate) delivery: Delivery,
}
impl Plan {
pub fn outcome(&self, status: &TreeStatus) -> OutcomeClass {
if status.complete || self.delivery.accept_partial {
OutcomeClass::Success
} else {
OutcomeClass::Partial
}
}
pub fn basis(&self) -> &crate::query::Basis {
&self.basis
}
pub const fn route(&self) -> Route {
self.route
}
pub const fn load(&self) -> Load {
self.load
}
pub const fn verify(&self) -> Verify {
self.verify
}
pub const fn persists(&self) -> bool {
self.persist
}
pub fn delivery(&self) -> &Delivery {
&self.delivery
}
}
pub(crate) struct StoreHeader<'a> {
pub(crate) root: &'a std::path::Path,
pub(crate) snapshot: crate::SnapshotIdentity,
pub(crate) content: Option<&'a crate::ContentTierIdentity>,
pub(crate) content_complete: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Admission {
Serve(crate::Serves),
NoLocation,
Missing,
WrongRoot,
WrongScope,
IncompleteContent,
}
impl Plan {
pub(crate) fn admit(
&self,
stored: Option<&StoreHeader<'_>>,
basis: &crate::query::Basis,
) -> Admission {
if self.delivery.cache_path.is_none() {
return Admission::NoLocation;
}
let Some(stored) = stored else {
return Admission::Missing;
};
if stored.root != basis.root {
return Admission::WrongRoot;
}
let relation = crate::serves_snapshot(stored.snapshot, basis.scope.snapshot_identity());
if relation == crate::Serves::Refuse {
return Admission::WrongScope;
}
if self.verify == Verify::None && basis.content.is_enabled() {
let wanted = crate::ContentTierIdentity::for_request(
basis.scope.snapshot_identity().entries,
basis.content,
);
if !stored.content_complete
|| stored.content.and_then(|identity| wanted.admit(identity)).is_none()
{
return Admission::IncompleteContent;
}
}
Admission::Serve(relation)
}
}
#[derive(Clone, Copy, Debug)]
#[allow(clippy::struct_excessive_bools)]
pub(crate) struct RunFacts {
pub(crate) entries_verified: bool,
pub(crate) entries_changed: bool,
pub(crate) content_changed: bool,
pub(crate) content_requested: bool,
pub(crate) projected: bool,
pub(crate) paired_entries: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct SaveTargets {
pub(crate) metadata: bool,
pub(crate) content: bool,
}
impl SaveTargets {
pub(crate) const fn none(self) -> bool {
!self.metadata && !self.content
}
}
impl Plan {
pub(crate) fn writes(&self, run: RunFacts) -> SaveTargets {
let allowed = self.persist && self.delivery.cache_path.is_some();
SaveTargets {
metadata: allowed && run.entries_verified && run.entries_changed && !run.projected,
content: allowed
&& run.content_requested
&& run.content_changed
&& (run.entries_verified || run.paired_entries),
}
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub struct PerformanceSummary {
pub walked_files: u64,
pub walked_bytes: u64,
pub walked_allocated: u64,
pub fresh_files: u64,
pub bytes_read: u64,
pub analysis_ns: u64,
pub cached_files: u64,
pub cached_bytes: u64,
pub source: ReportSource,
}
impl Default for PerformanceSummary {
fn default() -> Self {
Self {
walked_files: 0,
walked_bytes: 0,
walked_allocated: 0,
fresh_files: 0,
bytes_read: 0,
analysis_ns: 0,
cached_files: 0,
cached_bytes: 0,
source: ReportSource::ColdScan,
}
}
}
impl PerformanceSummary {
pub fn total_throughput(
self,
elapsed: std::time::Duration,
size: crate::query::SizeMetric,
) -> String {
let bytes = match size {
crate::query::SizeMetric::Apparent => self.walked_bytes,
crate::query::SizeMetric::Allocated => self.walked_allocated,
};
throughput_rates(self.walked_files, bytes, elapsed).map_or_else(
|| "throughput unavailable".to_owned(),
|(files, gib)| format!("{files} files/s ({gib} GiB/s)"),
)
}
fn from_open_report(report: &crate::OpenReport) -> Self {
let analysis = report.analysis.unwrap_or_default();
Self {
walked_files: report.scan.files_walked,
walked_bytes: report.scan.bytes_walked,
walked_allocated: report.scan.allocated_walked,
fresh_files: analysis.candidates,
bytes_read: analysis.bytes_read,
analysis_ns: analysis.elapsed_ns,
cached_files: report.content_cache.hits,
cached_bytes: report.content_cache.bytes,
source: match report.path_taken {
OpenPath::ColdScan => ReportSource::ColdScan,
OpenPath::WarmRevalidate => ReportSource::WarmRevalidate,
OpenPath::CacheOnly => ReportSource::CacheOnly,
},
}
}
}
pub fn throughput_rates(
files: u64,
bytes: u64,
elapsed: std::time::Duration,
) -> Option<(String, String)> {
let ns = elapsed.as_nanos();
if ns == 0 {
return None;
}
let files_per_second = u128::from(files) * 1_000_000_000 / ns;
let gib_denominator = ns * (1_u128 << 30);
let gib_thousandths =
(u128::from(bytes) * 1_000_000_000 * 1_000 + gib_denominator / 2) / gib_denominator;
Some((
crate::report_format::human_count_u128(files_per_second),
format!("{}.{:03}", gib_thousandths / 1_000, gib_thousandths % 1_000),
))
}
pub fn plan(
request: &Request,
delivery: &Delivery,
route: Route,
) -> std::result::Result<Plan, crate::query::RequestError> {
request.validate()?;
let mut normalized = delivery.clone();
if route == Route::Watch {
normalized.watch.get_or_insert_with(crate::query::WatchDelivery::default);
}
let delivery = &normalized;
request.validate_delivery(delivery)?;
if route == Route::Opened {
if delivery.cache != CachePolicy::Off
|| delivery.watch.is_some()
|| request.basis.content.is_enabled()
{
return Err(crate::query::RequestError::DeliveryUnsupported {
route: "opened",
reason: "progressive discovery requires cache off, no content analyzers, and observation configured through OpenOptions",
});
}
if delivery.workers.scan.is_some()
|| delivery.order != crate::ScanOrder::default()
|| delivery.accept_partial
{
return Err(crate::query::RequestError::DeliveryUnsupported {
route: "opened",
reason: "progressive discovery schedules one breadth-first producer and reports coverage as state, so it takes no scan worker count, traversal order, or partial-answer acceptance",
});
}
}
if route == Route::Refresh && delivery.stale_ok {
return Err(crate::query::RequestError::DeliveryUnsupported {
route: "refresh",
reason: "a stale answer cannot verify filesystem state",
});
}
let analysis_requested = request.basis.content.is_enabled();
let summary_is_sufficient = request.query.views.as_slice() == [ViewSpec::Summary]
&& request.query.selection.is_unfiltered()
&& request.basis.scope.population == crate::query::IgnoredEntries::Include;
let policy_requires_index =
delivery.stale_ok || (delivery.cache == CachePolicy::On && delivery.cache_path.is_some());
let stored_state_pays = route != Route::OneShot || analysis_requested;
let read_snapshot = delivery.stale_ok
|| match delivery.cache {
CachePolicy::Off => false,
CachePolicy::Auto | CachePolicy::On => stored_state_pays,
};
let persist = !delivery.stale_ok
&& match delivery.cache {
CachePolicy::Off => false,
CachePolicy::Auto => stored_state_pays,
CachePolicy::On => true,
};
let transient = route == Route::OneShot && !policy_requires_index && !analysis_requested;
let retained = if !transient {
RetainedState::FullIndex
} else if summary_is_sufficient {
RetainedState::Summary
} else if let Some(retention) = TreeRetention::for_request(request) {
RetainedState::Tree(retention)
} else {
RetainedState::FullIndex
};
Ok(Plan {
basis: request.basis.clone(),
route,
retained,
load: if read_snapshot { Load::Snapshot } else { Load::None },
verify: if delivery.stale_ok { Verify::None } else { Verify::Filesystem },
persist,
delivery: delivery.clone(),
})
}
pub fn prepare_report(
request: &Request,
delivery: &Delivery,
) -> Result<(Report, PendingSave, PerformanceSummary)> {
prepare_report_internal(request, delivery, false, None)
.map(|(report, pending, performance, _diagnostics)| (report, pending, performance))
}
pub fn prepare_report_with_progress(
request: &Request,
delivery: &Delivery,
progress: &Progress,
) -> Result<(Report, PendingSave, PerformanceSummary)> {
prepare_report_internal(request, delivery, false, Some(progress))
.map(|(report, pending, performance, _diagnostics)| (report, pending, performance))
}
pub fn prepare_report_with_scan_diagnostics(
request: &Request,
delivery: &Delivery,
) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)> {
prepare_report_internal(request, delivery, true, None)
}
fn prepare_report_internal(
request: &Request,
delivery: &Delivery,
collect_scan_diagnostics: bool,
progress: Option<&Progress>,
) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)> {
request.validate().map_err(Error::InvalidRequest)?;
let scan_config = crate::ScanConfig {
progress: progress.cloned(),
..request.basis.scope.scan_config(delivery)
};
let root = request.basis.root.as_path();
let scan_started_at = SystemTime::now();
let plan = plan(request, delivery, Route::OneShot).map_err(Error::InvalidRequest)?;
match plan.retained {
RetainedState::Summary => {
let root = root.canonicalize().map_err(|error| Error::io(root, error))?;
let mut fold = SummaryFold::new(&scan_config);
let mut reduce = |observed: &crate::ObservationOp| fold.observe(observed);
let (mut scan, scan_diagnostics) = if collect_scan_diagnostics {
let (scan, diagnostics) = crate::scan::scan_summary_fold_with_diagnostics(
&root,
&scan_config,
&mut reduce,
)?;
(scan, Some(diagnostics))
} else {
(crate::scan::scan_summary_fold(&root, &scan_config, &mut reduce)?, None)
};
let complete = scan.is_complete();
let generated_at = SystemTime::now();
let (summary, ignore_rules, ignored_unverified) = fold.finish(&root, &scan.errors)?;
let report = report_summary(
&root,
scan_config.scope(),
request,
summary,
ignore_rules,
ignored_unverified,
TreeStatus::of_walk(&root, &mut scan),
ReportProvenance::of_walk(scan_started_at, generated_at, complete),
);
let performance = PerformanceSummary {
walked_files: scan.files_walked,
walked_bytes: scan.bytes_walked,
walked_allocated: scan.allocated_walked,
source: ReportSource::ColdScan,
..PerformanceSummary::default()
};
Ok((report, PendingSave::none(), performance, scan_diagnostics))
}
RetainedState::Tree(retention) => {
let root = root.canonicalize().map_err(|error| Error::io(root, error))?;
let (index, scan, scan_diagnostics) = crate::scan::scan_into_folded_index(
&root,
&scan_config,
retention,
collect_scan_diagnostics,
)?;
let performance = PerformanceSummary {
walked_files: scan.files_walked,
walked_bytes: scan.bytes_walked,
walked_allocated: scan.allocated_walked,
source: ReportSource::ColdScan,
..PerformanceSummary::default()
};
if let Some(progress) = progress {
progress.enter(crate::ProgressPhase::Summarizing);
}
let answer = report(&index, request, SystemTime::now())?;
debug_assert_eq!(answer.scope, scan_config.scope());
crate::release_index(std::sync::Arc::new(index));
Ok((answer, PendingSave::none(), performance, scan_diagnostics))
}
RetainedState::FullIndex => {
let (index, open_report, pending_save, scan_diagnostics) =
execute(&plan, &request.basis, collect_scan_diagnostics, progress)?;
let performance = PerformanceSummary::from_open_report(&open_report);
if let Some(progress) = progress {
progress.enter(crate::ProgressPhase::Summarizing);
}
let answer = report(&index, request, SystemTime::now())?;
debug_assert_eq!(answer.scope, scan_config.scope());
crate::release_index(index);
Ok((answer, pending_save, performance, scan_diagnostics))
}
}
}
struct SummaryFold {
all: SummaryRow,
controls: Option<SummaryControls>,
unrepresentable: Option<Error>,
}
struct SummaryControls {
table: crate::control::ControlTable,
ignored_heads: std::collections::HashSet<std::ffi::OsString>,
parent: Option<(std::ffi::OsString, bool)>,
chain:
Option<(std::ffi::OsString, crate::control::ControlChain, crate::control::SplitDirectory)>,
unignored: crate::index::RollUpScalars,
rejected: Option<Error>,
}
impl SummaryFold {
fn new(config: &crate::ScanConfig) -> Self {
Self {
all: SummaryRow::default(),
controls: config.read_controls.then(|| SummaryControls {
table: crate::control::ControlTable::with_limits(config.control_limits),
ignored_heads: std::collections::HashSet::new(),
parent: None,
chain: None,
unignored: crate::index::RollUpScalars::default(),
rejected: None,
}),
unrepresentable: None,
}
}
fn observe(&mut self, observed: &crate::ObservationOp) {
match &observed.op {
crate::Op::Upsert { path, kind, attrs } => {
match kind {
EntryKind::File => {
if let Err(error) = crate::index::add_file_sizes(
&mut self.all.bytes,
&mut self.all.allocated,
attrs,
|| path.clone(),
) {
self.unrepresentable.get_or_insert(error);
return;
}
self.all.files += 1;
self.all.newest_mtime_ns = Some(
self.all
.newest_mtime_ns
.map_or(attrs.mtime_ns, |current| current.max(attrs.mtime_ns)),
);
}
EntryKind::Dir => self.all.dirs += 1,
EntryKind::Symlink | EntryKind::Other => {}
}
let Some(controls) = &mut self.controls else { return };
if controls.classify(path, *kind) {
return;
}
match kind {
EntryKind::File => {
controls.unignored.files += 1;
controls.unignored.bytes += attrs.size;
controls.unignored.allocated += attrs.allocated;
}
EntryKind::Dir => controls.unignored.dirs += 1,
EntryKind::Symlink | EntryKind::Other => {}
}
}
crate::Op::ControlUpsert { path, source } => {
if let Some(controls) = &mut self.controls {
controls.chain = None;
let admitted = controls.table.upsert(path, source.clone()).map(drop);
controls.record(admitted);
}
}
crate::Op::ControlRemove { path } => {
if let Some(controls) = &mut self.controls {
controls.chain = None;
let removed = controls.table.remove(path).map(drop);
controls.record(removed);
}
}
crate::Op::Remove { .. } | crate::Op::InvalidateSubtree { .. } => {}
}
}
fn finish(
self,
root: &std::path::Path,
errors: &[Error],
) -> Result<(SummaryRow, crate::control::ControlCoverage, bool)> {
if let Some(error) = self.unrepresentable {
return Err(error);
}
let Some(controls) = self.controls else {
return Ok((
SummaryRow { ignored: None, ..self.all },
crate::control::ControlCoverage::NotObserved,
false,
));
};
if let Some(error) = controls.rejected {
return Err(error);
}
let unreadable =
errors.iter().any(|error| crate::control::unreadable_control(root, error).is_some());
let verified = controls.table.refused_len() == 0 && !unreadable;
let all = crate::index::RollUpScalars {
files: self.all.files,
dirs: self.all.dirs,
bytes: self.all.bytes,
allocated: self.all.allocated,
newest_mtime_ns: self.all.newest_mtime_ns.unwrap_or_default(),
};
let summary = SummaryRow {
ignored: verified.then(|| crate::query::IgnoredTally::between(all, controls.unignored)),
..self.all
};
Ok((
summary,
crate::control::ControlCoverage::Observed(controls.table.observation()),
!verified,
))
}
}
impl SummaryControls {
fn classify(&mut self, path: &std::path::Path, kind: EntryKind) -> bool {
if self.ignored_heads.is_empty() && self.table.is_empty() {
return false;
}
let (parent, name) = crate::control::split_parent(path);
let parent_ignored = self.parent_ignored(parent);
let ignored = if parent_ignored || self.table.is_empty() {
parent_ignored
} else if !name.is_empty() {
if !matches!(&self.chain, Some((cached, ..)) if cached.as_os_str() == parent) {
let directory = std::path::Path::new(parent);
self.chain = Some((
parent.to_os_string(),
self.table.chain_for(directory),
crate::control::SplitDirectory::new(directory),
));
}
let (_, chain, split) = self.chain.as_ref().expect("the chain was just resolved");
!chain.is_empty()
&& split.with_components(|directory| {
chain.is_ignored_within(directory, name.as_encoded_bytes(), kind.is_dir())
})
} else {
self.table.matcher_for(path).is_ignored(kind.is_dir())
};
if ignored && !parent_ignored && kind.is_dir() {
self.ignored_heads.insert(path.as_os_str().to_os_string());
}
ignored
}
fn parent_ignored(&mut self, parent: &std::ffi::OsStr) -> bool {
if self.ignored_heads.is_empty() {
return false;
}
if let Some((cached, ignored)) = &self.parent {
if cached.as_os_str() == parent {
return *ignored;
}
}
let ignored =
crate::control::ancestors(parent).any(|ancestor| self.ignored_heads.contains(ancestor));
self.parent = Some((parent.to_os_string(), ignored));
ignored
}
fn record(&mut self, applied: Result<()>) {
if let Err(error) = applied {
self.rejected.get_or_insert(error);
}
}
}
#[cfg(test)]
mod tests {
use std::fs;
use std::path::{Path, PathBuf};
use super::*;
use crate::query::{IgnoredEntries, Pattern, Query, Section};
use crate::{OpenFixture, ScanConfig};
#[test]
fn byte_wise_path_splits_match_the_parsed_ones() {
use std::ffi::{OsStr, OsString};
let named = ["", "a", "a/b", "a/b/c.txt", ".gitignore", "d.d/.h", "x/..y"]
.into_iter()
.map(OsString::from);
#[cfg(unix)]
let paths: Vec<OsString> = {
use std::os::unix::ffi::OsStringExt as _;
named
.chain([
OsString::from_vec(b"d\xff/f\xfe".to_vec()),
OsString::from_vec(b"\xfe".to_vec()),
])
.collect()
};
#[cfg(not(unix))]
let paths: Vec<OsString> = named.collect();
for path in &paths {
let path = Path::new(path);
let (parent, name) = crate::control::split_parent(path);
assert_eq!(parent, path.parent().map_or(OsStr::new(""), Path::as_os_str), "{path:?}");
assert_eq!(name, path.file_name().unwrap_or(OsStr::new("")), "{path:?}");
let by_bytes: Vec<&OsStr> = crate::control::ancestors(parent).collect();
let parsed: Vec<&OsStr> = Path::new(parent).ancestors().map(Path::as_os_str).collect();
assert_eq!(by_bytes, parsed, "{path:?}");
}
}
#[test]
fn total_throughput_uses_one_elapsed_sample_and_selected_size() {
use crate::query::SizeMetric;
let work = PerformanceSummary {
walked_files: 200,
walked_bytes: 4_000_000_000,
walked_allocated: 1_000_000_000,
..PerformanceSummary::default()
};
assert_eq!(
work.total_throughput(std::time::Duration::from_secs(2), SizeMetric::Apparent),
"100 files/s (1.863 GiB/s)"
);
assert_eq!(
work.total_throughput(std::time::Duration::from_secs(2), SizeMetric::Allocated),
"100 files/s (0.466 GiB/s)"
);
assert_eq!(
work.total_throughput(std::time::Duration::ZERO, SizeMetric::Apparent),
"throughput unavailable"
);
assert_eq!(
PerformanceSummary::default()
.total_throughput(std::time::Duration::from_secs(1), SizeMetric::Apparent),
"0 files/s (0.000 GiB/s)"
);
assert_eq!(
throughput_rates(12_345, 3 * (1_u64 << 30), std::time::Duration::from_secs(2)),
Some(("6,172".to_owned(), "1.500".to_owned()))
);
}
#[test]
fn every_route_refuses_a_total_no_u64_can_hold_and_accepts_an_exact_fit() {
use crate::index::DetachedIndexBuilder;
use crate::scan::{DetachedChild, DetachedDirectory};
use crate::{Attrs, Index, Observation, ObservationOp, Op};
#[derive(Clone, Copy, Debug)]
enum Layout {
OneDirectory,
DirectoryEach,
}
let attrs = |size: u64, allocated: u64, inode: u64| Attrs {
size,
allocated,
mtime_ns: 1,
ctime_ns: 1,
inode,
dev: 1,
};
let files = |layout: Layout, count: usize| -> Vec<(PathBuf, Option<PathBuf>)> {
(0..count)
.map(|file| match layout {
Layout::OneDirectory => (PathBuf::from(format!("f{file}")), None),
Layout::DirectoryEach => {
let directory = PathBuf::from(format!("d{file}"));
(directory.join("f"), Some(directory))
}
})
.collect()
};
let upserts = |layout: Layout, sizes: &[(u64, u64)]| -> Vec<Op> {
let mut ops = Vec::new();
for ((path, directory), (inode, &(size, allocated))) in
files(layout, sizes.len()).into_iter().zip((1_u64..).zip(sizes))
{
if let Some(directory) = directory {
ops.push(Op::Upsert {
path: directory,
kind: EntryKind::Dir,
attrs: Attrs::default(),
});
}
ops.push(Op::Upsert {
path,
kind: EntryKind::File,
attrs: attrs(size, allocated, inode),
});
}
ops
};
let summary_fold = |layout: Layout, sizes: &[(u64, u64)], read_controls: bool| {
let config = ScanConfig { read_controls, ..ScanConfig::default() };
let mut fold = SummaryFold::new(&config);
for op in upserts(layout, sizes) {
fold.observe(&ObservationOp::unconditional(op));
}
fold.finish(Path::new("/root"), &[]).map(|(row, ..)| (row.bytes, row.allocated))
};
let builder = |layout: Layout, sizes: &[(u64, u64)], folding: bool| -> Result<(u64, u64)> {
let mut builder = DetachedIndexBuilder::new(
"/root",
crate::ScanScope::default(),
crate::classify::TypeRegistry::compiled_shared(),
);
if folding {
builder =
builder.folding(TreeRetention { largest_files: 1, size: SizeMetric::Apparent });
}
let child = |name: &Path, kind, attrs, position| DetachedChild {
name: name.as_os_str().to_owned(),
kind,
attrs,
position,
};
let mut root =
DetachedDirectory { path: PathBuf::new(), children: Vec::new(), control: None };
let mut nested = Vec::new();
for (position, ((path, directory), (inode, &(size, allocated)))) in
(0_u32..).zip(files(layout, sizes.len()).into_iter().zip((1_u64..).zip(sizes)))
{
let file = attrs(size, allocated, inode);
match directory {
None => root.children.push(child(&path, EntryKind::File, file, position)),
Some(directory) => {
root.children.push(child(
&directory,
EntryKind::Dir,
Attrs::default(),
position,
));
let name = path.file_name().expect("a file name");
nested.push(DetachedDirectory {
path: directory,
children: vec![child(Path::new(name), EntryKind::File, file, 0)],
control: None,
});
}
}
}
builder.push_directory(&mut root)?;
for listing in &mut nested {
builder.push_directory(listing)?;
}
let total = builder.finish().total();
Ok((total.bytes, total.allocated))
};
let apply_lane = |layout: Layout, sizes: &[(u64, u64)]| -> Result<(u64, u64)> {
let mut index = Index::new("/root");
index.apply(&Observation::new(upserts(layout, sizes)))?;
let total = index.total();
Ok((total.bytes, total.allocated))
};
let routes =
|layout: Layout, sizes: &[(u64, u64)]| -> Vec<(&'static str, Result<(u64, u64)>)> {
vec![
("the summary fold", summary_fold(layout, sizes, false)),
("the classifying summary fold", summary_fold(layout, sizes, true)),
("the detached builder", builder(layout, sizes, false)),
("the folding builder", builder(layout, sizes, true)),
("the apply lane", apply_lane(layout, sizes)),
]
};
let half = u64::MAX / 2 + 1;
for layout in [Layout::OneDirectory, Layout::DirectoryEach] {
for (sizes, counter) in [
(&[(half, 0), (half, 0), (half, 0)][..], "bytes"),
(&[(0, half), (0, half)][..], "allocated bytes"),
] {
for (route, outcome) in routes(layout, sizes) {
match outcome {
Err(Error::UnrepresentableTotal { counter: refused, .. }) => {
assert_eq!(refused, counter, "{route}, {layout:?}");
}
other => panic!(
"{route}, {layout:?}: expected the {counter} total refused, got {other:?}"
),
}
}
}
let exact = [(half, half), (half - 1, half - 1)];
for (route, outcome) in routes(layout, &exact) {
assert_eq!(
outcome.expect("an exact fit is representable"),
(u64::MAX, u64::MAX),
"{route}, {layout:?}"
);
}
}
}
#[test]
#[cfg(unix)]
fn an_unreadable_stored_header_never_authorizes_live_replacement() {
use std::os::unix::fs::PermissionsExt;
if !crate::test_support::require_permission_bits() {
return;
}
let root = tempfile::tempdir().expect("root");
let cache = tempfile::tempdir().expect("cache");
let snapshot = cache.path().join("snapshot.fdu");
fs::write(root.path().join(".gitignore"), b"ignored\n").expect("control");
let observed = crate::query::Basis {
root: root.path().into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE,
};
let delivery = Delivery::new(CachePolicy::Auto, Some(snapshot.clone()));
crate::open(&observed, &delivery).expect("stronger snapshot");
let original = fs::read(&snapshot).expect("original image");
let basis = crate::query::Basis {
scope: crate::query::Scope { read_controls: false, ..Default::default() },
..observed
};
let (mut index, _) =
crate::open(&basis, &Delivery::new(CachePolicy::Off, None)).expect("fresh blind index");
let request = Request::new(basis, Query::default(), SystemTime::now());
let plan = plan(&request, &delivery, Route::Refresh).expect("plan");
fs::set_permissions(&snapshot, fs::Permissions::from_mode(0o000))
.expect("deny header read");
let nonwriting = Delivery { cache: CachePolicy::Off, ..delivery.clone() };
let nonwriting_plan =
super::plan(&request, &nonwriting, Route::Refresh).expect("nonwriting plan");
assert!(
!crate::persist_index_changes(&index, &nonwriting_plan, true, true)
.expect("nonwriting policy never reads the header")
);
fs::write(root.path().join("fresh.txt"), b"fresh").expect("mutation");
crate::refresh(
&mut index,
&request.basis,
&Delivery { cache: CachePolicy::Off, ..delivery.clone() },
)
.expect("off refresh does not inspect cache state");
let result = crate::persist_index_changes(&index, &plan, true, true);
fs::set_permissions(&snapshot, fs::Permissions::from_mode(0o600)).expect("restore");
assert!(
result.is_err(),
"a writable parent must not let unknown identity authorize replacement"
);
assert_eq!(fs::read(&snapshot).expect("retained image"), original);
}
#[test]
fn refresh_rejects_another_root_before_mutating_or_persisting() {
let a = tempfile::tempdir().expect("root a");
let b = tempfile::tempdir().expect("root b");
let cache = tempfile::tempdir().expect("cache");
let basis = crate::query::Basis {
root: a.path().into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE,
};
let (mut index, _) =
crate::open(&basis, &Delivery::new(CachePolicy::Off, None)).expect("open a");
fs::write(a.path().join("new"), b"new facts").expect("mutation a");
let before = index.clock();
let snapshot = cache.path().join("snapshot.fdu");
let wrong = crate::query::Basis { root: b.path().into(), ..basis.clone() };
let delivery = Delivery::new(CachePolicy::Auto, Some(snapshot.clone()));
let error = crate::refresh(&mut index, &wrong, &delivery).expect_err("different root");
assert!(matches!(
error,
Error::InvalidRequest(crate::query::RequestError::RootMismatch { .. })
));
assert_eq!(index.clock(), before);
assert!(!snapshot.exists());
let alias = crate::query::Basis { root: a.path().join("."), ..basis };
crate::refresh(&mut index, &alias, &delivery).expect("same root spelling");
assert_eq!(index.total().files, 1);
#[cfg(unix)]
{
let link = cache.path().join("root-alias");
std::os::unix::fs::symlink(a.path(), &link).expect("root alias");
let symlink_basis = crate::query::Basis { root: link, ..alias };
crate::refresh(&mut index, &symlink_basis, &delivery)
.expect("same canonical root through symlink");
}
}
#[test]
fn unchanged_refresh_replaces_an_incompatible_stored_baseline() {
let root = tempfile::tempdir().expect("root");
let other = tempfile::tempdir().expect("other root");
let cache = tempfile::tempdir().expect("cache");
fs::write(root.path().join("file"), b"retained").expect("file");
let basis = crate::query::Basis {
root: root.path().into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE,
};
let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
for wrong_root in [false, true] {
let wrong = if wrong_root {
crate::query::Basis { root: other.path().into(), ..basis.clone() }
} else {
crate::query::Basis {
scope: crate::query::Scope { max_depth: Some(0), ..basis.scope.clone() },
..basis.clone()
}
};
crate::open(&wrong, &Delivery { cache: CachePolicy::On, ..delivery.clone() })
.expect("incompatible snapshot");
let (mut index, _) = crate::open(&basis, &Delivery::new(CachePolicy::Off, None))
.expect("retained index");
let refreshed =
crate::refresh(&mut index, &basis, &delivery).expect("refresh reseeds cache");
assert!(!refreshed.apply.mutated(), "the existing index was already current");
let (cached, _) = crate::open(&basis, &Delivery { stale_ok: true, ..delivery.clone() })
.expect("cache-only can now answer");
assert_eq!(cached.total().bytes, 8);
}
}
#[test]
fn refreshed_metadata_and_content_are_visible_to_a_later_cache_only_open() {
let root = tempfile::tempdir().expect("root");
let cache = tempfile::tempdir().expect("cache");
let path = root.path().join("note.txt");
fs::write(&path, b"old\n").expect("old file");
let basis = crate::query::Basis {
root: root.path().into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE.with_lines(),
};
let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
fs::write(&path, b"new longer text\nsecond line\n").expect("mutation");
let refreshed = crate::refresh(&mut index, &basis, &delivery).expect("refresh");
assert!(refreshed.is_complete());
let (cached, report) = crate::open(&basis, &Delivery { stale_ok: true, ..delivery })
.expect("cache-only sees refreshed tiers");
assert_eq!(report.path_taken, OpenPath::CacheOnly);
assert_eq!(cached.total(), index.total());
assert_eq!(
cached.total().bytes,
u64::try_from(b"new longer text\nsecond line\n".len()).expect("length")
);
assert_eq!(report.content_cache.hits, 1);
let original = index
.content()
.expect("fresh content")
.file(Path::new("note.txt"))
.expect("fresh file");
let restored = cached
.content()
.expect("restored content")
.file(Path::new("note.txt"))
.expect("restored file");
assert_eq!(restored, original);
}
#[cfg(unix)]
fn owed_persistence_fixture()
-> (tempfile::TempDir, tempfile::TempDir, crate::query::Basis, Delivery) {
let root = tempfile::tempdir().expect("root");
let cache = tempfile::tempdir().expect("cache");
fs::create_dir(root.path().join("locked")).expect("locked dir");
fs::write(root.path().join("first"), b"first").expect("first file");
let basis = crate::query::Basis {
root: root.path().into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE,
};
let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
(root, cache, basis, delivery)
}
#[cfg(unix)]
fn cached_files(basis: &crate::query::Basis, delivery: &Delivery) -> u64 {
let cache_only = Delivery { stale_ok: true, ..delivery.clone() };
crate::open(basis, &cache_only).expect("cache-only open").0.total().files
}
#[test]
#[cfg(unix)]
fn an_unchanged_refresh_repeats_the_metadata_write_a_failed_refresh_owed() {
use std::os::unix::fs::PermissionsExt;
if !crate::test_support::require_permission_bits() {
return;
}
let (root, cache, basis, delivery) = owed_persistence_fixture();
let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
assert_eq!(cached_files(&basis, &delivery), 1);
fs::write(root.path().join("second"), b"second").expect("second file");
fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o555)).expect("deny write");
let failed = crate::refresh(&mut index, &basis, &delivery);
fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o755)).expect("restore");
assert!(failed.is_err(), "a read-only cache directory fails the write");
assert_eq!(index.total().files, 2, "the index advanced before the write");
assert_eq!(cached_files(&basis, &delivery), 1, "the failed write left the old image");
let unchanged = crate::refresh(&mut index, &basis, &delivery).expect("unchanged refresh");
assert!(unchanged.is_complete());
assert!(!unchanged.apply.mutated(), "nothing changed between the passes");
assert_eq!(cached_files(&basis, &delivery), 2, "the owed write ran");
let snapshot = delivery.cache_path.as_deref().expect("path");
let written = fs::metadata(snapshot).expect("snapshot").modified().expect("mtime");
crate::refresh(&mut index, &basis, &delivery).expect("settled refresh");
assert_eq!(fs::metadata(snapshot).expect("snapshot").modified().expect("mtime"), written);
}
#[test]
#[cfg(unix)]
fn an_unchanged_refresh_repeats_the_metadata_write_a_failed_open_owed() {
use std::os::unix::fs::PermissionsExt;
if !crate::test_support::require_permission_bits() {
return;
}
let (root, cache, basis, delivery) = owed_persistence_fixture();
crate::open(&basis, &delivery).expect("complete open writes the snapshot");
fs::write(root.path().join("second"), b"second").expect("second file");
fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o555)).expect("deny write");
let opened = crate::open_with_pending_save(&basis, &delivery);
let outcome = opened.map(|(index, report, pending)| (index, report, pending.join()));
fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o755)).expect("restore");
let (index, report, joined) = outcome.expect("the open itself succeeds");
assert_eq!(report.path_taken, OpenPath::WarmRevalidate);
assert!(joined.is_err(), "the startup write failed");
let mut index = std::sync::Arc::into_inner(index).expect("the writer released the index");
assert_eq!(cached_files(&basis, &delivery), 1);
let unchanged = crate::refresh(&mut index, &basis, &delivery).expect("unchanged refresh");
assert!(!unchanged.apply.mutated(), "nothing changed between the passes");
assert_eq!(cached_files(&basis, &delivery), 2, "the owed write ran");
}
#[test]
#[cfg(unix)]
fn a_complete_refresh_persists_the_facts_a_partial_refresh_could_not() {
use std::os::unix::fs::PermissionsExt;
if !crate::test_support::require_permission_bits() {
return;
}
let (root, _cache, basis, delivery) = owed_persistence_fixture();
let locked = root.path().join("locked");
let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
assert_eq!(index.total().files, 1);
fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("deny read");
fs::write(root.path().join("second"), b"second").expect("second file");
let partial = crate::refresh(&mut index, &basis, &delivery);
fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).expect("restore");
let partial = partial.expect("partial refresh");
assert!(!partial.is_complete(), "the locked directory made the pass partial");
assert!(partial.apply.mutated(), "the second file was inserted");
assert_eq!(index.total().files, 2);
assert_eq!(cached_files(&basis, &delivery), 1, "a partial pass never writes entries");
let complete = crate::refresh(&mut index, &basis, &delivery).expect("complete refresh");
assert!(complete.is_complete(), "{:?}", complete.scan.errors);
assert_eq!(cached_files(&basis, &delivery), 2, "the complete pass wrote the facts");
}
#[test]
fn cache_only_refusals_name_location_root_and_absence_separately() {
let root = tempfile::tempdir().expect("root");
let other = tempfile::tempdir().expect("other root");
let cache = tempfile::tempdir().expect("cache");
let basis = crate::query::Basis {
root: root.path().into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE,
};
let snapshot = cache.path().join("snapshot.fdu");
let message = |delivery: &Delivery| {
crate::open(&basis, delivery).expect_err("cache-only refusal").to_string()
};
let no_location = message(&Delivery::stale_ok(None));
assert!(no_location.contains("no cache location"), "{no_location}");
assert!(!no_location.contains("`on`"), "no write can succeed without a location");
let missing = message(&Delivery::stale_ok(Some(snapshot.clone())));
assert!(missing.contains("no usable snapshot"), "{missing}");
assert!(missing.contains("with the `on` cache policy"), "{missing}");
let other_basis = crate::query::Basis { root: other.path().into(), ..basis.clone() };
crate::open(&other_basis, &Delivery::new(CachePolicy::Auto, Some(snapshot.clone())))
.expect("other snapshot");
let wrong_root = message(&Delivery::stale_ok(Some(snapshot)));
assert!(wrong_root.contains("different root"), "{wrong_root}");
}
#[test]
fn route_delivery_matrix_rejects_contracts_the_route_cannot_execute() {
let basis = crate::query::Basis {
root: ".".into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE,
};
let request = Request::new(basis, Query::default(), SystemTime::now());
for delivery in Delivery::enumerate() {
for route in
[Route::OneShot, Route::Retained, Route::Refresh, Route::Watch, Route::Opened]
{
let result = plan(&request, &delivery, route);
let forbidden = delivery.stale_ok
&& (delivery.watch.is_some()
|| matches!(route, Route::Watch | Route::Refresh)
|| delivery.cache == CachePolicy::Off)
|| route == Route::Opened
&& (delivery.cache != CachePolicy::Off
|| delivery.watch.is_some()
|| delivery.accept_partial);
assert_eq!(result.is_err(), forbidden, "{route:?} {delivery:?}");
if let Ok(plan) = result {
if route == Route::Opened {
assert_eq!(plan.load(), Load::None);
assert_eq!(plan.verify(), Verify::Filesystem);
}
}
}
}
}
#[test]
fn an_opened_root_refuses_the_scheduling_it_would_otherwise_drop() {
let basis = crate::query::Basis {
root: ".".into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE,
};
let request = Request::new(basis, Query::default(), SystemTime::now());
let default = Delivery::new(CachePolicy::Off, None);
let plan_opened = plan(&request, &default, Route::Opened).expect("defaults plan");
assert_eq!(plan_opened.delivery().batch_size, default.batch_size);
let unhonored = [
(
"scan workers",
Delivery {
workers: crate::query::Workers { scan: Some(4), ..default.workers },
..default.clone()
},
),
(
"depth-first order",
Delivery { order: crate::ScanOrder::DepthFirst, ..default.clone() },
),
("accept partial", Delivery { accept_partial: true, ..default.clone() }),
];
for (case, delivery) in unhonored {
let refused = plan(&request, &delivery, Route::Opened).expect_err(case);
assert!(
matches!(
refused,
crate::query::RequestError::DeliveryUnsupported { route: "opened", .. }
),
"{case}: {refused}"
);
plan(&request, &delivery, Route::Retained)
.unwrap_or_else(|error| panic!("{case} executes on a retained route: {error}"));
}
let batched = Delivery { batch_size: default.batch_size * 2, ..default };
let plan_batched = plan(&request, &batched, Route::Opened).expect("batch size plans");
assert_eq!(plan_batched.delivery().batch_size, batched.batch_size);
}
#[test]
fn tier_writes_depend_only_on_authorization_and_observed_facts() {
let routes = [Route::OneShot, Route::Retained, Route::Refresh, Route::Watch, Route::Opened];
for (delivery, persist) in
Delivery::enumerate().flat_map(|delivery| [(delivery.clone(), false), (delivery, true)])
{
for bits in 0_u8..64 {
let facts = RunFacts {
entries_verified: bits & 1 != 0,
entries_changed: bits & 2 != 0,
content_changed: bits & 4 != 0,
content_requested: bits & 8 != 0,
projected: bits & 16 != 0,
paired_entries: bits & 32 != 0,
};
let allowed = persist;
let expected = SaveTargets {
metadata: allowed
&& facts.entries_verified
&& facts.entries_changed
&& !facts.projected,
content: allowed
&& facts.content_requested
&& facts.content_changed
&& (facts.entries_verified || facts.paired_entries),
};
for route in routes {
let plan = Plan {
basis: crate::query::Basis {
root: ".".into(),
scope: crate::query::Scope::default(),
content: crate::content::AnalysisSet::NONE,
},
route,
retained: RetainedState::FullIndex,
load: Load::Snapshot,
verify: Verify::Filesystem,
persist,
delivery: delivery.clone(),
};
assert_eq!(plan.writes(facts), expected, "{route:?} {delivery:?} {facts:?}");
let unavailable = Plan {
delivery: Delivery { cache_path: None, ..delivery.clone() },
..plan
};
assert!(unavailable.writes(facts).none());
}
}
}
}
fn planned(config: &OpenFixture, query: &Query) -> Plan {
let (request, delivery) = split(Path::new("."), config, query);
plan(&request, &delivery, Route::OneShot).expect("valid plan")
}
fn summary_query() -> Query {
Query { views: vec![ViewSpec::Summary], ..Query::default() }
}
fn split(root: &Path, config: &OpenFixture, query: &Query) -> (Request, Delivery) {
let (basis, delivery) = config.split(root);
(Request::new(basis, query.clone(), std::time::UNIX_EPOCH), delivery)
}
fn prepared(
root: &Path,
config: &OpenFixture,
query: &Query,
) -> Result<(Report, PendingSave, PerformanceSummary)> {
let (request, delivery) = split(root, config, query);
prepare_report(&request, &delivery)
}
fn prepared_with_diagnostics(
root: &Path,
config: &OpenFixture,
query: &Query,
) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)>
{
let (request, delivery) = split(root, config, query);
prepare_report_with_scan_diagnostics(&request, &delivery)
}
fn config(policy: CachePolicy, cache_path: Option<PathBuf>) -> OpenFixture {
OpenFixture { scan: ScanConfig::default(), cache_path, policy, ..OpenFixture::default() }
}
fn blind(policy: CachePolicy, cache_path: Option<PathBuf>) -> OpenFixture {
OpenFixture {
scan: ScanConfig { read_controls: false, ..ScanConfig::default() },
..config(policy, cache_path)
}
}
fn controls_config(
policy: CachePolicy,
cache_path: PathBuf,
read_controls: bool,
) -> OpenFixture {
OpenFixture {
scan: ScanConfig { read_controls, ..ScanConfig::default() },
cache_path: Some(cache_path),
policy,
..OpenFixture::default()
}
}
fn stale(fixture: OpenFixture) -> OpenFixture {
OpenFixture { stale_ok: true, ..fixture }
}
fn seed_controls_snapshot(root: &Path, cache_path: PathBuf) {
fs::write(root.join(".gitignore"), b"ignored.log\n").expect("control file");
fs::write(root.join("ignored.log"), b"ignored").expect("ignored file");
crate::open_fixture(root, &controls_config(CachePolicy::Auto, cache_path, true))
.expect("seed controls-on snapshot");
}
#[test]
fn planner_uses_compact_state_only_when_the_request_proves_it_is_sufficient() {
let off = blind(CachePolicy::Off, Some(PathBuf::from("unused.fdu")));
assert_eq!(planned(&off, &summary_query()).retained, RetainedState::Summary);
for policy in [CachePolicy::Auto, CachePolicy::On] {
let unavailable = blind(policy, None);
assert_eq!(planned(&unavailable, &summary_query()).retained, RetainedState::Summary);
}
let mut several_views = summary_query();
several_views.views.push(ViewSpec::Types);
assert_eq!(planned(&off, &several_views).retained, RetainedState::FullIndex);
let mut filtered = summary_query();
filtered.selection.include.push(Pattern::parse("*.rs").expect("pattern"));
assert_eq!(planned(&off, &filtered).retained, RetainedState::FullIndex);
let observing = config(CachePolicy::Off, None);
assert!(observing.scan.read_controls, "observation is the default");
assert_eq!(planned(&observing, &summary_query()).retained, RetainedState::Summary);
let mut by_ignored = summary_query();
by_ignored.selection.ignored = IgnoredEntries::Exclude;
assert_eq!(planned(&observing, &by_ignored).retained, RetainedState::FullIndex);
for population in [IgnoredEntries::Exclude, IgnoredEntries::Only] {
let narrowed = OpenFixture {
scan: ScanConfig { population, ..ScanConfig::default() },
..config(CachePolicy::Off, None)
};
let mut query = summary_query();
query.selection.ignored = population;
assert_eq!(planned(&narrowed, &query).retained, RetainedState::FullIndex);
}
}
#[test]
fn an_available_snapshot_does_not_force_the_index_for_a_metadata_summary() {
let cached = blind(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")));
assert_eq!(
planned(&cached, &summary_query()).retained,
RetainedState::Summary,
"a present snapshot must not force the index"
);
}
#[test]
fn deliveries_whose_intent_is_the_snapshot_itself_still_retain_the_index() {
let cached = || blind(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")));
for (name, delivery) in [
("stale", stale(cached())),
("on", OpenFixture { policy: CachePolicy::On, ..cached() }),
] {
assert_eq!(
planned(&delivery, &summary_query()).retained,
RetainedState::FullIndex,
"{name} needs the index to honour its contract"
);
}
}
#[test]
fn a_one_shot_metadata_query_does_not_read_the_snapshot_it_cannot_use() {
let mut tree_query = summary_query();
tree_query.views = vec![ViewSpec::Tree];
for policy in [CachePolicy::Auto, CachePolicy::On] {
let cached = config(policy, Some(PathBuf::from("cache.fdu")));
assert!(
planned(&cached, &tree_query).load != Load::Snapshot,
"{policy:?} must not pay for a read that saves no work"
);
}
}
#[test]
fn the_snapshot_is_read_where_reading_pays_or_is_the_contract() {
let only = stale(config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))));
assert_eq!(planned(&only, &summary_query()).load, Load::Snapshot);
let analyzed = OpenFixture {
analysis: crate::content::AnalysisRequest {
profile: crate::content::AnalysisSet::NONE.with_code(),
..Default::default()
},
..config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")))
};
assert_eq!(planned(&analyzed, &summary_query()).load, Load::Snapshot);
let never = config(CachePolicy::Off, Some(PathBuf::from("cache.fdu")));
assert_eq!(planned(&never, &summary_query()).load, Load::None);
}
#[test]
fn auto_persists_where_a_later_request_reads_what_it_stores() {
let cache = Some(PathBuf::from("cache.fdu"));
let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
let persists = |fixture: &OpenFixture, query: &Query, route: Route| {
let (request, delivery) = split(Path::new("."), fixture, query);
plan(&request, &delivery, route).expect("valid plan").persists()
};
for (policy, one_shot, analysis, retained) in [
(CachePolicy::Auto, false, true, true),
(CachePolicy::On, true, true, true),
(CachePolicy::Off, false, false, false),
] {
let metadata = config(policy, cache.clone());
assert_eq!(persists(&metadata, &tree, Route::OneShot), one_shot, "{policy:?}");
assert_eq!(
persists(&analyzing(metadata.clone()), &tree, Route::OneShot),
analysis,
"{policy:?} with analysis"
);
for route in [Route::Retained, Route::Refresh, Route::Watch] {
assert_eq!(persists(&metadata, &tree, route), retained, "{policy:?} {route:?}");
}
}
let stale_answer = stale(config(CachePolicy::On, cache));
assert!(!persists(&stale_answer, &tree, Route::OneShot), "a stale answer writes nothing");
assert!(!persists(&stale_answer, &tree, Route::Retained), "a stale answer writes nothing");
}
#[test]
fn an_analysis_request_never_selects_the_compact_summary_tier() {
let off = blind(CachePolicy::Off, None);
assert_eq!(planned(&off, &summary_query()).retained, RetainedState::Summary);
for profile in [
crate::content::AnalysisSet::NONE.with_lines(),
crate::content::AnalysisSet::NONE.with_code(),
crate::content::AnalysisSet::NONE.with_words(),
crate::content::AnalysisSet::ALL,
] {
let analyzed = OpenFixture {
analysis: crate::content::AnalysisRequest { profile, ..Default::default() },
..blind(CachePolicy::Off, None)
};
assert_eq!(
planned(&analyzed, &summary_query()).retained,
RetainedState::FullIndex,
"{profile:?} must retain the index"
);
}
}
#[test]
fn a_repeated_one_shot_report_scans_cold_while_open_still_revalidates() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join("file.txt"), b"contents").expect("file");
let cache = tempfile::tempdir().expect("cache dir");
let auto = config(CachePolicy::Auto, Some(cache.path().join("cache.fdu")));
let on = OpenFixture { policy: CachePolicy::On, ..auto.clone() };
let mut tree_query = summary_query();
tree_query.views = vec![ViewSpec::Tree];
let (first, pending, _) = prepared(root.path(), &on, &tree_query).expect("first report");
pending.join().expect("first save");
assert_eq!(first.provenance.source, ReportSource::ColdScan);
assert!(on.cache_path.as_deref().expect("path").exists(), "`on` persists");
let (second, pending, performance) =
prepared(root.path(), &auto, &tree_query).expect("second report");
pending.join().expect("second save");
assert_eq!(
second.provenance.source,
ReportSource::ColdScan,
"a repeated one-shot must not pay for a read that saves no work"
);
assert_eq!(performance.walked_files, 1, "the walk still happened");
let (_, open_report) = crate::open_fixture(root.path(), &auto).expect("library open");
assert_eq!(
open_report.path_taken,
OpenPath::WarmRevalidate,
"a caller holding the index still amortises the load"
);
}
#[test]
fn a_stale_answer_reads_what_an_on_report_leaves_and_auto_leaves_nothing() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join("file.txt"), b"contents").expect("file");
let cache = tempfile::tempdir().expect("cache dir");
let auto = config(CachePolicy::Auto, Some(cache.path().join("cache.fdu")));
let mut tree_query = summary_query();
tree_query.views = vec![ViewSpec::Tree];
let (_, pending, _) = prepared(root.path(), &auto, &tree_query).expect("report");
assert!(!pending.writes_metadata(), "`auto` starts no metadata write");
pending.join().expect("nothing to save");
assert!(!cache.path().join("cache.fdu").exists(), "`auto` leaves nothing");
let only = stale(config(CachePolicy::Auto, Some(cache.path().join("cache.fdu"))));
let missing = prepared(root.path(), &only, &tree_query).expect_err("nothing to read");
assert!(missing.to_string().contains("with the `on` cache policy"), "{missing}");
let on = OpenFixture { policy: CachePolicy::On, ..auto };
let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
pending.join().expect("save");
let (from_cache, pending, performance, diagnostics) =
prepared_with_diagnostics(root.path(), &only, &tree_query).expect("cache-only report");
pending.join().expect("no save");
assert_eq!(from_cache.provenance.source, ReportSource::CacheOnly);
assert_eq!(performance.walked_files, 0, "cache-only never touches the tree");
assert!(diagnostics.is_none(), "a cache-only open has no scan trace");
}
#[test]
fn controls_on_snapshot_projects_to_an_equivalent_controls_off_cache_only_report() {
let root = tempfile::tempdir().expect("tempdir");
let cache = tempfile::tempdir().expect("cache dir");
let cache_path = cache.path().join("cache.fdu");
fs::create_dir(root.path().join("src")).expect("source dir");
fs::write(root.path().join("src/lib.rs"), b"library").expect("source file");
seed_controls_snapshot(root.path(), cache_path.clone());
let controls_off = stale(controls_config(CachePolicy::Auto, cache_path, false));
let query = Query {
views: vec![
ViewSpec::Summary,
ViewSpec::Tree,
ViewSpec::Families,
ViewSpec::Types,
ViewSpec::Extensions,
ViewSpec::Languages,
ViewSpec::Largest,
ViewSpec::Recent,
ViewSpec::Files,
],
..Query::default()
};
let (projected, pending, performance) =
prepared(root.path(), &controls_off, &query).expect("projected report");
pending.join().expect("no cache-only save");
let cold = OpenFixture {
policy: CachePolicy::Off,
stale_ok: false,
cache_path: None,
..controls_off
};
let (mut expected, pending, _) =
prepared(root.path(), &cold, &query).expect("controls-off cold report");
pending.join().expect("no cold save");
expected.provenance = projected.provenance.clone();
assert_eq!(performance.source, ReportSource::CacheOnly);
assert_eq!(projected.scope, cold.scan.scope());
assert_eq!(projected.ignore_rules, crate::control::ControlCoverage::NotObserved);
assert_eq!(
crate::report_format::render(&projected, crate::report_format::Format::Json, false,)
.expect("compatible report format"),
crate::report_format::render(&expected, crate::report_format::Format::Json, false,)
.expect("compatible report format"),
);
}
#[test]
fn controls_on_snapshot_projects_to_controls_off_auto_report() {
let root = tempfile::tempdir().expect("tempdir");
let cache = tempfile::tempdir().expect("cache dir");
let cache_path = cache.path().join("cache.fdu");
seed_controls_snapshot(root.path(), cache_path.clone());
let controls_off = OpenFixture {
analysis: crate::content::AnalysisRequest {
profile: crate::content::AnalysisSet::NONE.with_lines(),
..Default::default()
},
..controls_config(CachePolicy::Auto, cache_path, false)
};
let (report, pending, performance) = prepared(root.path(), &controls_off, &summary_query())
.expect("controls-off warm projection");
pending.join().expect("save content only");
assert_eq!(report.provenance.source, ReportSource::WarmRevalidate);
assert_eq!(report.scope, controls_off.scan.scope());
assert_eq!(performance.source, ReportSource::WarmRevalidate);
}
fn write_unobservable_controls(root: &Path) {
let mut rule = vec![b'a'; crate::control::DEFAULT_CONTROL_LINE_LIMIT + 1];
rule.push(b'\n');
fs::write(root.join(".gitignore"), rule).expect("oversized rule");
fs::create_dir(root.join("vendored")).expect("nested directory");
fs::write(
root.join("vendored/.gitignore"),
b"x\n".repeat(crate::control::DEFAULT_CONTROL_BUDGET / 2),
)
.expect("oversized source");
}
#[test]
fn a_one_shot_report_observes_control_state_as_its_caller_configures() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join("file.txt"), b"contents").expect("file");
write_unobservable_controls(root.path());
let mut tree_query = summary_query();
tree_query.views = vec![ViewSpec::Tree];
for read_controls in [true, false] {
let cache = tempfile::tempdir().expect("cache dir");
let cache_path = cache.path().join("cache.fdu");
let caller = controls_config(CachePolicy::On, cache_path.clone(), read_controls);
for query in [summary_query(), tree_query.clone()] {
let (report, pending, _) = prepared(root.path(), &caller, &query)
.expect("a refused control file ends nothing");
pending.join().expect("save");
assert!(
report.status.complete,
"a refusal is not a partial: {:?}",
report.status.errors
);
assert_eq!(report.scope, caller.scan.scope());
match &report.ignore_rules {
crate::control::ControlCoverage::Observed(coverage) => {
assert!(read_controls, "observed only when asked");
assert_eq!(coverage.refused, 2, "{coverage:?}");
assert_eq!(report.notes.len(), 2, "{:?}", report.notes);
assert!(
report.notes[0].contains("2 ignore files not applied"),
"{:?}",
report.notes
);
assert_eq!(
report.notes[1],
"note: gitignored subtotals are unavailable where governing rules could not be verified"
);
}
crate::control::ControlCoverage::NotObserved => {
assert!(!read_controls, "unobserved only when turned off");
assert!(report.notes.is_empty(), "{:?}", report.notes);
}
}
}
let saved = crate::snapshot::load(&cache_path)
.expect("load the snapshot")
.expect("the index tier persisted");
assert_eq!(saved.scope(), caller.scan.scope());
assert_eq!(saved.controls().is_ok(), read_controls);
}
}
#[test]
fn a_scope_this_build_cannot_honour_is_refused_before_any_snapshot_is_read() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join("file.txt"), b"contents").expect("file");
let cache = tempfile::tempdir().expect("cache dir");
let cache_path = cache.path().join("cache.fdu");
let warm = config(CachePolicy::On, Some(cache_path.clone()));
let (_, pending, _) = prepared(root.path(), &warm, &summary_query()).expect("warm");
pending.join().expect("save");
let (_, pending, _) = prepared(
root.path(),
&stale(config(CachePolicy::Auto, Some(cache_path.clone()))),
&summary_query(),
)
.expect("the snapshot answers a supported scope");
pending.join().expect("no save");
let unsupported = ScanConfig { follow_symlinks: true, ..ScanConfig::default() };
for (policy, stale_ok) in [
(CachePolicy::Auto, true),
(CachePolicy::Off, false),
(CachePolicy::Auto, false),
(CachePolicy::On, false),
] {
let asked = OpenFixture {
scan: unsupported.clone(),
stale_ok,
..config(policy, Some(cache_path.clone()))
};
let refused = prepared(root.path(), &asked, &summary_query())
.expect_err("a scope this build cannot honour has no answer at any policy");
assert!(
matches!(
refused,
Error::InvalidRequest(crate::query::RequestError::ScopeUnsupported {
axis: crate::query::ScopeAxis::FollowSymlinks,
..
})
),
"{policy:?} must refuse the request rather than fail the operation: {refused}"
);
assert_eq!(
refused.to_string(),
"unsupported scan configuration: follow_symlinks requires cycle, root-boundary, \
and filesystem-boundary semantics",
"{policy:?} must name the scope it cannot honour"
);
}
}
#[test]
fn a_report_that_reads_no_gitignore_refuses_to_select_by_ignored_state() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join(".gitignore"), b"*.log\n").expect("control file");
fs::write(root.path().join("debug.log"), b"ignored").expect("ignored file");
let mut only = summary_query();
only.selection.ignored = IgnoredEntries::Only;
assert!(matches!(
prepared(root.path(), &blind(CachePolicy::Off, None), &only),
Err(Error::InvalidRequest(crate::query::RequestError::IgnoredWithoutObservation(
IgnoredEntries::Only
)))
));
let (report, pending, _) =
prepared(root.path(), &config(CachePolicy::Off, None), &only).expect("observed");
pending.join().expect("no save");
let Section::Summary(row) = report.sections[0] else { panic!("a summary") };
assert_eq!((row.files, row.bytes), (1, 7), "only the ignored file is selected");
}
#[test]
fn an_on_reports_snapshot_serves_either_cache_only_report_but_not_the_reverse() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join("file.txt"), b"contents").expect("file");
let mut tree_query = summary_query();
tree_query.views = vec![ViewSpec::Tree];
for (writer, reader) in [(true, true), (true, false), (false, false), (false, true)] {
let cache = tempfile::tempdir().expect("cache dir");
let cache_path = cache.path().join("cache.fdu");
let write = controls_config(CachePolicy::On, cache_path.clone(), writer);
let (_, pending, _) =
prepared(root.path(), &write, &tree_query).expect("writing report");
pending.join().expect("save");
let read = stale(controls_config(CachePolicy::Auto, cache_path, reader));
match prepared(root.path(), &read, &tree_query) {
Ok((report, pending, _)) => {
pending.join().expect("no save");
assert!(writer || !reader, "writer {writer} served reader {reader}");
assert_eq!(report.provenance.source, ReportSource::CacheOnly);
assert_eq!(
matches!(report.ignore_rules, crate::control::ControlCoverage::Observed(_)),
reader,
"a report describes the scope it asked for"
);
}
Err(Error::Snapshot(message)) => {
assert!(!writer && reader, "writer {writer}, reader {reader}: {message}");
assert!(message.contains(".gitignore state"), "names the cause: {message}");
assert!(message.contains("verified answer"), "names the remedy: {message}");
}
Err(other) => panic!("writer {writer}, reader {reader}: {other}"),
}
}
}
#[test]
fn a_cache_only_open_answers_from_an_on_reports_snapshot() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join(".gitignore"), b"*.log\n").expect("control file");
fs::write(root.path().join("debug.log"), b"ignored").expect("ignored file");
let cache = tempfile::tempdir().expect("cache dir");
let cache_path = cache.path().join("cache.fdu");
let mut tree_query = summary_query();
tree_query.views = vec![ViewSpec::Tree];
let on = config(CachePolicy::On, Some(cache_path.clone()));
let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
pending.join().expect("save");
let only = stale(config(CachePolicy::Auto, Some(cache_path)));
let (index, report) = crate::open_fixture(root.path(), &only).expect("the shared snapshot");
assert_eq!(report.path_taken, OpenPath::CacheOnly);
assert_eq!(index.is_ignored(Path::new("debug.log")).ok(), Some(Some(true)));
}
#[test]
fn an_open_that_opts_out_of_control_state_shares_an_opted_out_reports_snapshot() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join("file.txt"), b"contents").expect("file");
let cache = tempfile::tempdir().expect("cache dir");
let cache_path = cache.path().join("cache.fdu");
let mut tree_query = summary_query();
tree_query.views = vec![ViewSpec::Tree];
let on = blind(CachePolicy::On, Some(cache_path.clone()));
let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
pending.join().expect("save");
assert!(cache_path.exists(), "the report left a snapshot");
let only = stale(blind(CachePolicy::Auto, Some(cache_path)));
let (index, report) = crate::open_fixture(root.path(), &only).expect("the shared snapshot");
assert_eq!(report.path_taken, OpenPath::CacheOnly);
assert!(matches!(
index.is_ignored(Path::new("file.txt")),
Err(Error::ControlStateNotObserved)
));
}
#[test]
fn compact_summary_matches_the_indexed_summary_exactly() {
let root = tempfile::tempdir().expect("tempdir");
fs::create_dir(root.path().join("src")).expect("directory");
fs::write(root.path().join("src/lib.rs"), b"library").expect("file");
fs::write(root.path().join("README.md"), b"read me").expect("file");
#[cfg(unix)]
std::os::unix::fs::symlink("README.md", root.path().join("readme-link")).expect("symlink");
crate::test_support::settle_allocations(root.path());
let query = summary_query();
let off = OpenFixture {
scan: ScanConfig { read_controls: false, threads: Some(2), ..ScanConfig::default() },
..blind(CachePolicy::Off, None)
};
let (compact, pending, performance) =
prepared(root.path(), &off, &query).expect("compact report");
pending.join().expect("no pending compact save");
assert_eq!(performance.walked_files, 2);
assert_eq!(performance.walked_bytes, 14);
let (index, _open_report) = crate::open_fixture(root.path(), &off).expect("indexed scan");
let indexed = report(
&index,
&crate::test_support::read_of(&index, query.clone()),
compact.provenance.generated_at,
)
.expect("report");
let Section::Summary(compact_row) = compact.sections[0] else {
panic!("compact plan did not return a summary")
};
let Section::Summary(indexed_row) = indexed.sections[0] else {
panic!("indexed plan did not return a summary")
};
assert_eq!(compact_row.files, indexed_row.files);
assert_eq!(compact_row.dirs, indexed_row.dirs);
assert_eq!(compact_row.bytes, indexed_row.bytes);
assert_eq!(compact_row.allocated, indexed_row.allocated);
assert_eq!(compact_row.newest_mtime_ns, indexed_row.newest_mtime_ns);
assert_eq!(compact.root, indexed.root);
assert_eq!(compact.scope, indexed.scope);
assert_eq!(compact.status.complete, indexed.status.complete);
assert_eq!(compact.provenance.freshness, indexed.provenance.freshness);
}
struct ControlCase {
name: &'static str,
#[cfg(unix)]
denied: Option<Unlocked>,
root: tempfile::TempDir,
scan: ScanConfig,
share: bool,
refused: u64,
order_dependent: bool,
errors: bool,
lookups: crate::test_support::CaseLookups,
variant_governs: Option<bool>,
}
#[cfg(unix)]
struct Unlocked(PathBuf);
#[cfg(unix)]
impl Drop for Unlocked {
fn drop(&mut self) {
use std::os::unix::fs::PermissionsExt;
let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o755));
}
}
#[cfg(unix)]
fn seal_search_denied(root: &Path) -> Unlocked {
use std::os::unix::fs::PermissionsExt;
let denied = root.join("denied");
put(root, "denied/sub/inner", b"inner");
put(root, "denied/f", b"f");
std::os::unix::fs::symlink("f", denied.join("link")).expect("symlink");
fs::set_permissions(&denied, fs::Permissions::from_mode(0o400)).expect("deny search");
Unlocked(denied)
}
fn put(root: &Path, path: &str, contents: &[u8]) {
let path = root.join(path);
fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories");
fs::write(path, contents).expect("fixture file");
}
fn rules_tree() -> tempfile::TempDir {
let root = tempfile::tempdir().expect("tempdir");
let root_path = root.path();
put(
root_path,
".gitignore",
b"*.log\n!keep.log\n/anchored.txt\ncache/\nnode_modules/\nvendor/*\n!vendor/keep/\n",
);
put(root_path, "a.log", b"alog");
put(root_path, "keep.log", b"keeplog");
put(root_path, "anchored.txt", b"anchored");
put(root_path, "cache", b"a file named like a directory rule");
put(root_path, "README.md", b"readme!");
put(root_path, "src/anchored.txt", b"not anchored here");
put(root_path, "src/x.log", b"xlog-");
put(root_path, "src/keep.log", b"kept everywhere");
put(root_path, "src/main.rs", b"fn main() {}");
put(root_path, "sub/.gitignore", b"!*.log\n*.tmp\n");
put(root_path, "sub/y.log", b"re-included");
put(root_path, "sub/z.tmp", b"tmp");
put(root_path, "sub/cache/data.bin", b"cached bytes");
put(root_path, "sub/deep/.gitignore", b"*\n!.gitignore\n");
put(root_path, "sub/deep/f.txt", b"deep");
put(root_path, "sub/deep/inner/g.txt", b"deeper");
put(root_path, "node_modules/pkg/.gitignore", b"!*\n");
put(root_path, "node_modules/pkg/index.js", b"module.exports = 1;");
put(root_path, "node_modules/pkg/lib/a.js", b"a");
put(root_path, "vendor/a.c", b"int a;");
put(root_path, "vendor/keep/k.c", b"int k;");
put(root_path, "vendor/drop/d.c", b"int d;");
put(root_path, "selfish/.gitignore", b".gitignore\n*.bak\n");
put(root_path, "selfish/x.bak", b"backup");
put(root_path, "selfish/y.txt", b"kept");
put(root_path, "many/.gitignore", b"*[02468].dat\n");
for file in 0..40 {
put(root_path, &format!("many/f{file:02}.dat"), &vec![b'.'; file + 1]);
}
root
}
const CASE_VARIANT: &str = "a case-variant control name";
const CASE_BOTH: &str = "both spellings of the control name";
fn control_cases() -> Vec<ControlCase> {
let case = |name, root, scan, share, refused| ControlCase {
name,
root,
scan,
share,
refused,
order_dependent: false,
errors: false,
lookups: crate::test_support::CaseLookups::Host,
variant_governs: None,
#[cfg(unix)]
denied: None,
};
let mut cases = vec![
case("rules", rules_tree(), ScanConfig::default(), true, 0),
case(
"rules with hidden entries pruned",
rules_tree(),
ScanConfig {
hidden: Some(std::sync::Arc::new(crate::HiddenPolicy::prune_hidden(Vec::<
std::ffi::OsString,
>::new(
)))),
..ScanConfig::default()
},
true,
0,
),
case(
"rules under a depth bound",
rules_tree(),
ScanConfig { max_depth: Some(1), ..ScanConfig::default() },
true,
0,
),
];
let empty = tempfile::tempdir().expect("tempdir");
put(empty.path(), "only.txt", b"no rules anywhere");
cases.push(case("no control file", empty, ScanConfig::default(), true, 0));
let long = tempfile::tempdir().expect("tempdir");
put(long.path(), ".gitignore", b"*.log\n");
put(long.path(), "x.log", b"ignored");
let mut line = vec![b'x'; crate::control::DEFAULT_CONTROL_LINE_LIMIT + 1];
line.push(b'\n');
put(long.path(), "long/.gitignore", &line);
put(long.path(), "long/kept.txt", b"kept");
put(long.path(), "long/y.log", b"unknown");
cases.push(case("line limit", long, ScanConfig::default(), false, 1));
let over = tempfile::tempdir().expect("tempdir");
put(over.path(), ".gitignore", b"*.log\n");
let mut oversized = vec![b'x'; 200];
oversized.push(b'\n');
for directory in 0..70 {
put(over.path(), &format!("d{directory:02}/.gitignore"), &oversized);
put(over.path(), &format!("d{directory:02}/f.log"), b"log");
}
let limits = crate::control::ControlLimits { budget: Some(256), ..Default::default() };
cases.push(case(
"every nested file over the budget",
over,
ScanConfig { control_limits: limits, ..ScanConfig::default() },
false,
70,
));
let competing = tempfile::tempdir().expect("tempdir");
for (position, directory) in ["a", "b", "c", "d"].into_iter().enumerate() {
let mut rules = format!("r{position}").into_bytes();
rules.extend(std::iter::repeat_n(b'y', 100));
rules.push(b'\n');
put(competing.path(), &format!("{directory}/.gitignore"), &rules);
put(competing.path(), &format!("{directory}/file.txt"), b"file");
}
let limits = crate::control::ControlLimits { budget: Some(1000), ..Default::default() };
let mut competing = case(
"competing for the budget",
competing,
ScanConfig { control_limits: limits, ..ScanConfig::default() },
false,
2,
);
competing.order_dependent = true;
cases.push(competing);
let probe = tempfile::tempdir().expect("tempdir");
for (lookups, governs) in crate::test_support::CaseLookups::on_this_host(probe.path()) {
for hidden in [None, Some(Vec::<std::ffi::OsString>::new())] {
let variant = tempfile::tempdir().expect("tempdir");
put(variant.path(), ".GITIGNORE", b"*.log\n");
for file in 0..1_200 {
put(variant.path(), &format!("f{file:04}.log"), b"log");
}
put(variant.path(), "kept.txt", b"kept");
let scan = ScanConfig {
hidden: hidden
.map(|allow| std::sync::Arc::new(crate::HiddenPolicy::prune_hidden(allow))),
..ScanConfig::default()
};
let mut variant = case(CASE_VARIANT, variant, scan, true, 0);
variant.lookups = lookups;
variant.variant_governs = Some(governs);
cases.push(variant);
}
}
for lookups in
[crate::test_support::CaseLookups::Host, crate::test_support::CaseLookups::Folded]
{
let both = tempfile::tempdir().expect("tempdir");
put(both.path(), ".gitignore", b"*.log\n");
put(both.path(), ".GITIGNORE", b"*.tmp\n");
if fs::read(both.path().join(".gitignore")).expect("read") != b"*.log\n" {
eprintln!("skipped {CASE_BOTH:?}: the temporary directory is case-insensitive");
break;
}
for file in 0..1_200 {
put(both.path(), &format!("f{file:04}.tmp"), b"tmp");
}
put(both.path(), "x.log", b"log");
let mut both = case(CASE_BOTH, both, ScanConfig::default(), true, 0);
both.lookups = lookups;
cases.push(both);
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let shapes = tempfile::tempdir().expect("tempdir");
put(shapes.path(), ".gitignore", b"*.tmp\n");
put(shapes.path(), "weird/.gitignore/inner.tmp", b"inner");
put(shapes.path(), "weird/kept.txt", b"kept");
put(shapes.path(), "rules.txt", b"*.txt\n");
fs::create_dir(shapes.path().join("linked")).expect("directory");
std::os::unix::fs::symlink("../rules.txt", shapes.path().join("linked/.gitignore"))
.expect("symlink");
put(shapes.path(), "linked/still.txt", b"still counted");
cases.push(case(
"control files that are not files",
shapes,
ScanConfig::default(),
true,
0,
));
if crate::test_support::require_permission_bits() {
let unreadable = tempfile::tempdir().expect("tempdir");
put(unreadable.path(), ".gitignore", b"*.log\n");
put(unreadable.path(), "x.log", b"ignored");
put(unreadable.path(), "sub/.gitignore", b"!*.log\n");
put(unreadable.path(), "sub/y.log", b"unknown");
let control = unreadable.path().join("sub/.gitignore");
let mut denied =
case("unreadable control file", unreadable, ScanConfig::default(), false, 0);
fs::set_permissions(&control, fs::Permissions::from_mode(0o000))
.expect("deny the control file");
denied.denied = Some(Unlocked(control));
denied.errors = true;
cases.push(denied);
let sealed = tempfile::tempdir().expect("tempdir");
put(sealed.path(), ".gitignore", b"*.log\n");
put(sealed.path(), "open/x.log", b"ignored");
put(sealed.path(), "open/y.txt", b"kept");
let denied = seal_search_denied(sealed.path());
let mut unsearchable = case(
"a directory that lists but refuses search",
sealed,
ScanConfig::default(),
true,
0,
);
unsearchable.denied = Some(denied);
unsearchable.errors = true;
cases.push(unsearchable);
}
}
for case in &cases {
crate::test_support::settle_allocations(case.root.path());
}
cases
}
fn transient_and_indexed(root: &Path, scan: &ScanConfig, label: &str) -> (Report, Report) {
let query = summary_query();
let transient = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
assert_eq!(planned(&transient, &query).retained, RetainedState::Summary, "{label}");
let (mut compact, pending, compact_performance) =
prepared(root, &transient, &query).expect("transient report");
pending.join().expect("the transient tier saves nothing");
let cache = tempfile::tempdir().expect("cache dir");
let indexed = OpenFixture {
scan: scan.clone(),
..config(CachePolicy::On, Some(cache.path().join("snapshot.fdu")))
};
assert_eq!(planned(&indexed, &query).retained, RetainedState::FullIndex, "{label}");
let (indexed, pending, indexed_performance) =
prepared(root, &indexed, &query).expect("indexed report");
pending.join().expect("save");
assert_eq!(
(
compact_performance.walked_files,
compact_performance.walked_bytes,
compact_performance.walked_allocated,
compact_performance.source,
),
(
indexed_performance.walked_files,
indexed_performance.walked_bytes,
indexed_performance.walked_allocated,
indexed_performance.source,
),
"{label}: walked totals"
);
assert_eq!(compact.provenance.source, indexed.provenance.source, "{label}");
assert_eq!(compact.provenance.freshness, indexed.provenance.freshness, "{label}");
assert_eq!(
(compact.provenance.tiers.entries.source, compact.provenance.tiers.entries.freshness),
(indexed.provenance.tiers.entries.source, indexed.provenance.tiers.entries.freshness),
"{label}"
);
compact.provenance = indexed.provenance.clone();
(compact, indexed)
}
#[test]
fn compact_summary_equals_the_indexed_summary_under_every_control_case() {
use crate::report_format::{Format, render};
let default_batch = ScanConfig::default().batch_size;
for case in control_cases() {
let _lookups = case.lookups.install(case.root.path());
let mut compared = 0;
for threads in [Some(1), Some(2), Some(4), None] {
if case.order_dependent && threads != Some(1) {
continue;
}
for batch_size in [default_batch, 1, 3] {
for order in [crate::ScanOrder::BreadthFirst, crate::ScanOrder::DepthFirst] {
let scan = ScanConfig { batch_size, threads, order, ..case.scan.clone() };
let label = format!(
"{} ({:?} lookups, {threads:?} workers, batch {batch_size}, {order:?})",
case.name, case.lookups
);
let (compact, indexed) =
transient_and_indexed(case.root.path(), &scan, &label);
assert_eq!(format!("{compact:#?}"), format!("{indexed:#?}"), "{label}");
for format in [Format::Text, Format::Json, Format::Yaml] {
assert_eq!(
render(&compact, format, false).expect("render"),
render(&indexed, format, false).expect("render"),
"{label}: {format:?}"
);
}
let Section::Summary(row) = indexed.sections[0] else {
panic!("{label}: a summary")
};
let crate::control::ControlCoverage::Observed(coverage) =
&indexed.ignore_rules
else {
panic!("{label}: the default scope observes .gitignore")
};
assert_eq!(coverage.refused, case.refused, "{label}");
assert_eq!(row.ignored.is_some(), case.share, "{label}");
assert_eq!(!indexed.status.errors.is_empty(), case.errors, "{label}");
if case.share && case.name.starts_with("rules") {
let ignored = row.ignored.expect("a share");
assert!(
ignored.files > 0 && ignored.files < row.files && ignored.dirs > 0,
"{label}: {ignored:?} of {row:?}"
);
}
if let Some(governs) = case.variant_governs {
assert_eq!(coverage.applied, u64::from(governs), "{label}");
assert_eq!(
row.ignored.map(|ignored| ignored.files),
Some(if governs { 1_200 } else { 0 }),
"{label}"
);
}
if case.name == CASE_BOTH {
assert_eq!(coverage.applied, 1, "{label}: the exact name alone");
assert_eq!(
row.ignored.map(|ignored| ignored.files),
Some(1),
"{label}: only `x.log` is ignored"
);
}
if !case.share {
assert!(
indexed
.notes
.iter()
.any(|note| note.contains("could not be verified")),
"{label}: {:?}",
indexed.notes
);
}
compared += 1;
}
}
}
assert!(compared > 0, "{} was compared", case.name);
}
}
#[test]
fn compact_summary_never_creates_the_configured_snapshot() {
let root = tempfile::tempdir().expect("tempdir");
fs::write(root.path().join("payload"), b"payload").expect("file");
let cache = root.path().join("must-not-exist.fdu");
let (report, pending, _) =
prepared(root.path(), &blind(CachePolicy::Off, Some(cache.clone())), &summary_query())
.expect("compact report");
pending.join().expect("no pending compact save");
assert!(report.status.complete);
assert!(!cache.exists());
}
fn tree_query(
views: Vec<ViewSpec>,
format: crate::report_format::Format,
share: &str,
size: crate::query::SizeMetric,
) -> Query {
let mut query = Query { views, format, ..Query::default() };
query.selection.min_share =
Some(crate::query::ShareThreshold::parse(share).expect("a share"));
query.selection.size = size;
query
}
#[test]
fn a_tree_takes_the_folded_index_only_when_the_request_proves_it_suffices() {
use crate::query::{Bound, SizeMetric, SortKey};
use crate::report_format::Format;
let off = config(CachePolicy::Off, None);
let list = Query { views: vec![ViewSpec::List], ..Query::default() };
let size = list.selection.size;
let kept = |largest_files, size| RetainedState::Tree(TreeRetention { largest_files, size });
assert_eq!(planned(&off, &list).retained, kept(100, size));
let mut folded = vec![
Query { format: Format::Tree, ..list.clone() },
Query { views: vec![ViewSpec::List, ViewSpec::Tree], ..list.clone() },
];
for format in [Format::Text, Format::Tree, Format::Json, Format::Jsonl, Format::Yaml] {
folded.push(Query { views: vec![ViewSpec::Tree], format, ..Query::default() });
}
for sort in [SortKey::Size, SortKey::Name, SortKey::Mtime, SortKey::Count] {
for reverse in [false, true] {
let mut query = list.clone();
query.selection.sort = Some(sort);
query.selection.reverse = reverse;
folded.push(query);
}
}
for bound in [Bound::Limit(0), Bound::Limit(2), Bound::All] {
let mut query = list.clone();
query.selection.depth = Some(bound);
query.selection.breadth = Some(bound);
query.selection.limit = Some(bound);
folded.push(query);
}
for query in &folded {
assert_eq!(planned(&off, query).retained, kept(100, size), "{query:?}");
}
for (share, largest_files) in
[("10%", 10), ("0.5%", 200), ("3%", 34), ("0.01%", 10_000), ("100%", 1)]
{
for metric in [SizeMetric::Apparent, SizeMetric::Allocated] {
let query = tree_query(vec![ViewSpec::List], Format::Text, share, metric);
assert_eq!(planned(&off, &query).retained, kept(largest_files, metric), "{share}");
}
}
let ceiling = tree_query(vec![ViewSpec::Tree], Format::Json, "0.00152587890625%", size);
assert_eq!(planned(&off, &ceiling).retained, kept(TreeRetention::MAX_FILES, size));
let past = tree_query(vec![ViewSpec::Tree], Format::Json, "0.0015258%", size);
assert_eq!(planned(&off, &past).retained, RetainedState::FullIndex);
for fixture in [
blind(CachePolicy::Off, None),
OpenFixture {
scan: ScanConfig { max_depth: Some(2), ..ScanConfig::default() },
..off.clone()
},
config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))),
config(CachePolicy::On, None),
] {
assert_eq!(planned(&fixture, &list).retained, kept(100, size), "{fixture:?}");
}
let mut full = vec![
Query { format: Format::Json, ..list.clone() },
Query { views: vec![ViewSpec::Tree], format: Format::Paths, ..list.clone() },
Query { views: vec![ViewSpec::Tree], format: Format::Long, ..list.clone() },
Query { views: vec![ViewSpec::Files], format: Format::Tree, ..list.clone() },
Query { views: vec![ViewSpec::List, ViewSpec::Types], ..list.clone() },
Query { views: vec![ViewSpec::Tree, ViewSpec::Extensions], ..list.clone() },
Query { views: vec![ViewSpec::Tree, ViewSpec::Summary], ..list.clone() },
Query { views: vec![ViewSpec::Tree, ViewSpec::Files], ..list.clone() },
Query { views: Vec::new(), ..list.clone() },
];
let mut filtered = list.clone();
filtered.selection.include.push(Pattern::parse("*.rs").expect("pattern"));
full.push(filtered);
let mut excluded = list.clone();
excluded.selection.exclude.push(Pattern::parse("target").expect("pattern"));
full.push(excluded);
let mut sized = list.clone();
sized.selection.min_size = Some(1);
full.push(sized);
let mut kinds = list.clone();
kinds.selection.kinds = vec![EntryKind::Dir];
full.push(kinds);
let mut files = list.clone();
files.selection.kinds = vec![EntryKind::File];
full.push(files);
let mut modified = list.clone();
modified.selection.modified.since = Some(0);
full.push(modified);
let mut by_ignored = list.clone();
by_ignored.selection.ignored = IgnoredEntries::Exclude;
full.push(by_ignored);
let mut reported = list.clone();
reported.selection.min_size = Some(300 << 20);
reported.selection.sort = Some(SortKey::Size);
reported.selection.depth = Some(Bound::Limit(2));
reported.selection.limit = Some(Bound::Limit(30));
full.push(reported);
full.push(tree_query(vec![ViewSpec::List], Format::Text, "0%", size));
full.push(tree_query(vec![ViewSpec::Tree], Format::Json, "0.000%", size));
for query in &full {
assert_eq!(planned(&off, query).retained, RetainedState::FullIndex, "{query:?}");
}
let cache = || Some(PathBuf::from("cache.fdu"));
for fixture in [
analyzing(off.clone()),
stale(config(CachePolicy::Auto, cache())),
config(CachePolicy::On, cache()),
] {
assert_eq!(planned(&fixture, &list).retained, RetainedState::FullIndex, "{fixture:?}");
}
for population in [IgnoredEntries::Exclude, IgnoredEntries::Only] {
let narrowed = OpenFixture {
scan: ScanConfig { population, ..ScanConfig::default() },
..off.clone()
};
let mut query = list.clone();
query.selection.ignored = population;
assert_eq!(planned(&narrowed, &query).retained, RetainedState::FullIndex);
}
let (request, delivery) = split(Path::new("."), &off, &list);
for route in [Route::Retained, Route::Refresh, Route::Watch, Route::Opened] {
let plan = plan(&request, &delivery, route).expect("a valid plan");
assert_eq!(plan.retained, RetainedState::FullIndex, "{route:?}");
}
}
#[test]
fn the_default_request_of_every_surface_plans_the_folded_tree() {
use crate::query::{AxisNames, ReadSpec, RequestSpec, SizeMetric};
let root = Path::new(".");
let default_tree =
RetainedState::Tree(TreeRetention { largest_files: 100, size: SizeMetric::Allocated });
let deliveries = [
Delivery::new(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))),
Delivery::new(CachePolicy::Off, None),
];
let planned = |spec: &RequestSpec<'_>, axes, delivery: &Delivery| {
let request =
Request::build(spec, SystemTime::UNIX_EPOCH, axes).expect("a valid request");
plan(&request, delivery, Route::OneShot).expect("a valid plan").retained
};
for axes in [&AxisNames::FLAGS, &AxisNames::FIELDS] {
for delivery in &deliveries {
assert_eq!(planned(&RequestSpec::new(root), axes, delivery), default_tree);
let tree_json = RequestSpec {
read: ReadSpec { views: Some("tree"), format: Some("json"), ..ReadSpec::new() },
..RequestSpec::new(root)
};
assert_eq!(planned(&tree_json, axes, delivery), default_tree);
let list_json = RequestSpec {
read: ReadSpec { format: Some("json"), ..ReadSpec::new() },
..RequestSpec::new(root)
};
assert_eq!(planned(&list_json, axes, delivery), RetainedState::FullIndex);
}
}
}
struct Shuffle(u64);
impl Shuffle {
fn below(&mut self, bound: usize) -> usize {
self.0 ^= self.0 >> 12;
self.0 ^= self.0 << 25;
self.0 ^= self.0 >> 27;
let drawn = self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 32;
usize::try_from(drawn).expect("32 bits") % bound
}
}
fn sized_file(path: &Path, size: u64, sparse: bool, minute: u64) {
use std::io::Write as _;
fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories");
let mut file = fs::File::create(path).expect("fixture file");
if sparse {
file.set_len(size).expect("sparse length");
} else {
file.write_all(&vec![b'.'; usize::try_from(size).expect("small")]).expect("contents");
}
file.set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(minute * 60))
.expect("mtime");
}
fn random_tree(seed: u64, files: usize) -> tempfile::TempDir {
let mut draw = Shuffle(seed);
let root = tempfile::tempdir().expect("tempdir");
let mut dirs = vec![PathBuf::new()];
for index in 0..files.div_ceil(12) {
let parent = dirs[draw.below(dirs.len())].clone();
if parent.components().count() < 5 {
let dir = parent.join(format!("d{index:03}"));
fs::create_dir(root.path().join(&dir)).expect("directory");
dirs.push(dir);
}
}
let extensions = ["log", "txt", "bin", "tmp", "rs"];
for index in 0..files {
let dir = &dirs[draw.below(dirs.len())];
let name = format!("f{index:05}.{}", extensions[draw.below(extensions.len())]);
let (size, sparse) = match draw.below(20) {
0..=2 => (0, false),
3..=10 => (draw.below(300) as u64, false),
11..=13 => (1_000, false),
14..=17 => (draw.below(40_000) as u64, false),
_ => (50_000 + draw.below(4_000_000) as u64, true),
};
let minute = FIXTURE_MINUTE + draw.below(4) as u64;
sized_file(&root.path().join(dir).join(name), size, sparse, minute);
}
put(root.path(), ".gitignore", b"*.log\n!f000*.log\nd002/\n");
if let Some(dir) = dirs.get(3) {
put(&root.path().join(dir), ".gitignore", b"!*.log\n*.tmp\n");
}
#[cfg(unix)]
for (index, dir) in dirs.iter().take(4).enumerate() {
std::os::unix::fs::symlink(
"elsewhere",
root.path().join(dir).join(format!("l{index}")),
)
.expect("symlink");
}
crate::test_support::settle_allocations(root.path());
root
}
struct BoundaryTree {
name: &'static str,
#[cfg(unix)]
_sealed: Option<Unlocked>,
tree: tempfile::TempDir,
}
fn boundary_trees() -> Vec<BoundaryTree> {
let mut trees = Vec::new();
let ties = tempfile::tempdir().expect("tempdir");
for index in 0..9 {
sized_file(&ties.path().join(format!("a/at{index}")), 1_000, false, index);
}
sized_file(&ties.path().join("a/half"), 500, false, 20);
sized_file(&ties.path().join("b/half"), 500, false, 20);
trees.push(("ties at the retention bound", ties));
let exact = tempfile::tempdir().expect("tempdir");
for index in 0..20 {
sized_file(&exact.path().join(format!("d{}/f{index:02}", index % 3)), 50, false, 1);
sized_file(&exact.path().join(format!("d{}/e{index:02}", index % 4)), 0, false, 2);
}
trees.push(("exactly the files a share shows", exact));
let zero = tempfile::tempdir().expect("tempdir");
for index in 0..60 {
sized_file(&zero.path().join(format!("d{}/z{index:02}", index % 5)), 0, false, 3);
}
trees.push(("all files empty", zero));
let empty = tempfile::tempdir().expect("tempdir");
fs::create_dir_all(empty.path().join("only/directories")).expect("directories");
trees.push(("no files", empty));
let single = tempfile::tempdir().expect("tempdir");
sized_file(&single.path().join("one"), 7, false, 4);
trees.push(("one file", single));
let ignored = tempfile::tempdir().expect("tempdir");
put(ignored.path(), ".gitignore", b"*.big\nbuild/\n");
sized_file(&ignored.path().join("src/at.big"), 1_000, false, 5);
sized_file(&ignored.path().join("src/below.big"), 999, false, 5);
for index in 0..150 {
sized_file(&ignored.path().join(format!("build/o{index:03}")), 30, false, 6);
sized_file(&ignored.path().join(format!("src/s{index:03}.rs")), 38, false, 7);
sized_file(&ignored.path().join(format!("src/x{index:03}.big")), 5, false, 8);
}
trees.push(("ignored files at the share boundary", ignored));
let linked = tempfile::tempdir().expect("tempdir");
sized_file(&linked.path().join("a/original"), 1_000, false, 9);
for link in ["b/link", "c/deeper/link"] {
fs::create_dir_all(linked.path().join(link).parent().expect("a parent"))
.expect("directories");
fs::hard_link(linked.path().join("a/original"), linked.path().join(link))
.expect("hard link");
}
for index in 0..7 {
sized_file(&linked.path().join(format!("d/f{index}")), 1_000, false, 10);
}
for index in 0..30 {
sized_file(&linked.path().join(format!("e/z{index:02}")), 0, false, 11);
}
trees.push(("hard links at the share boundary", linked));
#[cfg(unix)]
{
let special = tempfile::tempdir().expect("tempdir");
for index in 0..120 {
sized_file(&special.path().join(format!("s/f{index:03}")), 10, false, 12);
}
sized_file(&special.path().join("s/large"), 5_000, false, 13);
let fifo = special.path().join("s/pipe");
let status =
std::process::Command::new("mkfifo").arg(&fifo).status().expect("run mkfifo");
assert!(status.success(), "mkfifo exited with {status}");
drop(
std::os::unix::net::UnixListener::bind(special.path().join("s/socket"))
.expect("bind socket"),
);
trees.push(("special files beside folded files", special));
}
#[cfg(target_os = "linux")]
{
use std::os::unix::ffi::OsStrExt as _;
let lossy = tempfile::tempdir().expect("tempdir");
for byte in [0xfe_u8, 0xff] {
let bytes = [b't', b'i', b'e', byte];
let name = std::ffi::OsStr::from_bytes(&bytes);
sized_file(&lossy.path().join("n").join(name), 2_000, false, 14);
}
for index in 0..80 {
sized_file(&lossy.path().join(format!("n/small{index:02}")), 3, false, 15);
}
trees.push(("names equal once made readable", lossy));
}
let trees: Vec<BoundaryTree> = trees
.into_iter()
.map(|(name, tree)| BoundaryTree {
name,
#[cfg(unix)]
_sealed: None,
tree,
})
.chain(search_denied_boundary_tree())
.collect();
for tree in &trees {
crate::test_support::settle_allocations(tree.tree.path());
}
trees
}
#[cfg(unix)]
fn search_denied_boundary_tree() -> Option<BoundaryTree> {
if !crate::test_support::require_permission_bits() {
return None;
}
let unsearchable = tempfile::tempdir().expect("tempdir");
sized_file(&unsearchable.path().join("open/large"), 5_000, false, 18);
for index in 0..60 {
sized_file(&unsearchable.path().join(format!("open/s{index:02}")), 5, false, 19);
}
let sealed = seal_search_denied(unsearchable.path());
Some(BoundaryTree {
name: "a directory that lists but refuses search",
_sealed: Some(sealed),
tree: unsearchable,
})
}
#[cfg(not(unix))]
fn search_denied_boundary_tree() -> Option<BoundaryTree> {
None
}
const FIXTURE_MINUTE: u64 = 28_000_000;
fn crowded_tree() -> tempfile::TempDir {
let tree = tempfile::tempdir().expect("tempdir");
let root = tree.path();
let minute = |offset| FIXTURE_MINUTE + offset;
put(root, ".gitignore", b"*.skip\n");
sized_file(&root.join("hit/large"), 20_000, false, minute(3));
sized_file(&root.join("hit/covered"), 900, false, minute(3));
sized_file(&root.join("hit/x.skip"), 400, false, minute(2));
for index in 0..30 {
let path = root.join(format!("hit/many/m{index:02}.bin"));
sized_file(&path, 1_000, false, minute(index % 4));
}
for index in 0..110 {
sized_file(&root.join(format!("u{index:03}/f.rs")), 950, false, minute(1));
}
sized_file(&root.join("u000/y.skip"), 30, false, minute(1));
sized_file(&root.join("top-big"), 1_500, false, minute(2));
sized_file(&root.join("top-small"), 990, false, minute(0));
sized_file(&root.join("top.skip"), 20, false, minute(1));
for empty in ["empty", "hit/empty"] {
fs::create_dir_all(root.join(empty)).expect("empty directory");
}
#[cfg(unix)]
{
let mut stamps = vec![
("hit".to_string(), 3),
("hit/many".to_string(), 2),
("hit/empty".to_string(), 0),
("empty".to_string(), 0),
];
stamps.extend((0..110).map(|index| (format!("u{index:03}"), 1)));
for (directory, offset) in stamps {
fs::File::open(root.join(directory))
.expect("open directory")
.set_modified(
std::time::UNIX_EPOCH + std::time::Duration::from_secs(minute(offset) * 60),
)
.expect("directory mtime");
}
}
crate::test_support::settle_allocations(root);
tree
}
fn folded_walks(one_worker: bool) -> Vec<(usize, crate::ScanOrder, bool)> {
use crate::ScanOrder::{BreadthFirst, DepthFirst};
let mut walks = Vec::new();
for threads in [1, 2, 4, 8] {
if one_worker && threads != 1 {
continue;
}
for order in [BreadthFirst, DepthFirst] {
let every = matches!((threads, order), (1, BreadthFirst) | (8, DepthFirst))
|| (one_worker && order == DepthFirst);
walks.push((threads, order, every));
}
}
walks
}
fn folded_queries(share: &str, size: crate::query::SizeMetric, every: bool) -> Vec<Query> {
use crate::query::{Bound, SortKey};
use crate::report_format::Format;
let list = tree_query(vec![ViewSpec::List], Format::Text, share, size);
let json = tree_query(vec![ViewSpec::Tree], Format::Json, share, size);
let mut queries = vec![list.clone(), json.clone()];
if !every {
return queries;
}
queries.push(Query { format: Format::Tree, ..list.clone() });
for format in [Format::Jsonl, Format::Yaml, Format::Text] {
queries.push(Query { format, ..json.clone() });
}
queries.push(Query { views: vec![ViewSpec::List, ViewSpec::Tree], ..list.clone() });
for sort in [SortKey::Size, SortKey::Name, SortKey::Mtime, SortKey::Count] {
for reverse in [false, true] {
let mut query = list.clone();
query.selection.sort = Some(sort);
query.selection.reverse = reverse;
queries.push(query);
}
}
let bounds: [(Option<Bound>, Option<Bound>, Option<Bound>); 12] = [
(Some(Bound::Limit(0)), None, None),
(None, Some(Bound::Limit(0)), None),
(Some(Bound::Limit(0)), Some(Bound::Limit(0)), None),
(Some(Bound::Limit(1)), None, None),
(Some(Bound::Limit(2)), None, None),
(Some(Bound::All), None, None),
(None, Some(Bound::Limit(1)), None),
(Some(Bound::All), Some(Bound::Limit(3)), None),
(None, None, Some(Bound::Limit(0))),
(None, None, Some(Bound::Limit(1))),
(None, None, Some(Bound::Limit(5))),
(Some(Bound::All), Some(Bound::Limit(2)), Some(Bound::Limit(7))),
];
for (depth, breadth, limit) in bounds {
let mut query = json.clone();
query.selection.depth = depth;
query.selection.breadth = breadth;
query.selection.limit = limit;
queries.push(query.clone());
query.selection.sort = Some(SortKey::Name);
queries.push(query);
}
queries
}
const FOLDED_SHARES: [&str; 8] =
["1%", "0.5%", "5%", "10%", "0.01%", "33%", "100%", "0.00152587890625%"];
fn assert_folded_reports_match(
root: &Path,
scan: &ScanConfig,
shares: &[&str],
one_worker: bool,
label: &str,
) -> bool {
use crate::query::SizeMetric;
let canonical = root.canonicalize().expect("canonical root");
let mut folded_any = false;
for (threads, order, every) in folded_walks(one_worker) {
let fixture = OpenFixture {
scan: ScanConfig { threads: Some(threads), order, ..scan.clone() },
..config(CachePolicy::Off, None)
};
let mut full = None;
for share in shares.iter().take(if every { shares.len() } else { 4 }) {
for size in [SizeMetric::Apparent, SizeMetric::Allocated] {
let mut folded: Option<(TreeRetention, crate::Index)> = None;
for query in folded_queries(share, size, every) {
let (request, delivery) = split(root, &fixture, &query);
let scan_config = request.basis.scope.scan_config(&delivery);
let RetainedState::Tree(retention) =
plan(&request, &delivery, Route::OneShot).expect("plan").retained
else {
panic!("{label}: {query:?} takes the full index")
};
let full = full.get_or_insert_with(|| {
let (index, _) = crate::scan::scan_into_index(&canonical, &scan_config)
.expect("full index");
index
});
if folded.as_ref().is_none_or(|(held, _)| *held != retention) {
let (index, _, _) = crate::scan::scan_into_folded_index(
&canonical,
&scan_config,
retention,
false,
)
.expect("folded index");
assert!(index.is_folded() && !full.is_folded(), "{label}");
folded_any |= index.len() < full.len();
folded = Some((retention, index));
}
let (_, index) = folded.as_ref().expect("a folded index");
let mut from_folded =
report(index, &request, SystemTime::UNIX_EPOCH).expect("folded report");
let from_full =
report(full, &request, SystemTime::UNIX_EPOCH).expect("full report");
assert_eq!(
(from_folded.provenance.source, from_folded.provenance.freshness),
(from_full.provenance.source, from_full.provenance.freshness),
"{label}"
);
from_folded.provenance = from_full.provenance.clone();
assert_eq!(
format!("{from_folded:#?}"),
format!("{from_full:#?}"),
"{label} ({threads} workers, {order:?}, {share} of {size:?}): {query:?}"
);
assert_eq!(
crate::report_format::render(&from_folded, query.format, false)
.expect("render"),
crate::report_format::render(&from_full, query.format, false)
.expect("render"),
"{label} ({threads} workers, {order:?}, {share} of {size:?}): {query:?}"
);
}
}
}
}
folded_any
}
fn assert_folded_route_matches(root: &Path, scan: &ScanConfig, label: &str) {
use crate::report_format::{Format, render};
let list = Query { views: vec![ViewSpec::List], ..Query::default() };
for threads in [Some(1), Some(4)] {
let scan = ScanConfig { threads, ..scan.clone() };
let transient = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
assert!(matches!(planned(&transient, &list).retained, RetainedState::Tree(_)));
let (mut folded, pending, folded_performance) =
prepared(root, &transient, &list).expect("folded report");
pending.join().expect("the folded route saves nothing");
let cache = tempfile::tempdir().expect("cache dir");
let indexed = OpenFixture {
scan: scan.clone(),
..config(CachePolicy::On, Some(cache.path().join("snapshot.fdu")))
};
assert_eq!(planned(&indexed, &list).retained, RetainedState::FullIndex);
let (indexed, pending, indexed_performance) =
prepared(root, &indexed, &list).expect("indexed report");
pending.join().expect("save");
assert_eq!(folded_performance, indexed_performance, "{label}");
assert_eq!(folded.provenance.source, indexed.provenance.source, "{label}");
assert_eq!(folded.provenance.freshness, indexed.provenance.freshness, "{label}");
folded.provenance = indexed.provenance.clone();
assert_eq!(format!("{folded:#?}"), format!("{indexed:#?}"), "{label}");
for format in [Format::Text, Format::Json, Format::Yaml] {
assert_eq!(
render(&folded, format, false).expect("render"),
render(&indexed, format, false).expect("render"),
"{label}: {format:?}"
);
}
}
}
#[test]
fn transient_tree_equals_the_indexed_tree_under_every_bound_case() {
let mut folded_trees = 0;
for (seed, files, scan) in [
(0x9E37_79B9_7F4A_7C15_u64, 480, ScanConfig::default()),
(0xD1B5_4A32_D192_ED03, 700, ScanConfig::default()),
(
0x2545_F491_4F6C_DD1D,
360,
ScanConfig { max_depth: Some(3), ..ScanConfig::default() },
),
(
0x94D0_49BB_1331_11EB,
420,
ScanConfig { read_controls: false, ..ScanConfig::default() },
),
#[cfg(unix)]
(
0x7C15_9E37_79B9_7F4A,
390,
ScanConfig { one_filesystem: true, ..ScanConfig::default() },
),
] {
let tree = random_tree(seed, files);
let label = format!("random tree {seed:#x} of {files} files");
assert!(
assert_folded_reports_match(tree.path(), &scan, &FOLDED_SHARES, false, &label),
"{label} folds at some share"
);
assert_folded_route_matches(tree.path(), &scan, &label);
folded_trees += 1;
}
for boundary in boundary_trees() {
let (name, tree) = (boundary.name, boundary.tree.path());
let scan = ScanConfig::default();
let folds = assert_folded_reports_match(tree, &scan, &FOLDED_SHARES, false, name);
assert_eq!(
folds,
!matches!(name, "no files" | "one file"),
"{name}: whether the tree has files a share omits"
);
assert_folded_route_matches(tree, &scan, name);
folded_trees += usize::from(folds);
}
#[cfg(unix)]
if crate::test_support::require_permission_bits() {
use std::os::unix::fs::PermissionsExt;
let tree = random_tree(0x1405_7B7E_F767_814F, 300);
let locked = tree.path().join("locked");
sized_file(&locked.join("hidden-large"), 900_000, false, 16);
sized_file(&locked.join("inner/hidden-small"), 4, false, 16);
for index in 0..60 {
sized_file(&tree.path().join(format!("beside/b{index:02}")), 7, false, 17);
}
crate::test_support::settle_allocations(tree.path());
fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock");
let _unlocked = Unlocked(locked);
let (index, _) = crate::scan::scan_into_index(tree.path(), &ScanConfig::default())
.expect("a partial index");
assert_ne!(index.state().coverage, crate::Coverage::Complete, "the listing failed");
let label = "a failed listing beside folded files";
assert!(
assert_folded_reports_match(
tree.path(),
&ScanConfig::default(),
&FOLDED_SHARES,
false,
label
),
"{label}"
);
assert_folded_route_matches(tree.path(), &ScanConfig::default(), label);
folded_trees += 1;
}
for case in control_cases() {
let _lookups = case.lookups.install(case.root.path());
let label = format!("{} ({:?} lookups)", case.name, case.lookups);
let folds = assert_folded_reports_match(
case.root.path(),
&case.scan,
&["1%", "10%", "0.5%"],
case.order_dependent,
&label,
);
if !case.order_dependent {
assert_folded_route_matches(case.root.path(), &case.scan, &label);
}
folded_trees += usize::from(folds);
}
let large = random_tree(0x6C8E_9CF5_7093_2BD5, 12_000);
let label = "a tree larger than 0.01% keeps";
assert!(
assert_folded_reports_match(
large.path(),
&ScanConfig::default(),
&["0.01%"],
true,
label
),
"{label}"
);
assert!(folded_trees > 10, "the differential folded {folded_trees} trees");
}
#[test]
fn a_filtered_tree_is_measured_against_what_it_selects() {
use crate::query::{SizeMetric, TreeNode};
use crate::report_format::Format;
let tree = crowded_tree();
let off = config(CachePolicy::Off, None);
let unfiltered = tree_query(vec![ViewSpec::Tree], Format::Json, "1%", SizeMetric::Apparent);
let mut filtered = unfiltered.clone();
filtered.selection.min_size = Some(1_000);
assert_eq!(planned(&off, &filtered).retained, RetainedState::FullIndex);
let root_of = |query: &Query| {
let (report, pending, _) = prepared(tree.path(), &off, query).expect("a report");
pending.join().expect("a one-shot report saves nothing");
match report.sections.into_iter().next() {
Some(Section::Tree { root: Some(root), .. }) => *root,
other => panic!("expected a tree, got {other:?}"),
}
};
let rows = |node: &TreeNode| {
node.children.iter().map(|row| (row.name.clone(), row.bytes)).collect::<Vec<_>>()
};
let named = |rows: &[(&str, u64)]| {
rows.iter().map(|(name, bytes)| ((*name).to_string(), *bytes)).collect::<Vec<_>>()
};
let selected = root_of(&filtered);
assert_eq!(selected.bytes, 20_000 + 900 + 400 + 30 * 1_000 + 1_500);
assert_eq!(root_of(&unfiltered).bytes, selected.bytes + 990 + 20 + 110 * 950 + 30 + 7);
assert_eq!(rows(&selected), named(&[("hit", 51_300), ("top-big", 1_500)]));
assert_eq!(
rows(&selected.children[0]),
named(&[("many", 30_000), ("large", 20_000), ("covered", 900)])
);
}
fn filtered_selections() -> Vec<(&'static str, crate::query::Selection)> {
use crate::query::{ModifiedWindow, Selection};
let at = |offset: u64| {
i64::try_from((FIXTURE_MINUTE + offset) * 60 * 1_000_000_000).expect("nanoseconds")
};
let globs = |sources: &[&str]| -> Vec<Pattern> {
sources.iter().map(|source| Pattern::parse(source).expect("pattern")).collect()
};
let window = |since, before| ModifiedWindow { since, before };
let none = Selection::default;
vec![
("a minimum size", Selection { min_size: Some(1_000), ..none() }),
("a minimum every nonempty file passes", Selection { min_size: Some(1), ..none() }),
("included names", Selection { include: globs(&["*.rs", "*.skip"]), ..none() }),
("included directories", Selection { include: globs(&["hit", "d00*"]), ..none() }),
("excluded names", Selection { exclude: globs(&["*.log", "*.skip"]), ..none() }),
(
"excluded directories",
Selection { exclude: globs(&["u0*", "d001", "many"]), ..none() },
),
(
"included and excluded",
Selection {
include: globs(&["*.rs", "hit"]),
exclude: globs(&["many", "*.tmp"]),
..none()
},
),
("modified since", Selection { modified: window(Some(at(2)), None), ..none() }),
("modified before", Selection { modified: window(None, Some(at(2))), ..none() }),
(
"a modified window",
Selection { modified: window(Some(at(1)), Some(at(3))), ..none() },
),
("files", Selection { kinds: vec![EntryKind::File], ..none() }),
("directories", Selection { kinds: vec![EntryKind::Dir], ..none() }),
("symlinks", Selection { kinds: vec![EntryKind::Symlink], ..none() }),
("unignored entries", Selection { ignored: IgnoredEntries::Exclude, ..none() }),
("ignored entries", Selection { ignored: IgnoredEntries::Only, ..none() }),
(
"a minimum size among included names",
Selection {
min_size: Some(1_000),
include: globs(&["*.bin", "*.rs", "top-*"]),
..none()
},
),
]
}
fn filtered_queries(
selection: &crate::query::Selection,
share: &str,
size: crate::query::SizeMetric,
) -> Vec<Query> {
use crate::query::{Bound, SortKey};
use crate::report_format::Format;
let filtered = |views, format| {
let mut query = tree_query(views, format, share, size);
query.selection = crate::query::Selection {
min_share: query.selection.min_share.clone(),
size,
..selection.clone()
};
query
};
let list = filtered(vec![ViewSpec::List], Format::Text);
let json = filtered(vec![ViewSpec::Tree], Format::Json);
let mut queries = vec![list.clone(), json.clone()];
for query in [&list, &json] {
let mut named = query.clone();
named.selection.sort = Some(SortKey::Name);
queries.push(named);
}
let mut reported = list;
reported.selection.sort = Some(SortKey::Size);
reported.selection.depth = Some(Bound::Limit(2));
reported.selection.limit = Some(Bound::Limit(30));
queries.push(reported);
let mut bounded = json;
bounded.selection.sort = Some(SortKey::Name);
bounded.selection.reverse = true;
bounded.selection.depth = Some(Bound::Limit(1));
bounded.selection.breadth = Some(Bound::Limit(2));
queries.push(bounded);
queries
}
fn assert_same_answer(
mut actual: Report,
expected: &Report,
format: crate::report_format::Format,
context: &str,
) {
use crate::report_format::render;
assert_eq!(
(actual.provenance.source, actual.provenance.freshness),
(expected.provenance.source, expected.provenance.freshness),
"{context}"
);
actual.provenance = expected.provenance.clone();
assert_eq!(format!("{actual:#?}"), format!("{expected:#?}"), "{context}");
assert_eq!(
render(&actual, format, false).expect("render"),
render(expected, format, false).expect("render"),
"{context}"
);
}
fn assert_filtered_reports_match(root: &Path, scan: &ScanConfig, label: &str) {
use crate::query::{Selection, SizeMetric};
use crate::report_format::Format;
let root = root.canonicalize().expect("canonical root");
let fixture = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
let scan_config = {
let (request, delivery) = split(&root, &fixture, &Query::default());
request.basis.scope.scan_config(&delivery)
};
let (full, _) = crate::scan::scan_into_index(&root, &scan_config).expect("full index");
let answer = |query: &Query| {
let (request, _) = split(&root, &fixture, query);
report(&full, &request, SystemTime::UNIX_EPOCH)
};
let whole = tree_query(vec![ViewSpec::Tree], Format::Json, "0%", SizeMetric::Apparent);
let unfiltered = format!("{:#?}", answer(&whole).expect("unfiltered report").sections);
let mut vacuous = Vec::new();
for (name, selection) in filtered_selections() {
let refused = !scan.read_controls && selection.ignored != IgnoredEntries::Include;
let every_row = Query {
selection: Selection {
min_share: whole.selection.min_share.clone(),
size: whole.selection.size,
..selection.clone()
},
..whole.clone()
};
match answer(&every_row) {
Ok(filtered) => {
assert!(!refused, "{label}: {name} is answered");
if format!("{:#?}", filtered.sections) == unfiltered {
vacuous.push(name);
}
}
Err(error) => assert!(refused, "{label}: {name}: {error}"),
}
for share in ["1%", "10%"] {
for size in [SizeMetric::Apparent, SizeMetric::Allocated] {
for query in filtered_queries(&selection, share, size) {
let context = format!("{label}, {name}, {share} of {size:?}: {query:?}");
let (request, delivery) = split(&root, &fixture, &query);
let (expected, routed) =
match (answer(&query), prepare_report(&request, &delivery)) {
(Ok(expected), Ok((routed, pending, _))) => {
pending.join().expect("a one-shot report saves nothing");
(expected, routed)
}
(Err(_), Err(_)) if refused => continue,
(expected, routed) => panic!(
"{context}: {:?} beside {:?}",
expected.err(),
routed.err()
),
};
assert_same_answer(routed, &expected, query.format, &context);
let RetainedState::Tree(retention) =
plan(&request, &delivery, Route::OneShot).expect("plan").retained
else {
continue;
};
let (index, _, _) = crate::scan::scan_into_folded_index(
&root,
&scan_config,
retention,
false,
)
.expect("folded index");
assert!(index.is_folded(), "{context}");
let folded =
report(&index, &request, SystemTime::UNIX_EPOCH).expect("folded");
assert_same_answer(folded, &expected, query.format, &context);
}
}
}
}
assert!(vacuous.is_empty(), "{label}: these select the whole tree: {vacuous:?}");
}
#[test]
fn a_filtered_tree_answers_as_the_full_index_on_every_route() {
let random = random_tree(0x9E37_79B9_7F4A_7C15, 480);
let crowded = crowded_tree();
let deep = |max_depth| ScanConfig { max_depth: Some(max_depth), ..ScanConfig::default() };
let blind = ScanConfig { read_controls: false, ..ScanConfig::default() };
for (tree, scan, label) in [
(&random, ScanConfig::default(), "a random tree"),
(&random, deep(3), "a random tree scanned three deep"),
(&random, blind, "a random tree read without .gitignore"),
(&crowded, ScanConfig::default(), "the crowded tree"),
(&crowded, deep(1), "the crowded tree scanned one deep"),
] {
assert_filtered_reports_match(tree.path(), &scan, label);
}
}
#[test]
fn full_index_report_exposes_scan_diagnostics_when_requested() {
let root = tempfile::tempdir().expect("tempdir");
fs::create_dir(root.path().join("nested")).expect("directory");
fs::write(root.path().join("nested/file.txt"), b"trace me").expect("file");
let query = Query { views: vec![ViewSpec::Tree], ..Query::default() };
let (report, pending, performance, diagnostics) =
prepared_with_diagnostics(root.path(), &config(CachePolicy::Off, None), &query)
.expect("full-index report");
pending.join().expect("no pending save");
assert!(report.status.complete);
assert_eq!(performance.walked_files, 1);
let diagnostics = diagnostics.expect("full-index scan diagnostics");
assert_eq!(diagnostics.schema, crate::scan::SCAN_DIAGNOSTICS_SCHEMA);
assert_eq!(diagnostics.worker_policy.ready_directories_at_finish, 0);
assert_eq!(diagnostics.worker_policy.in_flight_directories_at_finish, 0);
}
fn wide_tree(dirs: usize, files: usize) -> (tempfile::TempDir, u64, u64) {
let root = tempfile::tempdir().expect("tempdir");
let mut bytes = 0;
for directory in 0..dirs {
let path = root.path().join(format!("d{directory:03}"));
fs::create_dir(&path).expect("directory");
for file in 0..files {
let size = directory * files + file + 1;
fs::write(path.join(format!("f{file}.txt")), vec![b'.'; size]).expect("file");
bytes += size as u64;
}
}
(root, (dirs * files) as u64, bytes)
}
fn prepared_with_progress(
root: &Path,
config: &OpenFixture,
query: &Query,
progress: &Progress,
) -> Result<(Report, PendingSave, PerformanceSummary)> {
let (request, delivery) = split(root, config, query);
prepare_report_with_progress(&request, &delivery, progress)
}
fn analyzing(fixture: OpenFixture) -> OpenFixture {
OpenFixture {
analysis: crate::content::AnalysisRequest {
profile: crate::content::AnalysisSet::LINES_ONLY,
workers: 0,
},
..fixture
}
}
#[test]
fn progress_ends_at_the_walked_totals_of_every_one_shot_route() {
use crate::ProgressPhase::{Scanning, Summarizing};
let (root, files, bytes) = wide_tree(6, 4);
let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
let progress = Progress::new();
let (_, pending, performance) =
prepared_with_progress(root.path(), &config(CachePolicy::Off, None), &tree, &progress)
.expect("cold full-index report");
pending.join().expect("no save");
let snapshot = progress.snapshot();
assert_eq!(performance.source, ReportSource::ColdScan);
assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "cold full index");
assert_eq!(snapshot.allocated, performance.walked_allocated);
assert!(snapshot.allocated > 0, "files with content occupy blocks");
assert_eq!(snapshot.directories, 7, "the root and its six children");
assert_eq!(
(snapshot.phase, snapshot.analysis),
(Summarizing, None),
"the walk ended, the index was assembled, then the answer was built"
);
let progress = Progress::new();
let (report, pending, performance) = prepared_with_progress(
root.path(),
&blind(CachePolicy::Off, None),
&summary_query(),
&progress,
)
.expect("compact summary report");
pending.join().expect("no save");
let Section::Summary(row) = report.sections[0] else { panic!("summary section") };
let snapshot = progress.snapshot();
assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "summary fold");
assert_eq!(snapshot.allocated, performance.walked_allocated);
assert_eq!(snapshot.allocated, row.allocated, "the progress figure is the answer's");
assert_eq!(snapshot.directories, row.dirs + 1, "the row's directories and the root");
assert_eq!((snapshot.phase, snapshot.analysis), (Scanning, None));
let cache_dir = tempfile::tempdir().expect("cache dir");
let cache = cache_dir.path().join("snapshot.fdu");
let progress = Progress::new();
let (_, pending, performance) = prepared_with_progress(
root.path(),
&analyzing(config(CachePolicy::Auto, Some(cache.clone()))),
&tree,
&progress,
)
.expect("cold analyzed report");
let snapshot = progress.snapshot();
assert_eq!(snapshot.phase, Summarizing, "the answer is built while the save runs");
pending.join().expect("save");
assert_eq!(performance.source, ReportSource::ColdScan);
assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "cold with analysis");
assert_eq!(snapshot.allocated, performance.walked_allocated);
assert_eq!(snapshot.directories, 7);
assert_eq!(performance.fresh_files, files, "every file is a lines candidate");
assert_eq!(snapshot.analysis, Some((files, files)));
let progress = Progress::new();
let (_, pending, performance) = prepared_with_progress(
root.path(),
&analyzing(config(CachePolicy::Auto, Some(cache))),
&tree,
&progress,
)
.expect("warm analyzed report");
pending.join().expect("nothing to save");
let snapshot = progress.snapshot();
assert_eq!(performance.source, ReportSource::WarmRevalidate);
assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "warm revalidation");
assert_eq!(snapshot.allocated, performance.walked_allocated);
assert_eq!(snapshot.directories, 7);
assert_eq!(performance.fresh_files, 0, "the sidecar answered every candidate");
assert_eq!((snapshot.phase, snapshot.analysis), (Summarizing, Some((0, 0))));
}
#[test]
fn a_cache_only_report_ends_summarizing_and_walks_nothing() {
use crate::ProgressPhase::Summarizing;
let (root, _, _) = wide_tree(3, 2);
let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
let cache_dir = tempfile::tempdir().expect("cache dir");
let cache = cache_dir.path().join("snapshot.fdu");
let (_, pending, _) =
prepared(root.path(), &config(CachePolicy::On, Some(cache.clone())), &tree)
.expect("a report that writes the snapshot");
pending.join().expect("save");
let progress = Progress::new();
let (_, pending, performance) = prepared_with_progress(
root.path(),
&stale(config(CachePolicy::Auto, Some(cache))),
&tree,
&progress,
)
.expect("cache-only report");
pending.join().expect("nothing to save");
let snapshot = progress.snapshot();
assert_eq!(performance.source, ReportSource::CacheOnly);
assert_eq!(snapshot.phase, Summarizing);
assert_eq!((snapshot.directories, snapshot.files, snapshot.bytes), (0, 0, 0));
}
fn rank(phase: crate::ProgressPhase, order: &[crate::ProgressPhase]) -> usize {
order
.iter()
.position(|expected| *expected == phase)
.unwrap_or_else(|| panic!("{phase:?} is not a phase of this route"))
}
#[test]
fn progress_is_monotonic_and_phases_advance_in_order_while_a_report_runs() {
use crate::ProgressPhase::{
Analyzing, Indexing, Loading, Revalidating, Saving, Scanning, Starting, Summarizing,
};
let (root, files, bytes) = wide_tree(48, 6);
let cache_dir = tempfile::tempdir().expect("cache dir");
let cache = cache_dir.path().join("snapshot.fdu");
let fixture = OpenFixture {
scan: ScanConfig { threads: Some(3), batch_size: 4, ..ScanConfig::default() },
..analyzing(config(CachePolicy::Auto, Some(cache)))
};
let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
let routes: [(&str, &[crate::ProgressPhase]); 2] = [
("cold", &[Starting, Loading, Scanning, Indexing, Analyzing, Saving, Summarizing]),
("warm", &[Starting, Loading, Revalidating, Analyzing, Saving, Summarizing]),
];
for (route, order) in routes {
let progress = Progress::new();
let initial = progress.snapshot();
let done = std::sync::atomic::AtomicBool::new(false);
let (performance, seen) = std::thread::scope(|scope| {
let poller = scope.spawn(|| {
let polled = progress.clone();
let mut previous = initial;
let mut seen = vec![previous.phase];
loop {
let finished = done.load(std::sync::atomic::Ordering::Acquire);
let current = polled.snapshot();
assert!(current.directories >= previous.directories, "{route}");
assert!(current.files >= previous.files, "{route}");
assert!(current.bytes >= previous.bytes, "{route}");
assert!(
rank(current.phase, order) >= rank(previous.phase, order),
"{route}: {:?} after {:?}",
current.phase,
previous.phase
);
if let (Some(before), Some(after)) = (previous.analysis, current.analysis) {
assert!(after.0 >= before.0 && after.0 <= after.1, "{route}");
assert_eq!(after.1, before.1, "{route}: the total is fixed");
}
if current.phase != previous.phase {
seen.push(current.phase);
}
previous = current;
if finished {
break;
}
std::thread::yield_now();
}
seen
});
let (_, pending, performance) =
prepared_with_progress(root.path(), &fixture, &tree, &progress)
.expect("report");
pending.join().expect("save");
done.store(true, std::sync::atomic::Ordering::Release);
(performance, poller.join().expect("poller"))
});
let snapshot = progress.snapshot();
assert_eq!(
(snapshot.files, snapshot.bytes),
(performance.walked_files, performance.walked_bytes),
"{route}"
);
assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "{route}");
assert_eq!(snapshot.directories, 49, "{route}");
assert_eq!(seen.first(), Some(&Starting), "{route}: {seen:?}");
assert_eq!(
seen.last().copied(),
Some(snapshot.phase),
"{route}: the poller saw the final phase"
);
assert_eq!(snapshot.phase, Summarizing, "{route}: the answer is built last");
if route == "cold" {
assert_eq!(snapshot.analysis, Some((files, files)));
} else {
assert_eq!(snapshot.analysis, Some((0, 0)));
assert!(!seen.contains(&Indexing), "{route}: a warm run assembles no index");
}
}
}
#[test]
fn a_report_prepared_with_a_handle_is_the_report_prepared_without() {
let (root, _, _) = wide_tree(5, 3);
let tree = Query { views: vec![ViewSpec::Tree, ViewSpec::Files], ..Query::default() };
let cases = [
("full index", config(CachePolicy::Off, None), tree),
("compact summary", blind(CachePolicy::Off, None), summary_query()),
];
for (route, fixture, query) in cases {
let (mut plain, pending, plain_performance) =
prepared(root.path(), &fixture, &query).expect("plain report");
pending.join().expect("no save");
let progress = Progress::new();
let (observed, pending, observed_performance) =
prepared_with_progress(root.path(), &fixture, &query, &progress)
.expect("observed report");
pending.join().expect("no save");
assert_eq!(plain_performance, observed_performance, "{route}");
plain.provenance = observed.provenance.clone();
let json = |report: &Report| {
crate::report_format::render(report, crate::report_format::Format::Json, false)
.expect("render")
};
assert_eq!(json(&plain), json(&observed), "{route}");
assert!(progress.snapshot().files > 0, "{route}: the handle did observe the run");
}
}
}