1use std::time::SystemTime;
13
14use crate::query::{
15 Delivery, Report, ReportProvenance, ReportSource, Request, SizeMetric, SortKey, SummaryRow,
16 TreeStatus, ViewSpec, report, report_summary,
17};
18use crate::{CachePolicy, EntryKind, Error, OpenPath, PendingSave, Progress, Result, execute};
19
20#[derive(Clone, Copy, PartialEq, Eq, Debug)]
26pub(crate) enum RetainedState {
27 Summary,
33 Tree(TreeRetention),
42 FullIndex,
44}
45
46#[derive(Clone, Copy, PartialEq, Eq, Debug)]
48pub(crate) struct TreeRetention {
49 pub(crate) largest_files: u32,
55 pub(crate) size: SizeMetric,
57}
58
59impl TreeRetention {
60 pub(crate) const MAX_FILES: u32 = 65_536;
67
68 fn for_request(request: &Request) -> Option<Self> {
79 let query = &request.query;
80 let tree_only = !query.views.is_empty()
81 && query.views.iter().all(|view| {
82 matches!(view, ViewSpec::List | ViewSpec::Tree) && query.tree_for(*view)
83 });
84 if !tree_only
85 || !query.selection.is_unfiltered()
86 || matches!(query.selection.sort, Some(SortKey::Metric(_)))
87 || request.basis.scope.population != crate::query::IgnoredEntries::Include
88 {
89 return None;
90 }
91 let largest_files = query
92 .min_share_for()
93 .admitted_parts_bound()
94 .and_then(|bound| u32::try_from(bound).ok())
95 .filter(|bound| *bound <= Self::MAX_FILES)?;
96 Some(Self { largest_files, size: query.selection.size })
97 }
98}
99
100#[derive(Clone, Copy, PartialEq, Eq, Debug)]
102pub enum Route {
103 OneShot,
105 Retained,
107 Refresh,
109 Watch,
111 Opened,
113}
114
115#[derive(Clone, Copy, PartialEq, Eq, Debug)]
117pub enum Load {
118 None,
120 Snapshot,
122}
123
124#[derive(Clone, Copy, PartialEq, Eq, Debug)]
126pub enum Verify {
127 None,
129 Filesystem,
131}
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq)]
135pub enum OutcomeClass {
136 Success,
138 Partial,
140}
141
142#[derive(Clone, Debug)]
144pub struct Plan {
145 pub(crate) basis: crate::query::Basis,
146 pub(crate) route: Route,
147 pub(crate) retained: RetainedState,
148 pub(crate) load: Load,
149 pub(crate) verify: Verify,
150 pub(crate) persist: bool,
151 pub(crate) delivery: Delivery,
152}
153
154impl Plan {
155 pub fn outcome(&self, status: &TreeStatus) -> OutcomeClass {
157 if status.complete || self.delivery.accept_partial {
158 OutcomeClass::Success
159 } else {
160 OutcomeClass::Partial
161 }
162 }
163 pub fn basis(&self) -> &crate::query::Basis {
165 &self.basis
166 }
167 pub const fn route(&self) -> Route {
169 self.route
170 }
171 pub const fn load(&self) -> Load {
173 self.load
174 }
175 pub const fn verify(&self) -> Verify {
177 self.verify
178 }
179 pub const fn persists(&self) -> bool {
186 self.persist
187 }
188 pub fn delivery(&self) -> &Delivery {
190 &self.delivery
191 }
192}
193
194pub(crate) struct StoreHeader<'a> {
196 pub(crate) root: &'a std::path::Path,
197 pub(crate) snapshot: crate::SnapshotIdentity,
198 pub(crate) content: Option<&'a crate::ContentTierIdentity>,
199 pub(crate) content_complete: bool,
200}
201
202#[derive(Clone, Copy, Debug, PartialEq, Eq)]
204pub(crate) enum Admission {
205 Serve(crate::Serves),
206 NoLocation,
207 Missing,
208 WrongRoot,
209 WrongScope,
210 IncompleteContent,
211}
212
213impl Plan {
214 pub(crate) fn admit(
221 &self,
222 stored: Option<&StoreHeader<'_>>,
223 basis: &crate::query::Basis,
224 ) -> Admission {
225 if self.delivery.cache_path.is_none() {
226 return Admission::NoLocation;
227 }
228 let Some(stored) = stored else {
229 return Admission::Missing;
230 };
231 if stored.root != basis.root {
232 return Admission::WrongRoot;
233 }
234 let relation = crate::serves_snapshot(stored.snapshot, basis.scope.snapshot_identity());
235 if relation == crate::Serves::Refuse {
236 return Admission::WrongScope;
237 }
238 if self.verify == Verify::None && basis.content.is_enabled() {
239 let wanted = crate::ContentTierIdentity::for_request(
240 basis.scope.snapshot_identity().entries,
241 basis.content,
242 );
243 if !stored.content_complete
244 || stored.content.and_then(|identity| wanted.admit(identity)).is_none()
245 {
246 return Admission::IncompleteContent;
247 }
248 }
249 Admission::Serve(relation)
250 }
251}
252
253#[derive(Clone, Copy, Debug)]
255#[allow(clippy::struct_excessive_bools)]
256pub(crate) struct RunFacts {
257 pub(crate) entries_verified: bool,
258 pub(crate) entries_changed: bool,
259 pub(crate) content_changed: bool,
260 pub(crate) content_requested: bool,
261 pub(crate) projected: bool,
262 pub(crate) paired_entries: bool,
263}
264
265#[derive(Clone, Copy, Debug, PartialEq, Eq)]
267pub(crate) struct SaveTargets {
268 pub(crate) metadata: bool,
269 pub(crate) content: bool,
270}
271
272impl SaveTargets {
273 pub(crate) const fn none(self) -> bool {
274 !self.metadata && !self.content
275 }
276}
277
278impl Plan {
279 pub(crate) fn writes(&self, run: RunFacts) -> SaveTargets {
280 let allowed = self.persist && self.delivery.cache_path.is_some();
281 SaveTargets {
282 metadata: allowed && run.entries_verified && run.entries_changed && !run.projected,
283 content: allowed
284 && run.content_requested
285 && run.content_changed
286 && (run.entries_verified || run.paired_entries),
287 }
288 }
289}
290
291#[derive(Clone, Copy, PartialEq, Eq, Debug)]
296pub struct PerformanceSummary {
297 pub walked_files: u64,
299 pub walked_bytes: u64,
301 pub walked_allocated: u64,
304 pub fresh_files: u64,
306 pub bytes_read: u64,
308 pub analysis_ns: u64,
310 pub cached_files: u64,
312 pub cached_bytes: u64,
314 pub source: ReportSource,
316}
317
318impl Default for PerformanceSummary {
319 fn default() -> Self {
320 Self {
321 walked_files: 0,
322 walked_bytes: 0,
323 walked_allocated: 0,
324 fresh_files: 0,
325 bytes_read: 0,
326 analysis_ns: 0,
327 cached_files: 0,
328 cached_bytes: 0,
329 source: ReportSource::ColdScan,
330 }
331 }
332}
333
334impl PerformanceSummary {
335 pub fn total_throughput(
338 self,
339 elapsed: std::time::Duration,
340 size: crate::query::SizeMetric,
341 ) -> String {
342 let bytes = match size {
343 crate::query::SizeMetric::Apparent => self.walked_bytes,
344 crate::query::SizeMetric::Allocated => self.walked_allocated,
345 };
346 throughput_rates(self.walked_files, bytes, elapsed).map_or_else(
347 || "throughput unavailable".to_owned(),
348 |(files, gib)| format!("{files} files/s ({gib} GiB/s)"),
349 )
350 }
351
352 fn from_open_report(report: &crate::OpenReport) -> Self {
353 let analysis = report.analysis.unwrap_or_default();
354 Self {
355 walked_files: report.scan.files_walked,
356 walked_bytes: report.scan.bytes_walked,
357 walked_allocated: report.scan.allocated_walked,
358 fresh_files: analysis.candidates,
359 bytes_read: analysis.bytes_read,
360 analysis_ns: analysis.elapsed_ns,
361 cached_files: report.content_cache.hits,
362 cached_bytes: report.content_cache.bytes,
363 source: match report.path_taken {
364 OpenPath::ColdScan => ReportSource::ColdScan,
365 OpenPath::WarmRevalidate => ReportSource::WarmRevalidate,
366 OpenPath::CacheOnly => ReportSource::CacheOnly,
367 },
368 }
369 }
370}
371
372pub fn throughput_rates(
377 files: u64,
378 bytes: u64,
379 elapsed: std::time::Duration,
380) -> Option<(String, String)> {
381 let ns = elapsed.as_nanos();
382 if ns == 0 {
383 return None;
384 }
385 let files_per_second = u128::from(files) * 1_000_000_000 / ns;
386 let gib_denominator = ns * (1_u128 << 30);
387 let gib_thousandths =
388 (u128::from(bytes) * 1_000_000_000 * 1_000 + gib_denominator / 2) / gib_denominator;
389 Some((
390 crate::report_format::human_count_u128(files_per_second),
391 format!("{}.{:03}", gib_thousandths / 1_000, gib_thousandths % 1_000),
392 ))
393}
394
395pub fn plan(
444 request: &Request,
445 delivery: &Delivery,
446 route: Route,
447) -> std::result::Result<Plan, crate::query::RequestError> {
448 request.validate()?;
449 let mut normalized = delivery.clone();
450 if route == Route::Watch {
451 normalized.watch.get_or_insert_with(crate::query::WatchDelivery::default);
452 }
453 let delivery = &normalized;
454 request.validate_delivery(delivery)?;
455 if route == Route::Opened {
456 if delivery.cache != CachePolicy::Off
457 || delivery.watch.is_some()
458 || request.basis.content.is_enabled()
459 {
460 return Err(crate::query::RequestError::DeliveryUnsupported {
461 route: "opened",
462 reason: "progressive discovery requires cache off, no content analyzers, and observation configured through OpenOptions",
463 });
464 }
465 if delivery.workers.scan.is_some()
469 || delivery.order != crate::ScanOrder::default()
470 || delivery.accept_partial
471 {
472 return Err(crate::query::RequestError::DeliveryUnsupported {
473 route: "opened",
474 reason: "progressive discovery schedules one breadth-first producer and reports coverage as state, so it takes no scan worker count, traversal order, or partial-answer acceptance",
475 });
476 }
477 }
478 if route == Route::Refresh && delivery.stale_ok {
479 return Err(crate::query::RequestError::DeliveryUnsupported {
480 route: "refresh",
481 reason: "a stale answer cannot verify filesystem state",
482 });
483 }
484 let analysis_requested = request.basis.content.is_enabled();
485 let summary_is_sufficient = request.query.views.as_slice() == [ViewSpec::Summary]
486 && request.query.selection.is_unfiltered()
487 && request.basis.scope.population == crate::query::IgnoredEntries::Include;
488 let policy_requires_index =
489 delivery.stale_ok || (delivery.cache == CachePolicy::On && delivery.cache_path.is_some());
490 let stored_state_pays = route != Route::OneShot || analysis_requested;
496 let read_snapshot = delivery.stale_ok
497 || match delivery.cache {
498 CachePolicy::Off => false,
499 CachePolicy::Auto | CachePolicy::On => stored_state_pays,
500 };
501 let persist = !delivery.stale_ok
502 && match delivery.cache {
503 CachePolicy::Off => false,
504 CachePolicy::Auto => stored_state_pays,
505 CachePolicy::On => true,
506 };
507 let transient = route == Route::OneShot && !policy_requires_index && !analysis_requested;
508 let retained = if !transient {
509 RetainedState::FullIndex
510 } else if summary_is_sufficient {
511 RetainedState::Summary
512 } else if let Some(retention) = TreeRetention::for_request(request) {
513 RetainedState::Tree(retention)
514 } else {
515 RetainedState::FullIndex
516 };
517 Ok(Plan {
518 basis: request.basis.clone(),
519 route,
520 retained,
521 load: if read_snapshot { Load::Snapshot } else { Load::None },
522 verify: if delivery.stale_ok { Verify::None } else { Verify::Filesystem },
523 persist,
524 delivery: delivery.clone(),
525 })
526}
527
528pub fn prepare_report(
556 request: &Request,
557 delivery: &Delivery,
558) -> Result<(Report, PendingSave, PerformanceSummary)> {
559 prepare_report_internal(request, delivery, false, None)
560 .map(|(report, pending, performance, _diagnostics)| (report, pending, performance))
561}
562
563pub fn prepare_report_with_progress(
580 request: &Request,
581 delivery: &Delivery,
582 progress: &Progress,
583) -> Result<(Report, PendingSave, PerformanceSummary)> {
584 prepare_report_internal(request, delivery, false, Some(progress))
585 .map(|(report, pending, performance, _diagnostics)| (report, pending, performance))
586}
587
588pub fn prepare_report_with_scan_diagnostics(
597 request: &Request,
598 delivery: &Delivery,
599) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)> {
600 prepare_report_internal(request, delivery, true, None)
601}
602
603fn prepare_report_internal(
604 request: &Request,
605 delivery: &Delivery,
606 collect_scan_diagnostics: bool,
607 progress: Option<&Progress>,
608) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)> {
609 request.validate().map_err(Error::InvalidRequest)?;
616 let scan_config = crate::ScanConfig {
617 progress: progress.cloned(),
618 ..request.basis.scope.scan_config(delivery)
619 };
620 let root = request.basis.root.as_path();
621 let scan_started_at = SystemTime::now();
622 let plan = plan(request, delivery, Route::OneShot).map_err(Error::InvalidRequest)?;
623 match plan.retained {
624 RetainedState::Summary => {
625 let root = root.canonicalize().map_err(|error| Error::io(root, error))?;
626 let mut fold = SummaryFold::new(&scan_config);
627 let mut reduce = |observed: &crate::ObservationOp| fold.observe(observed);
628 let (mut scan, scan_diagnostics) = if collect_scan_diagnostics {
629 let (scan, diagnostics) = crate::scan::scan_summary_fold_with_diagnostics(
630 &root,
631 &scan_config,
632 &mut reduce,
633 )?;
634 (scan, Some(diagnostics))
635 } else {
636 (crate::scan::scan_summary_fold(&root, &scan_config, &mut reduce)?, None)
637 };
638 let complete = scan.is_complete();
639 let generated_at = SystemTime::now();
640 let (summary, ignore_rules, ignored_unverified) = fold.finish(&root, &scan.errors)?;
641 let report = report_summary(
642 &root,
643 scan_config.scope(),
644 request,
645 summary,
646 ignore_rules,
647 ignored_unverified,
648 TreeStatus::of_walk(&root, &mut scan),
649 ReportProvenance::of_walk(scan_started_at, generated_at, complete),
650 );
651 let performance = PerformanceSummary {
652 walked_files: scan.files_walked,
653 walked_bytes: scan.bytes_walked,
654 walked_allocated: scan.allocated_walked,
655 source: ReportSource::ColdScan,
656 ..PerformanceSummary::default()
657 };
658 Ok((report, PendingSave::none(), performance, scan_diagnostics))
659 }
660 RetainedState::Tree(retention) => {
661 let root = root.canonicalize().map_err(|error| Error::io(root, error))?;
666 let (index, scan, scan_diagnostics) = crate::scan::scan_into_folded_index(
667 &root,
668 &scan_config,
669 retention,
670 collect_scan_diagnostics,
671 )?;
672 let performance = PerformanceSummary {
673 walked_files: scan.files_walked,
674 walked_bytes: scan.bytes_walked,
675 walked_allocated: scan.allocated_walked,
676 source: ReportSource::ColdScan,
677 ..PerformanceSummary::default()
678 };
679 if let Some(progress) = progress {
680 progress.enter(crate::ProgressPhase::Summarizing);
681 }
682 let answer = report(&index, request, SystemTime::now())?;
683 debug_assert_eq!(answer.scope, scan_config.scope());
684 crate::release_index(std::sync::Arc::new(index));
685 Ok((answer, PendingSave::none(), performance, scan_diagnostics))
686 }
687 RetainedState::FullIndex => {
688 let (index, open_report, pending_save, scan_diagnostics) =
689 execute(&plan, &request.basis, collect_scan_diagnostics, progress)?;
690 let performance = PerformanceSummary::from_open_report(&open_report);
691 if let Some(progress) = progress {
692 progress.enter(crate::ProgressPhase::Summarizing);
693 }
694 let answer = report(&index, request, SystemTime::now())?;
695 debug_assert_eq!(answer.scope, scan_config.scope());
696 crate::release_index(index);
699 Ok((answer, pending_save, performance, scan_diagnostics))
700 }
701 }
702}
703
704struct SummaryFold {
748 all: SummaryRow,
750 controls: Option<SummaryControls>,
752 unrepresentable: Option<Error>,
754}
755
756struct SummaryControls {
759 table: crate::control::ControlTable,
760 ignored_heads: std::collections::HashSet<std::ffi::OsString>,
772 parent: Option<(std::ffi::OsString, bool)>,
775 chain:
779 Option<(std::ffi::OsString, crate::control::ControlChain, crate::control::SplitDirectory)>,
780 unignored: crate::index::RollUpScalars,
782 rejected: Option<Error>,
785}
786
787impl SummaryFold {
788 fn new(config: &crate::ScanConfig) -> Self {
789 Self {
790 all: SummaryRow::default(),
791 controls: config.read_controls.then(|| SummaryControls {
792 table: crate::control::ControlTable::with_limits(config.control_limits),
793 ignored_heads: std::collections::HashSet::new(),
794 parent: None,
795 chain: None,
796 unignored: crate::index::RollUpScalars::default(),
797 rejected: None,
798 }),
799 unrepresentable: None,
800 }
801 }
802
803 fn observe(&mut self, observed: &crate::ObservationOp) {
804 match &observed.op {
805 crate::Op::Upsert { path, kind, attrs } => {
806 match kind {
807 EntryKind::File => {
808 if let Err(error) = crate::index::add_file_sizes(
809 &mut self.all.bytes,
810 &mut self.all.allocated,
811 attrs,
812 || path.clone(),
813 ) {
814 self.unrepresentable.get_or_insert(error);
815 return;
816 }
817 self.all.files += 1;
818 self.all.newest_mtime_ns = Some(
819 self.all
820 .newest_mtime_ns
821 .map_or(attrs.mtime_ns, |current| current.max(attrs.mtime_ns)),
822 );
823 }
824 EntryKind::Dir => self.all.dirs += 1,
825 EntryKind::Symlink | EntryKind::Other => {}
826 }
827 let Some(controls) = &mut self.controls else { return };
828 if controls.classify(path, *kind) {
829 return;
830 }
831 match kind {
834 EntryKind::File => {
835 controls.unignored.files += 1;
836 controls.unignored.bytes += attrs.size;
837 controls.unignored.allocated += attrs.allocated;
838 }
839 EntryKind::Dir => controls.unignored.dirs += 1,
840 EntryKind::Symlink | EntryKind::Other => {}
841 }
842 }
843 crate::Op::ControlUpsert { path, source } => {
844 if let Some(controls) = &mut self.controls {
845 controls.chain = None;
846 let admitted = controls.table.upsert(path, source.clone()).map(drop);
847 controls.record(admitted);
848 }
849 }
850 crate::Op::ControlRemove { path } => {
851 if let Some(controls) = &mut self.controls {
852 controls.chain = None;
853 let removed = controls.table.remove(path).map(drop);
854 controls.record(removed);
855 }
856 }
857 crate::Op::Remove { .. } | crate::Op::InvalidateSubtree { .. } => {}
859 }
860 }
861
862 fn finish(
867 self,
868 root: &std::path::Path,
869 errors: &[Error],
870 ) -> Result<(SummaryRow, crate::control::ControlCoverage, bool)> {
871 if let Some(error) = self.unrepresentable {
872 return Err(error);
873 }
874 let Some(controls) = self.controls else {
875 return Ok((
876 SummaryRow { ignored: None, ..self.all },
877 crate::control::ControlCoverage::NotObserved,
878 false,
879 ));
880 };
881 if let Some(error) = controls.rejected {
882 return Err(error);
883 }
884 let unreadable =
885 errors.iter().any(|error| crate::control::unreadable_control(root, error).is_some());
886 let verified = controls.table.refused_len() == 0 && !unreadable;
887 let all = crate::index::RollUpScalars {
888 files: self.all.files,
889 dirs: self.all.dirs,
890 bytes: self.all.bytes,
891 allocated: self.all.allocated,
892 newest_mtime_ns: self.all.newest_mtime_ns.unwrap_or_default(),
893 };
894 let summary = SummaryRow {
895 ignored: verified.then(|| crate::query::IgnoredTally::between(all, controls.unignored)),
896 ..self.all
897 };
898 Ok((
899 summary,
900 crate::control::ControlCoverage::Observed(controls.table.observation()),
901 !verified,
902 ))
903 }
904}
905
906impl SummaryControls {
907 fn classify(&mut self, path: &std::path::Path, kind: EntryKind) -> bool {
912 if self.ignored_heads.is_empty() && self.table.is_empty() {
915 return false;
916 }
917 let (parent, name) = crate::control::split_parent(path);
924 let parent_ignored = self.parent_ignored(parent);
925 let ignored = if parent_ignored || self.table.is_empty() {
926 parent_ignored
927 } else if !name.is_empty() {
928 if !matches!(&self.chain, Some((cached, ..)) if cached.as_os_str() == parent) {
929 let directory = std::path::Path::new(parent);
930 self.chain = Some((
931 parent.to_os_string(),
932 self.table.chain_for(directory),
933 crate::control::SplitDirectory::new(directory),
934 ));
935 }
936 let (_, chain, split) = self.chain.as_ref().expect("the chain was just resolved");
937 !chain.is_empty()
938 && split.with_components(|directory| {
939 chain.is_ignored_within(directory, name.as_encoded_bytes(), kind.is_dir())
940 })
941 } else {
942 self.table.matcher_for(path).is_ignored(kind.is_dir())
943 };
944 if ignored && !parent_ignored && kind.is_dir() {
945 self.ignored_heads.insert(path.as_os_str().to_os_string());
946 }
947 ignored
948 }
949
950 fn parent_ignored(&mut self, parent: &std::ffi::OsStr) -> bool {
951 if self.ignored_heads.is_empty() {
952 return false;
953 }
954 if let Some((cached, ignored)) = &self.parent {
955 if cached.as_os_str() == parent {
956 return *ignored;
957 }
958 }
959 let ignored =
960 crate::control::ancestors(parent).any(|ancestor| self.ignored_heads.contains(ancestor));
961 self.parent = Some((parent.to_os_string(), ignored));
962 ignored
963 }
964
965 fn record(&mut self, applied: Result<()>) {
966 if let Err(error) = applied {
967 self.rejected.get_or_insert(error);
968 }
969 }
970}
971
972#[cfg(test)]
973mod tests {
974 use std::fs;
975 use std::path::{Path, PathBuf};
976
977 use super::*;
978 use crate::query::{IgnoredEntries, Pattern, Query, Section};
979 use crate::{OpenFixture, ScanConfig};
980
981 #[test]
985 fn byte_wise_path_splits_match_the_parsed_ones() {
986 use std::ffi::{OsStr, OsString};
987 let named = ["", "a", "a/b", "a/b/c.txt", ".gitignore", "d.d/.h", "x/..y"]
988 .into_iter()
989 .map(OsString::from);
990 #[cfg(unix)]
991 let paths: Vec<OsString> = {
992 use std::os::unix::ffi::OsStringExt as _;
993 named
994 .chain([
995 OsString::from_vec(b"d\xff/f\xfe".to_vec()),
996 OsString::from_vec(b"\xfe".to_vec()),
997 ])
998 .collect()
999 };
1000 #[cfg(not(unix))]
1001 let paths: Vec<OsString> = named.collect();
1002 for path in &paths {
1003 let path = Path::new(path);
1004 let (parent, name) = crate::control::split_parent(path);
1005 assert_eq!(parent, path.parent().map_or(OsStr::new(""), Path::as_os_str), "{path:?}");
1006 assert_eq!(name, path.file_name().unwrap_or(OsStr::new("")), "{path:?}");
1007 let by_bytes: Vec<&OsStr> = crate::control::ancestors(parent).collect();
1008 let parsed: Vec<&OsStr> = Path::new(parent).ancestors().map(Path::as_os_str).collect();
1009 assert_eq!(by_bytes, parsed, "{path:?}");
1010 }
1011 }
1012
1013 #[test]
1014 fn total_throughput_uses_one_elapsed_sample_and_selected_size() {
1015 use crate::query::SizeMetric;
1016 let work = PerformanceSummary {
1017 walked_files: 200,
1018 walked_bytes: 4_000_000_000,
1019 walked_allocated: 1_000_000_000,
1020 ..PerformanceSummary::default()
1021 };
1022 assert_eq!(
1023 work.total_throughput(std::time::Duration::from_secs(2), SizeMetric::Apparent),
1024 "100 files/s (1.863 GiB/s)"
1025 );
1026 assert_eq!(
1027 work.total_throughput(std::time::Duration::from_secs(2), SizeMetric::Allocated),
1028 "100 files/s (0.466 GiB/s)"
1029 );
1030 assert_eq!(
1031 work.total_throughput(std::time::Duration::ZERO, SizeMetric::Apparent),
1032 "throughput unavailable"
1033 );
1034 assert_eq!(
1035 PerformanceSummary::default()
1036 .total_throughput(std::time::Duration::from_secs(1), SizeMetric::Apparent),
1037 "0 files/s (0.000 GiB/s)"
1038 );
1039 assert_eq!(
1040 throughput_rates(12_345, 3 * (1_u64 << 30), std::time::Duration::from_secs(2)),
1041 Some(("6,172".to_owned(), "1.500".to_owned()))
1042 );
1043 }
1044
1045 #[test]
1056 fn every_route_refuses_a_total_no_u64_can_hold_and_accepts_an_exact_fit() {
1057 use crate::index::DetachedIndexBuilder;
1058 use crate::scan::{DetachedChild, DetachedDirectory};
1059 use crate::{Attrs, Index, Observation, ObservationOp, Op};
1060
1061 #[derive(Clone, Copy, Debug)]
1062 enum Layout {
1063 OneDirectory,
1064 DirectoryEach,
1065 }
1066 let attrs = |size: u64, allocated: u64, inode: u64| Attrs {
1067 size,
1068 allocated,
1069 mtime_ns: 1,
1070 ctime_ns: 1,
1071 inode,
1072 dev: 1,
1073 };
1074 let files = |layout: Layout, count: usize| -> Vec<(PathBuf, Option<PathBuf>)> {
1076 (0..count)
1077 .map(|file| match layout {
1078 Layout::OneDirectory => (PathBuf::from(format!("f{file}")), None),
1079 Layout::DirectoryEach => {
1080 let directory = PathBuf::from(format!("d{file}"));
1081 (directory.join("f"), Some(directory))
1082 }
1083 })
1084 .collect()
1085 };
1086 let upserts = |layout: Layout, sizes: &[(u64, u64)]| -> Vec<Op> {
1087 let mut ops = Vec::new();
1088 for ((path, directory), (inode, &(size, allocated))) in
1089 files(layout, sizes.len()).into_iter().zip((1_u64..).zip(sizes))
1090 {
1091 if let Some(directory) = directory {
1092 ops.push(Op::Upsert {
1093 path: directory,
1094 kind: EntryKind::Dir,
1095 attrs: Attrs::default(),
1096 });
1097 }
1098 ops.push(Op::Upsert {
1099 path,
1100 kind: EntryKind::File,
1101 attrs: attrs(size, allocated, inode),
1102 });
1103 }
1104 ops
1105 };
1106 let summary_fold = |layout: Layout, sizes: &[(u64, u64)], read_controls: bool| {
1107 let config = ScanConfig { read_controls, ..ScanConfig::default() };
1108 let mut fold = SummaryFold::new(&config);
1109 for op in upserts(layout, sizes) {
1110 fold.observe(&ObservationOp::unconditional(op));
1111 }
1112 fold.finish(Path::new("/root"), &[]).map(|(row, ..)| (row.bytes, row.allocated))
1113 };
1114 let builder = |layout: Layout, sizes: &[(u64, u64)], folding: bool| -> Result<(u64, u64)> {
1115 let mut builder = DetachedIndexBuilder::new(
1116 "/root",
1117 crate::ScanScope::default(),
1118 crate::classify::TypeRegistry::compiled_shared(),
1119 );
1120 if folding {
1121 builder =
1122 builder.folding(TreeRetention { largest_files: 1, size: SizeMetric::Apparent });
1123 }
1124 let child = |name: &Path, kind, attrs, position| DetachedChild {
1125 name: name.as_os_str().to_owned(),
1126 kind,
1127 attrs,
1128 position,
1129 };
1130 let mut root =
1131 DetachedDirectory { path: PathBuf::new(), children: Vec::new(), control: None };
1132 let mut nested = Vec::new();
1133 for (position, ((path, directory), (inode, &(size, allocated)))) in
1134 (0_u32..).zip(files(layout, sizes.len()).into_iter().zip((1_u64..).zip(sizes)))
1135 {
1136 let file = attrs(size, allocated, inode);
1137 match directory {
1138 None => root.children.push(child(&path, EntryKind::File, file, position)),
1139 Some(directory) => {
1140 root.children.push(child(
1141 &directory,
1142 EntryKind::Dir,
1143 Attrs::default(),
1144 position,
1145 ));
1146 let name = path.file_name().expect("a file name");
1147 nested.push(DetachedDirectory {
1148 path: directory,
1149 children: vec![child(Path::new(name), EntryKind::File, file, 0)],
1150 control: None,
1151 });
1152 }
1153 }
1154 }
1155 builder.push_directory(&mut root)?;
1156 for listing in &mut nested {
1157 builder.push_directory(listing)?;
1158 }
1159 let total = builder.finish().total();
1160 Ok((total.bytes, total.allocated))
1161 };
1162 let apply_lane = |layout: Layout, sizes: &[(u64, u64)]| -> Result<(u64, u64)> {
1163 let mut index = Index::new("/root");
1164 index.apply(&Observation::new(upserts(layout, sizes)))?;
1165 let total = index.total();
1166 Ok((total.bytes, total.allocated))
1167 };
1168 let routes =
1169 |layout: Layout, sizes: &[(u64, u64)]| -> Vec<(&'static str, Result<(u64, u64)>)> {
1170 vec![
1171 ("the summary fold", summary_fold(layout, sizes, false)),
1172 ("the classifying summary fold", summary_fold(layout, sizes, true)),
1173 ("the detached builder", builder(layout, sizes, false)),
1174 ("the folding builder", builder(layout, sizes, true)),
1175 ("the apply lane", apply_lane(layout, sizes)),
1176 ]
1177 };
1178
1179 let half = u64::MAX / 2 + 1;
1180 for layout in [Layout::OneDirectory, Layout::DirectoryEach] {
1181 for (sizes, counter) in [
1182 (&[(half, 0), (half, 0), (half, 0)][..], "bytes"),
1183 (&[(0, half), (0, half)][..], "allocated bytes"),
1184 ] {
1185 for (route, outcome) in routes(layout, sizes) {
1186 match outcome {
1187 Err(Error::UnrepresentableTotal { counter: refused, .. }) => {
1188 assert_eq!(refused, counter, "{route}, {layout:?}");
1189 }
1190 other => panic!(
1191 "{route}, {layout:?}: expected the {counter} total refused, got {other:?}"
1192 ),
1193 }
1194 }
1195 }
1196 let exact = [(half, half), (half - 1, half - 1)];
1197 for (route, outcome) in routes(layout, &exact) {
1198 assert_eq!(
1199 outcome.expect("an exact fit is representable"),
1200 (u64::MAX, u64::MAX),
1201 "{route}, {layout:?}"
1202 );
1203 }
1204 }
1205 }
1206
1207 #[test]
1208 #[cfg(unix)]
1209 fn an_unreadable_stored_header_never_authorizes_live_replacement() {
1210 use std::os::unix::fs::PermissionsExt;
1211 if !crate::test_support::require_permission_bits() {
1212 return;
1213 }
1214 let root = tempfile::tempdir().expect("root");
1215 let cache = tempfile::tempdir().expect("cache");
1216 let snapshot = cache.path().join("snapshot.fdu");
1217 fs::write(root.path().join(".gitignore"), b"ignored\n").expect("control");
1218 let observed = crate::query::Basis {
1219 root: root.path().into(),
1220 scope: crate::query::Scope::default(),
1221 content: crate::content::AnalysisSet::NONE,
1222 };
1223 let delivery = Delivery::new(CachePolicy::Auto, Some(snapshot.clone()));
1224 crate::open(&observed, &delivery).expect("stronger snapshot");
1225 let original = fs::read(&snapshot).expect("original image");
1226 let basis = crate::query::Basis {
1227 scope: crate::query::Scope { read_controls: false, ..Default::default() },
1228 ..observed
1229 };
1230 let (mut index, _) =
1231 crate::open(&basis, &Delivery::new(CachePolicy::Off, None)).expect("fresh blind index");
1232 let request = Request::new(basis, Query::default(), SystemTime::now());
1233 let plan = plan(&request, &delivery, Route::Refresh).expect("plan");
1234 fs::set_permissions(&snapshot, fs::Permissions::from_mode(0o000))
1235 .expect("deny header read");
1236 let nonwriting = Delivery { cache: CachePolicy::Off, ..delivery.clone() };
1237 let nonwriting_plan =
1238 super::plan(&request, &nonwriting, Route::Refresh).expect("nonwriting plan");
1239 assert!(
1240 !crate::persist_index_changes(&index, &nonwriting_plan, true, true)
1241 .expect("nonwriting policy never reads the header")
1242 );
1243 fs::write(root.path().join("fresh.txt"), b"fresh").expect("mutation");
1244 crate::refresh(
1245 &mut index,
1246 &request.basis,
1247 &Delivery { cache: CachePolicy::Off, ..delivery.clone() },
1248 )
1249 .expect("off refresh does not inspect cache state");
1250 let result = crate::persist_index_changes(&index, &plan, true, true);
1251 fs::set_permissions(&snapshot, fs::Permissions::from_mode(0o600)).expect("restore");
1252 assert!(
1253 result.is_err(),
1254 "a writable parent must not let unknown identity authorize replacement"
1255 );
1256 assert_eq!(fs::read(&snapshot).expect("retained image"), original);
1257 }
1258
1259 #[test]
1260 fn refresh_rejects_another_root_before_mutating_or_persisting() {
1261 let a = tempfile::tempdir().expect("root a");
1262 let b = tempfile::tempdir().expect("root b");
1263 let cache = tempfile::tempdir().expect("cache");
1264 let basis = crate::query::Basis {
1265 root: a.path().into(),
1266 scope: crate::query::Scope::default(),
1267 content: crate::content::AnalysisSet::NONE,
1268 };
1269 let (mut index, _) =
1270 crate::open(&basis, &Delivery::new(CachePolicy::Off, None)).expect("open a");
1271 fs::write(a.path().join("new"), b"new facts").expect("mutation a");
1272 let before = index.clock();
1273 let snapshot = cache.path().join("snapshot.fdu");
1274 let wrong = crate::query::Basis { root: b.path().into(), ..basis.clone() };
1275 let delivery = Delivery::new(CachePolicy::Auto, Some(snapshot.clone()));
1276 let error = crate::refresh(&mut index, &wrong, &delivery).expect_err("different root");
1277 assert!(matches!(
1278 error,
1279 Error::InvalidRequest(crate::query::RequestError::RootMismatch { .. })
1280 ));
1281 assert_eq!(index.clock(), before);
1282 assert!(!snapshot.exists());
1283 let alias = crate::query::Basis { root: a.path().join("."), ..basis };
1284 crate::refresh(&mut index, &alias, &delivery).expect("same root spelling");
1285 assert_eq!(index.total().files, 1);
1286 #[cfg(unix)]
1287 {
1288 let link = cache.path().join("root-alias");
1289 std::os::unix::fs::symlink(a.path(), &link).expect("root alias");
1290 let symlink_basis = crate::query::Basis { root: link, ..alias };
1291 crate::refresh(&mut index, &symlink_basis, &delivery)
1292 .expect("same canonical root through symlink");
1293 }
1294 }
1295
1296 #[test]
1297 fn unchanged_refresh_replaces_an_incompatible_stored_baseline() {
1298 let root = tempfile::tempdir().expect("root");
1299 let other = tempfile::tempdir().expect("other root");
1300 let cache = tempfile::tempdir().expect("cache");
1301 fs::write(root.path().join("file"), b"retained").expect("file");
1302 let basis = crate::query::Basis {
1303 root: root.path().into(),
1304 scope: crate::query::Scope::default(),
1305 content: crate::content::AnalysisSet::NONE,
1306 };
1307 let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
1308 for wrong_root in [false, true] {
1309 let wrong = if wrong_root {
1310 crate::query::Basis { root: other.path().into(), ..basis.clone() }
1311 } else {
1312 crate::query::Basis {
1313 scope: crate::query::Scope { max_depth: Some(0), ..basis.scope.clone() },
1314 ..basis.clone()
1315 }
1316 };
1317 crate::open(&wrong, &Delivery { cache: CachePolicy::On, ..delivery.clone() })
1318 .expect("incompatible snapshot");
1319 let (mut index, _) = crate::open(&basis, &Delivery::new(CachePolicy::Off, None))
1320 .expect("retained index");
1321 let refreshed =
1322 crate::refresh(&mut index, &basis, &delivery).expect("refresh reseeds cache");
1323 assert!(!refreshed.apply.mutated(), "the existing index was already current");
1324 let (cached, _) = crate::open(&basis, &Delivery { stale_ok: true, ..delivery.clone() })
1325 .expect("cache-only can now answer");
1326 assert_eq!(cached.total().bytes, 8);
1327 }
1328 }
1329
1330 #[test]
1331 fn refreshed_metadata_and_content_are_visible_to_a_later_cache_only_open() {
1332 let root = tempfile::tempdir().expect("root");
1333 let cache = tempfile::tempdir().expect("cache");
1334 let path = root.path().join("note.txt");
1335 fs::write(&path, b"old\n").expect("old file");
1336 let basis = crate::query::Basis {
1337 root: root.path().into(),
1338 scope: crate::query::Scope::default(),
1339 content: crate::content::AnalysisSet::NONE.with_lines(),
1340 };
1341 let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
1342 let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
1343 fs::write(&path, b"new longer text\nsecond line\n").expect("mutation");
1344 let refreshed = crate::refresh(&mut index, &basis, &delivery).expect("refresh");
1345 assert!(refreshed.is_complete());
1346 let (cached, report) = crate::open(&basis, &Delivery { stale_ok: true, ..delivery })
1347 .expect("cache-only sees refreshed tiers");
1348 assert_eq!(report.path_taken, OpenPath::CacheOnly);
1349 assert_eq!(cached.total(), index.total());
1350 assert_eq!(
1351 cached.total().bytes,
1352 u64::try_from(b"new longer text\nsecond line\n".len()).expect("length")
1353 );
1354 assert_eq!(report.content_cache.hits, 1);
1355 let original = index
1356 .content()
1357 .expect("fresh content")
1358 .file(Path::new("note.txt"))
1359 .expect("fresh file");
1360 let restored = cached
1361 .content()
1362 .expect("restored content")
1363 .file(Path::new("note.txt"))
1364 .expect("restored file");
1365 assert_eq!(restored, original);
1366 }
1367
1368 #[cfg(unix)]
1371 fn owed_persistence_fixture()
1372 -> (tempfile::TempDir, tempfile::TempDir, crate::query::Basis, Delivery) {
1373 let root = tempfile::tempdir().expect("root");
1374 let cache = tempfile::tempdir().expect("cache");
1375 fs::create_dir(root.path().join("locked")).expect("locked dir");
1376 fs::write(root.path().join("first"), b"first").expect("first file");
1377 let basis = crate::query::Basis {
1378 root: root.path().into(),
1379 scope: crate::query::Scope::default(),
1380 content: crate::content::AnalysisSet::NONE,
1381 };
1382 let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
1383 (root, cache, basis, delivery)
1384 }
1385
1386 #[cfg(unix)]
1388 fn cached_files(basis: &crate::query::Basis, delivery: &Delivery) -> u64 {
1389 let cache_only = Delivery { stale_ok: true, ..delivery.clone() };
1390 crate::open(basis, &cache_only).expect("cache-only open").0.total().files
1391 }
1392
1393 #[test]
1400 #[cfg(unix)]
1401 fn an_unchanged_refresh_repeats_the_metadata_write_a_failed_refresh_owed() {
1402 use std::os::unix::fs::PermissionsExt;
1403 if !crate::test_support::require_permission_bits() {
1404 return;
1405 }
1406 let (root, cache, basis, delivery) = owed_persistence_fixture();
1407 let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
1408 assert_eq!(cached_files(&basis, &delivery), 1);
1409
1410 fs::write(root.path().join("second"), b"second").expect("second file");
1411 fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o555)).expect("deny write");
1412 let failed = crate::refresh(&mut index, &basis, &delivery);
1413 fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o755)).expect("restore");
1414 assert!(failed.is_err(), "a read-only cache directory fails the write");
1415 assert_eq!(index.total().files, 2, "the index advanced before the write");
1416 assert_eq!(cached_files(&basis, &delivery), 1, "the failed write left the old image");
1417
1418 let unchanged = crate::refresh(&mut index, &basis, &delivery).expect("unchanged refresh");
1419 assert!(unchanged.is_complete());
1420 assert!(!unchanged.apply.mutated(), "nothing changed between the passes");
1421 assert_eq!(cached_files(&basis, &delivery), 2, "the owed write ran");
1422
1423 let snapshot = delivery.cache_path.as_deref().expect("path");
1425 let written = fs::metadata(snapshot).expect("snapshot").modified().expect("mtime");
1426 crate::refresh(&mut index, &basis, &delivery).expect("settled refresh");
1427 assert_eq!(fs::metadata(snapshot).expect("snapshot").modified().expect("mtime"), written);
1428 }
1429
1430 #[test]
1433 #[cfg(unix)]
1434 fn an_unchanged_refresh_repeats_the_metadata_write_a_failed_open_owed() {
1435 use std::os::unix::fs::PermissionsExt;
1436 if !crate::test_support::require_permission_bits() {
1437 return;
1438 }
1439 let (root, cache, basis, delivery) = owed_persistence_fixture();
1440 crate::open(&basis, &delivery).expect("complete open writes the snapshot");
1441 fs::write(root.path().join("second"), b"second").expect("second file");
1442 fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o555)).expect("deny write");
1443 let opened = crate::open_with_pending_save(&basis, &delivery);
1444 let outcome = opened.map(|(index, report, pending)| (index, report, pending.join()));
1446 fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o755)).expect("restore");
1447 let (index, report, joined) = outcome.expect("the open itself succeeds");
1448 assert_eq!(report.path_taken, OpenPath::WarmRevalidate);
1449 assert!(joined.is_err(), "the startup write failed");
1450 let mut index = std::sync::Arc::into_inner(index).expect("the writer released the index");
1451 assert_eq!(cached_files(&basis, &delivery), 1);
1452
1453 let unchanged = crate::refresh(&mut index, &basis, &delivery).expect("unchanged refresh");
1454 assert!(!unchanged.apply.mutated(), "nothing changed between the passes");
1455 assert_eq!(cached_files(&basis, &delivery), 2, "the owed write ran");
1456 }
1457
1458 #[test]
1466 #[cfg(unix)]
1467 fn a_complete_refresh_persists_the_facts_a_partial_refresh_could_not() {
1468 use std::os::unix::fs::PermissionsExt;
1469 if !crate::test_support::require_permission_bits() {
1470 return;
1471 }
1472 let (root, _cache, basis, delivery) = owed_persistence_fixture();
1473 let locked = root.path().join("locked");
1474 let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
1475 assert_eq!(index.total().files, 1);
1476
1477 fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("deny read");
1478 fs::write(root.path().join("second"), b"second").expect("second file");
1479 let partial = crate::refresh(&mut index, &basis, &delivery);
1480 fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).expect("restore");
1481 let partial = partial.expect("partial refresh");
1482 assert!(!partial.is_complete(), "the locked directory made the pass partial");
1483 assert!(partial.apply.mutated(), "the second file was inserted");
1484 assert_eq!(index.total().files, 2);
1485 assert_eq!(cached_files(&basis, &delivery), 1, "a partial pass never writes entries");
1486
1487 let complete = crate::refresh(&mut index, &basis, &delivery).expect("complete refresh");
1488 assert!(complete.is_complete(), "{:?}", complete.scan.errors);
1489 assert_eq!(cached_files(&basis, &delivery), 2, "the complete pass wrote the facts");
1490 }
1491
1492 #[test]
1493 fn cache_only_refusals_name_location_root_and_absence_separately() {
1494 let root = tempfile::tempdir().expect("root");
1495 let other = tempfile::tempdir().expect("other root");
1496 let cache = tempfile::tempdir().expect("cache");
1497 let basis = crate::query::Basis {
1498 root: root.path().into(),
1499 scope: crate::query::Scope::default(),
1500 content: crate::content::AnalysisSet::NONE,
1501 };
1502 let snapshot = cache.path().join("snapshot.fdu");
1503 let message = |delivery: &Delivery| {
1504 crate::open(&basis, delivery).expect_err("cache-only refusal").to_string()
1505 };
1506 let no_location = message(&Delivery::stale_ok(None));
1507 assert!(no_location.contains("no cache location"), "{no_location}");
1508 assert!(!no_location.contains("`on`"), "no write can succeed without a location");
1509 let missing = message(&Delivery::stale_ok(Some(snapshot.clone())));
1510 assert!(missing.contains("no usable snapshot"), "{missing}");
1511 assert!(missing.contains("with the `on` cache policy"), "{missing}");
1512 let other_basis = crate::query::Basis { root: other.path().into(), ..basis.clone() };
1513 crate::open(&other_basis, &Delivery::new(CachePolicy::Auto, Some(snapshot.clone())))
1514 .expect("other snapshot");
1515 let wrong_root = message(&Delivery::stale_ok(Some(snapshot)));
1516 assert!(wrong_root.contains("different root"), "{wrong_root}");
1517 }
1518
1519 #[test]
1520 fn route_delivery_matrix_rejects_contracts_the_route_cannot_execute() {
1521 let basis = crate::query::Basis {
1522 root: ".".into(),
1523 scope: crate::query::Scope::default(),
1524 content: crate::content::AnalysisSet::NONE,
1525 };
1526 let request = Request::new(basis, Query::default(), SystemTime::now());
1527 for delivery in Delivery::enumerate() {
1528 for route in
1529 [Route::OneShot, Route::Retained, Route::Refresh, Route::Watch, Route::Opened]
1530 {
1531 let result = plan(&request, &delivery, route);
1532 let forbidden = delivery.stale_ok
1533 && (delivery.watch.is_some()
1534 || matches!(route, Route::Watch | Route::Refresh)
1535 || delivery.cache == CachePolicy::Off)
1536 || route == Route::Opened
1537 && (delivery.cache != CachePolicy::Off
1538 || delivery.watch.is_some()
1539 || delivery.accept_partial);
1540 assert_eq!(result.is_err(), forbidden, "{route:?} {delivery:?}");
1541 if let Ok(plan) = result {
1542 if route == Route::Opened {
1543 assert_eq!(plan.load(), Load::None);
1544 assert_eq!(plan.verify(), Verify::Filesystem);
1545 }
1546 }
1547 }
1548 }
1549 }
1550
1551 #[test]
1552 fn an_opened_root_refuses_the_scheduling_it_would_otherwise_drop() {
1553 let basis = crate::query::Basis {
1558 root: ".".into(),
1559 scope: crate::query::Scope::default(),
1560 content: crate::content::AnalysisSet::NONE,
1561 };
1562 let request = Request::new(basis, Query::default(), SystemTime::now());
1563 let default = Delivery::new(CachePolicy::Off, None);
1564 let plan_opened = plan(&request, &default, Route::Opened).expect("defaults plan");
1565 assert_eq!(plan_opened.delivery().batch_size, default.batch_size);
1566 let unhonored = [
1567 (
1568 "scan workers",
1569 Delivery {
1570 workers: crate::query::Workers { scan: Some(4), ..default.workers },
1571 ..default.clone()
1572 },
1573 ),
1574 (
1575 "depth-first order",
1576 Delivery { order: crate::ScanOrder::DepthFirst, ..default.clone() },
1577 ),
1578 ("accept partial", Delivery { accept_partial: true, ..default.clone() }),
1579 ];
1580 for (case, delivery) in unhonored {
1581 let refused = plan(&request, &delivery, Route::Opened).expect_err(case);
1582 assert!(
1583 matches!(
1584 refused,
1585 crate::query::RequestError::DeliveryUnsupported { route: "opened", .. }
1586 ),
1587 "{case}: {refused}"
1588 );
1589 plan(&request, &delivery, Route::Retained)
1590 .unwrap_or_else(|error| panic!("{case} executes on a retained route: {error}"));
1591 }
1592 let batched = Delivery { batch_size: default.batch_size * 2, ..default };
1594 let plan_batched = plan(&request, &batched, Route::Opened).expect("batch size plans");
1595 assert_eq!(plan_batched.delivery().batch_size, batched.batch_size);
1596 }
1597
1598 #[test]
1599 fn tier_writes_depend_only_on_authorization_and_observed_facts() {
1600 let routes = [Route::OneShot, Route::Retained, Route::Refresh, Route::Watch, Route::Opened];
1604 for (delivery, persist) in
1605 Delivery::enumerate().flat_map(|delivery| [(delivery.clone(), false), (delivery, true)])
1606 {
1607 for bits in 0_u8..64 {
1608 let facts = RunFacts {
1609 entries_verified: bits & 1 != 0,
1610 entries_changed: bits & 2 != 0,
1611 content_changed: bits & 4 != 0,
1612 content_requested: bits & 8 != 0,
1613 projected: bits & 16 != 0,
1614 paired_entries: bits & 32 != 0,
1615 };
1616 let allowed = persist;
1617 let expected = SaveTargets {
1618 metadata: allowed
1619 && facts.entries_verified
1620 && facts.entries_changed
1621 && !facts.projected,
1622 content: allowed
1623 && facts.content_requested
1624 && facts.content_changed
1625 && (facts.entries_verified || facts.paired_entries),
1626 };
1627 for route in routes {
1628 let plan = Plan {
1629 basis: crate::query::Basis {
1630 root: ".".into(),
1631 scope: crate::query::Scope::default(),
1632 content: crate::content::AnalysisSet::NONE,
1633 },
1634 route,
1635 retained: RetainedState::FullIndex,
1636 load: Load::Snapshot,
1637 verify: Verify::Filesystem,
1638 persist,
1639 delivery: delivery.clone(),
1640 };
1641 assert_eq!(plan.writes(facts), expected, "{route:?} {delivery:?} {facts:?}");
1642 let unavailable = Plan {
1643 delivery: Delivery { cache_path: None, ..delivery.clone() },
1644 ..plan
1645 };
1646 assert!(unavailable.writes(facts).none());
1647 }
1648 }
1649 }
1650 }
1651
1652 fn planned(config: &OpenFixture, query: &Query) -> Plan {
1653 let (request, delivery) = split(Path::new("."), config, query);
1654 plan(&request, &delivery, Route::OneShot).expect("valid plan")
1655 }
1656
1657 fn summary_query() -> Query {
1658 Query { views: vec![ViewSpec::Summary], ..Query::default() }
1659 }
1660
1661 fn split(root: &Path, config: &OpenFixture, query: &Query) -> (Request, Delivery) {
1664 let (basis, delivery) = config.split(root);
1665 (Request::new(basis, query.clone(), std::time::UNIX_EPOCH), delivery)
1666 }
1667
1668 fn prepared(
1670 root: &Path,
1671 config: &OpenFixture,
1672 query: &Query,
1673 ) -> Result<(Report, PendingSave, PerformanceSummary)> {
1674 let (request, delivery) = split(root, config, query);
1675 prepare_report(&request, &delivery)
1676 }
1677
1678 fn prepared_with_diagnostics(
1680 root: &Path,
1681 config: &OpenFixture,
1682 query: &Query,
1683 ) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)>
1684 {
1685 let (request, delivery) = split(root, config, query);
1686 prepare_report_with_scan_diagnostics(&request, &delivery)
1687 }
1688
1689 fn config(policy: CachePolicy, cache_path: Option<PathBuf>) -> OpenFixture {
1690 OpenFixture { scan: ScanConfig::default(), cache_path, policy, ..OpenFixture::default() }
1691 }
1692
1693 fn blind(policy: CachePolicy, cache_path: Option<PathBuf>) -> OpenFixture {
1695 OpenFixture {
1696 scan: ScanConfig { read_controls: false, ..ScanConfig::default() },
1697 ..config(policy, cache_path)
1698 }
1699 }
1700
1701 fn controls_config(
1702 policy: CachePolicy,
1703 cache_path: PathBuf,
1704 read_controls: bool,
1705 ) -> OpenFixture {
1706 OpenFixture {
1707 scan: ScanConfig { read_controls, ..ScanConfig::default() },
1708 cache_path: Some(cache_path),
1709 policy,
1710 ..OpenFixture::default()
1711 }
1712 }
1713
1714 fn stale(fixture: OpenFixture) -> OpenFixture {
1716 OpenFixture { stale_ok: true, ..fixture }
1717 }
1718
1719 fn seed_controls_snapshot(root: &Path, cache_path: PathBuf) {
1720 fs::write(root.join(".gitignore"), b"ignored.log\n").expect("control file");
1721 fs::write(root.join("ignored.log"), b"ignored").expect("ignored file");
1722 crate::open_fixture(root, &controls_config(CachePolicy::Auto, cache_path, true))
1723 .expect("seed controls-on snapshot");
1724 }
1725
1726 #[test]
1727 fn planner_uses_compact_state_only_when_the_request_proves_it_is_sufficient() {
1728 let off = blind(CachePolicy::Off, Some(PathBuf::from("unused.fdu")));
1729 assert_eq!(planned(&off, &summary_query()).retained, RetainedState::Summary);
1730
1731 for policy in [CachePolicy::Auto, CachePolicy::On] {
1732 let unavailable = blind(policy, None);
1733 assert_eq!(planned(&unavailable, &summary_query()).retained, RetainedState::Summary);
1734 }
1735
1736 let mut several_views = summary_query();
1737 several_views.views.push(ViewSpec::Types);
1738 assert_eq!(planned(&off, &several_views).retained, RetainedState::FullIndex);
1739
1740 let mut filtered = summary_query();
1741 filtered.selection.include.push(Pattern::parse("*.rs").expect("pattern"));
1742 assert_eq!(planned(&off, &filtered).retained, RetainedState::FullIndex);
1743
1744 let observing = config(CachePolicy::Off, None);
1750 assert!(observing.scan.read_controls, "observation is the default");
1751 assert_eq!(planned(&observing, &summary_query()).retained, RetainedState::Summary);
1752 let mut by_ignored = summary_query();
1755 by_ignored.selection.ignored = IgnoredEntries::Exclude;
1756 assert_eq!(planned(&observing, &by_ignored).retained, RetainedState::FullIndex);
1757 for population in [IgnoredEntries::Exclude, IgnoredEntries::Only] {
1758 let narrowed = OpenFixture {
1759 scan: ScanConfig { population, ..ScanConfig::default() },
1760 ..config(CachePolicy::Off, None)
1761 };
1762 let mut query = summary_query();
1763 query.selection.ignored = population;
1764 assert_eq!(planned(&narrowed, &query).retained, RetainedState::FullIndex);
1765 }
1766 }
1767
1768 #[test]
1769 fn an_available_snapshot_does_not_force_the_index_for_a_metadata_summary() {
1770 let cached = blind(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")));
1776 assert_eq!(
1777 planned(&cached, &summary_query()).retained,
1778 RetainedState::Summary,
1779 "a present snapshot must not force the index"
1780 );
1781 }
1782
1783 #[test]
1784 fn deliveries_whose_intent_is_the_snapshot_itself_still_retain_the_index() {
1785 let cached = || blind(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")));
1789 for (name, delivery) in [
1790 ("stale", stale(cached())),
1791 ("on", OpenFixture { policy: CachePolicy::On, ..cached() }),
1792 ] {
1793 assert_eq!(
1794 planned(&delivery, &summary_query()).retained,
1795 RetainedState::FullIndex,
1796 "{name} needs the index to honour its contract"
1797 );
1798 }
1799 }
1800
1801 #[test]
1802 fn a_one_shot_metadata_query_does_not_read_the_snapshot_it_cannot_use() {
1803 let mut tree_query = summary_query();
1809 tree_query.views = vec![ViewSpec::Tree];
1810 for policy in [CachePolicy::Auto, CachePolicy::On] {
1811 let cached = config(policy, Some(PathBuf::from("cache.fdu")));
1812 assert!(
1813 planned(&cached, &tree_query).load != Load::Snapshot,
1814 "{policy:?} must not pay for a read that saves no work"
1815 );
1816 }
1817 }
1818
1819 #[test]
1820 fn the_snapshot_is_read_where_reading_pays_or_is_the_contract() {
1821 let only = stale(config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))));
1823 assert_eq!(planned(&only, &summary_query()).load, Load::Snapshot);
1824
1825 let analyzed = OpenFixture {
1828 analysis: crate::content::AnalysisRequest {
1829 profile: crate::content::AnalysisSet::NONE.with_code(),
1830 ..Default::default()
1831 },
1832 ..config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")))
1833 };
1834 assert_eq!(planned(&analyzed, &summary_query()).load, Load::Snapshot);
1835
1836 let never = config(CachePolicy::Off, Some(PathBuf::from("cache.fdu")));
1838 assert_eq!(planned(&never, &summary_query()).load, Load::None);
1839 }
1840
1841 #[test]
1842 fn auto_persists_where_a_later_request_reads_what_it_stores() {
1843 let cache = Some(PathBuf::from("cache.fdu"));
1849 let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
1850 let persists = |fixture: &OpenFixture, query: &Query, route: Route| {
1851 let (request, delivery) = split(Path::new("."), fixture, query);
1852 plan(&request, &delivery, route).expect("valid plan").persists()
1853 };
1854 for (policy, one_shot, analysis, retained) in [
1855 (CachePolicy::Auto, false, true, true),
1856 (CachePolicy::On, true, true, true),
1857 (CachePolicy::Off, false, false, false),
1858 ] {
1859 let metadata = config(policy, cache.clone());
1860 assert_eq!(persists(&metadata, &tree, Route::OneShot), one_shot, "{policy:?}");
1861 assert_eq!(
1862 persists(&analyzing(metadata.clone()), &tree, Route::OneShot),
1863 analysis,
1864 "{policy:?} with analysis"
1865 );
1866 for route in [Route::Retained, Route::Refresh, Route::Watch] {
1867 assert_eq!(persists(&metadata, &tree, route), retained, "{policy:?} {route:?}");
1868 }
1869 }
1870 let stale_answer = stale(config(CachePolicy::On, cache));
1871 assert!(!persists(&stale_answer, &tree, Route::OneShot), "a stale answer writes nothing");
1872 assert!(!persists(&stale_answer, &tree, Route::Retained), "a stale answer writes nothing");
1873 }
1874
1875 #[test]
1876 fn an_analysis_request_never_selects_the_compact_summary_tier() {
1877 let off = blind(CachePolicy::Off, None);
1881 assert_eq!(planned(&off, &summary_query()).retained, RetainedState::Summary);
1882
1883 for profile in [
1884 crate::content::AnalysisSet::NONE.with_lines(),
1885 crate::content::AnalysisSet::NONE.with_code(),
1886 crate::content::AnalysisSet::NONE.with_words(),
1887 crate::content::AnalysisSet::ALL,
1888 ] {
1889 let analyzed = OpenFixture {
1890 analysis: crate::content::AnalysisRequest { profile, ..Default::default() },
1891 ..blind(CachePolicy::Off, None)
1892 };
1893 assert_eq!(
1894 planned(&analyzed, &summary_query()).retained,
1895 RetainedState::FullIndex,
1896 "{profile:?} must retain the index"
1897 );
1898 }
1899 }
1900
1901 #[test]
1902 fn a_repeated_one_shot_report_scans_cold_while_open_still_revalidates() {
1903 let root = tempfile::tempdir().expect("tempdir");
1908 fs::write(root.path().join("file.txt"), b"contents").expect("file");
1909 let cache = tempfile::tempdir().expect("cache dir");
1910 let auto = config(CachePolicy::Auto, Some(cache.path().join("cache.fdu")));
1911 let on = OpenFixture { policy: CachePolicy::On, ..auto.clone() };
1912 let mut tree_query = summary_query();
1913 tree_query.views = vec![ViewSpec::Tree];
1914
1915 let (first, pending, _) = prepared(root.path(), &on, &tree_query).expect("first report");
1916 pending.join().expect("first save");
1917 assert_eq!(first.provenance.source, ReportSource::ColdScan);
1918 assert!(on.cache_path.as_deref().expect("path").exists(), "`on` persists");
1919
1920 let (second, pending, performance) =
1921 prepared(root.path(), &auto, &tree_query).expect("second report");
1922 pending.join().expect("second save");
1923 assert_eq!(
1924 second.provenance.source,
1925 ReportSource::ColdScan,
1926 "a repeated one-shot must not pay for a read that saves no work"
1927 );
1928 assert_eq!(performance.walked_files, 1, "the walk still happened");
1929
1930 let (_, open_report) = crate::open_fixture(root.path(), &auto).expect("library open");
1933 assert_eq!(
1934 open_report.path_taken,
1935 OpenPath::WarmRevalidate,
1936 "a caller holding the index still amortises the load"
1937 );
1938 }
1939
1940 #[test]
1941 fn a_stale_answer_reads_what_an_on_report_leaves_and_auto_leaves_nothing() {
1942 let root = tempfile::tempdir().expect("tempdir");
1946 fs::write(root.path().join("file.txt"), b"contents").expect("file");
1947 let cache = tempfile::tempdir().expect("cache dir");
1948 let auto = config(CachePolicy::Auto, Some(cache.path().join("cache.fdu")));
1949 let mut tree_query = summary_query();
1950 tree_query.views = vec![ViewSpec::Tree];
1951
1952 let (_, pending, _) = prepared(root.path(), &auto, &tree_query).expect("report");
1953 assert!(!pending.writes_metadata(), "`auto` starts no metadata write");
1954 pending.join().expect("nothing to save");
1955 assert!(!cache.path().join("cache.fdu").exists(), "`auto` leaves nothing");
1956 let only = stale(config(CachePolicy::Auto, Some(cache.path().join("cache.fdu"))));
1957 let missing = prepared(root.path(), &only, &tree_query).expect_err("nothing to read");
1958 assert!(missing.to_string().contains("with the `on` cache policy"), "{missing}");
1959
1960 let on = OpenFixture { policy: CachePolicy::On, ..auto };
1961 let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
1962 pending.join().expect("save");
1963
1964 let (from_cache, pending, performance, diagnostics) =
1965 prepared_with_diagnostics(root.path(), &only, &tree_query).expect("cache-only report");
1966 pending.join().expect("no save");
1967 assert_eq!(from_cache.provenance.source, ReportSource::CacheOnly);
1968 assert_eq!(performance.walked_files, 0, "cache-only never touches the tree");
1969 assert!(diagnostics.is_none(), "a cache-only open has no scan trace");
1970 }
1971
1972 #[test]
1973 fn controls_on_snapshot_projects_to_an_equivalent_controls_off_cache_only_report() {
1974 let root = tempfile::tempdir().expect("tempdir");
1975 let cache = tempfile::tempdir().expect("cache dir");
1976 let cache_path = cache.path().join("cache.fdu");
1977 fs::create_dir(root.path().join("src")).expect("source dir");
1978 fs::write(root.path().join("src/lib.rs"), b"library").expect("source file");
1979 seed_controls_snapshot(root.path(), cache_path.clone());
1980
1981 let controls_off = stale(controls_config(CachePolicy::Auto, cache_path, false));
1982 let query = Query {
1983 views: vec![
1984 ViewSpec::Summary,
1985 ViewSpec::Tree,
1986 ViewSpec::Families,
1987 ViewSpec::Types,
1988 ViewSpec::Extensions,
1989 ViewSpec::Languages,
1990 ViewSpec::Largest,
1991 ViewSpec::Recent,
1992 ViewSpec::Files,
1993 ],
1994 ..Query::default()
1995 };
1996 let (projected, pending, performance) =
1997 prepared(root.path(), &controls_off, &query).expect("projected report");
1998 pending.join().expect("no cache-only save");
1999
2000 let cold = OpenFixture {
2001 policy: CachePolicy::Off,
2002 stale_ok: false,
2003 cache_path: None,
2004 ..controls_off
2005 };
2006 let (mut expected, pending, _) =
2007 prepared(root.path(), &cold, &query).expect("controls-off cold report");
2008 pending.join().expect("no cold save");
2009 expected.provenance = projected.provenance.clone();
2010
2011 assert_eq!(performance.source, ReportSource::CacheOnly);
2012 assert_eq!(projected.scope, cold.scan.scope());
2013 assert_eq!(projected.ignore_rules, crate::control::ControlCoverage::NotObserved);
2014 assert_eq!(
2015 crate::report_format::render(&projected, crate::report_format::Format::Json, false,)
2016 .expect("compatible report format"),
2017 crate::report_format::render(&expected, crate::report_format::Format::Json, false,)
2018 .expect("compatible report format"),
2019 );
2020 }
2021
2022 #[test]
2023 fn controls_on_snapshot_projects_to_controls_off_auto_report() {
2024 let root = tempfile::tempdir().expect("tempdir");
2025 let cache = tempfile::tempdir().expect("cache dir");
2026 let cache_path = cache.path().join("cache.fdu");
2027 seed_controls_snapshot(root.path(), cache_path.clone());
2028
2029 let controls_off = OpenFixture {
2030 analysis: crate::content::AnalysisRequest {
2031 profile: crate::content::AnalysisSet::NONE.with_lines(),
2032 ..Default::default()
2033 },
2034 ..controls_config(CachePolicy::Auto, cache_path, false)
2035 };
2036 let (report, pending, performance) = prepared(root.path(), &controls_off, &summary_query())
2037 .expect("controls-off warm projection");
2038 pending.join().expect("save content only");
2039
2040 assert_eq!(report.provenance.source, ReportSource::WarmRevalidate);
2041 assert_eq!(report.scope, controls_off.scan.scope());
2042 assert_eq!(performance.source, ReportSource::WarmRevalidate);
2043 }
2044
2045 fn write_unobservable_controls(root: &Path) {
2051 let mut rule = vec![b'a'; crate::control::DEFAULT_CONTROL_LINE_LIMIT + 1];
2052 rule.push(b'\n');
2053 fs::write(root.join(".gitignore"), rule).expect("oversized rule");
2054 fs::create_dir(root.join("vendored")).expect("nested directory");
2055 fs::write(
2056 root.join("vendored/.gitignore"),
2057 b"x\n".repeat(crate::control::DEFAULT_CONTROL_BUDGET / 2),
2058 )
2059 .expect("oversized source");
2060 }
2061
2062 #[test]
2063 fn a_one_shot_report_observes_control_state_as_its_caller_configures() {
2064 let root = tempfile::tempdir().expect("tempdir");
2069 fs::write(root.path().join("file.txt"), b"contents").expect("file");
2070 write_unobservable_controls(root.path());
2071 let mut tree_query = summary_query();
2072 tree_query.views = vec![ViewSpec::Tree];
2073
2074 for read_controls in [true, false] {
2075 let cache = tempfile::tempdir().expect("cache dir");
2076 let cache_path = cache.path().join("cache.fdu");
2077 let caller = controls_config(CachePolicy::On, cache_path.clone(), read_controls);
2078 for query in [summary_query(), tree_query.clone()] {
2079 let (report, pending, _) = prepared(root.path(), &caller, &query)
2080 .expect("a refused control file ends nothing");
2081 pending.join().expect("save");
2082 assert!(
2083 report.status.complete,
2084 "a refusal is not a partial: {:?}",
2085 report.status.errors
2086 );
2087 assert_eq!(report.scope, caller.scan.scope());
2088 match &report.ignore_rules {
2089 crate::control::ControlCoverage::Observed(coverage) => {
2090 assert!(read_controls, "observed only when asked");
2091 assert_eq!(coverage.refused, 2, "{coverage:?}");
2092 assert_eq!(report.notes.len(), 2, "{:?}", report.notes);
2093 assert!(
2094 report.notes[0].contains("2 ignore files not applied"),
2095 "{:?}",
2096 report.notes
2097 );
2098 assert_eq!(
2099 report.notes[1],
2100 "note: gitignored subtotals are unavailable where governing rules could not be verified"
2101 );
2102 }
2103 crate::control::ControlCoverage::NotObserved => {
2104 assert!(!read_controls, "unobserved only when turned off");
2105 assert!(report.notes.is_empty(), "{:?}", report.notes);
2106 }
2107 }
2108 }
2109
2110 let saved = crate::snapshot::load(&cache_path)
2111 .expect("load the snapshot")
2112 .expect("the index tier persisted");
2113 assert_eq!(saved.scope(), caller.scan.scope());
2114 assert_eq!(saved.controls().is_ok(), read_controls);
2115 }
2116 }
2117
2118 #[test]
2132 fn a_scope_this_build_cannot_honour_is_refused_before_any_snapshot_is_read() {
2133 let root = tempfile::tempdir().expect("tempdir");
2134 fs::write(root.path().join("file.txt"), b"contents").expect("file");
2135 let cache = tempfile::tempdir().expect("cache dir");
2136 let cache_path = cache.path().join("cache.fdu");
2137
2138 let warm = config(CachePolicy::On, Some(cache_path.clone()));
2141 let (_, pending, _) = prepared(root.path(), &warm, &summary_query()).expect("warm");
2142 pending.join().expect("save");
2143 let (_, pending, _) = prepared(
2144 root.path(),
2145 &stale(config(CachePolicy::Auto, Some(cache_path.clone()))),
2146 &summary_query(),
2147 )
2148 .expect("the snapshot answers a supported scope");
2149 pending.join().expect("no save");
2150
2151 let unsupported = ScanConfig { follow_symlinks: true, ..ScanConfig::default() };
2152 for (policy, stale_ok) in [
2153 (CachePolicy::Auto, true),
2154 (CachePolicy::Off, false),
2155 (CachePolicy::Auto, false),
2156 (CachePolicy::On, false),
2157 ] {
2158 let asked = OpenFixture {
2159 scan: unsupported.clone(),
2160 stale_ok,
2161 ..config(policy, Some(cache_path.clone()))
2162 };
2163 let refused = prepared(root.path(), &asked, &summary_query())
2164 .expect_err("a scope this build cannot honour has no answer at any policy");
2165 assert!(
2166 matches!(
2167 refused,
2168 Error::InvalidRequest(crate::query::RequestError::ScopeUnsupported {
2169 axis: crate::query::ScopeAxis::FollowSymlinks,
2170 ..
2171 })
2172 ),
2173 "{policy:?} must refuse the request rather than fail the operation: {refused}"
2174 );
2175 assert_eq!(
2176 refused.to_string(),
2177 "unsupported scan configuration: follow_symlinks requires cycle, root-boundary, \
2178 and filesystem-boundary semantics",
2179 "{policy:?} must name the scope it cannot honour"
2180 );
2181 }
2182 }
2183
2184 #[test]
2185 fn a_report_that_reads_no_gitignore_refuses_to_select_by_ignored_state() {
2186 let root = tempfile::tempdir().expect("tempdir");
2189 fs::write(root.path().join(".gitignore"), b"*.log\n").expect("control file");
2190 fs::write(root.path().join("debug.log"), b"ignored").expect("ignored file");
2191 let mut only = summary_query();
2192 only.selection.ignored = IgnoredEntries::Only;
2193
2194 assert!(matches!(
2198 prepared(root.path(), &blind(CachePolicy::Off, None), &only),
2199 Err(Error::InvalidRequest(crate::query::RequestError::IgnoredWithoutObservation(
2200 IgnoredEntries::Only
2201 )))
2202 ));
2203
2204 let (report, pending, _) =
2205 prepared(root.path(), &config(CachePolicy::Off, None), &only).expect("observed");
2206 pending.join().expect("no save");
2207 let Section::Summary(row) = report.sections[0] else { panic!("a summary") };
2208 assert_eq!((row.files, row.bytes), (1, 7), "only the ignored file is selected");
2209 }
2210
2211 #[test]
2212 fn an_on_reports_snapshot_serves_either_cache_only_report_but_not_the_reverse() {
2213 let root = tempfile::tempdir().expect("tempdir");
2219 fs::write(root.path().join("file.txt"), b"contents").expect("file");
2220 let mut tree_query = summary_query();
2221 tree_query.views = vec![ViewSpec::Tree];
2222
2223 for (writer, reader) in [(true, true), (true, false), (false, false), (false, true)] {
2224 let cache = tempfile::tempdir().expect("cache dir");
2225 let cache_path = cache.path().join("cache.fdu");
2226 let write = controls_config(CachePolicy::On, cache_path.clone(), writer);
2227 let (_, pending, _) =
2228 prepared(root.path(), &write, &tree_query).expect("writing report");
2229 pending.join().expect("save");
2230
2231 let read = stale(controls_config(CachePolicy::Auto, cache_path, reader));
2232 match prepared(root.path(), &read, &tree_query) {
2233 Ok((report, pending, _)) => {
2234 pending.join().expect("no save");
2235 assert!(writer || !reader, "writer {writer} served reader {reader}");
2236 assert_eq!(report.provenance.source, ReportSource::CacheOnly);
2237 assert_eq!(
2238 matches!(report.ignore_rules, crate::control::ControlCoverage::Observed(_)),
2239 reader,
2240 "a report describes the scope it asked for"
2241 );
2242 }
2243 Err(Error::Snapshot(message)) => {
2244 assert!(!writer && reader, "writer {writer}, reader {reader}: {message}");
2245 assert!(message.contains(".gitignore state"), "names the cause: {message}");
2246 assert!(message.contains("verified answer"), "names the remedy: {message}");
2247 }
2248 Err(other) => panic!("writer {writer}, reader {reader}: {other}"),
2249 }
2250 }
2251 }
2252
2253 #[test]
2254 fn a_cache_only_open_answers_from_an_on_reports_snapshot() {
2255 let root = tempfile::tempdir().expect("tempdir");
2259 fs::write(root.path().join(".gitignore"), b"*.log\n").expect("control file");
2260 fs::write(root.path().join("debug.log"), b"ignored").expect("ignored file");
2261 let cache = tempfile::tempdir().expect("cache dir");
2262 let cache_path = cache.path().join("cache.fdu");
2263 let mut tree_query = summary_query();
2264 tree_query.views = vec![ViewSpec::Tree];
2265
2266 let on = config(CachePolicy::On, Some(cache_path.clone()));
2267 let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
2268 pending.join().expect("save");
2269
2270 let only = stale(config(CachePolicy::Auto, Some(cache_path)));
2271 let (index, report) = crate::open_fixture(root.path(), &only).expect("the shared snapshot");
2272 assert_eq!(report.path_taken, OpenPath::CacheOnly);
2273 assert_eq!(index.is_ignored(Path::new("debug.log")).ok(), Some(Some(true)));
2274 }
2275
2276 #[test]
2277 fn an_open_that_opts_out_of_control_state_shares_an_opted_out_reports_snapshot() {
2278 let root = tempfile::tempdir().expect("tempdir");
2283 fs::write(root.path().join("file.txt"), b"contents").expect("file");
2284 let cache = tempfile::tempdir().expect("cache dir");
2285 let cache_path = cache.path().join("cache.fdu");
2286 let mut tree_query = summary_query();
2287 tree_query.views = vec![ViewSpec::Tree];
2288
2289 let on = blind(CachePolicy::On, Some(cache_path.clone()));
2290 let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
2291 pending.join().expect("save");
2292 assert!(cache_path.exists(), "the report left a snapshot");
2293
2294 let only = stale(blind(CachePolicy::Auto, Some(cache_path)));
2295 let (index, report) = crate::open_fixture(root.path(), &only).expect("the shared snapshot");
2296 assert_eq!(report.path_taken, OpenPath::CacheOnly);
2297 assert!(matches!(
2298 index.is_ignored(Path::new("file.txt")),
2299 Err(Error::ControlStateNotObserved)
2300 ));
2301 }
2302
2303 #[test]
2304 fn compact_summary_matches_the_indexed_summary_exactly() {
2305 let root = tempfile::tempdir().expect("tempdir");
2306 fs::create_dir(root.path().join("src")).expect("directory");
2307 fs::write(root.path().join("src/lib.rs"), b"library").expect("file");
2308 fs::write(root.path().join("README.md"), b"read me").expect("file");
2309 #[cfg(unix)]
2310 std::os::unix::fs::symlink("README.md", root.path().join("readme-link")).expect("symlink");
2311 crate::test_support::settle_allocations(root.path());
2312
2313 let query = summary_query();
2314 let off = OpenFixture {
2317 scan: ScanConfig { read_controls: false, threads: Some(2), ..ScanConfig::default() },
2318 ..blind(CachePolicy::Off, None)
2319 };
2320 let (compact, pending, performance) =
2321 prepared(root.path(), &off, &query).expect("compact report");
2322 pending.join().expect("no pending compact save");
2323 assert_eq!(performance.walked_files, 2);
2324 assert_eq!(performance.walked_bytes, 14);
2325
2326 let (index, _open_report) = crate::open_fixture(root.path(), &off).expect("indexed scan");
2329 let indexed = report(
2330 &index,
2331 &crate::test_support::read_of(&index, query.clone()),
2332 compact.provenance.generated_at,
2333 )
2334 .expect("report");
2335
2336 let Section::Summary(compact_row) = compact.sections[0] else {
2337 panic!("compact plan did not return a summary")
2338 };
2339 let Section::Summary(indexed_row) = indexed.sections[0] else {
2340 panic!("indexed plan did not return a summary")
2341 };
2342 assert_eq!(compact_row.files, indexed_row.files);
2343 assert_eq!(compact_row.dirs, indexed_row.dirs);
2344 assert_eq!(compact_row.bytes, indexed_row.bytes);
2345 assert_eq!(compact_row.allocated, indexed_row.allocated);
2346 assert_eq!(compact_row.newest_mtime_ns, indexed_row.newest_mtime_ns);
2347 assert_eq!(compact.root, indexed.root);
2348 assert_eq!(compact.scope, indexed.scope);
2349 assert_eq!(compact.status.complete, indexed.status.complete);
2350 assert_eq!(compact.provenance.freshness, indexed.provenance.freshness);
2351 }
2352
2353 struct ControlCase {
2357 name: &'static str,
2358 #[cfg(unix)]
2363 denied: Option<Unlocked>,
2364 root: tempfile::TempDir,
2365 scan: ScanConfig,
2366 share: bool,
2369 refused: u64,
2371 order_dependent: bool,
2374 errors: bool,
2376 lookups: crate::test_support::CaseLookups,
2378 variant_governs: Option<bool>,
2381 }
2382
2383 #[cfg(unix)]
2387 struct Unlocked(PathBuf);
2388
2389 #[cfg(unix)]
2390 impl Drop for Unlocked {
2391 fn drop(&mut self) {
2392 use std::os::unix::fs::PermissionsExt;
2393 let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o755));
2394 }
2395 }
2396
2397 #[cfg(unix)]
2402 fn seal_search_denied(root: &Path) -> Unlocked {
2403 use std::os::unix::fs::PermissionsExt;
2404 let denied = root.join("denied");
2405 put(root, "denied/sub/inner", b"inner");
2406 put(root, "denied/f", b"f");
2407 std::os::unix::fs::symlink("f", denied.join("link")).expect("symlink");
2408 fs::set_permissions(&denied, fs::Permissions::from_mode(0o400)).expect("deny search");
2409 Unlocked(denied)
2410 }
2411
2412 fn put(root: &Path, path: &str, contents: &[u8]) {
2413 let path = root.join(path);
2414 fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories");
2415 fs::write(path, contents).expect("fixture file");
2416 }
2417
2418 fn rules_tree() -> tempfile::TempDir {
2422 let root = tempfile::tempdir().expect("tempdir");
2423 let root_path = root.path();
2424 put(
2425 root_path,
2426 ".gitignore",
2427 b"*.log\n!keep.log\n/anchored.txt\ncache/\nnode_modules/\nvendor/*\n!vendor/keep/\n",
2428 );
2429 put(root_path, "a.log", b"alog");
2430 put(root_path, "keep.log", b"keeplog");
2431 put(root_path, "anchored.txt", b"anchored");
2432 put(root_path, "cache", b"a file named like a directory rule");
2433 put(root_path, "README.md", b"readme!");
2434 put(root_path, "src/anchored.txt", b"not anchored here");
2435 put(root_path, "src/x.log", b"xlog-");
2436 put(root_path, "src/keep.log", b"kept everywhere");
2437 put(root_path, "src/main.rs", b"fn main() {}");
2438 put(root_path, "sub/.gitignore", b"!*.log\n*.tmp\n");
2439 put(root_path, "sub/y.log", b"re-included");
2440 put(root_path, "sub/z.tmp", b"tmp");
2441 put(root_path, "sub/cache/data.bin", b"cached bytes");
2442 put(root_path, "sub/deep/.gitignore", b"*\n!.gitignore\n");
2443 put(root_path, "sub/deep/f.txt", b"deep");
2444 put(root_path, "sub/deep/inner/g.txt", b"deeper");
2445 put(root_path, "node_modules/pkg/.gitignore", b"!*\n");
2446 put(root_path, "node_modules/pkg/index.js", b"module.exports = 1;");
2447 put(root_path, "node_modules/pkg/lib/a.js", b"a");
2448 put(root_path, "vendor/a.c", b"int a;");
2449 put(root_path, "vendor/keep/k.c", b"int k;");
2450 put(root_path, "vendor/drop/d.c", b"int d;");
2451 put(root_path, "selfish/.gitignore", b".gitignore\n*.bak\n");
2452 put(root_path, "selfish/x.bak", b"backup");
2453 put(root_path, "selfish/y.txt", b"kept");
2454 put(root_path, "many/.gitignore", b"*[02468].dat\n");
2455 for file in 0..40 {
2456 put(root_path, &format!("many/f{file:02}.dat"), &vec![b'.'; file + 1]);
2457 }
2458 root
2459 }
2460
2461 const CASE_VARIANT: &str = "a case-variant control name";
2463
2464 const CASE_BOTH: &str = "both spellings of the control name";
2466
2467 fn control_cases() -> Vec<ControlCase> {
2468 let case = |name, root, scan, share, refused| ControlCase {
2469 name,
2470 root,
2471 scan,
2472 share,
2473 refused,
2474 order_dependent: false,
2475 errors: false,
2476 lookups: crate::test_support::CaseLookups::Host,
2477 variant_governs: None,
2478 #[cfg(unix)]
2479 denied: None,
2480 };
2481 let mut cases = vec![
2482 case("rules", rules_tree(), ScanConfig::default(), true, 0),
2483 case(
2484 "rules with hidden entries pruned",
2485 rules_tree(),
2486 ScanConfig {
2487 hidden: Some(std::sync::Arc::new(crate::HiddenPolicy::prune_hidden(Vec::<
2488 std::ffi::OsString,
2489 >::new(
2490 )))),
2491 ..ScanConfig::default()
2492 },
2493 true,
2494 0,
2495 ),
2496 case(
2497 "rules under a depth bound",
2498 rules_tree(),
2499 ScanConfig { max_depth: Some(1), ..ScanConfig::default() },
2500 true,
2501 0,
2502 ),
2503 ];
2504
2505 let empty = tempfile::tempdir().expect("tempdir");
2506 put(empty.path(), "only.txt", b"no rules anywhere");
2507 cases.push(case("no control file", empty, ScanConfig::default(), true, 0));
2508
2509 let long = tempfile::tempdir().expect("tempdir");
2511 put(long.path(), ".gitignore", b"*.log\n");
2512 put(long.path(), "x.log", b"ignored");
2513 let mut line = vec![b'x'; crate::control::DEFAULT_CONTROL_LINE_LIMIT + 1];
2514 line.push(b'\n');
2515 put(long.path(), "long/.gitignore", &line);
2516 put(long.path(), "long/kept.txt", b"kept");
2517 put(long.path(), "long/y.log", b"unknown");
2518 cases.push(case("line limit", long, ScanConfig::default(), false, 1));
2519
2520 let over = tempfile::tempdir().expect("tempdir");
2523 put(over.path(), ".gitignore", b"*.log\n");
2524 let mut oversized = vec![b'x'; 200];
2525 oversized.push(b'\n');
2526 for directory in 0..70 {
2527 put(over.path(), &format!("d{directory:02}/.gitignore"), &oversized);
2528 put(over.path(), &format!("d{directory:02}/f.log"), b"log");
2529 }
2530 let limits = crate::control::ControlLimits { budget: Some(256), ..Default::default() };
2531 cases.push(case(
2532 "every nested file over the budget",
2533 over,
2534 ScanConfig { control_limits: limits, ..ScanConfig::default() },
2535 false,
2536 70,
2537 ));
2538
2539 let competing = tempfile::tempdir().expect("tempdir");
2541 for (position, directory) in ["a", "b", "c", "d"].into_iter().enumerate() {
2542 let mut rules = format!("r{position}").into_bytes();
2543 rules.extend(std::iter::repeat_n(b'y', 100));
2544 rules.push(b'\n');
2545 put(competing.path(), &format!("{directory}/.gitignore"), &rules);
2546 put(competing.path(), &format!("{directory}/file.txt"), b"file");
2547 }
2548 let limits = crate::control::ControlLimits { budget: Some(1000), ..Default::default() };
2549 let mut competing = case(
2550 "competing for the budget",
2551 competing,
2552 ScanConfig { control_limits: limits, ..ScanConfig::default() },
2553 false,
2554 2,
2555 );
2556 competing.order_dependent = true;
2557 cases.push(competing);
2558
2559 let probe = tempfile::tempdir().expect("tempdir");
2566 for (lookups, governs) in crate::test_support::CaseLookups::on_this_host(probe.path()) {
2567 for hidden in [None, Some(Vec::<std::ffi::OsString>::new())] {
2568 let variant = tempfile::tempdir().expect("tempdir");
2569 put(variant.path(), ".GITIGNORE", b"*.log\n");
2570 for file in 0..1_200 {
2571 put(variant.path(), &format!("f{file:04}.log"), b"log");
2572 }
2573 put(variant.path(), "kept.txt", b"kept");
2574 let scan = ScanConfig {
2575 hidden: hidden
2576 .map(|allow| std::sync::Arc::new(crate::HiddenPolicy::prune_hidden(allow))),
2577 ..ScanConfig::default()
2578 };
2579 let mut variant = case(CASE_VARIANT, variant, scan, true, 0);
2580 variant.lookups = lookups;
2581 variant.variant_governs = Some(governs);
2582 cases.push(variant);
2583 }
2584 }
2585
2586 for lookups in
2590 [crate::test_support::CaseLookups::Host, crate::test_support::CaseLookups::Folded]
2591 {
2592 let both = tempfile::tempdir().expect("tempdir");
2593 put(both.path(), ".gitignore", b"*.log\n");
2594 put(both.path(), ".GITIGNORE", b"*.tmp\n");
2595 if fs::read(both.path().join(".gitignore")).expect("read") != b"*.log\n" {
2596 eprintln!("skipped {CASE_BOTH:?}: the temporary directory is case-insensitive");
2597 break;
2598 }
2599 for file in 0..1_200 {
2600 put(both.path(), &format!("f{file:04}.tmp"), b"tmp");
2601 }
2602 put(both.path(), "x.log", b"log");
2603 let mut both = case(CASE_BOTH, both, ScanConfig::default(), true, 0);
2604 both.lookups = lookups;
2605 cases.push(both);
2606 }
2607
2608 #[cfg(unix)]
2609 {
2610 use std::os::unix::fs::PermissionsExt;
2611
2612 let shapes = tempfile::tempdir().expect("tempdir");
2614 put(shapes.path(), ".gitignore", b"*.tmp\n");
2615 put(shapes.path(), "weird/.gitignore/inner.tmp", b"inner");
2616 put(shapes.path(), "weird/kept.txt", b"kept");
2617 put(shapes.path(), "rules.txt", b"*.txt\n");
2618 fs::create_dir(shapes.path().join("linked")).expect("directory");
2619 std::os::unix::fs::symlink("../rules.txt", shapes.path().join("linked/.gitignore"))
2620 .expect("symlink");
2621 put(shapes.path(), "linked/still.txt", b"still counted");
2622 cases.push(case(
2623 "control files that are not files",
2624 shapes,
2625 ScanConfig::default(),
2626 true,
2627 0,
2628 ));
2629
2630 if crate::test_support::require_permission_bits() {
2631 let unreadable = tempfile::tempdir().expect("tempdir");
2632 put(unreadable.path(), ".gitignore", b"*.log\n");
2633 put(unreadable.path(), "x.log", b"ignored");
2634 put(unreadable.path(), "sub/.gitignore", b"!*.log\n");
2635 put(unreadable.path(), "sub/y.log", b"unknown");
2636 let control = unreadable.path().join("sub/.gitignore");
2637 let mut denied =
2638 case("unreadable control file", unreadable, ScanConfig::default(), false, 0);
2639 fs::set_permissions(&control, fs::Permissions::from_mode(0o000))
2640 .expect("deny the control file");
2641 denied.denied = Some(Unlocked(control));
2642 denied.errors = true;
2643 cases.push(denied);
2644
2645 let sealed = tempfile::tempdir().expect("tempdir");
2651 put(sealed.path(), ".gitignore", b"*.log\n");
2652 put(sealed.path(), "open/x.log", b"ignored");
2653 put(sealed.path(), "open/y.txt", b"kept");
2654 let denied = seal_search_denied(sealed.path());
2655 let mut unsearchable = case(
2656 "a directory that lists but refuses search",
2657 sealed,
2658 ScanConfig::default(),
2659 true,
2660 0,
2661 );
2662 unsearchable.denied = Some(denied);
2663 unsearchable.errors = true;
2664 cases.push(unsearchable);
2665 }
2666 }
2667 for case in &cases {
2668 crate::test_support::settle_allocations(case.root.path());
2669 }
2670 cases
2671 }
2672
2673 fn transient_and_indexed(root: &Path, scan: &ScanConfig, label: &str) -> (Report, Report) {
2677 let query = summary_query();
2678 let transient = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
2679 assert_eq!(planned(&transient, &query).retained, RetainedState::Summary, "{label}");
2680 let (mut compact, pending, compact_performance) =
2681 prepared(root, &transient, &query).expect("transient report");
2682 pending.join().expect("the transient tier saves nothing");
2683
2684 let cache = tempfile::tempdir().expect("cache dir");
2687 let indexed = OpenFixture {
2688 scan: scan.clone(),
2689 ..config(CachePolicy::On, Some(cache.path().join("snapshot.fdu")))
2690 };
2691 assert_eq!(planned(&indexed, &query).retained, RetainedState::FullIndex, "{label}");
2692 let (indexed, pending, indexed_performance) =
2693 prepared(root, &indexed, &query).expect("indexed report");
2694 pending.join().expect("save");
2695
2696 assert_eq!(
2697 (
2698 compact_performance.walked_files,
2699 compact_performance.walked_bytes,
2700 compact_performance.walked_allocated,
2701 compact_performance.source,
2702 ),
2703 (
2704 indexed_performance.walked_files,
2705 indexed_performance.walked_bytes,
2706 indexed_performance.walked_allocated,
2707 indexed_performance.source,
2708 ),
2709 "{label}: walked totals"
2710 );
2711 assert_eq!(compact.provenance.source, indexed.provenance.source, "{label}");
2712 assert_eq!(compact.provenance.freshness, indexed.provenance.freshness, "{label}");
2713 assert_eq!(
2714 (compact.provenance.tiers.entries.source, compact.provenance.tiers.entries.freshness),
2715 (indexed.provenance.tiers.entries.source, indexed.provenance.tiers.entries.freshness),
2716 "{label}"
2717 );
2718 compact.provenance = indexed.provenance.clone();
2719 (compact, indexed)
2720 }
2721
2722 #[test]
2730 fn compact_summary_equals_the_indexed_summary_under_every_control_case() {
2731 use crate::report_format::{Format, render};
2732
2733 let default_batch = ScanConfig::default().batch_size;
2734 for case in control_cases() {
2735 let _lookups = case.lookups.install(case.root.path());
2736 let mut compared = 0;
2737 for threads in [Some(1), Some(2), Some(4), None] {
2738 if case.order_dependent && threads != Some(1) {
2739 continue;
2740 }
2741 for batch_size in [default_batch, 1, 3] {
2742 for order in [crate::ScanOrder::BreadthFirst, crate::ScanOrder::DepthFirst] {
2743 let scan = ScanConfig { batch_size, threads, order, ..case.scan.clone() };
2744 let label = format!(
2745 "{} ({:?} lookups, {threads:?} workers, batch {batch_size}, {order:?})",
2746 case.name, case.lookups
2747 );
2748 let (compact, indexed) =
2749 transient_and_indexed(case.root.path(), &scan, &label);
2750
2751 assert_eq!(format!("{compact:#?}"), format!("{indexed:#?}"), "{label}");
2752 for format in [Format::Text, Format::Json, Format::Yaml] {
2753 assert_eq!(
2754 render(&compact, format, false).expect("render"),
2755 render(&indexed, format, false).expect("render"),
2756 "{label}: {format:?}"
2757 );
2758 }
2759
2760 let Section::Summary(row) = indexed.sections[0] else {
2762 panic!("{label}: a summary")
2763 };
2764 let crate::control::ControlCoverage::Observed(coverage) =
2765 &indexed.ignore_rules
2766 else {
2767 panic!("{label}: the default scope observes .gitignore")
2768 };
2769 assert_eq!(coverage.refused, case.refused, "{label}");
2770 assert_eq!(row.ignored.is_some(), case.share, "{label}");
2771 assert_eq!(!indexed.status.errors.is_empty(), case.errors, "{label}");
2772 if case.share && case.name.starts_with("rules") {
2773 let ignored = row.ignored.expect("a share");
2774 assert!(
2775 ignored.files > 0 && ignored.files < row.files && ignored.dirs > 0,
2776 "{label}: {ignored:?} of {row:?}"
2777 );
2778 }
2779 if let Some(governs) = case.variant_governs {
2780 assert_eq!(coverage.applied, u64::from(governs), "{label}");
2781 assert_eq!(
2782 row.ignored.map(|ignored| ignored.files),
2783 Some(if governs { 1_200 } else { 0 }),
2784 "{label}"
2785 );
2786 }
2787 if case.name == CASE_BOTH {
2788 assert_eq!(coverage.applied, 1, "{label}: the exact name alone");
2789 assert_eq!(
2790 row.ignored.map(|ignored| ignored.files),
2791 Some(1),
2792 "{label}: only `x.log` is ignored"
2793 );
2794 }
2795 if !case.share {
2796 assert!(
2797 indexed
2798 .notes
2799 .iter()
2800 .any(|note| note.contains("could not be verified")),
2801 "{label}: {:?}",
2802 indexed.notes
2803 );
2804 }
2805 compared += 1;
2806 }
2807 }
2808 }
2809 assert!(compared > 0, "{} was compared", case.name);
2810 }
2811 }
2812
2813 #[test]
2814 fn compact_summary_never_creates_the_configured_snapshot() {
2815 let root = tempfile::tempdir().expect("tempdir");
2816 fs::write(root.path().join("payload"), b"payload").expect("file");
2817 let cache = root.path().join("must-not-exist.fdu");
2818
2819 let (report, pending, _) =
2820 prepared(root.path(), &blind(CachePolicy::Off, Some(cache.clone())), &summary_query())
2821 .expect("compact report");
2822 pending.join().expect("no pending compact save");
2823
2824 assert!(report.status.complete);
2825 assert!(!cache.exists());
2826 }
2827
2828 fn tree_query(
2832 views: Vec<ViewSpec>,
2833 format: crate::report_format::Format,
2834 share: &str,
2835 size: crate::query::SizeMetric,
2836 ) -> Query {
2837 let mut query = Query { views, format, ..Query::default() };
2838 query.selection.min_share =
2839 Some(crate::query::ShareThreshold::parse(share).expect("a share"));
2840 query.selection.size = size;
2841 query
2842 }
2843
2844 #[test]
2847 fn a_tree_takes_the_folded_index_only_when_the_request_proves_it_suffices() {
2848 use crate::query::{Bound, SizeMetric, SortKey};
2849 use crate::report_format::Format;
2850
2851 let off = config(CachePolicy::Off, None);
2852 let list = Query { views: vec![ViewSpec::List], ..Query::default() };
2853 let size = list.selection.size;
2854 let kept = |largest_files, size| RetainedState::Tree(TreeRetention { largest_files, size });
2855
2856 assert_eq!(planned(&off, &list).retained, kept(100, size));
2858 let mut folded = vec![
2861 Query { format: Format::Tree, ..list.clone() },
2862 Query { views: vec![ViewSpec::List, ViewSpec::Tree], ..list.clone() },
2863 ];
2864 for format in [Format::Text, Format::Tree, Format::Json, Format::Jsonl, Format::Yaml] {
2865 folded.push(Query { views: vec![ViewSpec::Tree], format, ..Query::default() });
2866 }
2867 for sort in [SortKey::Size, SortKey::Name, SortKey::Mtime, SortKey::Count] {
2868 for reverse in [false, true] {
2869 let mut query = list.clone();
2870 query.selection.sort = Some(sort);
2871 query.selection.reverse = reverse;
2872 folded.push(query);
2873 }
2874 }
2875 for bound in [Bound::Limit(0), Bound::Limit(2), Bound::All] {
2876 let mut query = list.clone();
2877 query.selection.depth = Some(bound);
2878 query.selection.breadth = Some(bound);
2879 query.selection.limit = Some(bound);
2880 folded.push(query);
2881 }
2882 for query in &folded {
2883 assert_eq!(planned(&off, query).retained, kept(100, size), "{query:?}");
2884 }
2885 for (share, largest_files) in
2887 [("10%", 10), ("0.5%", 200), ("3%", 34), ("0.01%", 10_000), ("100%", 1)]
2888 {
2889 for metric in [SizeMetric::Apparent, SizeMetric::Allocated] {
2890 let query = tree_query(vec![ViewSpec::List], Format::Text, share, metric);
2891 assert_eq!(planned(&off, &query).retained, kept(largest_files, metric), "{share}");
2892 }
2893 }
2894 let ceiling = tree_query(vec![ViewSpec::Tree], Format::Json, "0.00152587890625%", size);
2897 assert_eq!(planned(&off, &ceiling).retained, kept(TreeRetention::MAX_FILES, size));
2898 let past = tree_query(vec![ViewSpec::Tree], Format::Json, "0.0015258%", size);
2899 assert_eq!(planned(&off, &past).retained, RetainedState::FullIndex);
2900 for fixture in [
2903 blind(CachePolicy::Off, None),
2904 OpenFixture {
2905 scan: ScanConfig { max_depth: Some(2), ..ScanConfig::default() },
2906 ..off.clone()
2907 },
2908 config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))),
2909 config(CachePolicy::On, None),
2910 ] {
2911 assert_eq!(planned(&fixture, &list).retained, kept(100, size), "{fixture:?}");
2912 }
2913
2914 let mut full = vec![
2917 Query { format: Format::Json, ..list.clone() },
2918 Query { views: vec![ViewSpec::Tree], format: Format::Paths, ..list.clone() },
2919 Query { views: vec![ViewSpec::Tree], format: Format::Long, ..list.clone() },
2920 Query { views: vec![ViewSpec::Files], format: Format::Tree, ..list.clone() },
2921 Query { views: vec![ViewSpec::List, ViewSpec::Types], ..list.clone() },
2922 Query { views: vec![ViewSpec::Tree, ViewSpec::Extensions], ..list.clone() },
2923 Query { views: vec![ViewSpec::Tree, ViewSpec::Summary], ..list.clone() },
2924 Query { views: vec![ViewSpec::Tree, ViewSpec::Files], ..list.clone() },
2925 Query { views: Vec::new(), ..list.clone() },
2926 ];
2927 let mut filtered = list.clone();
2929 filtered.selection.include.push(Pattern::parse("*.rs").expect("pattern"));
2930 full.push(filtered);
2931 let mut excluded = list.clone();
2932 excluded.selection.exclude.push(Pattern::parse("target").expect("pattern"));
2933 full.push(excluded);
2934 let mut sized = list.clone();
2935 sized.selection.min_size = Some(1);
2936 full.push(sized);
2937 let mut kinds = list.clone();
2938 kinds.selection.kinds = vec![EntryKind::Dir];
2939 full.push(kinds);
2940 let mut files = list.clone();
2941 files.selection.kinds = vec![EntryKind::File];
2942 full.push(files);
2943 let mut modified = list.clone();
2944 modified.selection.modified.since = Some(0);
2945 full.push(modified);
2946 let mut by_ignored = list.clone();
2947 by_ignored.selection.ignored = IgnoredEntries::Exclude;
2948 full.push(by_ignored);
2949 let mut reported = list.clone();
2952 reported.selection.min_size = Some(300 << 20);
2953 reported.selection.sort = Some(SortKey::Size);
2954 reported.selection.depth = Some(Bound::Limit(2));
2955 reported.selection.limit = Some(Bound::Limit(30));
2956 full.push(reported);
2957 full.push(tree_query(vec![ViewSpec::List], Format::Text, "0%", size));
2959 full.push(tree_query(vec![ViewSpec::Tree], Format::Json, "0.000%", size));
2960 for query in &full {
2961 assert_eq!(planned(&off, query).retained, RetainedState::FullIndex, "{query:?}");
2962 }
2963
2964 let cache = || Some(PathBuf::from("cache.fdu"));
2966 for fixture in [
2967 analyzing(off.clone()),
2968 stale(config(CachePolicy::Auto, cache())),
2969 config(CachePolicy::On, cache()),
2970 ] {
2971 assert_eq!(planned(&fixture, &list).retained, RetainedState::FullIndex, "{fixture:?}");
2972 }
2973 for population in [IgnoredEntries::Exclude, IgnoredEntries::Only] {
2974 let narrowed = OpenFixture {
2975 scan: ScanConfig { population, ..ScanConfig::default() },
2976 ..off.clone()
2977 };
2978 let mut query = list.clone();
2979 query.selection.ignored = population;
2980 assert_eq!(planned(&narrowed, &query).retained, RetainedState::FullIndex);
2981 }
2982 let (request, delivery) = split(Path::new("."), &off, &list);
2984 for route in [Route::Retained, Route::Refresh, Route::Watch, Route::Opened] {
2985 let plan = plan(&request, &delivery, route).expect("a valid plan");
2986 assert_eq!(plan.retained, RetainedState::FullIndex, "{route:?}");
2987 }
2988 }
2989
2990 #[test]
2996 fn the_default_request_of_every_surface_plans_the_folded_tree() {
2997 use crate::query::{AxisNames, ReadSpec, RequestSpec, SizeMetric};
2998 let root = Path::new(".");
2999 let default_tree =
3000 RetainedState::Tree(TreeRetention { largest_files: 100, size: SizeMetric::Allocated });
3001 let deliveries = [
3002 Delivery::new(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))),
3003 Delivery::new(CachePolicy::Off, None),
3004 ];
3005 let planned = |spec: &RequestSpec<'_>, axes, delivery: &Delivery| {
3006 let request =
3007 Request::build(spec, SystemTime::UNIX_EPOCH, axes).expect("a valid request");
3008 plan(&request, delivery, Route::OneShot).expect("a valid plan").retained
3009 };
3010 for axes in [&AxisNames::FLAGS, &AxisNames::FIELDS] {
3011 for delivery in &deliveries {
3012 assert_eq!(planned(&RequestSpec::new(root), axes, delivery), default_tree);
3013 let tree_json = RequestSpec {
3015 read: ReadSpec { views: Some("tree"), format: Some("json"), ..ReadSpec::new() },
3016 ..RequestSpec::new(root)
3017 };
3018 assert_eq!(planned(&tree_json, axes, delivery), default_tree);
3019 let list_json = RequestSpec {
3020 read: ReadSpec { format: Some("json"), ..ReadSpec::new() },
3021 ..RequestSpec::new(root)
3022 };
3023 assert_eq!(planned(&list_json, axes, delivery), RetainedState::FullIndex);
3024 }
3025 }
3026 }
3027
3028 struct Shuffle(u64);
3031
3032 impl Shuffle {
3033 fn below(&mut self, bound: usize) -> usize {
3034 self.0 ^= self.0 >> 12;
3035 self.0 ^= self.0 << 25;
3036 self.0 ^= self.0 >> 27;
3037 let drawn = self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 32;
3038 usize::try_from(drawn).expect("32 bits") % bound
3039 }
3040 }
3041
3042 fn sized_file(path: &Path, size: u64, sparse: bool, minute: u64) {
3044 use std::io::Write as _;
3045 fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories");
3046 let mut file = fs::File::create(path).expect("fixture file");
3047 if sparse {
3048 file.set_len(size).expect("sparse length");
3049 } else {
3050 file.write_all(&vec![b'.'; usize::try_from(size).expect("small")]).expect("contents");
3051 }
3052 file.set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(minute * 60))
3053 .expect("mtime");
3054 }
3055
3056 fn random_tree(seed: u64, files: usize) -> tempfile::TempDir {
3062 let mut draw = Shuffle(seed);
3063 let root = tempfile::tempdir().expect("tempdir");
3064 let mut dirs = vec![PathBuf::new()];
3065 for index in 0..files.div_ceil(12) {
3066 let parent = dirs[draw.below(dirs.len())].clone();
3067 if parent.components().count() < 5 {
3068 let dir = parent.join(format!("d{index:03}"));
3069 fs::create_dir(root.path().join(&dir)).expect("directory");
3070 dirs.push(dir);
3071 }
3072 }
3073 let extensions = ["log", "txt", "bin", "tmp", "rs"];
3074 for index in 0..files {
3075 let dir = &dirs[draw.below(dirs.len())];
3076 let name = format!("f{index:05}.{}", extensions[draw.below(extensions.len())]);
3077 let (size, sparse) = match draw.below(20) {
3078 0..=2 => (0, false),
3079 3..=10 => (draw.below(300) as u64, false),
3080 11..=13 => (1_000, false),
3081 14..=17 => (draw.below(40_000) as u64, false),
3082 _ => (50_000 + draw.below(4_000_000) as u64, true),
3083 };
3084 let minute = FIXTURE_MINUTE + draw.below(4) as u64;
3085 sized_file(&root.path().join(dir).join(name), size, sparse, minute);
3086 }
3087 put(root.path(), ".gitignore", b"*.log\n!f000*.log\nd002/\n");
3088 if let Some(dir) = dirs.get(3) {
3089 put(&root.path().join(dir), ".gitignore", b"!*.log\n*.tmp\n");
3090 }
3091 #[cfg(unix)]
3092 for (index, dir) in dirs.iter().take(4).enumerate() {
3093 std::os::unix::fs::symlink(
3094 "elsewhere",
3095 root.path().join(dir).join(format!("l{index}")),
3096 )
3097 .expect("symlink");
3098 }
3099 crate::test_support::settle_allocations(root.path());
3100 root
3101 }
3102
3103 struct BoundaryTree {
3106 name: &'static str,
3107 #[cfg(unix)]
3108 _sealed: Option<Unlocked>,
3109 tree: tempfile::TempDir,
3110 }
3111
3112 fn boundary_trees() -> Vec<BoundaryTree> {
3114 let mut trees = Vec::new();
3115
3116 let ties = tempfile::tempdir().expect("tempdir");
3120 for index in 0..9 {
3121 sized_file(&ties.path().join(format!("a/at{index}")), 1_000, false, index);
3122 }
3123 sized_file(&ties.path().join("a/half"), 500, false, 20);
3124 sized_file(&ties.path().join("b/half"), 500, false, 20);
3125 trees.push(("ties at the retention bound", ties));
3126
3127 let exact = tempfile::tempdir().expect("tempdir");
3129 for index in 0..20 {
3130 sized_file(&exact.path().join(format!("d{}/f{index:02}", index % 3)), 50, false, 1);
3131 sized_file(&exact.path().join(format!("d{}/e{index:02}", index % 4)), 0, false, 2);
3132 }
3133 trees.push(("exactly the files a share shows", exact));
3134
3135 let zero = tempfile::tempdir().expect("tempdir");
3137 for index in 0..60 {
3138 sized_file(&zero.path().join(format!("d{}/z{index:02}", index % 5)), 0, false, 3);
3139 }
3140 trees.push(("all files empty", zero));
3141
3142 let empty = tempfile::tempdir().expect("tempdir");
3143 fs::create_dir_all(empty.path().join("only/directories")).expect("directories");
3144 trees.push(("no files", empty));
3145
3146 let single = tempfile::tempdir().expect("tempdir");
3147 sized_file(&single.path().join("one"), 7, false, 4);
3148 trees.push(("one file", single));
3149
3150 let ignored = tempfile::tempdir().expect("tempdir");
3153 put(ignored.path(), ".gitignore", b"*.big\nbuild/\n");
3154 sized_file(&ignored.path().join("src/at.big"), 1_000, false, 5);
3155 sized_file(&ignored.path().join("src/below.big"), 999, false, 5);
3156 for index in 0..150 {
3157 sized_file(&ignored.path().join(format!("build/o{index:03}")), 30, false, 6);
3158 sized_file(&ignored.path().join(format!("src/s{index:03}.rs")), 38, false, 7);
3159 sized_file(&ignored.path().join(format!("src/x{index:03}.big")), 5, false, 8);
3160 }
3161 trees.push(("ignored files at the share boundary", ignored));
3162
3163 let linked = tempfile::tempdir().expect("tempdir");
3166 sized_file(&linked.path().join("a/original"), 1_000, false, 9);
3167 for link in ["b/link", "c/deeper/link"] {
3168 fs::create_dir_all(linked.path().join(link).parent().expect("a parent"))
3169 .expect("directories");
3170 fs::hard_link(linked.path().join("a/original"), linked.path().join(link))
3171 .expect("hard link");
3172 }
3173 for index in 0..7 {
3174 sized_file(&linked.path().join(format!("d/f{index}")), 1_000, false, 10);
3175 }
3176 for index in 0..30 {
3177 sized_file(&linked.path().join(format!("e/z{index:02}")), 0, false, 11);
3178 }
3179 trees.push(("hard links at the share boundary", linked));
3180
3181 #[cfg(unix)]
3183 {
3184 let special = tempfile::tempdir().expect("tempdir");
3185 for index in 0..120 {
3186 sized_file(&special.path().join(format!("s/f{index:03}")), 10, false, 12);
3187 }
3188 sized_file(&special.path().join("s/large"), 5_000, false, 13);
3189 let fifo = special.path().join("s/pipe");
3190 let status =
3191 std::process::Command::new("mkfifo").arg(&fifo).status().expect("run mkfifo");
3192 assert!(status.success(), "mkfifo exited with {status}");
3193 drop(
3194 std::os::unix::net::UnixListener::bind(special.path().join("s/socket"))
3195 .expect("bind socket"),
3196 );
3197 trees.push(("special files beside folded files", special));
3198 }
3199
3200 #[cfg(target_os = "linux")]
3203 {
3204 use std::os::unix::ffi::OsStrExt as _;
3205 let lossy = tempfile::tempdir().expect("tempdir");
3206 for byte in [0xfe_u8, 0xff] {
3207 let bytes = [b't', b'i', b'e', byte];
3208 let name = std::ffi::OsStr::from_bytes(&bytes);
3209 sized_file(&lossy.path().join("n").join(name), 2_000, false, 14);
3210 }
3211 for index in 0..80 {
3212 sized_file(&lossy.path().join(format!("n/small{index:02}")), 3, false, 15);
3213 }
3214 trees.push(("names equal once made readable", lossy));
3215 }
3216 let trees: Vec<BoundaryTree> = trees
3217 .into_iter()
3218 .map(|(name, tree)| BoundaryTree {
3219 name,
3220 #[cfg(unix)]
3221 _sealed: None,
3222 tree,
3223 })
3224 .chain(search_denied_boundary_tree())
3225 .collect();
3226 for tree in &trees {
3227 crate::test_support::settle_allocations(tree.tree.path());
3228 }
3229 trees
3230 }
3231
3232 #[cfg(unix)]
3236 fn search_denied_boundary_tree() -> Option<BoundaryTree> {
3237 if !crate::test_support::require_permission_bits() {
3238 return None;
3239 }
3240 let unsearchable = tempfile::tempdir().expect("tempdir");
3241 sized_file(&unsearchable.path().join("open/large"), 5_000, false, 18);
3242 for index in 0..60 {
3243 sized_file(&unsearchable.path().join(format!("open/s{index:02}")), 5, false, 19);
3244 }
3245 let sealed = seal_search_denied(unsearchable.path());
3246 Some(BoundaryTree {
3247 name: "a directory that lists but refuses search",
3248 _sealed: Some(sealed),
3249 tree: unsearchable,
3250 })
3251 }
3252
3253 #[cfg(not(unix))]
3254 fn search_denied_boundary_tree() -> Option<BoundaryTree> {
3255 None
3256 }
3257
3258 const FIXTURE_MINUTE: u64 = 28_000_000;
3261
3262 fn crowded_tree() -> tempfile::TempDir {
3274 let tree = tempfile::tempdir().expect("tempdir");
3275 let root = tree.path();
3276 let minute = |offset| FIXTURE_MINUTE + offset;
3277 put(root, ".gitignore", b"*.skip\n");
3278 sized_file(&root.join("hit/large"), 20_000, false, minute(3));
3279 sized_file(&root.join("hit/covered"), 900, false, minute(3));
3280 sized_file(&root.join("hit/x.skip"), 400, false, minute(2));
3281 for index in 0..30 {
3282 let path = root.join(format!("hit/many/m{index:02}.bin"));
3283 sized_file(&path, 1_000, false, minute(index % 4));
3284 }
3285 for index in 0..110 {
3286 sized_file(&root.join(format!("u{index:03}/f.rs")), 950, false, minute(1));
3287 }
3288 sized_file(&root.join("u000/y.skip"), 30, false, minute(1));
3289 sized_file(&root.join("top-big"), 1_500, false, minute(2));
3290 sized_file(&root.join("top-small"), 990, false, minute(0));
3291 sized_file(&root.join("top.skip"), 20, false, minute(1));
3292 for empty in ["empty", "hit/empty"] {
3293 fs::create_dir_all(root.join(empty)).expect("empty directory");
3294 }
3295 #[cfg(unix)]
3298 {
3299 let mut stamps = vec![
3300 ("hit".to_string(), 3),
3301 ("hit/many".to_string(), 2),
3302 ("hit/empty".to_string(), 0),
3303 ("empty".to_string(), 0),
3304 ];
3305 stamps.extend((0..110).map(|index| (format!("u{index:03}"), 1)));
3306 for (directory, offset) in stamps {
3307 fs::File::open(root.join(directory))
3308 .expect("open directory")
3309 .set_modified(
3310 std::time::UNIX_EPOCH + std::time::Duration::from_secs(minute(offset) * 60),
3311 )
3312 .expect("directory mtime");
3313 }
3314 }
3315 crate::test_support::settle_allocations(root);
3316 tree
3317 }
3318
3319 fn folded_walks(one_worker: bool) -> Vec<(usize, crate::ScanOrder, bool)> {
3322 use crate::ScanOrder::{BreadthFirst, DepthFirst};
3323 let mut walks = Vec::new();
3324 for threads in [1, 2, 4, 8] {
3325 if one_worker && threads != 1 {
3326 continue;
3327 }
3328 for order in [BreadthFirst, DepthFirst] {
3329 let every = matches!((threads, order), (1, BreadthFirst) | (8, DepthFirst))
3330 || (one_worker && order == DepthFirst);
3331 walks.push((threads, order, every));
3332 }
3333 }
3334 walks
3335 }
3336
3337 fn folded_queries(share: &str, size: crate::query::SizeMetric, every: bool) -> Vec<Query> {
3340 use crate::query::{Bound, SortKey};
3341 use crate::report_format::Format;
3342 let list = tree_query(vec![ViewSpec::List], Format::Text, share, size);
3343 let json = tree_query(vec![ViewSpec::Tree], Format::Json, share, size);
3344 let mut queries = vec![list.clone(), json.clone()];
3345 if !every {
3346 return queries;
3347 }
3348 queries.push(Query { format: Format::Tree, ..list.clone() });
3349 for format in [Format::Jsonl, Format::Yaml, Format::Text] {
3350 queries.push(Query { format, ..json.clone() });
3351 }
3352 queries.push(Query { views: vec![ViewSpec::List, ViewSpec::Tree], ..list.clone() });
3353 for sort in [SortKey::Size, SortKey::Name, SortKey::Mtime, SortKey::Count] {
3354 for reverse in [false, true] {
3355 let mut query = list.clone();
3356 query.selection.sort = Some(sort);
3357 query.selection.reverse = reverse;
3358 queries.push(query);
3359 }
3360 }
3361 let bounds: [(Option<Bound>, Option<Bound>, Option<Bound>); 12] = [
3362 (Some(Bound::Limit(0)), None, None),
3363 (None, Some(Bound::Limit(0)), None),
3364 (Some(Bound::Limit(0)), Some(Bound::Limit(0)), None),
3365 (Some(Bound::Limit(1)), None, None),
3366 (Some(Bound::Limit(2)), None, None),
3367 (Some(Bound::All), None, None),
3368 (None, Some(Bound::Limit(1)), None),
3369 (Some(Bound::All), Some(Bound::Limit(3)), None),
3370 (None, None, Some(Bound::Limit(0))),
3371 (None, None, Some(Bound::Limit(1))),
3372 (None, None, Some(Bound::Limit(5))),
3373 (Some(Bound::All), Some(Bound::Limit(2)), Some(Bound::Limit(7))),
3374 ];
3375 for (depth, breadth, limit) in bounds {
3376 let mut query = json.clone();
3377 query.selection.depth = depth;
3378 query.selection.breadth = breadth;
3379 query.selection.limit = limit;
3380 queries.push(query.clone());
3381 query.selection.sort = Some(SortKey::Name);
3382 queries.push(query);
3383 }
3384 queries
3385 }
3386
3387 const FOLDED_SHARES: [&str; 8] =
3390 ["1%", "0.5%", "5%", "10%", "0.01%", "33%", "100%", "0.00152587890625%"];
3391
3392 fn assert_folded_reports_match(
3398 root: &Path,
3399 scan: &ScanConfig,
3400 shares: &[&str],
3401 one_worker: bool,
3402 label: &str,
3403 ) -> bool {
3404 use crate::query::SizeMetric;
3405 let canonical = root.canonicalize().expect("canonical root");
3406 let mut folded_any = false;
3407 for (threads, order, every) in folded_walks(one_worker) {
3408 let fixture = OpenFixture {
3409 scan: ScanConfig { threads: Some(threads), order, ..scan.clone() },
3410 ..config(CachePolicy::Off, None)
3411 };
3412 let mut full = None;
3413 for share in shares.iter().take(if every { shares.len() } else { 4 }) {
3415 for size in [SizeMetric::Apparent, SizeMetric::Allocated] {
3416 let mut folded: Option<(TreeRetention, crate::Index)> = None;
3417 for query in folded_queries(share, size, every) {
3418 let (request, delivery) = split(root, &fixture, &query);
3419 let scan_config = request.basis.scope.scan_config(&delivery);
3420 let RetainedState::Tree(retention) =
3421 plan(&request, &delivery, Route::OneShot).expect("plan").retained
3422 else {
3423 panic!("{label}: {query:?} takes the full index")
3424 };
3425 let full = full.get_or_insert_with(|| {
3426 let (index, _) = crate::scan::scan_into_index(&canonical, &scan_config)
3427 .expect("full index");
3428 index
3429 });
3430 if folded.as_ref().is_none_or(|(held, _)| *held != retention) {
3431 let (index, _, _) = crate::scan::scan_into_folded_index(
3432 &canonical,
3433 &scan_config,
3434 retention,
3435 false,
3436 )
3437 .expect("folded index");
3438 assert!(index.is_folded() && !full.is_folded(), "{label}");
3439 folded_any |= index.len() < full.len();
3440 folded = Some((retention, index));
3441 }
3442 let (_, index) = folded.as_ref().expect("a folded index");
3443 let mut from_folded =
3444 report(index, &request, SystemTime::UNIX_EPOCH).expect("folded report");
3445 let from_full =
3446 report(full, &request, SystemTime::UNIX_EPOCH).expect("full report");
3447 assert_eq!(
3449 (from_folded.provenance.source, from_folded.provenance.freshness),
3450 (from_full.provenance.source, from_full.provenance.freshness),
3451 "{label}"
3452 );
3453 from_folded.provenance = from_full.provenance.clone();
3454 assert_eq!(
3455 format!("{from_folded:#?}"),
3456 format!("{from_full:#?}"),
3457 "{label} ({threads} workers, {order:?}, {share} of {size:?}): {query:?}"
3458 );
3459 assert_eq!(
3460 crate::report_format::render(&from_folded, query.format, false)
3461 .expect("render"),
3462 crate::report_format::render(&from_full, query.format, false)
3463 .expect("render"),
3464 "{label} ({threads} workers, {order:?}, {share} of {size:?}): {query:?}"
3465 );
3466 }
3467 }
3468 }
3469 }
3470 folded_any
3471 }
3472
3473 fn assert_folded_route_matches(root: &Path, scan: &ScanConfig, label: &str) {
3477 use crate::report_format::{Format, render};
3478 let list = Query { views: vec![ViewSpec::List], ..Query::default() };
3479 for threads in [Some(1), Some(4)] {
3480 let scan = ScanConfig { threads, ..scan.clone() };
3481 let transient = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
3482 assert!(matches!(planned(&transient, &list).retained, RetainedState::Tree(_)));
3483 let (mut folded, pending, folded_performance) =
3484 prepared(root, &transient, &list).expect("folded report");
3485 pending.join().expect("the folded route saves nothing");
3486
3487 let cache = tempfile::tempdir().expect("cache dir");
3488 let indexed = OpenFixture {
3489 scan: scan.clone(),
3490 ..config(CachePolicy::On, Some(cache.path().join("snapshot.fdu")))
3491 };
3492 assert_eq!(planned(&indexed, &list).retained, RetainedState::FullIndex);
3493 let (indexed, pending, indexed_performance) =
3494 prepared(root, &indexed, &list).expect("indexed report");
3495 pending.join().expect("save");
3496
3497 assert_eq!(folded_performance, indexed_performance, "{label}");
3498 assert_eq!(folded.provenance.source, indexed.provenance.source, "{label}");
3499 assert_eq!(folded.provenance.freshness, indexed.provenance.freshness, "{label}");
3500 folded.provenance = indexed.provenance.clone();
3501 assert_eq!(format!("{folded:#?}"), format!("{indexed:#?}"), "{label}");
3502 for format in [Format::Text, Format::Json, Format::Yaml] {
3503 assert_eq!(
3504 render(&folded, format, false).expect("render"),
3505 render(&indexed, format, false).expect("render"),
3506 "{label}: {format:?}"
3507 );
3508 }
3509 }
3510 }
3511
3512 #[test]
3522 fn transient_tree_equals_the_indexed_tree_under_every_bound_case() {
3523 let mut folded_trees = 0;
3524 for (seed, files, scan) in [
3525 (0x9E37_79B9_7F4A_7C15_u64, 480, ScanConfig::default()),
3526 (0xD1B5_4A32_D192_ED03, 700, ScanConfig::default()),
3527 (
3528 0x2545_F491_4F6C_DD1D,
3529 360,
3530 ScanConfig { max_depth: Some(3), ..ScanConfig::default() },
3531 ),
3532 (
3533 0x94D0_49BB_1331_11EB,
3534 420,
3535 ScanConfig { read_controls: false, ..ScanConfig::default() },
3536 ),
3537 #[cfg(unix)]
3542 (
3543 0x7C15_9E37_79B9_7F4A,
3544 390,
3545 ScanConfig { one_filesystem: true, ..ScanConfig::default() },
3546 ),
3547 ] {
3548 let tree = random_tree(seed, files);
3549 let label = format!("random tree {seed:#x} of {files} files");
3550 assert!(
3551 assert_folded_reports_match(tree.path(), &scan, &FOLDED_SHARES, false, &label),
3552 "{label} folds at some share"
3553 );
3554 assert_folded_route_matches(tree.path(), &scan, &label);
3555 folded_trees += 1;
3556 }
3557
3558 for boundary in boundary_trees() {
3559 let (name, tree) = (boundary.name, boundary.tree.path());
3560 let scan = ScanConfig::default();
3561 let folds = assert_folded_reports_match(tree, &scan, &FOLDED_SHARES, false, name);
3562 assert_eq!(
3563 folds,
3564 !matches!(name, "no files" | "one file"),
3565 "{name}: whether the tree has files a share omits"
3566 );
3567 assert_folded_route_matches(tree, &scan, name);
3568 folded_trees += usize::from(folds);
3569 }
3570
3571 #[cfg(unix)]
3575 if crate::test_support::require_permission_bits() {
3576 use std::os::unix::fs::PermissionsExt;
3577 let tree = random_tree(0x1405_7B7E_F767_814F, 300);
3578 let locked = tree.path().join("locked");
3579 sized_file(&locked.join("hidden-large"), 900_000, false, 16);
3580 sized_file(&locked.join("inner/hidden-small"), 4, false, 16);
3581 for index in 0..60 {
3582 sized_file(&tree.path().join(format!("beside/b{index:02}")), 7, false, 17);
3583 }
3584 crate::test_support::settle_allocations(tree.path());
3585 fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock");
3586 let _unlocked = Unlocked(locked);
3587 let (index, _) = crate::scan::scan_into_index(tree.path(), &ScanConfig::default())
3588 .expect("a partial index");
3589 assert_ne!(index.state().coverage, crate::Coverage::Complete, "the listing failed");
3590 let label = "a failed listing beside folded files";
3591 assert!(
3592 assert_folded_reports_match(
3593 tree.path(),
3594 &ScanConfig::default(),
3595 &FOLDED_SHARES,
3596 false,
3597 label
3598 ),
3599 "{label}"
3600 );
3601 assert_folded_route_matches(tree.path(), &ScanConfig::default(), label);
3602 folded_trees += 1;
3603 }
3604
3605 for case in control_cases() {
3606 let _lookups = case.lookups.install(case.root.path());
3607 let label = format!("{} ({:?} lookups)", case.name, case.lookups);
3608 let folds = assert_folded_reports_match(
3609 case.root.path(),
3610 &case.scan,
3611 &["1%", "10%", "0.5%"],
3612 case.order_dependent,
3613 &label,
3614 );
3615 if !case.order_dependent {
3616 assert_folded_route_matches(case.root.path(), &case.scan, &label);
3617 }
3618 folded_trees += usize::from(folds);
3619 }
3620
3621 let large = random_tree(0x6C8E_9CF5_7093_2BD5, 12_000);
3624 let label = "a tree larger than 0.01% keeps";
3625 assert!(
3626 assert_folded_reports_match(
3627 large.path(),
3628 &ScanConfig::default(),
3629 &["0.01%"],
3630 true,
3631 label
3632 ),
3633 "{label}"
3634 );
3635 assert!(folded_trees > 10, "the differential folded {folded_trees} trees");
3636 }
3637
3638 #[test]
3648 fn a_filtered_tree_is_measured_against_what_it_selects() {
3649 use crate::query::{SizeMetric, TreeNode};
3650 use crate::report_format::Format;
3651 let tree = crowded_tree();
3652 let off = config(CachePolicy::Off, None);
3653 let unfiltered = tree_query(vec![ViewSpec::Tree], Format::Json, "1%", SizeMetric::Apparent);
3654 let mut filtered = unfiltered.clone();
3655 filtered.selection.min_size = Some(1_000);
3656 assert_eq!(planned(&off, &filtered).retained, RetainedState::FullIndex);
3657 let root_of = |query: &Query| {
3658 let (report, pending, _) = prepared(tree.path(), &off, query).expect("a report");
3659 pending.join().expect("a one-shot report saves nothing");
3660 match report.sections.into_iter().next() {
3661 Some(Section::Tree { root: Some(root), .. }) => *root,
3662 other => panic!("expected a tree, got {other:?}"),
3663 }
3664 };
3665 let rows = |node: &TreeNode| {
3666 node.children.iter().map(|row| (row.name.clone(), row.bytes)).collect::<Vec<_>>()
3667 };
3668 let named = |rows: &[(&str, u64)]| {
3669 rows.iter().map(|(name, bytes)| ((*name).to_string(), *bytes)).collect::<Vec<_>>()
3670 };
3671
3672 let selected = root_of(&filtered);
3673 assert_eq!(selected.bytes, 20_000 + 900 + 400 + 30 * 1_000 + 1_500);
3676 assert_eq!(root_of(&unfiltered).bytes, selected.bytes + 990 + 20 + 110 * 950 + 30 + 7);
3677 assert_eq!(rows(&selected), named(&[("hit", 51_300), ("top-big", 1_500)]));
3678 assert_eq!(
3679 rows(&selected.children[0]),
3680 named(&[("many", 30_000), ("large", 20_000), ("covered", 900)])
3681 );
3682 }
3683
3684 fn filtered_selections() -> Vec<(&'static str, crate::query::Selection)> {
3689 use crate::query::{ModifiedWindow, Selection};
3690 let at = |offset: u64| {
3691 i64::try_from((FIXTURE_MINUTE + offset) * 60 * 1_000_000_000).expect("nanoseconds")
3692 };
3693 let globs = |sources: &[&str]| -> Vec<Pattern> {
3694 sources.iter().map(|source| Pattern::parse(source).expect("pattern")).collect()
3695 };
3696 let window = |since, before| ModifiedWindow { since, before };
3697 let none = Selection::default;
3698 vec![
3699 ("a minimum size", Selection { min_size: Some(1_000), ..none() }),
3700 ("a minimum every nonempty file passes", Selection { min_size: Some(1), ..none() }),
3701 ("included names", Selection { include: globs(&["*.rs", "*.skip"]), ..none() }),
3702 ("included directories", Selection { include: globs(&["hit", "d00*"]), ..none() }),
3703 ("excluded names", Selection { exclude: globs(&["*.log", "*.skip"]), ..none() }),
3704 (
3705 "excluded directories",
3706 Selection { exclude: globs(&["u0*", "d001", "many"]), ..none() },
3707 ),
3708 (
3709 "included and excluded",
3710 Selection {
3711 include: globs(&["*.rs", "hit"]),
3712 exclude: globs(&["many", "*.tmp"]),
3713 ..none()
3714 },
3715 ),
3716 ("modified since", Selection { modified: window(Some(at(2)), None), ..none() }),
3717 ("modified before", Selection { modified: window(None, Some(at(2))), ..none() }),
3718 (
3719 "a modified window",
3720 Selection { modified: window(Some(at(1)), Some(at(3))), ..none() },
3721 ),
3722 ("files", Selection { kinds: vec![EntryKind::File], ..none() }),
3723 ("directories", Selection { kinds: vec![EntryKind::Dir], ..none() }),
3724 ("symlinks", Selection { kinds: vec![EntryKind::Symlink], ..none() }),
3725 ("unignored entries", Selection { ignored: IgnoredEntries::Exclude, ..none() }),
3726 ("ignored entries", Selection { ignored: IgnoredEntries::Only, ..none() }),
3727 (
3728 "a minimum size among included names",
3729 Selection {
3730 min_size: Some(1_000),
3731 include: globs(&["*.bin", "*.rs", "top-*"]),
3732 ..none()
3733 },
3734 ),
3735 ]
3736 }
3737
3738 fn filtered_queries(
3743 selection: &crate::query::Selection,
3744 share: &str,
3745 size: crate::query::SizeMetric,
3746 ) -> Vec<Query> {
3747 use crate::query::{Bound, SortKey};
3748 use crate::report_format::Format;
3749 let filtered = |views, format| {
3750 let mut query = tree_query(views, format, share, size);
3751 query.selection = crate::query::Selection {
3752 min_share: query.selection.min_share.clone(),
3753 size,
3754 ..selection.clone()
3755 };
3756 query
3757 };
3758 let list = filtered(vec![ViewSpec::List], Format::Text);
3759 let json = filtered(vec![ViewSpec::Tree], Format::Json);
3760 let mut queries = vec![list.clone(), json.clone()];
3761 for query in [&list, &json] {
3762 let mut named = query.clone();
3763 named.selection.sort = Some(SortKey::Name);
3764 queries.push(named);
3765 }
3766 let mut reported = list;
3767 reported.selection.sort = Some(SortKey::Size);
3768 reported.selection.depth = Some(Bound::Limit(2));
3769 reported.selection.limit = Some(Bound::Limit(30));
3770 queries.push(reported);
3771 let mut bounded = json;
3772 bounded.selection.sort = Some(SortKey::Name);
3773 bounded.selection.reverse = true;
3774 bounded.selection.depth = Some(Bound::Limit(1));
3775 bounded.selection.breadth = Some(Bound::Limit(2));
3776 queries.push(bounded);
3777 queries
3778 }
3779
3780 fn assert_same_answer(
3783 mut actual: Report,
3784 expected: &Report,
3785 format: crate::report_format::Format,
3786 context: &str,
3787 ) {
3788 use crate::report_format::render;
3789 assert_eq!(
3790 (actual.provenance.source, actual.provenance.freshness),
3791 (expected.provenance.source, expected.provenance.freshness),
3792 "{context}"
3793 );
3794 actual.provenance = expected.provenance.clone();
3795 assert_eq!(format!("{actual:#?}"), format!("{expected:#?}"), "{context}");
3796 assert_eq!(
3797 render(&actual, format, false).expect("render"),
3798 render(expected, format, false).expect("render"),
3799 "{context}"
3800 );
3801 }
3802
3803 fn assert_filtered_reports_match(root: &Path, scan: &ScanConfig, label: &str) {
3810 use crate::query::{Selection, SizeMetric};
3811 use crate::report_format::Format;
3812 let root = root.canonicalize().expect("canonical root");
3813 let fixture = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
3814 let scan_config = {
3815 let (request, delivery) = split(&root, &fixture, &Query::default());
3816 request.basis.scope.scan_config(&delivery)
3817 };
3818 let (full, _) = crate::scan::scan_into_index(&root, &scan_config).expect("full index");
3819 let answer = |query: &Query| {
3820 let (request, _) = split(&root, &fixture, query);
3821 report(&full, &request, SystemTime::UNIX_EPOCH)
3822 };
3823 let whole = tree_query(vec![ViewSpec::Tree], Format::Json, "0%", SizeMetric::Apparent);
3824 let unfiltered = format!("{:#?}", answer(&whole).expect("unfiltered report").sections);
3825 let mut vacuous = Vec::new();
3826 for (name, selection) in filtered_selections() {
3827 let refused = !scan.read_controls && selection.ignored != IgnoredEntries::Include;
3830 let every_row = Query {
3831 selection: Selection {
3832 min_share: whole.selection.min_share.clone(),
3833 size: whole.selection.size,
3834 ..selection.clone()
3835 },
3836 ..whole.clone()
3837 };
3838 match answer(&every_row) {
3839 Ok(filtered) => {
3840 assert!(!refused, "{label}: {name} is answered");
3841 if format!("{:#?}", filtered.sections) == unfiltered {
3842 vacuous.push(name);
3843 }
3844 }
3845 Err(error) => assert!(refused, "{label}: {name}: {error}"),
3846 }
3847 for share in ["1%", "10%"] {
3848 for size in [SizeMetric::Apparent, SizeMetric::Allocated] {
3849 for query in filtered_queries(&selection, share, size) {
3850 let context = format!("{label}, {name}, {share} of {size:?}: {query:?}");
3851 let (request, delivery) = split(&root, &fixture, &query);
3852 let (expected, routed) =
3853 match (answer(&query), prepare_report(&request, &delivery)) {
3854 (Ok(expected), Ok((routed, pending, _))) => {
3855 pending.join().expect("a one-shot report saves nothing");
3856 (expected, routed)
3857 }
3858 (Err(_), Err(_)) if refused => continue,
3859 (expected, routed) => panic!(
3860 "{context}: {:?} beside {:?}",
3861 expected.err(),
3862 routed.err()
3863 ),
3864 };
3865 assert_same_answer(routed, &expected, query.format, &context);
3866 let RetainedState::Tree(retention) =
3867 plan(&request, &delivery, Route::OneShot).expect("plan").retained
3868 else {
3869 continue;
3870 };
3871 let (index, _, _) = crate::scan::scan_into_folded_index(
3872 &root,
3873 &scan_config,
3874 retention,
3875 false,
3876 )
3877 .expect("folded index");
3878 assert!(index.is_folded(), "{context}");
3879 let folded =
3880 report(&index, &request, SystemTime::UNIX_EPOCH).expect("folded");
3881 assert_same_answer(folded, &expected, query.format, &context);
3882 }
3883 }
3884 }
3885 }
3886 assert!(vacuous.is_empty(), "{label}: these select the whole tree: {vacuous:?}");
3887 }
3888
3889 #[test]
3899 fn a_filtered_tree_answers_as_the_full_index_on_every_route() {
3900 let random = random_tree(0x9E37_79B9_7F4A_7C15, 480);
3901 let crowded = crowded_tree();
3902 let deep = |max_depth| ScanConfig { max_depth: Some(max_depth), ..ScanConfig::default() };
3903 let blind = ScanConfig { read_controls: false, ..ScanConfig::default() };
3904 for (tree, scan, label) in [
3905 (&random, ScanConfig::default(), "a random tree"),
3906 (&random, deep(3), "a random tree scanned three deep"),
3907 (&random, blind, "a random tree read without .gitignore"),
3908 (&crowded, ScanConfig::default(), "the crowded tree"),
3909 (&crowded, deep(1), "the crowded tree scanned one deep"),
3910 ] {
3911 assert_filtered_reports_match(tree.path(), &scan, label);
3912 }
3913 }
3914
3915 #[test]
3916 fn full_index_report_exposes_scan_diagnostics_when_requested() {
3917 let root = tempfile::tempdir().expect("tempdir");
3918 fs::create_dir(root.path().join("nested")).expect("directory");
3919 fs::write(root.path().join("nested/file.txt"), b"trace me").expect("file");
3920 let query = Query { views: vec![ViewSpec::Tree], ..Query::default() };
3921
3922 let (report, pending, performance, diagnostics) =
3923 prepared_with_diagnostics(root.path(), &config(CachePolicy::Off, None), &query)
3924 .expect("full-index report");
3925 pending.join().expect("no pending save");
3926
3927 assert!(report.status.complete);
3928 assert_eq!(performance.walked_files, 1);
3929 let diagnostics = diagnostics.expect("full-index scan diagnostics");
3930 assert_eq!(diagnostics.schema, crate::scan::SCAN_DIAGNOSTICS_SCHEMA);
3931 assert_eq!(diagnostics.worker_policy.ready_directories_at_finish, 0);
3932 assert_eq!(diagnostics.worker_policy.in_flight_directories_at_finish, 0);
3933 }
3934
3935 fn wide_tree(dirs: usize, files: usize) -> (tempfile::TempDir, u64, u64) {
3938 let root = tempfile::tempdir().expect("tempdir");
3939 let mut bytes = 0;
3940 for directory in 0..dirs {
3941 let path = root.path().join(format!("d{directory:03}"));
3942 fs::create_dir(&path).expect("directory");
3943 for file in 0..files {
3944 let size = directory * files + file + 1;
3945 fs::write(path.join(format!("f{file}.txt")), vec![b'.'; size]).expect("file");
3946 bytes += size as u64;
3947 }
3948 }
3949 (root, (dirs * files) as u64, bytes)
3950 }
3951
3952 fn prepared_with_progress(
3954 root: &Path,
3955 config: &OpenFixture,
3956 query: &Query,
3957 progress: &Progress,
3958 ) -> Result<(Report, PendingSave, PerformanceSummary)> {
3959 let (request, delivery) = split(root, config, query);
3960 prepare_report_with_progress(&request, &delivery, progress)
3961 }
3962
3963 fn analyzing(fixture: OpenFixture) -> OpenFixture {
3964 OpenFixture {
3965 analysis: crate::content::AnalysisRequest {
3966 profile: crate::content::AnalysisSet::LINES_ONLY,
3967 workers: 0,
3968 },
3969 ..fixture
3970 }
3971 }
3972
3973 #[test]
3981 fn progress_ends_at_the_walked_totals_of_every_one_shot_route() {
3982 use crate::ProgressPhase::{Scanning, Summarizing};
3983 let (root, files, bytes) = wide_tree(6, 4);
3984 let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
3985
3986 let progress = Progress::new();
3987 let (_, pending, performance) =
3988 prepared_with_progress(root.path(), &config(CachePolicy::Off, None), &tree, &progress)
3989 .expect("cold full-index report");
3990 pending.join().expect("no save");
3991 let snapshot = progress.snapshot();
3992 assert_eq!(performance.source, ReportSource::ColdScan);
3993 assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
3994 assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "cold full index");
3995 assert_eq!(snapshot.allocated, performance.walked_allocated);
3996 assert!(snapshot.allocated > 0, "files with content occupy blocks");
3997 assert_eq!(snapshot.directories, 7, "the root and its six children");
3998 assert_eq!(
3999 (snapshot.phase, snapshot.analysis),
4000 (Summarizing, None),
4001 "the walk ended, the index was assembled, then the answer was built"
4002 );
4003
4004 let progress = Progress::new();
4005 let (report, pending, performance) = prepared_with_progress(
4006 root.path(),
4007 &blind(CachePolicy::Off, None),
4008 &summary_query(),
4009 &progress,
4010 )
4011 .expect("compact summary report");
4012 pending.join().expect("no save");
4013 let Section::Summary(row) = report.sections[0] else { panic!("summary section") };
4014 let snapshot = progress.snapshot();
4015 assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
4016 assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "summary fold");
4017 assert_eq!(snapshot.allocated, performance.walked_allocated);
4018 assert_eq!(snapshot.allocated, row.allocated, "the progress figure is the answer's");
4019 assert_eq!(snapshot.directories, row.dirs + 1, "the row's directories and the root");
4020 assert_eq!((snapshot.phase, snapshot.analysis), (Scanning, None));
4021
4022 let cache_dir = tempfile::tempdir().expect("cache dir");
4024 let cache = cache_dir.path().join("snapshot.fdu");
4025 let progress = Progress::new();
4026 let (_, pending, performance) = prepared_with_progress(
4027 root.path(),
4028 &analyzing(config(CachePolicy::Auto, Some(cache.clone()))),
4029 &tree,
4030 &progress,
4031 )
4032 .expect("cold analyzed report");
4033 let snapshot = progress.snapshot();
4034 assert_eq!(snapshot.phase, Summarizing, "the answer is built while the save runs");
4035 pending.join().expect("save");
4036 assert_eq!(performance.source, ReportSource::ColdScan);
4037 assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "cold with analysis");
4038 assert_eq!(snapshot.allocated, performance.walked_allocated);
4039 assert_eq!(snapshot.directories, 7);
4040 assert_eq!(performance.fresh_files, files, "every file is a lines candidate");
4041 assert_eq!(snapshot.analysis, Some((files, files)));
4042
4043 let progress = Progress::new();
4044 let (_, pending, performance) = prepared_with_progress(
4045 root.path(),
4046 &analyzing(config(CachePolicy::Auto, Some(cache))),
4047 &tree,
4048 &progress,
4049 )
4050 .expect("warm analyzed report");
4051 pending.join().expect("nothing to save");
4052 let snapshot = progress.snapshot();
4053 assert_eq!(performance.source, ReportSource::WarmRevalidate);
4054 assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
4055 assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "warm revalidation");
4056 assert_eq!(snapshot.allocated, performance.walked_allocated);
4057 assert_eq!(snapshot.directories, 7);
4058 assert_eq!(performance.fresh_files, 0, "the sidecar answered every candidate");
4059 assert_eq!((snapshot.phase, snapshot.analysis), (Summarizing, Some((0, 0))));
4060 }
4061
4062 #[test]
4065 fn a_cache_only_report_ends_summarizing_and_walks_nothing() {
4066 use crate::ProgressPhase::Summarizing;
4067 let (root, _, _) = wide_tree(3, 2);
4068 let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
4069 let cache_dir = tempfile::tempdir().expect("cache dir");
4070 let cache = cache_dir.path().join("snapshot.fdu");
4071 let (_, pending, _) =
4072 prepared(root.path(), &config(CachePolicy::On, Some(cache.clone())), &tree)
4073 .expect("a report that writes the snapshot");
4074 pending.join().expect("save");
4075
4076 let progress = Progress::new();
4077 let (_, pending, performance) = prepared_with_progress(
4078 root.path(),
4079 &stale(config(CachePolicy::Auto, Some(cache))),
4080 &tree,
4081 &progress,
4082 )
4083 .expect("cache-only report");
4084 pending.join().expect("nothing to save");
4085 let snapshot = progress.snapshot();
4086 assert_eq!(performance.source, ReportSource::CacheOnly);
4087 assert_eq!(snapshot.phase, Summarizing);
4088 assert_eq!((snapshot.directories, snapshot.files, snapshot.bytes), (0, 0, 0));
4089 }
4090
4091 fn rank(phase: crate::ProgressPhase, order: &[crate::ProgressPhase]) -> usize {
4093 order
4094 .iter()
4095 .position(|expected| *expected == phase)
4096 .unwrap_or_else(|| panic!("{phase:?} is not a phase of this route"))
4097 }
4098
4099 #[test]
4106 fn progress_is_monotonic_and_phases_advance_in_order_while_a_report_runs() {
4107 use crate::ProgressPhase::{
4108 Analyzing, Indexing, Loading, Revalidating, Saving, Scanning, Starting, Summarizing,
4109 };
4110 let (root, files, bytes) = wide_tree(48, 6);
4111 let cache_dir = tempfile::tempdir().expect("cache dir");
4112 let cache = cache_dir.path().join("snapshot.fdu");
4113 let fixture = OpenFixture {
4114 scan: ScanConfig { threads: Some(3), batch_size: 4, ..ScanConfig::default() },
4115 ..analyzing(config(CachePolicy::Auto, Some(cache)))
4116 };
4117 let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
4118
4119 let routes: [(&str, &[crate::ProgressPhase]); 2] = [
4120 ("cold", &[Starting, Loading, Scanning, Indexing, Analyzing, Saving, Summarizing]),
4121 ("warm", &[Starting, Loading, Revalidating, Analyzing, Saving, Summarizing]),
4122 ];
4123 for (route, order) in routes {
4124 let progress = Progress::new();
4125 let initial = progress.snapshot();
4128 let done = std::sync::atomic::AtomicBool::new(false);
4129 let (performance, seen) = std::thread::scope(|scope| {
4130 let poller = scope.spawn(|| {
4131 let polled = progress.clone();
4132 let mut previous = initial;
4133 let mut seen = vec![previous.phase];
4134 loop {
4135 let finished = done.load(std::sync::atomic::Ordering::Acquire);
4136 let current = polled.snapshot();
4137 assert!(current.directories >= previous.directories, "{route}");
4138 assert!(current.files >= previous.files, "{route}");
4139 assert!(current.bytes >= previous.bytes, "{route}");
4140 assert!(
4141 rank(current.phase, order) >= rank(previous.phase, order),
4142 "{route}: {:?} after {:?}",
4143 current.phase,
4144 previous.phase
4145 );
4146 if let (Some(before), Some(after)) = (previous.analysis, current.analysis) {
4147 assert!(after.0 >= before.0 && after.0 <= after.1, "{route}");
4148 assert_eq!(after.1, before.1, "{route}: the total is fixed");
4149 }
4150 if current.phase != previous.phase {
4151 seen.push(current.phase);
4152 }
4153 previous = current;
4154 if finished {
4157 break;
4158 }
4159 std::thread::yield_now();
4160 }
4161 seen
4162 });
4163 let (_, pending, performance) =
4164 prepared_with_progress(root.path(), &fixture, &tree, &progress)
4165 .expect("report");
4166 pending.join().expect("save");
4167 done.store(true, std::sync::atomic::Ordering::Release);
4168 (performance, poller.join().expect("poller"))
4169 });
4170 let snapshot = progress.snapshot();
4171 assert_eq!(
4172 (snapshot.files, snapshot.bytes),
4173 (performance.walked_files, performance.walked_bytes),
4174 "{route}"
4175 );
4176 assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "{route}");
4177 assert_eq!(snapshot.directories, 49, "{route}");
4178 assert_eq!(seen.first(), Some(&Starting), "{route}: {seen:?}");
4179 assert_eq!(
4180 seen.last().copied(),
4181 Some(snapshot.phase),
4182 "{route}: the poller saw the final phase"
4183 );
4184 assert_eq!(snapshot.phase, Summarizing, "{route}: the answer is built last");
4185 if route == "cold" {
4186 assert_eq!(snapshot.analysis, Some((files, files)));
4187 } else {
4188 assert_eq!(snapshot.analysis, Some((0, 0)));
4189 assert!(!seen.contains(&Indexing), "{route}: a warm run assembles no index");
4190 }
4191 }
4192 }
4193
4194 #[test]
4198 fn a_report_prepared_with_a_handle_is_the_report_prepared_without() {
4199 let (root, _, _) = wide_tree(5, 3);
4200 let tree = Query { views: vec![ViewSpec::Tree, ViewSpec::Files], ..Query::default() };
4201 let cases = [
4202 ("full index", config(CachePolicy::Off, None), tree),
4203 ("compact summary", blind(CachePolicy::Off, None), summary_query()),
4204 ];
4205 for (route, fixture, query) in cases {
4206 let (mut plain, pending, plain_performance) =
4207 prepared(root.path(), &fixture, &query).expect("plain report");
4208 pending.join().expect("no save");
4209 let progress = Progress::new();
4210 let (observed, pending, observed_performance) =
4211 prepared_with_progress(root.path(), &fixture, &query, &progress)
4212 .expect("observed report");
4213 pending.join().expect("no save");
4214
4215 assert_eq!(plain_performance, observed_performance, "{route}");
4216 plain.provenance = observed.provenance.clone();
4217 let json = |report: &Report| {
4218 crate::report_format::render(report, crate::report_format::Format::Json, false)
4219 .expect("render")
4220 };
4221 assert_eq!(json(&plain), json(&observed), "{route}");
4222 assert!(progress.snapshot().files > 0, "{route}: the handle did observe the run");
4223 }
4224 }
4225}