Skip to main content

fdu_core/
execution.rs

1//! Planning and executing one-shot reports with the least retained state they require.
2//!
3//! The command surface stays composable: callers describe cache policy and a query, not
4//! an implementation strategy.  This module derives that strategy.  Most reports need
5//! the complete [`Index`](crate::Index), either because another view needs hierarchy or
6//! paths, or because the cache must retain reusable state.  An unfiltered summary needs
7//! only its aggregate values and, when it observes `.gitignore`, the ignored share of
8//! them, so that one plan reduces the scan's observations directly and never builds an
9//! index.  An unfiltered tree with a positive share threshold needs every directory but
10//! only the files large enough to be shown, so that plan builds an index of those alone.
11
12use std::time::SystemTime;
13
14use crate::query::{
15    Delivery, Report, ReportProvenance, ReportSource, Request, SizeMetric, SortKey, SummaryRow,
16    TreeStatus, ViewSpec, report, report_summary,
17};
18use crate::{CachePolicy, EntryKind, Error, OpenPath, PendingSave, Progress, Result, execute};
19
20/// The minimum state a one-shot report plan retains while scanning.
21///
22/// This is deliberately a small closed set.  Add another tier only when a measured view
23/// can be answered exactly from materially less state than an index; callers should not
24/// have to select it themselves.
25#[derive(Clone, Copy, PartialEq, Eq, Debug)]
26pub(crate) enum RetainedState {
27    /// One aggregate row; no path or hierarchy records survive the scan.
28    ///
29    /// A scan that observes control state keeps the control table and the few ignored
30    /// directories that head an ignored subtree, which is all it needs to classify each
31    /// entry as the index would ([`SummaryFold`]).
32    Summary,
33    /// A folded index: every directory, symlink, and other entry, but only the largest
34    /// regular files, and only the reducers a tree reads.
35    ///
36    /// Every other file still counts in its directory's roll-ups, and in a folded tally
37    /// that stands for the rows a share threshold would have omitted
38    /// ([`Index::folded_children`](crate::Index)). Such an index answers its one tree
39    /// report exactly and nothing else, so the plan that builds it reports from it and
40    /// frees it: it is never returned, persisted, or mutated ([`Index::is_folded`]).
41    Tree(TreeRetention),
42    /// The complete reusable metadata index.
43    FullIndex,
44}
45
46/// What a folded index ([`RetainedState::Tree`]) keeps of the regular files it walks.
47#[derive(Clone, Copy, PartialEq, Eq, Debug)]
48pub(crate) struct TreeRetention {
49    /// How many of the largest files it keeps as entries: at least as many as the share
50    /// threshold can show as rows ([`ShareThreshold::admitted_parts_bound`]), since every
51    /// file is a part of the root total the threshold measures it against.
52    ///
53    /// [`ShareThreshold::admitted_parts_bound`]: crate::query::ShareThreshold
54    pub(crate) largest_files: u32,
55    /// The metric the share threshold measures, which is the one that ranks them.
56    pub(crate) size: SizeMetric,
57}
58
59impl TreeRetention {
60    /// The most files a folded index keeps.
61    ///
62    /// Beyond this a request takes the full index. The bound is on the memory the kept
63    /// files hold at once, each a heap slot with its name and attributes until the walk
64    /// ends, and on the cost of ranking against them. It admits every share of at least
65    /// `100 / 65,536` percent, about 0.0015%.
66    pub(crate) const MAX_FILES: u32 = 65_536;
67
68    /// The retention that answers `request` exactly, when a folded index can.
69    ///
70    /// That is when the tree projection is the report's only reader of the index: every
71    /// requested view is a List or Tree rendered as a tree, whose rows [`report`] builds
72    /// from roll-ups and direct children alone, never from a flat inventory or extension
73    /// tallies; the selection filters nothing, so every row is measured against the
74    /// root's maintained total; the scan retains its whole population; nothing is sorted
75    /// by a content metric; and the share threshold is positive and admits at most
76    /// [`Self::MAX_FILES`] rows. A files view is a flat inventory even where it renders as
77    /// a tree, so it takes the index.
78    fn for_request(request: &Request) -> Option<Self> {
79        let query = &request.query;
80        let tree_only = !query.views.is_empty()
81            && query.views.iter().all(|view| {
82                matches!(view, ViewSpec::List | ViewSpec::Tree) && query.tree_for(*view)
83            });
84        if !tree_only
85            || !query.selection.is_unfiltered()
86            || matches!(query.selection.sort, Some(SortKey::Metric(_)))
87            || request.basis.scope.population != crate::query::IgnoredEntries::Include
88        {
89            return None;
90        }
91        let largest_files = query
92            .min_share_for()
93            .admitted_parts_bound()
94            .and_then(|bound| u32::try_from(bound).ok())
95            .filter(|bound| *bound <= Self::MAX_FILES)?;
96        Some(Self { largest_files, size: query.selection.size })
97    }
98}
99
100/// The engine lifecycle that will deliver an answer.
101#[derive(Clone, Copy, PartialEq, Eq, Debug)]
102pub enum Route {
103    /// A single report may retain only an aggregate.
104    OneShot,
105    /// A reusable index returned to the caller.
106    Retained,
107    /// Verification of an index already held by the caller.
108    Refresh,
109    /// A continuing observation session.
110    Watch,
111    /// Progressive discovery and serving of an opened root.
112    Opened,
113}
114
115/// Which persisted state execution may read.
116#[derive(Clone, Copy, PartialEq, Eq, Debug)]
117pub enum Load {
118    /// Start without a metadata snapshot.
119    None,
120    /// Attempt to restore a serving metadata snapshot.
121    Snapshot,
122}
123
124/// Whether execution must verify the filesystem.
125#[derive(Clone, Copy, PartialEq, Eq, Debug)]
126pub enum Verify {
127    /// Answer only from persisted facts, marked unverified.
128    None,
129    /// Observe the requested filesystem scope.
130    Filesystem,
131}
132
133/// Whether an answer fulfills the caller's delivery contract.
134#[derive(Clone, Copy, Debug, PartialEq, Eq)]
135pub enum OutcomeClass {
136    /// A complete answer, or a partial answer the caller explicitly accepts.
137    Success,
138    /// An incomplete answer the caller did not accept.
139    Partial,
140}
141
142/// Validated policy shared by all engine execution routes.
143#[derive(Clone, Debug)]
144pub struct Plan {
145    pub(crate) basis: crate::query::Basis,
146    pub(crate) route: Route,
147    pub(crate) retained: RetainedState,
148    pub(crate) load: Load,
149    pub(crate) verify: Verify,
150    pub(crate) persist: bool,
151    pub(crate) delivery: Delivery,
152}
153
154impl Plan {
155    /// Classify the answer using the caller's partial-answer policy.
156    pub fn outcome(&self, status: &TreeStatus) -> OutcomeClass {
157        if status.complete || self.delivery.accept_partial {
158            OutcomeClass::Success
159        } else {
160            OutcomeClass::Partial
161        }
162    }
163    /// The semantic basis validated when the plan was constructed.
164    pub fn basis(&self) -> &crate::query::Basis {
165        &self.basis
166    }
167    /// The lifecycle this plan executes.
168    pub const fn route(&self) -> Route {
169        self.route
170    }
171    /// The persisted state this plan may read.
172    pub const fn load(&self) -> Load {
173        self.load
174    }
175    /// The verification this plan performs.
176    pub const fn verify(&self) -> Verify {
177        self.verify
178    }
179    /// Whether this plan may write the snapshot and its content sidecar.
180    ///
181    /// Each tier still writes only under its own rule (`Plan::writes`); this is the
182    /// authorization those rules start from, and a caller deciding whether to warn that a
183    /// run leaves nothing behind asks it here rather than of the policy, whose meaning
184    /// under `Auto` depends on the route and the analysis.
185    pub const fn persists(&self) -> bool {
186        self.persist
187    }
188    /// The caller's operational choices after validation.
189    pub fn delivery(&self) -> &Delivery {
190        &self.delivery
191    }
192}
193
194/// Stored tier declarations and restoration evidence presented to a plan.
195pub(crate) struct StoreHeader<'a> {
196    pub(crate) root: &'a std::path::Path,
197    pub(crate) snapshot: crate::SnapshotIdentity,
198    pub(crate) content: Option<&'a crate::ContentTierIdentity>,
199    pub(crate) content_complete: bool,
200}
201
202/// Why persisted state cannot deliver a planned answer.
203#[derive(Clone, Copy, Debug, PartialEq, Eq)]
204pub(crate) enum Admission {
205    Serve(crate::Serves),
206    NoLocation,
207    Missing,
208    WrongRoot,
209    WrongScope,
210    IncompleteContent,
211}
212
213impl Plan {
214    /// The one decision of whether stored state answers this plan's basis.
215    ///
216    /// Every route that reads a snapshot admits it here, warm and cache-only alike, and
217    /// persistence asks the same question of the header on disk before it decides what an
218    /// unchanged pass owes the store. The content arm applies only to a plan that verifies
219    /// nothing, because a verifying route re-reads what its sidecar lacks.
220    pub(crate) fn admit(
221        &self,
222        stored: Option<&StoreHeader<'_>>,
223        basis: &crate::query::Basis,
224    ) -> Admission {
225        if self.delivery.cache_path.is_none() {
226            return Admission::NoLocation;
227        }
228        let Some(stored) = stored else {
229            return Admission::Missing;
230        };
231        if stored.root != basis.root {
232            return Admission::WrongRoot;
233        }
234        let relation = crate::serves_snapshot(stored.snapshot, basis.scope.snapshot_identity());
235        if relation == crate::Serves::Refuse {
236            return Admission::WrongScope;
237        }
238        if self.verify == Verify::None && basis.content.is_enabled() {
239            let wanted = crate::ContentTierIdentity::for_request(
240                basis.scope.snapshot_identity().entries,
241                basis.content,
242            );
243            if !stored.content_complete
244                || stored.content.and_then(|identity| wanted.admit(identity)).is_none()
245            {
246                return Admission::IncompleteContent;
247            }
248        }
249        Admission::Serve(relation)
250    }
251}
252
253/// Facts observed by execution, independent of the route that observed them.
254#[derive(Clone, Copy, Debug)]
255#[allow(clippy::struct_excessive_bools)]
256pub(crate) struct RunFacts {
257    pub(crate) entries_verified: bool,
258    pub(crate) entries_changed: bool,
259    pub(crate) content_changed: bool,
260    pub(crate) content_requested: bool,
261    pub(crate) projected: bool,
262    pub(crate) paired_entries: bool,
263}
264
265/// Artifacts the plan authorizes its executor to write.
266#[derive(Clone, Copy, Debug, PartialEq, Eq)]
267pub(crate) struct SaveTargets {
268    pub(crate) metadata: bool,
269    pub(crate) content: bool,
270}
271
272impl SaveTargets {
273    pub(crate) const fn none(self) -> bool {
274        !self.metadata && !self.content
275    }
276}
277
278impl Plan {
279    pub(crate) fn writes(&self, run: RunFacts) -> SaveTargets {
280        let allowed = self.persist && self.delivery.cache_path.is_some();
281        SaveTargets {
282            metadata: allowed && run.entries_verified && run.entries_changed && !run.projected,
283            content: allowed
284                && run.content_requested
285                && run.content_changed
286                && (run.entries_verified || run.paired_entries),
287        }
288    }
289}
290
291/// Operational work behind one one-shot report.
292///
293/// This is deliberately separate from [`Report`]: it is transient CLI telemetry, not
294/// part of the stable machine-report schema or the pure query result.
295#[derive(Clone, Copy, PartialEq, Eq, Debug)]
296pub struct PerformanceSummary {
297    /// Regular files whose metadata was observed during this run.
298    pub walked_files: u64,
299    /// Apparent bytes represented by those walked files.
300    pub walked_bytes: u64,
301    /// Allocated bytes of those walked files, the figure to show beside an answer
302    /// measured in allocated bytes.
303    pub walked_allocated: u64,
304    /// Fresh content-analysis candidates processed.
305    pub fresh_files: u64,
306    /// Bytes actually returned by fresh content reads.
307    pub bytes_read: u64,
308    /// Wall time spent processing fresh analysis candidates.
309    pub analysis_ns: u64,
310    /// Content-analysis records restored from the sidecar.
311    pub cached_files: u64,
312    /// Apparent bytes represented by restored content records.
313    pub cached_bytes: u64,
314    /// Metadata cache tier used for this report.
315    pub source: ReportSource,
316}
317
318impl Default for PerformanceSummary {
319    fn default() -> Self {
320        Self {
321            walked_files: 0,
322            walked_bytes: 0,
323            walked_allocated: 0,
324            fresh_files: 0,
325            bytes_read: 0,
326            analysis_ns: 0,
327            cached_files: 0,
328            cached_bytes: 0,
329            source: ReportSource::ColdScan,
330        }
331    }
332}
333
334impl PerformanceSummary {
335    /// Total metadata throughput over the same elapsed sample as the report duration.
336    /// GiB/s represents walked size, not storage read bandwidth.
337    pub fn total_throughput(
338        self,
339        elapsed: std::time::Duration,
340        size: crate::query::SizeMetric,
341    ) -> String {
342        let bytes = match size {
343            crate::query::SizeMetric::Apparent => self.walked_bytes,
344            crate::query::SizeMetric::Allocated => self.walked_allocated,
345        };
346        throughput_rates(self.walked_files, bytes, elapsed).map_or_else(
347            || "throughput unavailable".to_owned(),
348            |(files, gib)| format!("{files} files/s ({gib} GiB/s)"),
349        )
350    }
351
352    fn from_open_report(report: &crate::OpenReport) -> Self {
353        let analysis = report.analysis.unwrap_or_default();
354        Self {
355            walked_files: report.scan.files_walked,
356            walked_bytes: report.scan.bytes_walked,
357            walked_allocated: report.scan.allocated_walked,
358            fresh_files: analysis.candidates,
359            bytes_read: analysis.bytes_read,
360            analysis_ns: analysis.elapsed_ns,
361            cached_files: report.content_cache.hits,
362            cached_bytes: report.content_cache.bytes,
363            source: match report.path_taken {
364                OpenPath::ColdScan => ReportSource::ColdScan,
365                OpenPath::WarmRevalidate => ReportSource::WarmRevalidate,
366                OpenPath::CacheOnly => ReportSource::CacheOnly,
367            },
368        }
369    }
370}
371
372/// Cumulative walk rates over one actual elapsed sample. The byte rate is binary GiB/s,
373/// rounded to three decimals; the grouped file rate counts complete files per second.
374/// Returns `(files_per_second, gib_per_second)` without unit labels, or `None` when
375/// elapsed time is zero. Neither rate estimates storage read bandwidth.
376pub fn throughput_rates(
377    files: u64,
378    bytes: u64,
379    elapsed: std::time::Duration,
380) -> Option<(String, String)> {
381    let ns = elapsed.as_nanos();
382    if ns == 0 {
383        return None;
384    }
385    let files_per_second = u128::from(files) * 1_000_000_000 / ns;
386    let gib_denominator = ns * (1_u128 << 30);
387    let gib_thousandths =
388        (u128::from(bytes) * 1_000_000_000 * 1_000 + gib_denominator / 2) / gib_denominator;
389    Some((
390        crate::report_format::human_count_u128(files_per_second),
391        format!("{}.{:03}", gib_thousandths / 1_000, gib_thousandths % 1_000),
392    ))
393}
394
395/// Validate a request and derive the least-retention plan for its delivery and route.
396///
397/// A summary reducer is legal when no content analysis is requested, the sole requested
398/// view is an unfiltered summary, the scan retains its whole population, and the policy
399/// does not require the snapshot to participate.  [`crate::open`] and live sessions still
400/// promise an index and therefore always plan full retention. Any future requirement the
401/// compact tier cannot prove falls closed to `RetainedState::FullIndex`.
402///
403/// Control observation is the caller's decision, not this planner's: a report's rows carry
404/// the ignored share of every size they show (fdu-elnn), so a scan that reads `.gitignore`
405/// displays what it paid for, and one that turned it off shows no share rather than a zero.
406/// The summary reducer classifies each entry against the control table the index would
407/// hold and folds the ignored share without retaining the entry (fdu-1ovb), so the default
408/// `fdu --view summary` takes it as well. A narrowed population (`--ignored=exclude|only`)
409/// is also a selection by ignored state, which `is_unfiltered` sends to the index; the
410/// planner checks the scope's population as well rather than rest on validation pairing
411/// the two.
412///
413/// The compact tier is not gated on the cache being unavailable, because for an
414/// unfiltered metadata summary the snapshot cannot save the work the scan is doing.
415/// Revalidating a loaded snapshot stats every entry anyway, so the reusable index and its
416/// write are additive cost with nothing to amortise them: measured on Linux/ext4 over
417/// 84,539 entries, the compact tier answered in 71 ms against 161 ms for a warm
418/// revalidating `Auto` run, and even a no-scan stale answer cost 81 ms because
419/// deserialisation is about as expensive per record as a warm walk.  A snapshot earns its
420/// keep when it avoids expensive work — re-reading file bodies for content analysis, or a
421/// cold filesystem walk — not when it merely mirrors a walk that still has to happen.
422///
423/// Two deliveries still require the index, for reasons that are about intent rather than
424/// cost.  [`Delivery::stale_ok`] must answer from the snapshot without touching the tree,
425/// so it has no scan to reduce.  [`CachePolicy::On`] is an explicit request to leave a
426/// current snapshot, and honouring it means materialising the index that gets written —
427/// though with no cache path configured there is nothing to write, and the compact tier
428/// answers it like any other summary.
429///
430/// A tree takes a folded index under the same route, policy, and analysis conditions,
431/// when `TreeRetention::for_request` finds the tree projection is its only reader and
432/// its share threshold bounds the rows it can show. Every directory is kept, since the
433/// tree shows directories by their subtree totals; only files too small to be shown are
434/// folded into their directory. The default `fdu PATH` is such a tree: at its 1% share
435/// the index keeps at most a hundred files.
436///
437/// Persistence follows the same reasoning as the read.  Under [`CachePolicy::Auto`] a
438/// one-shot metadata report writes nothing, because no later one-shot report reads what
439/// it would store: on a million-entry Linux tree the write was 0.26 s of a 1.51 s default
440/// run.  Content analysis writes, because its sidecar spares re-reading unchanged files
441/// and is paired with the snapshot beside it; sessions, watches, and refreshes write,
442/// because they are the later reader.
443pub fn plan(
444    request: &Request,
445    delivery: &Delivery,
446    route: Route,
447) -> std::result::Result<Plan, crate::query::RequestError> {
448    request.validate()?;
449    let mut normalized = delivery.clone();
450    if route == Route::Watch {
451        normalized.watch.get_or_insert_with(crate::query::WatchDelivery::default);
452    }
453    let delivery = &normalized;
454    request.validate_delivery(delivery)?;
455    if route == Route::Opened {
456        if delivery.cache != CachePolicy::Off
457            || delivery.watch.is_some()
458            || request.basis.content.is_enabled()
459        {
460            return Err(crate::query::RequestError::DeliveryUnsupported {
461                route: "opened",
462                reason: "progressive discovery requires cache off, no content analyzers, and observation configured through OpenOptions",
463            });
464        }
465        // An opened root runs one breadth-first producer and publishes coverage as state
466        // rather than as one answer, so these fields have no effect there. Refused rather
467        // than dropped: a delivery the route accepts is one it executes.
468        if delivery.workers.scan.is_some()
469            || delivery.order != crate::ScanOrder::default()
470            || delivery.accept_partial
471        {
472            return Err(crate::query::RequestError::DeliveryUnsupported {
473                route: "opened",
474                reason: "progressive discovery schedules one breadth-first producer and reports coverage as state, so it takes no scan worker count, traversal order, or partial-answer acceptance",
475            });
476        }
477    }
478    if route == Route::Refresh && delivery.stale_ok {
479        return Err(crate::query::RequestError::DeliveryUnsupported {
480            route: "refresh",
481            reason: "a stale answer cannot verify filesystem state",
482        });
483    }
484    let analysis_requested = request.basis.content.is_enabled();
485    let summary_is_sufficient = request.query.views.as_slice() == [ViewSpec::Summary]
486        && request.query.selection.is_unfiltered()
487        && request.basis.scope.population == crate::query::IgnoredEntries::Include;
488    let policy_requires_index =
489        delivery.stale_ok || (delivery.cache == CachePolicy::On && delivery.cache_path.is_some());
490    // What a later request can reuse decides both directions. A one-shot metadata query
491    // cannot amortize loading and reconciling a snapshot: both paths stat every entry. On
492    // macOS/APFS (494,031 entries), warm revalidation cost 4.8 s versus 3.6 s cold. Nor
493    // does a later one-shot metadata query read what it would write. Content avoids body
494    // reads, and retained routes are their own later reader.
495    let stored_state_pays = route != Route::OneShot || analysis_requested;
496    let read_snapshot = delivery.stale_ok
497        || match delivery.cache {
498            CachePolicy::Off => false,
499            CachePolicy::Auto | CachePolicy::On => stored_state_pays,
500        };
501    let persist = !delivery.stale_ok
502        && match delivery.cache {
503            CachePolicy::Off => false,
504            CachePolicy::Auto => stored_state_pays,
505            CachePolicy::On => true,
506        };
507    let transient = route == Route::OneShot && !policy_requires_index && !analysis_requested;
508    let retained = if !transient {
509        RetainedState::FullIndex
510    } else if summary_is_sufficient {
511        RetainedState::Summary
512    } else if let Some(retention) = TreeRetention::for_request(request) {
513        RetainedState::Tree(retention)
514    } else {
515        RetainedState::FullIndex
516    };
517    Ok(Plan {
518        basis: request.basis.clone(),
519        route,
520        retained,
521        load: if read_snapshot { Load::Snapshot } else { Load::None },
522        verify: if delivery.stale_ok { Verify::None } else { Verify::Filesystem },
523        persist,
524        delivery: delivery.clone(),
525    })
526}
527
528/// Execute a one-shot report, retaining the least state the request needs.
529///
530/// The returned report is complete as a value even while the optional save runs.  The
531/// caller must join the handle before exit; dropping it also joins defensively.
532///
533/// This is the contract the command line has always run under, and until now the only way
534/// to get it was to be the command line. `open` takes the session path: it retains an
535/// index and writes a snapshot, which is right for a caller asking many questions and
536/// wrong for one asking a single question -- an unfiltered summary is answered by a
537/// transient tier that retains no index, and writing a snapshot for it caches state the
538/// walk did not save. A Python caller therefore left
539/// cache state on a tree that the same command would not have, which a later cache-only
540/// read could see (fdu-4msv).
541///
542/// The report observes `.gitignore` control state as the request's
543/// [`ScanConfig::read_controls`](crate::ScanConfig) says, on by default as for
544/// [`crate::open`], so the two share one snapshot scope. Observing,
545/// every tree, summary, extension, and file row carries its ignored share, and
546/// [`Report::ignore_rules`](crate::query::Report::ignore_rules) names any file a control
547/// limit refused, by the budget or by the line limit. Turned off, no `.gitignore` is
548/// read, every share is `None`, and a selection by ignored state is refused with
549/// [`Error::InvalidRequest`] before anything is scanned. Such a report reads a default
550/// snapshot under every reading policy by constructing a requested-scope index from its
551/// all-entry facts and discarding its classification.
552///
553/// The caller owns the returned [`PendingSave`] and decides when to join it, exactly as
554/// the command line does, so a renderer can run while the snapshot is still being written.
555pub fn prepare_report(
556    request: &Request,
557    delivery: &Delivery,
558) -> Result<(Report, PendingSave, PerformanceSummary)> {
559    prepare_report_internal(request, delivery, false, None)
560        .map(|(report, pending, performance, _diagnostics)| (report, pending, performance))
561}
562
563/// Execute a one-shot report, reporting its progress through `progress` as it runs.
564///
565/// The same contract and the same answer as [`prepare_report`]: the handle observes the
566/// run and changes nothing about it, so a report prepared with one is byte-for-byte the
567/// report prepared without. The caller polls [`Progress::snapshot`] from another thread
568/// while this blocks, typically to draw a wait indicator. Which phases the run passes
569/// through, what the counters mean, and what holds when this returns are documented on
570/// [`Progress`]; in short, the walk counters equal the returned
571/// [`PerformanceSummary`]'s walked totals, and a run that requested content analysis
572/// leaves `analysis` at `(fresh_files, fresh_files)`.
573///
574/// A run over a full index ends in [`ProgressPhase::Summarizing`](crate::ProgressPhase)
575/// while it builds the answer; a save it started continues in the background, and the
576/// caller decides when to join it, as with [`prepare_report`]. `Saving` is therefore
577/// shown only for the moment between the save's start and the answer's, however long
578/// the write takes.
579pub fn prepare_report_with_progress(
580    request: &Request,
581    delivery: &Delivery,
582    progress: &Progress,
583) -> Result<(Report, PendingSave, PerformanceSummary)> {
584    prepare_report_internal(request, delivery, false, Some(progress))
585        .map(|(report, pending, performance, _diagnostics)| (report, pending, performance))
586}
587
588/// Execute a one-shot report and retain scan diagnostics.
589///
590/// Public because the command line needs it and the command line is an ordinary consumer:
591/// it drives repository-controlled measurement of the installed binary. Kept separate
592/// from [`prepare_report`] so callers who do not want traces pay for neither collection
593/// nor serialization. The diagnostic value is present only when the report performs a
594/// cold scan; cache-only opens do not scan, and warm reconciliation has a different
595/// execution contract.
596pub fn prepare_report_with_scan_diagnostics(
597    request: &Request,
598    delivery: &Delivery,
599) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)> {
600    prepare_report_internal(request, delivery, true, None)
601}
602
603fn prepare_report_internal(
604    request: &Request,
605    delivery: &Delivery,
606    collect_scan_diagnostics: bool,
607    progress: Option<&Progress>,
608) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)> {
609    // Before anything is scanned, loaded, or reduced: a request its own basis cannot answer
610    // has no answer at any cost, and the compact summary tier below never reaches a reader,
611    // so a check made there would not cover this route at all. A scope this build cannot
612    // honour is part of that one check rather than a second one beside it, which is what
613    // keeps the refusal independent of the delivery: the cache-only tier never scans and
614    // the cold tier never loads, so a rule stated at either would hold for one of them.
615    request.validate().map_err(Error::InvalidRequest)?;
616    let scan_config = crate::ScanConfig {
617        progress: progress.cloned(),
618        ..request.basis.scope.scan_config(delivery)
619    };
620    let root = request.basis.root.as_path();
621    let scan_started_at = SystemTime::now();
622    let plan = plan(request, delivery, Route::OneShot).map_err(Error::InvalidRequest)?;
623    match plan.retained {
624        RetainedState::Summary => {
625            let root = root.canonicalize().map_err(|error| Error::io(root, error))?;
626            let mut fold = SummaryFold::new(&scan_config);
627            let mut reduce = |observed: &crate::ObservationOp| fold.observe(observed);
628            let (mut scan, scan_diagnostics) = if collect_scan_diagnostics {
629                let (scan, diagnostics) = crate::scan::scan_summary_fold_with_diagnostics(
630                    &root,
631                    &scan_config,
632                    &mut reduce,
633                )?;
634                (scan, Some(diagnostics))
635            } else {
636                (crate::scan::scan_summary_fold(&root, &scan_config, &mut reduce)?, None)
637            };
638            let complete = scan.is_complete();
639            let generated_at = SystemTime::now();
640            let (summary, ignore_rules, ignored_unverified) = fold.finish(&root, &scan.errors)?;
641            let report = report_summary(
642                &root,
643                scan_config.scope(),
644                request,
645                summary,
646                ignore_rules,
647                ignored_unverified,
648                TreeStatus::of_walk(&root, &mut scan),
649                ReportProvenance::of_walk(scan_started_at, generated_at, complete),
650            );
651            let performance = PerformanceSummary {
652                walked_files: scan.files_walked,
653                walked_bytes: scan.bytes_walked,
654                walked_allocated: scan.allocated_walked,
655                source: ReportSource::ColdScan,
656                ..PerformanceSummary::default()
657            };
658            Ok((report, PendingSave::none(), performance, scan_diagnostics))
659        }
660        RetainedState::Tree(retention) => {
661            // The cold walk `execute` would run for this plan, which neither loads nor
662            // writes a snapshot and analyzes nothing, into a folded index. This arm is the
663            // only one that builds such an index, and it never lets it go: it reports from
664            // it and frees it here.
665            let root = root.canonicalize().map_err(|error| Error::io(root, error))?;
666            let (index, scan, scan_diagnostics) = crate::scan::scan_into_folded_index(
667                &root,
668                &scan_config,
669                retention,
670                collect_scan_diagnostics,
671            )?;
672            let performance = PerformanceSummary {
673                walked_files: scan.files_walked,
674                walked_bytes: scan.bytes_walked,
675                walked_allocated: scan.allocated_walked,
676                source: ReportSource::ColdScan,
677                ..PerformanceSummary::default()
678            };
679            if let Some(progress) = progress {
680                progress.enter(crate::ProgressPhase::Summarizing);
681            }
682            let answer = report(&index, request, SystemTime::now())?;
683            debug_assert_eq!(answer.scope, scan_config.scope());
684            crate::release_index(std::sync::Arc::new(index));
685            Ok((answer, PendingSave::none(), performance, scan_diagnostics))
686        }
687        RetainedState::FullIndex => {
688            let (index, open_report, pending_save, scan_diagnostics) =
689                execute(&plan, &request.basis, collect_scan_diagnostics, progress)?;
690            let performance = PerformanceSummary::from_open_report(&open_report);
691            if let Some(progress) = progress {
692                progress.enter(crate::ProgressPhase::Summarizing);
693            }
694            let answer = report(&index, request, SystemTime::now())?;
695            debug_assert_eq!(answer.scope, scan_config.scope());
696            // The answer is complete and owns no part of the index, so freeing it is no
697            // longer the caller's wait.
698            crate::release_index(index);
699            Ok((answer, pending_save, performance, scan_diagnostics))
700        }
701    }
702}
703
704/// The transient summary tier's reducer: the root roll-up the index would report, folded
705/// from the walk's observations as they arrive, with nothing retained per entry.
706///
707/// Observing `.gitignore`, it also classifies every entry as the detached index builder
708/// does, and tallies the entries no rule ignores beside every entry, the index's
709/// `unignored` and `all` partitions; the share it reports is
710/// [`IgnoredTally::between`](crate::query::IgnoredTally) the two, the index's own formula.
711///
712/// **Why each classification is the index's.** The builder classifies a child from two
713/// inputs: its parent's classification, since nothing below an ignored directory can be
714/// re-included, and the admitted control files of the directories above it, deepest
715/// first. The fold has both when each entry arrives:
716///
717/// - A worker publishes a listing before any directory in it becomes claimable, so an
718///   entry arrives after its parent's own observation, as a listing reaches the builder
719///   after its parent's.
720/// - A classifying walk sends each directory's control ahead of its entries
721///   (`SinkMode::groups_directories` in the scanner), so it is applied before any entry
722///   in the directory is classified, as the builder applies a listing's control before
723///   its children. A listing that fills a batch before its `.gitignore` is listed has
724///   that file read directly, only under the exact name a listing accepts, and the read
725///   stands for the listing; on a tree nothing modifies during the walk it is the same
726///   file with the same bytes.
727/// - One consumer applies every control in arrival order, as the builder's one consumer
728///   does. Which files a budget refuses when several compete for it depends on that
729///   order on both routes; with one worker the order is the same on both, and with
730///   several it is an order the index could also have met. Whether any file is refused
731///   does not depend on it, because a cold walk only adds charges.
732///
733/// **Why it keeps no entries.** A parent's classification is final once its own
734/// observation is folded, since every ancestor's was folded first. So the fold keeps only
735/// the ignored directories whose parent is not ignored, which head every ignored subtree,
736/// and a parent is ignored exactly when one of them is its ancestor or itself.
737///
738/// **What it withholds.** The share, where the index withholds the root's: when a control
739/// file was refused, because it may have held negations as well as ignore rules, or could
740/// not be read. The coverage it reports is the table's, refusals included.
741///
742/// **What it refuses.** A tree whose apparent or allocated bytes no `u64` can hold, as the
743/// index refuses it on every route ([`crate::index::add_file_sizes`]): the first file that
744/// would carry `all` past `u64::MAX` is not counted, and the report fails with
745/// [`Error::UnrepresentableTotal`]. The unignored tally is a part of `all`, so it cannot
746/// overflow while `all` does not.
747struct SummaryFold {
748    /// Every entry the walk retained.
749    all: SummaryRow,
750    /// The classifier, when the scan observes `.gitignore`.
751    controls: Option<SummaryControls>,
752    /// The first file the root total could not hold, which fails the report.
753    unrepresentable: Option<Error>,
754}
755
756/// What a classifying [`SummaryFold`] keeps: the control table, the heads of ignored
757/// subtrees, and the tallies of what no rule ignores.
758struct SummaryControls {
759    table: crate::control::ControlTable,
760    /// Ignored directories whose parent is not ignored.
761    ///
762    /// Unbounded by design: no head lies below another, so the set holds one path per
763    /// separately ignored subtree, which is what classifying an entry needs. It grows with
764    /// such subtrees, not with the entries in them: a tree of many small ignored
765    /// directories, each under a directory that is not ignored, is the case that makes it
766    /// large. The RSS evidence so far (exp-170, exp-171) is on trees with few heads.
767    ///
768    /// Keyed by the path's bytes: an `OsString` hashes and compares as bytes, where a
769    /// `PathBuf` does both component by component, which cost this fold 28M instructions
770    /// of hashing on `linux-v6.12` (H188).
771    ignored_heads: std::collections::HashSet<std::ffi::OsString>,
772    /// The parent last looked up, and whether it is ignored. A listing's entries mostly
773    /// arrive together, so this answers nearly all of them. Compared as bytes (H188).
774    parent: Option<(std::ffi::OsString, bool)>,
775    /// The controls governing the parent last classified under, resolved once for its
776    /// listing (H163) with the parent's components split once for it too (H171), and
777    /// dropped whenever the table changes. Compared as bytes (H188).
778    chain:
779        Option<(std::ffi::OsString, crate::control::ControlChain, crate::control::SplitDirectory)>,
780    /// Every entry no rule ignores, as the index's `unignored` partition.
781    unignored: crate::index::RollUpScalars,
782    /// The first control observation the table rejected, which fails the report as it
783    /// fails the index build.
784    rejected: Option<Error>,
785}
786
787impl SummaryFold {
788    fn new(config: &crate::ScanConfig) -> Self {
789        Self {
790            all: SummaryRow::default(),
791            controls: config.read_controls.then(|| SummaryControls {
792                table: crate::control::ControlTable::with_limits(config.control_limits),
793                ignored_heads: std::collections::HashSet::new(),
794                parent: None,
795                chain: None,
796                unignored: crate::index::RollUpScalars::default(),
797                rejected: None,
798            }),
799            unrepresentable: None,
800        }
801    }
802
803    fn observe(&mut self, observed: &crate::ObservationOp) {
804        match &observed.op {
805            crate::Op::Upsert { path, kind, attrs } => {
806                match kind {
807                    EntryKind::File => {
808                        if let Err(error) = crate::index::add_file_sizes(
809                            &mut self.all.bytes,
810                            &mut self.all.allocated,
811                            attrs,
812                            || path.clone(),
813                        ) {
814                            self.unrepresentable.get_or_insert(error);
815                            return;
816                        }
817                        self.all.files += 1;
818                        self.all.newest_mtime_ns = Some(
819                            self.all
820                                .newest_mtime_ns
821                                .map_or(attrs.mtime_ns, |current| current.max(attrs.mtime_ns)),
822                        );
823                    }
824                    EntryKind::Dir => self.all.dirs += 1,
825                    EntryKind::Symlink | EntryKind::Other => {}
826                }
827                let Some(controls) = &mut self.controls else { return };
828                if controls.classify(path, *kind) {
829                    return;
830                }
831                // The index's contribution of an unignored entry: a file's sizes, one
832                // directory, nothing for any other kind.
833                match kind {
834                    EntryKind::File => {
835                        controls.unignored.files += 1;
836                        controls.unignored.bytes += attrs.size;
837                        controls.unignored.allocated += attrs.allocated;
838                    }
839                    EntryKind::Dir => controls.unignored.dirs += 1,
840                    EntryKind::Symlink | EntryKind::Other => {}
841                }
842            }
843            crate::Op::ControlUpsert { path, source } => {
844                if let Some(controls) = &mut self.controls {
845                    controls.chain = None;
846                    let admitted = controls.table.upsert(path, source.clone()).map(drop);
847                    controls.record(admitted);
848                }
849            }
850            crate::Op::ControlRemove { path } => {
851                if let Some(controls) = &mut self.controls {
852                    controls.chain = None;
853                    let removed = controls.table.remove(path).map(drop);
854                    controls.record(removed);
855                }
856            }
857            // A cold walk observes what is there; it neither removes nor invalidates.
858            crate::Op::Remove { .. } | crate::Op::InvalidateSubtree { .. } => {}
859        }
860    }
861
862    /// The summary row, the control coverage, and whether the row withholds its ignored
863    /// share because a governing rule could not be verified.
864    ///
865    /// `errors` are the walk's, normalized, as the index records them.
866    fn finish(
867        self,
868        root: &std::path::Path,
869        errors: &[Error],
870    ) -> Result<(SummaryRow, crate::control::ControlCoverage, bool)> {
871        if let Some(error) = self.unrepresentable {
872            return Err(error);
873        }
874        let Some(controls) = self.controls else {
875            return Ok((
876                SummaryRow { ignored: None, ..self.all },
877                crate::control::ControlCoverage::NotObserved,
878                false,
879            ));
880        };
881        if let Some(error) = controls.rejected {
882            return Err(error);
883        }
884        let unreadable =
885            errors.iter().any(|error| crate::control::unreadable_control(root, error).is_some());
886        let verified = controls.table.refused_len() == 0 && !unreadable;
887        let all = crate::index::RollUpScalars {
888            files: self.all.files,
889            dirs: self.all.dirs,
890            bytes: self.all.bytes,
891            allocated: self.all.allocated,
892            newest_mtime_ns: self.all.newest_mtime_ns.unwrap_or_default(),
893        };
894        let summary = SummaryRow {
895            ignored: verified.then(|| crate::query::IgnoredTally::between(all, controls.unignored)),
896            ..self.all
897        };
898        Ok((
899            summary,
900            crate::control::ControlCoverage::Observed(controls.table.observation()),
901            !verified,
902        ))
903    }
904}
905
906impl SummaryControls {
907    /// Whether the entry at `path` is ignored, decided as
908    /// `DetachedIndexBuilder::push_directory` decides it: an entry below an ignored
909    /// directory is ignored, one in a table with no rules is not, and otherwise the
910    /// deepest control with an opinion decides.
911    fn classify(&mut self, path: &std::path::Path, kind: EntryKind) -> bool {
912        // No rule and no ignored subtree yet, as in every tree without a `.gitignore`:
913        // nothing is ignored, and the parent need not even be derived.
914        if self.ignored_heads.is_empty() && self.table.is_empty() {
915            return false;
916        }
917        // The parent and the name are the bytes before and after the last separator
918        // (H188): `Path::parent` and `Path::file_name` each parse the components, and the
919        // fold asked for both on every entry, then compared the parent with the cached one
920        // component by component. On `linux-v6.12` that was 180M of the fold's 373M
921        // instructions; the bytes say the same thing for the normalized relative paths a
922        // walk emits.
923        let (parent, name) = crate::control::split_parent(path);
924        let parent_ignored = self.parent_ignored(parent);
925        let ignored = if parent_ignored || self.table.is_empty() {
926            parent_ignored
927        } else if !name.is_empty() {
928            if !matches!(&self.chain, Some((cached, ..)) if cached.as_os_str() == parent) {
929                let directory = std::path::Path::new(parent);
930                self.chain = Some((
931                    parent.to_os_string(),
932                    self.table.chain_for(directory),
933                    crate::control::SplitDirectory::new(directory),
934                ));
935            }
936            let (_, chain, split) = self.chain.as_ref().expect("the chain was just resolved");
937            !chain.is_empty()
938                && split.with_components(|directory| {
939                    chain.is_ignored_within(directory, name.as_encoded_bytes(), kind.is_dir())
940                })
941        } else {
942            self.table.matcher_for(path).is_ignored(kind.is_dir())
943        };
944        if ignored && !parent_ignored && kind.is_dir() {
945            self.ignored_heads.insert(path.as_os_str().to_os_string());
946        }
947        ignored
948    }
949
950    fn parent_ignored(&mut self, parent: &std::ffi::OsStr) -> bool {
951        if self.ignored_heads.is_empty() {
952            return false;
953        }
954        if let Some((cached, ignored)) = &self.parent {
955            if cached.as_os_str() == parent {
956                return *ignored;
957            }
958        }
959        let ignored =
960            crate::control::ancestors(parent).any(|ancestor| self.ignored_heads.contains(ancestor));
961        self.parent = Some((parent.to_os_string(), ignored));
962        ignored
963    }
964
965    fn record(&mut self, applied: Result<()>) {
966        if let Err(error) = applied {
967            self.rejected.get_or_insert(error);
968        }
969    }
970}
971
972#[cfg(test)]
973mod tests {
974    use std::fs;
975    use std::path::{Path, PathBuf};
976
977    use super::*;
978    use crate::query::{IgnoredEntries, Pattern, Query, Section};
979    use crate::{OpenFixture, ScanConfig};
980
981    /// H188: the byte-wise parent, name, and ancestors of a walked path are the ones
982    /// `Path` parses, for every shape a normalized relative path takes, including the
983    /// root itself, names with dots, and bytes that are not UTF-8.
984    #[test]
985    fn byte_wise_path_splits_match_the_parsed_ones() {
986        use std::ffi::{OsStr, OsString};
987        let named = ["", "a", "a/b", "a/b/c.txt", ".gitignore", "d.d/.h", "x/..y"]
988            .into_iter()
989            .map(OsString::from);
990        #[cfg(unix)]
991        let paths: Vec<OsString> = {
992            use std::os::unix::ffi::OsStringExt as _;
993            named
994                .chain([
995                    OsString::from_vec(b"d\xff/f\xfe".to_vec()),
996                    OsString::from_vec(b"\xfe".to_vec()),
997                ])
998                .collect()
999        };
1000        #[cfg(not(unix))]
1001        let paths: Vec<OsString> = named.collect();
1002        for path in &paths {
1003            let path = Path::new(path);
1004            let (parent, name) = crate::control::split_parent(path);
1005            assert_eq!(parent, path.parent().map_or(OsStr::new(""), Path::as_os_str), "{path:?}");
1006            assert_eq!(name, path.file_name().unwrap_or(OsStr::new("")), "{path:?}");
1007            let by_bytes: Vec<&OsStr> = crate::control::ancestors(parent).collect();
1008            let parsed: Vec<&OsStr> = Path::new(parent).ancestors().map(Path::as_os_str).collect();
1009            assert_eq!(by_bytes, parsed, "{path:?}");
1010        }
1011    }
1012
1013    #[test]
1014    fn total_throughput_uses_one_elapsed_sample_and_selected_size() {
1015        use crate::query::SizeMetric;
1016        let work = PerformanceSummary {
1017            walked_files: 200,
1018            walked_bytes: 4_000_000_000,
1019            walked_allocated: 1_000_000_000,
1020            ..PerformanceSummary::default()
1021        };
1022        assert_eq!(
1023            work.total_throughput(std::time::Duration::from_secs(2), SizeMetric::Apparent),
1024            "100 files/s (1.863 GiB/s)"
1025        );
1026        assert_eq!(
1027            work.total_throughput(std::time::Duration::from_secs(2), SizeMetric::Allocated),
1028            "100 files/s (0.466 GiB/s)"
1029        );
1030        assert_eq!(
1031            work.total_throughput(std::time::Duration::ZERO, SizeMetric::Apparent),
1032            "throughput unavailable"
1033        );
1034        assert_eq!(
1035            PerformanceSummary::default()
1036                .total_throughput(std::time::Duration::from_secs(1), SizeMetric::Apparent),
1037            "0 files/s (0.000 GiB/s)"
1038        );
1039        assert_eq!(
1040            throughput_rates(12_345, 3 * (1_u64 << 30), std::time::Duration::from_secs(2)),
1041            Some(("6,172".to_owned(), "1.500".to_owned()))
1042        );
1043    }
1044
1045    /// Every route that counts a tree refuses a total no `u64` can hold with the same
1046    /// error, and accepts one that fits exactly (fdu-sqyk).
1047    ///
1048    /// The one-shot routes are the summary fold and the detached builder, full or folded,
1049    /// which the default `fdu PATH` takes; a narrowed population, `open`, and a watch
1050    /// commit through the apply lane. Three sparse files of `u64::MAX / 2 + 1` apparent
1051    /// bytes, which tmpfs lets anyone create, are what each is given here, in the shape
1052    /// each takes them: as observations, as listings, and as a batch. Each file sits in a
1053    /// directory of its own as well as all in one, so no directory overflows where the
1054    /// root does, and the allocated total is checked on its own.
1055    #[test]
1056    fn every_route_refuses_a_total_no_u64_can_hold_and_accepts_an_exact_fit() {
1057        use crate::index::DetachedIndexBuilder;
1058        use crate::scan::{DetachedChild, DetachedDirectory};
1059        use crate::{Attrs, Index, Observation, ObservationOp, Op};
1060
1061        #[derive(Clone, Copy, Debug)]
1062        enum Layout {
1063            OneDirectory,
1064            DirectoryEach,
1065        }
1066        let attrs = |size: u64, allocated: u64, inode: u64| Attrs {
1067            size,
1068            allocated,
1069            mtime_ns: 1,
1070            ctime_ns: 1,
1071            inode,
1072            dev: 1,
1073        };
1074        // Each file's path, relative to the root, and the directories above it.
1075        let files = |layout: Layout, count: usize| -> Vec<(PathBuf, Option<PathBuf>)> {
1076            (0..count)
1077                .map(|file| match layout {
1078                    Layout::OneDirectory => (PathBuf::from(format!("f{file}")), None),
1079                    Layout::DirectoryEach => {
1080                        let directory = PathBuf::from(format!("d{file}"));
1081                        (directory.join("f"), Some(directory))
1082                    }
1083                })
1084                .collect()
1085        };
1086        let upserts = |layout: Layout, sizes: &[(u64, u64)]| -> Vec<Op> {
1087            let mut ops = Vec::new();
1088            for ((path, directory), (inode, &(size, allocated))) in
1089                files(layout, sizes.len()).into_iter().zip((1_u64..).zip(sizes))
1090            {
1091                if let Some(directory) = directory {
1092                    ops.push(Op::Upsert {
1093                        path: directory,
1094                        kind: EntryKind::Dir,
1095                        attrs: Attrs::default(),
1096                    });
1097                }
1098                ops.push(Op::Upsert {
1099                    path,
1100                    kind: EntryKind::File,
1101                    attrs: attrs(size, allocated, inode),
1102                });
1103            }
1104            ops
1105        };
1106        let summary_fold = |layout: Layout, sizes: &[(u64, u64)], read_controls: bool| {
1107            let config = ScanConfig { read_controls, ..ScanConfig::default() };
1108            let mut fold = SummaryFold::new(&config);
1109            for op in upserts(layout, sizes) {
1110                fold.observe(&ObservationOp::unconditional(op));
1111            }
1112            fold.finish(Path::new("/root"), &[]).map(|(row, ..)| (row.bytes, row.allocated))
1113        };
1114        let builder = |layout: Layout, sizes: &[(u64, u64)], folding: bool| -> Result<(u64, u64)> {
1115            let mut builder = DetachedIndexBuilder::new(
1116                "/root",
1117                crate::ScanScope::default(),
1118                crate::classify::TypeRegistry::compiled_shared(),
1119            );
1120            if folding {
1121                builder =
1122                    builder.folding(TreeRetention { largest_files: 1, size: SizeMetric::Apparent });
1123            }
1124            let child = |name: &Path, kind, attrs, position| DetachedChild {
1125                name: name.as_os_str().to_owned(),
1126                kind,
1127                attrs,
1128                position,
1129            };
1130            let mut root =
1131                DetachedDirectory { path: PathBuf::new(), children: Vec::new(), control: None };
1132            let mut nested = Vec::new();
1133            for (position, ((path, directory), (inode, &(size, allocated)))) in
1134                (0_u32..).zip(files(layout, sizes.len()).into_iter().zip((1_u64..).zip(sizes)))
1135            {
1136                let file = attrs(size, allocated, inode);
1137                match directory {
1138                    None => root.children.push(child(&path, EntryKind::File, file, position)),
1139                    Some(directory) => {
1140                        root.children.push(child(
1141                            &directory,
1142                            EntryKind::Dir,
1143                            Attrs::default(),
1144                            position,
1145                        ));
1146                        let name = path.file_name().expect("a file name");
1147                        nested.push(DetachedDirectory {
1148                            path: directory,
1149                            children: vec![child(Path::new(name), EntryKind::File, file, 0)],
1150                            control: None,
1151                        });
1152                    }
1153                }
1154            }
1155            builder.push_directory(&mut root)?;
1156            for listing in &mut nested {
1157                builder.push_directory(listing)?;
1158            }
1159            let total = builder.finish().total();
1160            Ok((total.bytes, total.allocated))
1161        };
1162        let apply_lane = |layout: Layout, sizes: &[(u64, u64)]| -> Result<(u64, u64)> {
1163            let mut index = Index::new("/root");
1164            index.apply(&Observation::new(upserts(layout, sizes)))?;
1165            let total = index.total();
1166            Ok((total.bytes, total.allocated))
1167        };
1168        let routes =
1169            |layout: Layout, sizes: &[(u64, u64)]| -> Vec<(&'static str, Result<(u64, u64)>)> {
1170                vec![
1171                    ("the summary fold", summary_fold(layout, sizes, false)),
1172                    ("the classifying summary fold", summary_fold(layout, sizes, true)),
1173                    ("the detached builder", builder(layout, sizes, false)),
1174                    ("the folding builder", builder(layout, sizes, true)),
1175                    ("the apply lane", apply_lane(layout, sizes)),
1176                ]
1177            };
1178
1179        let half = u64::MAX / 2 + 1;
1180        for layout in [Layout::OneDirectory, Layout::DirectoryEach] {
1181            for (sizes, counter) in [
1182                (&[(half, 0), (half, 0), (half, 0)][..], "bytes"),
1183                (&[(0, half), (0, half)][..], "allocated bytes"),
1184            ] {
1185                for (route, outcome) in routes(layout, sizes) {
1186                    match outcome {
1187                        Err(Error::UnrepresentableTotal { counter: refused, .. }) => {
1188                            assert_eq!(refused, counter, "{route}, {layout:?}");
1189                        }
1190                        other => panic!(
1191                            "{route}, {layout:?}: expected the {counter} total refused, got {other:?}"
1192                        ),
1193                    }
1194                }
1195            }
1196            let exact = [(half, half), (half - 1, half - 1)];
1197            for (route, outcome) in routes(layout, &exact) {
1198                assert_eq!(
1199                    outcome.expect("an exact fit is representable"),
1200                    (u64::MAX, u64::MAX),
1201                    "{route}, {layout:?}"
1202                );
1203            }
1204        }
1205    }
1206
1207    #[test]
1208    #[cfg(unix)]
1209    fn an_unreadable_stored_header_never_authorizes_live_replacement() {
1210        use std::os::unix::fs::PermissionsExt;
1211        if !crate::test_support::require_permission_bits() {
1212            return;
1213        }
1214        let root = tempfile::tempdir().expect("root");
1215        let cache = tempfile::tempdir().expect("cache");
1216        let snapshot = cache.path().join("snapshot.fdu");
1217        fs::write(root.path().join(".gitignore"), b"ignored\n").expect("control");
1218        let observed = crate::query::Basis {
1219            root: root.path().into(),
1220            scope: crate::query::Scope::default(),
1221            content: crate::content::AnalysisSet::NONE,
1222        };
1223        let delivery = Delivery::new(CachePolicy::Auto, Some(snapshot.clone()));
1224        crate::open(&observed, &delivery).expect("stronger snapshot");
1225        let original = fs::read(&snapshot).expect("original image");
1226        let basis = crate::query::Basis {
1227            scope: crate::query::Scope { read_controls: false, ..Default::default() },
1228            ..observed
1229        };
1230        let (mut index, _) =
1231            crate::open(&basis, &Delivery::new(CachePolicy::Off, None)).expect("fresh blind index");
1232        let request = Request::new(basis, Query::default(), SystemTime::now());
1233        let plan = plan(&request, &delivery, Route::Refresh).expect("plan");
1234        fs::set_permissions(&snapshot, fs::Permissions::from_mode(0o000))
1235            .expect("deny header read");
1236        let nonwriting = Delivery { cache: CachePolicy::Off, ..delivery.clone() };
1237        let nonwriting_plan =
1238            super::plan(&request, &nonwriting, Route::Refresh).expect("nonwriting plan");
1239        assert!(
1240            !crate::persist_index_changes(&index, &nonwriting_plan, true, true)
1241                .expect("nonwriting policy never reads the header")
1242        );
1243        fs::write(root.path().join("fresh.txt"), b"fresh").expect("mutation");
1244        crate::refresh(
1245            &mut index,
1246            &request.basis,
1247            &Delivery { cache: CachePolicy::Off, ..delivery.clone() },
1248        )
1249        .expect("off refresh does not inspect cache state");
1250        let result = crate::persist_index_changes(&index, &plan, true, true);
1251        fs::set_permissions(&snapshot, fs::Permissions::from_mode(0o600)).expect("restore");
1252        assert!(
1253            result.is_err(),
1254            "a writable parent must not let unknown identity authorize replacement"
1255        );
1256        assert_eq!(fs::read(&snapshot).expect("retained image"), original);
1257    }
1258
1259    #[test]
1260    fn refresh_rejects_another_root_before_mutating_or_persisting() {
1261        let a = tempfile::tempdir().expect("root a");
1262        let b = tempfile::tempdir().expect("root b");
1263        let cache = tempfile::tempdir().expect("cache");
1264        let basis = crate::query::Basis {
1265            root: a.path().into(),
1266            scope: crate::query::Scope::default(),
1267            content: crate::content::AnalysisSet::NONE,
1268        };
1269        let (mut index, _) =
1270            crate::open(&basis, &Delivery::new(CachePolicy::Off, None)).expect("open a");
1271        fs::write(a.path().join("new"), b"new facts").expect("mutation a");
1272        let before = index.clock();
1273        let snapshot = cache.path().join("snapshot.fdu");
1274        let wrong = crate::query::Basis { root: b.path().into(), ..basis.clone() };
1275        let delivery = Delivery::new(CachePolicy::Auto, Some(snapshot.clone()));
1276        let error = crate::refresh(&mut index, &wrong, &delivery).expect_err("different root");
1277        assert!(matches!(
1278            error,
1279            Error::InvalidRequest(crate::query::RequestError::RootMismatch { .. })
1280        ));
1281        assert_eq!(index.clock(), before);
1282        assert!(!snapshot.exists());
1283        let alias = crate::query::Basis { root: a.path().join("."), ..basis };
1284        crate::refresh(&mut index, &alias, &delivery).expect("same root spelling");
1285        assert_eq!(index.total().files, 1);
1286        #[cfg(unix)]
1287        {
1288            let link = cache.path().join("root-alias");
1289            std::os::unix::fs::symlink(a.path(), &link).expect("root alias");
1290            let symlink_basis = crate::query::Basis { root: link, ..alias };
1291            crate::refresh(&mut index, &symlink_basis, &delivery)
1292                .expect("same canonical root through symlink");
1293        }
1294    }
1295
1296    #[test]
1297    fn unchanged_refresh_replaces_an_incompatible_stored_baseline() {
1298        let root = tempfile::tempdir().expect("root");
1299        let other = tempfile::tempdir().expect("other root");
1300        let cache = tempfile::tempdir().expect("cache");
1301        fs::write(root.path().join("file"), b"retained").expect("file");
1302        let basis = crate::query::Basis {
1303            root: root.path().into(),
1304            scope: crate::query::Scope::default(),
1305            content: crate::content::AnalysisSet::NONE,
1306        };
1307        let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
1308        for wrong_root in [false, true] {
1309            let wrong = if wrong_root {
1310                crate::query::Basis { root: other.path().into(), ..basis.clone() }
1311            } else {
1312                crate::query::Basis {
1313                    scope: crate::query::Scope { max_depth: Some(0), ..basis.scope.clone() },
1314                    ..basis.clone()
1315                }
1316            };
1317            crate::open(&wrong, &Delivery { cache: CachePolicy::On, ..delivery.clone() })
1318                .expect("incompatible snapshot");
1319            let (mut index, _) = crate::open(&basis, &Delivery::new(CachePolicy::Off, None))
1320                .expect("retained index");
1321            let refreshed =
1322                crate::refresh(&mut index, &basis, &delivery).expect("refresh reseeds cache");
1323            assert!(!refreshed.apply.mutated(), "the existing index was already current");
1324            let (cached, _) = crate::open(&basis, &Delivery { stale_ok: true, ..delivery.clone() })
1325                .expect("cache-only can now answer");
1326            assert_eq!(cached.total().bytes, 8);
1327        }
1328    }
1329
1330    #[test]
1331    fn refreshed_metadata_and_content_are_visible_to_a_later_cache_only_open() {
1332        let root = tempfile::tempdir().expect("root");
1333        let cache = tempfile::tempdir().expect("cache");
1334        let path = root.path().join("note.txt");
1335        fs::write(&path, b"old\n").expect("old file");
1336        let basis = crate::query::Basis {
1337            root: root.path().into(),
1338            scope: crate::query::Scope::default(),
1339            content: crate::content::AnalysisSet::NONE.with_lines(),
1340        };
1341        let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
1342        let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
1343        fs::write(&path, b"new longer text\nsecond line\n").expect("mutation");
1344        let refreshed = crate::refresh(&mut index, &basis, &delivery).expect("refresh");
1345        assert!(refreshed.is_complete());
1346        let (cached, report) = crate::open(&basis, &Delivery { stale_ok: true, ..delivery })
1347            .expect("cache-only sees refreshed tiers");
1348        assert_eq!(report.path_taken, OpenPath::CacheOnly);
1349        assert_eq!(cached.total(), index.total());
1350        assert_eq!(
1351            cached.total().bytes,
1352            u64::try_from(b"new longer text\nsecond line\n".len()).expect("length")
1353        );
1354        assert_eq!(report.content_cache.hits, 1);
1355        let original = index
1356            .content()
1357            .expect("fresh content")
1358            .file(Path::new("note.txt"))
1359            .expect("fresh file");
1360        let restored = cached
1361            .content()
1362            .expect("restored content")
1363            .file(Path::new("note.txt"))
1364            .expect("restored file");
1365        assert_eq!(restored, original);
1366    }
1367
1368    /// One root with one file and one empty directory, and a writing delivery whose
1369    /// snapshot lives in its own directory so a test can make that directory read-only.
1370    #[cfg(unix)]
1371    fn owed_persistence_fixture()
1372    -> (tempfile::TempDir, tempfile::TempDir, crate::query::Basis, Delivery) {
1373        let root = tempfile::tempdir().expect("root");
1374        let cache = tempfile::tempdir().expect("cache");
1375        fs::create_dir(root.path().join("locked")).expect("locked dir");
1376        fs::write(root.path().join("first"), b"first").expect("first file");
1377        let basis = crate::query::Basis {
1378            root: root.path().into(),
1379            scope: crate::query::Scope::default(),
1380            content: crate::content::AnalysisSet::NONE,
1381        };
1382        let delivery = Delivery::new(CachePolicy::Auto, Some(cache.path().join("snapshot.fdu")));
1383        (root, cache, basis, delivery)
1384    }
1385
1386    /// The files a cache-only open of `delivery`'s snapshot answers with.
1387    #[cfg(unix)]
1388    fn cached_files(basis: &crate::query::Basis, delivery: &Delivery) -> u64 {
1389        let cache_only = Delivery { stale_ok: true, ..delivery.clone() };
1390        crate::open(basis, &cache_only).expect("cache-only open").0.total().files
1391    }
1392
1393    /// A refresh whose metadata write failed leaves the index holding facts the snapshot
1394    /// lacks; the next refresh must write them even though it changes nothing itself.
1395    ///
1396    /// The metadata write used to be keyed to the pass that ran it: a later pass that
1397    /// mutated nothing wrote nothing, so a snapshot that missed one write missed the
1398    /// facts for good, and cache-only reads answered older facts than the index held.
1399    #[test]
1400    #[cfg(unix)]
1401    fn an_unchanged_refresh_repeats_the_metadata_write_a_failed_refresh_owed() {
1402        use std::os::unix::fs::PermissionsExt;
1403        if !crate::test_support::require_permission_bits() {
1404            return;
1405        }
1406        let (root, cache, basis, delivery) = owed_persistence_fixture();
1407        let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
1408        assert_eq!(cached_files(&basis, &delivery), 1);
1409
1410        fs::write(root.path().join("second"), b"second").expect("second file");
1411        fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o555)).expect("deny write");
1412        let failed = crate::refresh(&mut index, &basis, &delivery);
1413        fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o755)).expect("restore");
1414        assert!(failed.is_err(), "a read-only cache directory fails the write");
1415        assert_eq!(index.total().files, 2, "the index advanced before the write");
1416        assert_eq!(cached_files(&basis, &delivery), 1, "the failed write left the old image");
1417
1418        let unchanged = crate::refresh(&mut index, &basis, &delivery).expect("unchanged refresh");
1419        assert!(unchanged.is_complete());
1420        assert!(!unchanged.apply.mutated(), "nothing changed between the passes");
1421        assert_eq!(cached_files(&basis, &delivery), 2, "the owed write ran");
1422
1423        // Paid once: the next unchanged pass has nothing to write.
1424        let snapshot = delivery.cache_path.as_deref().expect("path");
1425        let written = fs::metadata(snapshot).expect("snapshot").modified().expect("mtime");
1426        crate::refresh(&mut index, &basis, &delivery).expect("settled refresh");
1427        assert_eq!(fs::metadata(snapshot).expect("snapshot").modified().expect("mtime"), written);
1428    }
1429
1430    /// The same debt when the failed write is the one a warm `open` started: a caller
1431    /// keeping the index through [`crate::open_with_pending_save`] keeps the debt too.
1432    #[test]
1433    #[cfg(unix)]
1434    fn an_unchanged_refresh_repeats_the_metadata_write_a_failed_open_owed() {
1435        use std::os::unix::fs::PermissionsExt;
1436        if !crate::test_support::require_permission_bits() {
1437            return;
1438        }
1439        let (root, cache, basis, delivery) = owed_persistence_fixture();
1440        crate::open(&basis, &delivery).expect("complete open writes the snapshot");
1441        fs::write(root.path().join("second"), b"second").expect("second file");
1442        fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o555)).expect("deny write");
1443        let opened = crate::open_with_pending_save(&basis, &delivery);
1444        // Joined before the directory is writable again: the write runs in the background.
1445        let outcome = opened.map(|(index, report, pending)| (index, report, pending.join()));
1446        fs::set_permissions(cache.path(), fs::Permissions::from_mode(0o755)).expect("restore");
1447        let (index, report, joined) = outcome.expect("the open itself succeeds");
1448        assert_eq!(report.path_taken, OpenPath::WarmRevalidate);
1449        assert!(joined.is_err(), "the startup write failed");
1450        let mut index = std::sync::Arc::into_inner(index).expect("the writer released the index");
1451        assert_eq!(cached_files(&basis, &delivery), 1);
1452
1453        let unchanged = crate::refresh(&mut index, &basis, &delivery).expect("unchanged refresh");
1454        assert!(!unchanged.apply.mutated(), "nothing changed between the passes");
1455        assert_eq!(cached_files(&basis, &delivery), 2, "the owed write ran");
1456    }
1457
1458    /// A partial refresh cannot write the entry tier; once the tree is readable again a
1459    /// complete refresh delivers the partial pass's facts to the snapshot.
1460    ///
1461    /// Restoring the directory's permissions updates its change time, so on a POSIX host
1462    /// the recovering pass reports that directory as updated and would write on its own
1463    /// account. The failed-write tests above are the ones that prove the debt is carried;
1464    /// this one guards that a partial pass's verified facts reach the snapshot at all.
1465    #[test]
1466    #[cfg(unix)]
1467    fn a_complete_refresh_persists_the_facts_a_partial_refresh_could_not() {
1468        use std::os::unix::fs::PermissionsExt;
1469        if !crate::test_support::require_permission_bits() {
1470            return;
1471        }
1472        let (root, _cache, basis, delivery) = owed_persistence_fixture();
1473        let locked = root.path().join("locked");
1474        let (mut index, _) = crate::open(&basis, &delivery).expect("initial open");
1475        assert_eq!(index.total().files, 1);
1476
1477        fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("deny read");
1478        fs::write(root.path().join("second"), b"second").expect("second file");
1479        let partial = crate::refresh(&mut index, &basis, &delivery);
1480        fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).expect("restore");
1481        let partial = partial.expect("partial refresh");
1482        assert!(!partial.is_complete(), "the locked directory made the pass partial");
1483        assert!(partial.apply.mutated(), "the second file was inserted");
1484        assert_eq!(index.total().files, 2);
1485        assert_eq!(cached_files(&basis, &delivery), 1, "a partial pass never writes entries");
1486
1487        let complete = crate::refresh(&mut index, &basis, &delivery).expect("complete refresh");
1488        assert!(complete.is_complete(), "{:?}", complete.scan.errors);
1489        assert_eq!(cached_files(&basis, &delivery), 2, "the complete pass wrote the facts");
1490    }
1491
1492    #[test]
1493    fn cache_only_refusals_name_location_root_and_absence_separately() {
1494        let root = tempfile::tempdir().expect("root");
1495        let other = tempfile::tempdir().expect("other root");
1496        let cache = tempfile::tempdir().expect("cache");
1497        let basis = crate::query::Basis {
1498            root: root.path().into(),
1499            scope: crate::query::Scope::default(),
1500            content: crate::content::AnalysisSet::NONE,
1501        };
1502        let snapshot = cache.path().join("snapshot.fdu");
1503        let message = |delivery: &Delivery| {
1504            crate::open(&basis, delivery).expect_err("cache-only refusal").to_string()
1505        };
1506        let no_location = message(&Delivery::stale_ok(None));
1507        assert!(no_location.contains("no cache location"), "{no_location}");
1508        assert!(!no_location.contains("`on`"), "no write can succeed without a location");
1509        let missing = message(&Delivery::stale_ok(Some(snapshot.clone())));
1510        assert!(missing.contains("no usable snapshot"), "{missing}");
1511        assert!(missing.contains("with the `on` cache policy"), "{missing}");
1512        let other_basis = crate::query::Basis { root: other.path().into(), ..basis.clone() };
1513        crate::open(&other_basis, &Delivery::new(CachePolicy::Auto, Some(snapshot.clone())))
1514            .expect("other snapshot");
1515        let wrong_root = message(&Delivery::stale_ok(Some(snapshot)));
1516        assert!(wrong_root.contains("different root"), "{wrong_root}");
1517    }
1518
1519    #[test]
1520    fn route_delivery_matrix_rejects_contracts_the_route_cannot_execute() {
1521        let basis = crate::query::Basis {
1522            root: ".".into(),
1523            scope: crate::query::Scope::default(),
1524            content: crate::content::AnalysisSet::NONE,
1525        };
1526        let request = Request::new(basis, Query::default(), SystemTime::now());
1527        for delivery in Delivery::enumerate() {
1528            for route in
1529                [Route::OneShot, Route::Retained, Route::Refresh, Route::Watch, Route::Opened]
1530            {
1531                let result = plan(&request, &delivery, route);
1532                let forbidden = delivery.stale_ok
1533                    && (delivery.watch.is_some()
1534                        || matches!(route, Route::Watch | Route::Refresh)
1535                        || delivery.cache == CachePolicy::Off)
1536                    || route == Route::Opened
1537                        && (delivery.cache != CachePolicy::Off
1538                            || delivery.watch.is_some()
1539                            || delivery.accept_partial);
1540                assert_eq!(result.is_err(), forbidden, "{route:?} {delivery:?}");
1541                if let Ok(plan) = result {
1542                    if route == Route::Opened {
1543                        assert_eq!(plan.load(), Load::None);
1544                        assert_eq!(plan.verify(), Verify::Filesystem);
1545                    }
1546                }
1547            }
1548        }
1549    }
1550
1551    #[test]
1552    fn an_opened_root_refuses_the_scheduling_it_would_otherwise_drop() {
1553        // `OpenOptions::into_parts` runs one breadth-first producer whatever the delivery
1554        // says, and an opened root has no single answer for `accept_partial` to classify.
1555        // A value the route would silently ignore is refused at planning instead, and the
1556        // same values plan on a route that executes them.
1557        let basis = crate::query::Basis {
1558            root: ".".into(),
1559            scope: crate::query::Scope::default(),
1560            content: crate::content::AnalysisSet::NONE,
1561        };
1562        let request = Request::new(basis, Query::default(), SystemTime::now());
1563        let default = Delivery::new(CachePolicy::Off, None);
1564        let plan_opened = plan(&request, &default, Route::Opened).expect("defaults plan");
1565        assert_eq!(plan_opened.delivery().batch_size, default.batch_size);
1566        let unhonored = [
1567            (
1568                "scan workers",
1569                Delivery {
1570                    workers: crate::query::Workers { scan: Some(4), ..default.workers },
1571                    ..default.clone()
1572                },
1573            ),
1574            (
1575                "depth-first order",
1576                Delivery { order: crate::ScanOrder::DepthFirst, ..default.clone() },
1577            ),
1578            ("accept partial", Delivery { accept_partial: true, ..default.clone() }),
1579        ];
1580        for (case, delivery) in unhonored {
1581            let refused = plan(&request, &delivery, Route::Opened).expect_err(case);
1582            assert!(
1583                matches!(
1584                    refused,
1585                    crate::query::RequestError::DeliveryUnsupported { route: "opened", .. }
1586                ),
1587                "{case}: {refused}"
1588            );
1589            plan(&request, &delivery, Route::Retained)
1590                .unwrap_or_else(|error| panic!("{case} executes on a retained route: {error}"));
1591        }
1592        // A larger batch is honored, so it is not refused.
1593        let batched = Delivery { batch_size: default.batch_size * 2, ..default };
1594        let plan_batched = plan(&request, &batched, Route::Opened).expect("batch size plans");
1595        assert_eq!(plan_batched.delivery().batch_size, batched.batch_size);
1596    }
1597
1598    #[test]
1599    fn tier_writes_depend_only_on_authorization_and_observed_facts() {
1600        // Which routes are authorized is `plan`'s decision, pinned by
1601        // `auto_persists_where_a_later_request_reads_what_it_stores`; this pins what each
1602        // tier does with the authorization it was given.
1603        let routes = [Route::OneShot, Route::Retained, Route::Refresh, Route::Watch, Route::Opened];
1604        for (delivery, persist) in
1605            Delivery::enumerate().flat_map(|delivery| [(delivery.clone(), false), (delivery, true)])
1606        {
1607            for bits in 0_u8..64 {
1608                let facts = RunFacts {
1609                    entries_verified: bits & 1 != 0,
1610                    entries_changed: bits & 2 != 0,
1611                    content_changed: bits & 4 != 0,
1612                    content_requested: bits & 8 != 0,
1613                    projected: bits & 16 != 0,
1614                    paired_entries: bits & 32 != 0,
1615                };
1616                let allowed = persist;
1617                let expected = SaveTargets {
1618                    metadata: allowed
1619                        && facts.entries_verified
1620                        && facts.entries_changed
1621                        && !facts.projected,
1622                    content: allowed
1623                        && facts.content_requested
1624                        && facts.content_changed
1625                        && (facts.entries_verified || facts.paired_entries),
1626                };
1627                for route in routes {
1628                    let plan = Plan {
1629                        basis: crate::query::Basis {
1630                            root: ".".into(),
1631                            scope: crate::query::Scope::default(),
1632                            content: crate::content::AnalysisSet::NONE,
1633                        },
1634                        route,
1635                        retained: RetainedState::FullIndex,
1636                        load: Load::Snapshot,
1637                        verify: Verify::Filesystem,
1638                        persist,
1639                        delivery: delivery.clone(),
1640                    };
1641                    assert_eq!(plan.writes(facts), expected, "{route:?} {delivery:?} {facts:?}");
1642                    let unavailable = Plan {
1643                        delivery: Delivery { cache_path: None, ..delivery.clone() },
1644                        ..plan
1645                    };
1646                    assert!(unavailable.writes(facts).none());
1647                }
1648            }
1649        }
1650    }
1651
1652    fn planned(config: &OpenFixture, query: &Query) -> Plan {
1653        let (request, delivery) = split(Path::new("."), config, query);
1654        plan(&request, &delivery, Route::OneShot).expect("valid plan")
1655    }
1656
1657    fn summary_query() -> Query {
1658        Query { views: vec![ViewSpec::Summary], ..Query::default() }
1659    }
1660
1661    /// The request and the delivery a test's `OpenFixture` spells, split the way the two
1662    /// models now divide it: what the answer says, and how it is carried out.
1663    fn split(root: &Path, config: &OpenFixture, query: &Query) -> (Request, Delivery) {
1664        let (basis, delivery) = config.split(root);
1665        (Request::new(basis, query.clone(), std::time::UNIX_EPOCH), delivery)
1666    }
1667
1668    /// [`prepare_report`] as these tests ask for it: one configuration, one query.
1669    fn prepared(
1670        root: &Path,
1671        config: &OpenFixture,
1672        query: &Query,
1673    ) -> Result<(Report, PendingSave, PerformanceSummary)> {
1674        let (request, delivery) = split(root, config, query);
1675        prepare_report(&request, &delivery)
1676    }
1677
1678    /// [`prepared`], keeping the scan diagnostics.
1679    fn prepared_with_diagnostics(
1680        root: &Path,
1681        config: &OpenFixture,
1682        query: &Query,
1683    ) -> Result<(Report, PendingSave, PerformanceSummary, Option<crate::scan::ScanDiagnostics>)>
1684    {
1685        let (request, delivery) = split(root, config, query);
1686        prepare_report_with_scan_diagnostics(&request, &delivery)
1687    }
1688
1689    fn config(policy: CachePolicy, cache_path: Option<PathBuf>) -> OpenFixture {
1690        OpenFixture { scan: ScanConfig::default(), cache_path, policy, ..OpenFixture::default() }
1691    }
1692
1693    /// [`config`] with `.gitignore` observation turned off.
1694    fn blind(policy: CachePolicy, cache_path: Option<PathBuf>) -> OpenFixture {
1695        OpenFixture {
1696            scan: ScanConfig { read_controls: false, ..ScanConfig::default() },
1697            ..config(policy, cache_path)
1698        }
1699    }
1700
1701    fn controls_config(
1702        policy: CachePolicy,
1703        cache_path: PathBuf,
1704        read_controls: bool,
1705    ) -> OpenFixture {
1706        OpenFixture {
1707            scan: ScanConfig { read_controls, ..ScanConfig::default() },
1708            cache_path: Some(cache_path),
1709            policy,
1710            ..OpenFixture::default()
1711        }
1712    }
1713
1714    /// `fixture`, answered from its snapshot alone.
1715    fn stale(fixture: OpenFixture) -> OpenFixture {
1716        OpenFixture { stale_ok: true, ..fixture }
1717    }
1718
1719    fn seed_controls_snapshot(root: &Path, cache_path: PathBuf) {
1720        fs::write(root.join(".gitignore"), b"ignored.log\n").expect("control file");
1721        fs::write(root.join("ignored.log"), b"ignored").expect("ignored file");
1722        crate::open_fixture(root, &controls_config(CachePolicy::Auto, cache_path, true))
1723            .expect("seed controls-on snapshot");
1724    }
1725
1726    #[test]
1727    fn planner_uses_compact_state_only_when_the_request_proves_it_is_sufficient() {
1728        let off = blind(CachePolicy::Off, Some(PathBuf::from("unused.fdu")));
1729        assert_eq!(planned(&off, &summary_query()).retained, RetainedState::Summary);
1730
1731        for policy in [CachePolicy::Auto, CachePolicy::On] {
1732            let unavailable = blind(policy, None);
1733            assert_eq!(planned(&unavailable, &summary_query()).retained, RetainedState::Summary);
1734        }
1735
1736        let mut several_views = summary_query();
1737        several_views.views.push(ViewSpec::Types);
1738        assert_eq!(planned(&off, &several_views).retained, RetainedState::FullIndex);
1739
1740        let mut filtered = summary_query();
1741        filtered.selection.include.push(Pattern::parse("*.rs").expect("pattern"));
1742        assert_eq!(planned(&off, &filtered).retained, RetainedState::FullIndex);
1743
1744        // Changed deliberately by fdu-1ovb. The reducer classifies each entry with the
1745        // control table the index would hold, so the default summary, whose row carries an
1746        // ignored share, no longer needs the index; this assertion used to expect
1747        // `FullIndex`. `compact_summary_equals_the_indexed_summary_under_every_control_case`
1748        // is what licenses the change.
1749        let observing = config(CachePolicy::Off, None);
1750        assert!(observing.scan.read_controls, "observation is the default");
1751        assert_eq!(planned(&observing, &summary_query()).retained, RetainedState::Summary);
1752        // Selecting by ignored state is a filter, and a narrowed population is one retained
1753        // in the scope as well: both still need the index, whose traversal answers them.
1754        let mut by_ignored = summary_query();
1755        by_ignored.selection.ignored = IgnoredEntries::Exclude;
1756        assert_eq!(planned(&observing, &by_ignored).retained, RetainedState::FullIndex);
1757        for population in [IgnoredEntries::Exclude, IgnoredEntries::Only] {
1758            let narrowed = OpenFixture {
1759                scan: ScanConfig { population, ..ScanConfig::default() },
1760                ..config(CachePolicy::Off, None)
1761            };
1762            let mut query = summary_query();
1763            query.selection.ignored = population;
1764            assert_eq!(planned(&narrowed, &query).retained, RetainedState::FullIndex);
1765        }
1766    }
1767
1768    #[test]
1769    fn an_available_snapshot_does_not_force_the_index_for_a_metadata_summary() {
1770        // A loaded snapshot cannot save the work an unfiltered metadata summary is
1771        // already doing: revalidation stats every entry regardless, so retaining the
1772        // index and writing it back is additive cost with nothing to amortise it. The
1773        // compact tier stays selected so the common one-shot totals request pays for a
1774        // walk and nothing else.
1775        let cached = blind(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")));
1776        assert_eq!(
1777            planned(&cached, &summary_query()).retained,
1778            RetainedState::Summary,
1779            "a present snapshot must not force the index"
1780        );
1781    }
1782
1783    #[test]
1784    fn deliveries_whose_intent_is_the_snapshot_itself_still_retain_the_index() {
1785        // These two are not cost decisions. A stale answer must come without touching the
1786        // tree, so it has no scan to reduce; `On` is an explicit request to leave a
1787        // current snapshot, which means materialising the index that gets written.
1788        let cached = || blind(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")));
1789        for (name, delivery) in [
1790            ("stale", stale(cached())),
1791            ("on", OpenFixture { policy: CachePolicy::On, ..cached() }),
1792        ] {
1793            assert_eq!(
1794                planned(&delivery, &summary_query()).retained,
1795                RetainedState::FullIndex,
1796                "{name} needs the index to honour its contract"
1797            );
1798        }
1799    }
1800
1801    #[test]
1802    fn a_one_shot_metadata_query_does_not_read_the_snapshot_it_cannot_use() {
1803        // Revalidation stats every entry regardless of what the snapshot holds, so for
1804        // a metadata query the load and the reconciliation against it are additive cost:
1805        // measured on macOS/APFS over 494,031 entries, warm revalidation cost 4.8 s
1806        // against 3.6 s for the cold path. This holds for every view, not just the
1807        // compact summary — the tree default was the measured case.
1808        let mut tree_query = summary_query();
1809        tree_query.views = vec![ViewSpec::Tree];
1810        for policy in [CachePolicy::Auto, CachePolicy::On] {
1811            let cached = config(policy, Some(PathBuf::from("cache.fdu")));
1812            assert!(
1813                planned(&cached, &tree_query).load != Load::Snapshot,
1814                "{policy:?} must not pay for a read that saves no work"
1815            );
1816        }
1817    }
1818
1819    #[test]
1820    fn the_snapshot_is_read_where_reading_pays_or_is_the_contract() {
1821        // A stale answer comes from the snapshot; reading it is the request itself.
1822        let only = stale(config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))));
1823        assert_eq!(planned(&only, &summary_query()).load, Load::Snapshot);
1824
1825        // Analysis reuses the content sidecar, which avoids re-reading file bodies —
1826        // the one measured case where a warm read wins (639 ms to 325 ms).
1827        let analyzed = OpenFixture {
1828            analysis: crate::content::AnalysisRequest {
1829                profile: crate::content::AnalysisSet::NONE.with_code(),
1830                ..Default::default()
1831            },
1832            ..config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu")))
1833        };
1834        assert_eq!(planned(&analyzed, &summary_query()).load, Load::Snapshot);
1835
1836        // `Off` never reads by definition.
1837        let never = config(CachePolicy::Off, Some(PathBuf::from("cache.fdu")));
1838        assert_eq!(planned(&never, &summary_query()).load, Load::None);
1839    }
1840
1841    #[test]
1842    fn auto_persists_where_a_later_request_reads_what_it_stores() {
1843        // The policy table in one place. A one-shot metadata report under `Auto` leaves
1844        // nothing: no later one-shot report reads it, and it cost 0.26 s of a 1.51 s
1845        // default run on a million-entry Linux tree. Analysis keeps its sidecar and the
1846        // snapshot it pairs with; retained routes are their own later reader. `On` writes
1847        // everywhere a verified answer is produced, `Off` and a stale answer nowhere.
1848        let cache = Some(PathBuf::from("cache.fdu"));
1849        let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
1850        let persists = |fixture: &OpenFixture, query: &Query, route: Route| {
1851            let (request, delivery) = split(Path::new("."), fixture, query);
1852            plan(&request, &delivery, route).expect("valid plan").persists()
1853        };
1854        for (policy, one_shot, analysis, retained) in [
1855            (CachePolicy::Auto, false, true, true),
1856            (CachePolicy::On, true, true, true),
1857            (CachePolicy::Off, false, false, false),
1858        ] {
1859            let metadata = config(policy, cache.clone());
1860            assert_eq!(persists(&metadata, &tree, Route::OneShot), one_shot, "{policy:?}");
1861            assert_eq!(
1862                persists(&analyzing(metadata.clone()), &tree, Route::OneShot),
1863                analysis,
1864                "{policy:?} with analysis"
1865            );
1866            for route in [Route::Retained, Route::Refresh, Route::Watch] {
1867                assert_eq!(persists(&metadata, &tree, route), retained, "{policy:?} {route:?}");
1868            }
1869        }
1870        let stale_answer = stale(config(CachePolicy::On, cache));
1871        assert!(!persists(&stale_answer, &tree, Route::OneShot), "a stale answer writes nothing");
1872        assert!(!persists(&stale_answer, &tree, Route::Retained), "a stale answer writes nothing");
1873    }
1874
1875    #[test]
1876    fn an_analysis_request_never_selects_the_compact_summary_tier() {
1877        // Analysis reads file contents keyed by retained entries and writes its own
1878        // sidecar, so the aggregate-only tier cannot answer it even though the request
1879        // otherwise looks like the uncached unfiltered summary the planner compacts.
1880        let off = blind(CachePolicy::Off, None);
1881        assert_eq!(planned(&off, &summary_query()).retained, RetainedState::Summary);
1882
1883        for profile in [
1884            crate::content::AnalysisSet::NONE.with_lines(),
1885            crate::content::AnalysisSet::NONE.with_code(),
1886            crate::content::AnalysisSet::NONE.with_words(),
1887            crate::content::AnalysisSet::ALL,
1888        ] {
1889            let analyzed = OpenFixture {
1890                analysis: crate::content::AnalysisRequest { profile, ..Default::default() },
1891                ..blind(CachePolicy::Off, None)
1892            };
1893            assert_eq!(
1894                planned(&analyzed, &summary_query()).retained,
1895                RetainedState::FullIndex,
1896                "{profile:?} must retain the index"
1897            );
1898        }
1899    }
1900
1901    #[test]
1902    fn a_repeated_one_shot_report_scans_cold_while_open_still_revalidates() {
1903        // The same snapshot, two consumers, two right answers. A one-shot report cannot
1904        // amortise a snapshot load, so its second run scans cold again; a caller holding
1905        // the index through `open` amortises it across everything that follows, so its
1906        // second open still takes the warm path. Both report truthfully.
1907        let root = tempfile::tempdir().expect("tempdir");
1908        fs::write(root.path().join("file.txt"), b"contents").expect("file");
1909        let cache = tempfile::tempdir().expect("cache dir");
1910        let auto = config(CachePolicy::Auto, Some(cache.path().join("cache.fdu")));
1911        let on = OpenFixture { policy: CachePolicy::On, ..auto.clone() };
1912        let mut tree_query = summary_query();
1913        tree_query.views = vec![ViewSpec::Tree];
1914
1915        let (first, pending, _) = prepared(root.path(), &on, &tree_query).expect("first report");
1916        pending.join().expect("first save");
1917        assert_eq!(first.provenance.source, ReportSource::ColdScan);
1918        assert!(on.cache_path.as_deref().expect("path").exists(), "`on` persists");
1919
1920        let (second, pending, performance) =
1921            prepared(root.path(), &auto, &tree_query).expect("second report");
1922        pending.join().expect("second save");
1923        assert_eq!(
1924            second.provenance.source,
1925            ReportSource::ColdScan,
1926            "a repeated one-shot must not pay for a read that saves no work"
1927        );
1928        assert_eq!(performance.walked_files, 1, "the walk still happened");
1929
1930        // A default report and a default `open` both observe control state, so they share
1931        // one snapshot scope and the `open` starts from the report's snapshot.
1932        let (_, open_report) = crate::open_fixture(root.path(), &auto).expect("library open");
1933        assert_eq!(
1934            open_report.path_taken,
1935            OpenPath::WarmRevalidate,
1936            "a caller holding the index still amortises the load"
1937        );
1938    }
1939
1940    #[test]
1941    fn a_stale_answer_reads_what_an_on_report_leaves_and_auto_leaves_nothing() {
1942        // `auto` skips the write a one-shot metadata report cannot use, so a stale answer
1943        // after it has nothing to read and says how to leave something; `on` is that way,
1944        // and what it leaves is answered from without touching the tree.
1945        let root = tempfile::tempdir().expect("tempdir");
1946        fs::write(root.path().join("file.txt"), b"contents").expect("file");
1947        let cache = tempfile::tempdir().expect("cache dir");
1948        let auto = config(CachePolicy::Auto, Some(cache.path().join("cache.fdu")));
1949        let mut tree_query = summary_query();
1950        tree_query.views = vec![ViewSpec::Tree];
1951
1952        let (_, pending, _) = prepared(root.path(), &auto, &tree_query).expect("report");
1953        assert!(!pending.writes_metadata(), "`auto` starts no metadata write");
1954        pending.join().expect("nothing to save");
1955        assert!(!cache.path().join("cache.fdu").exists(), "`auto` leaves nothing");
1956        let only = stale(config(CachePolicy::Auto, Some(cache.path().join("cache.fdu"))));
1957        let missing = prepared(root.path(), &only, &tree_query).expect_err("nothing to read");
1958        assert!(missing.to_string().contains("with the `on` cache policy"), "{missing}");
1959
1960        let on = OpenFixture { policy: CachePolicy::On, ..auto };
1961        let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
1962        pending.join().expect("save");
1963
1964        let (from_cache, pending, performance, diagnostics) =
1965            prepared_with_diagnostics(root.path(), &only, &tree_query).expect("cache-only report");
1966        pending.join().expect("no save");
1967        assert_eq!(from_cache.provenance.source, ReportSource::CacheOnly);
1968        assert_eq!(performance.walked_files, 0, "cache-only never touches the tree");
1969        assert!(diagnostics.is_none(), "a cache-only open has no scan trace");
1970    }
1971
1972    #[test]
1973    fn controls_on_snapshot_projects_to_an_equivalent_controls_off_cache_only_report() {
1974        let root = tempfile::tempdir().expect("tempdir");
1975        let cache = tempfile::tempdir().expect("cache dir");
1976        let cache_path = cache.path().join("cache.fdu");
1977        fs::create_dir(root.path().join("src")).expect("source dir");
1978        fs::write(root.path().join("src/lib.rs"), b"library").expect("source file");
1979        seed_controls_snapshot(root.path(), cache_path.clone());
1980
1981        let controls_off = stale(controls_config(CachePolicy::Auto, cache_path, false));
1982        let query = Query {
1983            views: vec![
1984                ViewSpec::Summary,
1985                ViewSpec::Tree,
1986                ViewSpec::Families,
1987                ViewSpec::Types,
1988                ViewSpec::Extensions,
1989                ViewSpec::Languages,
1990                ViewSpec::Largest,
1991                ViewSpec::Recent,
1992                ViewSpec::Files,
1993            ],
1994            ..Query::default()
1995        };
1996        let (projected, pending, performance) =
1997            prepared(root.path(), &controls_off, &query).expect("projected report");
1998        pending.join().expect("no cache-only save");
1999
2000        let cold = OpenFixture {
2001            policy: CachePolicy::Off,
2002            stale_ok: false,
2003            cache_path: None,
2004            ..controls_off
2005        };
2006        let (mut expected, pending, _) =
2007            prepared(root.path(), &cold, &query).expect("controls-off cold report");
2008        pending.join().expect("no cold save");
2009        expected.provenance = projected.provenance.clone();
2010
2011        assert_eq!(performance.source, ReportSource::CacheOnly);
2012        assert_eq!(projected.scope, cold.scan.scope());
2013        assert_eq!(projected.ignore_rules, crate::control::ControlCoverage::NotObserved);
2014        assert_eq!(
2015            crate::report_format::render(&projected, crate::report_format::Format::Json, false,)
2016                .expect("compatible report format"),
2017            crate::report_format::render(&expected, crate::report_format::Format::Json, false,)
2018                .expect("compatible report format"),
2019        );
2020    }
2021
2022    #[test]
2023    fn controls_on_snapshot_projects_to_controls_off_auto_report() {
2024        let root = tempfile::tempdir().expect("tempdir");
2025        let cache = tempfile::tempdir().expect("cache dir");
2026        let cache_path = cache.path().join("cache.fdu");
2027        seed_controls_snapshot(root.path(), cache_path.clone());
2028
2029        let controls_off = OpenFixture {
2030            analysis: crate::content::AnalysisRequest {
2031                profile: crate::content::AnalysisSet::NONE.with_lines(),
2032                ..Default::default()
2033            },
2034            ..controls_config(CachePolicy::Auto, cache_path, false)
2035        };
2036        let (report, pending, performance) = prepared(root.path(), &controls_off, &summary_query())
2037            .expect("controls-off warm projection");
2038        pending.join().expect("save content only");
2039
2040        assert_eq!(report.provenance.source, ReportSource::WarmRevalidate);
2041        assert_eq!(report.scope, controls_off.scan.scope());
2042        assert_eq!(performance.source, ReportSource::WarmRevalidate);
2043    }
2044
2045    /// Control sources past both limits, so no scan can observe them without saying so.
2046    ///
2047    /// The root rule is longer than the line limit and the nested source is past the
2048    /// table budget. An observing scan refuses both and records it in its control coverage;
2049    /// a report whose scope observes no control state read neither.
2050    fn write_unobservable_controls(root: &Path) {
2051        let mut rule = vec![b'a'; crate::control::DEFAULT_CONTROL_LINE_LIMIT + 1];
2052        rule.push(b'\n');
2053        fs::write(root.join(".gitignore"), rule).expect("oversized rule");
2054        fs::create_dir(root.join("vendored")).expect("nested directory");
2055        fs::write(
2056            root.join("vendored/.gitignore"),
2057            b"x\n".repeat(crate::control::DEFAULT_CONTROL_BUDGET / 2),
2058        )
2059        .expect("oversized source");
2060    }
2061
2062    #[test]
2063    fn a_one_shot_report_observes_control_state_as_its_caller_configures() {
2064        // A default report reads every `.gitignore`, and a file past a bound is refused and
2065        // named without ending the report or its snapshot. A report that turns observation
2066        // off reads neither file, and says so in its report and in any snapshot it writes;
2067        // `on` makes each one write.
2068        let root = tempfile::tempdir().expect("tempdir");
2069        fs::write(root.path().join("file.txt"), b"contents").expect("file");
2070        write_unobservable_controls(root.path());
2071        let mut tree_query = summary_query();
2072        tree_query.views = vec![ViewSpec::Tree];
2073
2074        for read_controls in [true, false] {
2075            let cache = tempfile::tempdir().expect("cache dir");
2076            let cache_path = cache.path().join("cache.fdu");
2077            let caller = controls_config(CachePolicy::On, cache_path.clone(), read_controls);
2078            for query in [summary_query(), tree_query.clone()] {
2079                let (report, pending, _) = prepared(root.path(), &caller, &query)
2080                    .expect("a refused control file ends nothing");
2081                pending.join().expect("save");
2082                assert!(
2083                    report.status.complete,
2084                    "a refusal is not a partial: {:?}",
2085                    report.status.errors
2086                );
2087                assert_eq!(report.scope, caller.scan.scope());
2088                match &report.ignore_rules {
2089                    crate::control::ControlCoverage::Observed(coverage) => {
2090                        assert!(read_controls, "observed only when asked");
2091                        assert_eq!(coverage.refused, 2, "{coverage:?}");
2092                        assert_eq!(report.notes.len(), 2, "{:?}", report.notes);
2093                        assert!(
2094                            report.notes[0].contains("2 ignore files not applied"),
2095                            "{:?}",
2096                            report.notes
2097                        );
2098                        assert_eq!(
2099                            report.notes[1],
2100                            "note: gitignored subtotals are unavailable where governing rules could not be verified"
2101                        );
2102                    }
2103                    crate::control::ControlCoverage::NotObserved => {
2104                        assert!(!read_controls, "unobserved only when turned off");
2105                        assert!(report.notes.is_empty(), "{:?}", report.notes);
2106                    }
2107                }
2108            }
2109
2110            let saved = crate::snapshot::load(&cache_path)
2111                .expect("load the snapshot")
2112                .expect("the index tier persisted");
2113            assert_eq!(saved.scope(), caller.scan.scope());
2114            assert_eq!(saved.controls().is_ok(), read_controls);
2115        }
2116    }
2117
2118    /// Which failure a run names, and what kind of failure it is, must not depend on how it
2119    /// was delivered.
2120    ///
2121    /// A scope this build cannot honour is refused by every policy, and by a stale answer,
2122    /// which never scans: under `--stale-ok` the scan that would have refused it never runs,
2123    /// so the run used to report a snapshot miss instead -- the same request naming two
2124    /// different failures depending on its delivery, which the path-independence registry
2125    /// records as `refusal-order` for `--one-filesystem` on Windows. `follow_symlinks` is
2126    /// the same rule on every platform, so this test runs where the Windows case cannot.
2127    ///
2128    /// The refusal is the request model's typed one, not an engine error the surfaces then
2129    /// classify differently: reporting it as an engine error made the command line exit 1
2130    /// where Python raised `ValueError`, one request with two kinds of outcome.
2131    #[test]
2132    fn a_scope_this_build_cannot_honour_is_refused_before_any_snapshot_is_read() {
2133        let root = tempfile::tempdir().expect("tempdir");
2134        fs::write(root.path().join("file.txt"), b"contents").expect("file");
2135        let cache = tempfile::tempdir().expect("cache dir");
2136        let cache_path = cache.path().join("cache.fdu");
2137
2138        // A usable snapshot exists, so a stale read of a scope this build supports answers
2139        // from it.
2140        let warm = config(CachePolicy::On, Some(cache_path.clone()));
2141        let (_, pending, _) = prepared(root.path(), &warm, &summary_query()).expect("warm");
2142        pending.join().expect("save");
2143        let (_, pending, _) = prepared(
2144            root.path(),
2145            &stale(config(CachePolicy::Auto, Some(cache_path.clone()))),
2146            &summary_query(),
2147        )
2148        .expect("the snapshot answers a supported scope");
2149        pending.join().expect("no save");
2150
2151        let unsupported = ScanConfig { follow_symlinks: true, ..ScanConfig::default() };
2152        for (policy, stale_ok) in [
2153            (CachePolicy::Auto, true),
2154            (CachePolicy::Off, false),
2155            (CachePolicy::Auto, false),
2156            (CachePolicy::On, false),
2157        ] {
2158            let asked = OpenFixture {
2159                scan: unsupported.clone(),
2160                stale_ok,
2161                ..config(policy, Some(cache_path.clone()))
2162            };
2163            let refused = prepared(root.path(), &asked, &summary_query())
2164                .expect_err("a scope this build cannot honour has no answer at any policy");
2165            assert!(
2166                matches!(
2167                    refused,
2168                    Error::InvalidRequest(crate::query::RequestError::ScopeUnsupported {
2169                        axis: crate::query::ScopeAxis::FollowSymlinks,
2170                        ..
2171                    })
2172                ),
2173                "{policy:?} must refuse the request rather than fail the operation: {refused}"
2174            );
2175            assert_eq!(
2176                refused.to_string(),
2177                "unsupported scan configuration: follow_symlinks requires cycle, root-boundary, \
2178                 and filesystem-boundary semantics",
2179                "{policy:?} must name the scope it cannot honour"
2180            );
2181        }
2182    }
2183
2184    #[test]
2185    fn a_report_that_reads_no_gitignore_refuses_to_select_by_ignored_state() {
2186        // No entry of a scan that read no rule can be shown to be ignored or not, so the
2187        // request is refused rather than answered with every entry or none.
2188        let root = tempfile::tempdir().expect("tempdir");
2189        fs::write(root.path().join(".gitignore"), b"*.log\n").expect("control file");
2190        fs::write(root.path().join("debug.log"), b"ignored").expect("ignored file");
2191        let mut only = summary_query();
2192        only.selection.ignored = IgnoredEntries::Only;
2193
2194        // Refused by the request model before anything is scanned: the compact summary
2195        // tier this request would take reaches no reader, so a check made there would not
2196        // cover this route at all.
2197        assert!(matches!(
2198            prepared(root.path(), &blind(CachePolicy::Off, None), &only),
2199            Err(Error::InvalidRequest(crate::query::RequestError::IgnoredWithoutObservation(
2200                IgnoredEntries::Only
2201            )))
2202        ));
2203
2204        let (report, pending, _) =
2205            prepared(root.path(), &config(CachePolicy::Off, None), &only).expect("observed");
2206        pending.join().expect("no save");
2207        let Section::Summary(row) = report.sections[0] else { panic!("a summary") };
2208        assert_eq!((row.files, row.bytes), (1, 7), "only the ignored file is selected");
2209    }
2210
2211    #[test]
2212    fn an_on_reports_snapshot_serves_either_cache_only_report_but_not_the_reverse() {
2213        // Every surface reaches this planner observing control state by default, so a
2214        // snapshot a default-scope report wrote under `on` serves the next one. A cache-only report that
2215        // turns observation off also answers from it, reading only the all-entry facts. An
2216        // opted-out snapshot holds no classification, so it cannot serve a default report,
2217        // and the refusal says why and what recovers.
2218        let root = tempfile::tempdir().expect("tempdir");
2219        fs::write(root.path().join("file.txt"), b"contents").expect("file");
2220        let mut tree_query = summary_query();
2221        tree_query.views = vec![ViewSpec::Tree];
2222
2223        for (writer, reader) in [(true, true), (true, false), (false, false), (false, true)] {
2224            let cache = tempfile::tempdir().expect("cache dir");
2225            let cache_path = cache.path().join("cache.fdu");
2226            let write = controls_config(CachePolicy::On, cache_path.clone(), writer);
2227            let (_, pending, _) =
2228                prepared(root.path(), &write, &tree_query).expect("writing report");
2229            pending.join().expect("save");
2230
2231            let read = stale(controls_config(CachePolicy::Auto, cache_path, reader));
2232            match prepared(root.path(), &read, &tree_query) {
2233                Ok((report, pending, _)) => {
2234                    pending.join().expect("no save");
2235                    assert!(writer || !reader, "writer {writer} served reader {reader}");
2236                    assert_eq!(report.provenance.source, ReportSource::CacheOnly);
2237                    assert_eq!(
2238                        matches!(report.ignore_rules, crate::control::ControlCoverage::Observed(_)),
2239                        reader,
2240                        "a report describes the scope it asked for"
2241                    );
2242                }
2243                Err(Error::Snapshot(message)) => {
2244                    assert!(!writer && reader, "writer {writer}, reader {reader}: {message}");
2245                    assert!(message.contains(".gitignore state"), "names the cause: {message}");
2246                    assert!(message.contains("verified answer"), "names the remedy: {message}");
2247                }
2248                Err(other) => panic!("writer {writer}, reader {reader}: {other}"),
2249            }
2250        }
2251    }
2252
2253    #[test]
2254    fn a_cache_only_open_answers_from_an_on_reports_snapshot() {
2255        // A default-scope report and a default `open` share one scope, so the one delivery
2256        // that forbids a scan answers the `open` from the snapshot the report left under
2257        // `on`, with the classification the report observed.
2258        let root = tempfile::tempdir().expect("tempdir");
2259        fs::write(root.path().join(".gitignore"), b"*.log\n").expect("control file");
2260        fs::write(root.path().join("debug.log"), b"ignored").expect("ignored file");
2261        let cache = tempfile::tempdir().expect("cache dir");
2262        let cache_path = cache.path().join("cache.fdu");
2263        let mut tree_query = summary_query();
2264        tree_query.views = vec![ViewSpec::Tree];
2265
2266        let on = config(CachePolicy::On, Some(cache_path.clone()));
2267        let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
2268        pending.join().expect("save");
2269
2270        let only = stale(config(CachePolicy::Auto, Some(cache_path)));
2271        let (index, report) = crate::open_fixture(root.path(), &only).expect("the shared snapshot");
2272        assert_eq!(report.path_taken, OpenPath::CacheOnly);
2273        assert_eq!(index.is_ignored(Path::new("debug.log")).ok(), Some(Some(true)));
2274    }
2275
2276    #[test]
2277    fn an_open_that_opts_out_of_control_state_shares_an_opted_out_reports_snapshot() {
2278        // A report and an `open` that both turn observation off write and want one scope,
2279        // so that open answers from the snapshot the report left under `on` without
2280        // touching the tree, as the one delivery that forbids a scan, and says it cannot
2281        // classify ignored entries.
2282        let root = tempfile::tempdir().expect("tempdir");
2283        fs::write(root.path().join("file.txt"), b"contents").expect("file");
2284        let cache = tempfile::tempdir().expect("cache dir");
2285        let cache_path = cache.path().join("cache.fdu");
2286        let mut tree_query = summary_query();
2287        tree_query.views = vec![ViewSpec::Tree];
2288
2289        let on = blind(CachePolicy::On, Some(cache_path.clone()));
2290        let (_, pending, _) = prepared(root.path(), &on, &tree_query).expect("report");
2291        pending.join().expect("save");
2292        assert!(cache_path.exists(), "the report left a snapshot");
2293
2294        let only = stale(blind(CachePolicy::Auto, Some(cache_path)));
2295        let (index, report) = crate::open_fixture(root.path(), &only).expect("the shared snapshot");
2296        assert_eq!(report.path_taken, OpenPath::CacheOnly);
2297        assert!(matches!(
2298            index.is_ignored(Path::new("file.txt")),
2299            Err(Error::ControlStateNotObserved)
2300        ));
2301    }
2302
2303    #[test]
2304    fn compact_summary_matches_the_indexed_summary_exactly() {
2305        let root = tempfile::tempdir().expect("tempdir");
2306        fs::create_dir(root.path().join("src")).expect("directory");
2307        fs::write(root.path().join("src/lib.rs"), b"library").expect("file");
2308        fs::write(root.path().join("README.md"), b"read me").expect("file");
2309        #[cfg(unix)]
2310        std::os::unix::fs::symlink("README.md", root.path().join("readme-link")).expect("symlink");
2311        crate::test_support::settle_allocations(root.path());
2312
2313        let query = summary_query();
2314        // Two workers so the compact fold exercises StreamingEmission recycle even on
2315        // a one-vCPU runner (`threads: None` would take the serial walker there).
2316        let off = OpenFixture {
2317            scan: ScanConfig { read_controls: false, threads: Some(2), ..ScanConfig::default() },
2318            ..blind(CachePolicy::Off, None)
2319        };
2320        let (compact, pending, performance) =
2321            prepared(root.path(), &off, &query).expect("compact report");
2322        pending.join().expect("no pending compact save");
2323        assert_eq!(performance.walked_files, 2);
2324        assert_eq!(performance.walked_bytes, 14);
2325
2326        // This report turns control observation off, so the index it must match exactly is
2327        // opened under that scope too.
2328        let (index, _open_report) = crate::open_fixture(root.path(), &off).expect("indexed scan");
2329        let indexed = report(
2330            &index,
2331            &crate::test_support::read_of(&index, query.clone()),
2332            compact.provenance.generated_at,
2333        )
2334        .expect("report");
2335
2336        let Section::Summary(compact_row) = compact.sections[0] else {
2337            panic!("compact plan did not return a summary")
2338        };
2339        let Section::Summary(indexed_row) = indexed.sections[0] else {
2340            panic!("indexed plan did not return a summary")
2341        };
2342        assert_eq!(compact_row.files, indexed_row.files);
2343        assert_eq!(compact_row.dirs, indexed_row.dirs);
2344        assert_eq!(compact_row.bytes, indexed_row.bytes);
2345        assert_eq!(compact_row.allocated, indexed_row.allocated);
2346        assert_eq!(compact_row.newest_mtime_ns, indexed_row.newest_mtime_ns);
2347        assert_eq!(compact.root, indexed.root);
2348        assert_eq!(compact.scope, indexed.scope);
2349        assert_eq!(compact.status.complete, indexed.status.complete);
2350        assert_eq!(compact.provenance.freshness, indexed.provenance.freshness);
2351    }
2352
2353    /// One tree for the transient-versus-indexed differential, the scope it is summarized
2354    /// under, and what the index must say of it, so a case that stopped exercising what it
2355    /// names fails instead of agreeing vacuously.
2356    struct ControlCase {
2357        name: &'static str,
2358        /// A file the case made unreadable, or a directory it made unsearchable, restored
2359        /// when the case is dropped so its tree can be removed even after a failed
2360        /// assertion. Declared before `root`: fields drop in order, and the restore must
2361        /// precede the removal. Only Unix can make one.
2362        #[cfg(unix)]
2363        denied: Option<Unlocked>,
2364        root: tempfile::TempDir,
2365        scan: ScanConfig,
2366        /// Whether the row carries an ignored share, which a refused or unreadable control
2367        /// file withholds.
2368        share: bool,
2369        /// Control files refused.
2370        refused: u64,
2371        /// Which files a budget refuses depends on the order controls arrive in, so the two
2372        /// routes are compared only where that order is fixed: with one worker.
2373        order_dependent: bool,
2374        /// Walk errors the case induces.
2375        errors: bool,
2376        /// How control lookups under the case's root resolve a case variant of the name.
2377        lookups: crate::test_support::CaseLookups,
2378        /// For a tree whose rules sit in a case variant of the name: whether they govern,
2379        /// which is whether a lookup of `.gitignore` resolves to the variant.
2380        variant_governs: Option<bool>,
2381    }
2382
2383    /// A path a fixture sealed, a file made unreadable or a directory made unsearchable,
2384    /// restored when dropped so the tree holding it can be removed even after a failed
2385    /// assertion.
2386    #[cfg(unix)]
2387    struct Unlocked(PathBuf);
2388
2389    #[cfg(unix)]
2390    impl Drop for Unlocked {
2391        fn drop(&mut self) {
2392            use std::os::unix::fs::PermissionsExt;
2393            let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o755));
2394        }
2395    }
2396
2397    /// A directory that is readable but not searchable, so it lists and then every
2398    /// child's stat fails with `EACCES`: a subdirectory, a file, and a symlink, whose
2399    /// `d_type` names them where no stat may (R163-1). The walk reports each child as
2400    /// an error and holds no entry for any of them, on every route.
2401    #[cfg(unix)]
2402    fn seal_search_denied(root: &Path) -> Unlocked {
2403        use std::os::unix::fs::PermissionsExt;
2404        let denied = root.join("denied");
2405        put(root, "denied/sub/inner", b"inner");
2406        put(root, "denied/f", b"f");
2407        std::os::unix::fs::symlink("f", denied.join("link")).expect("symlink");
2408        fs::set_permissions(&denied, fs::Permissions::from_mode(0o400)).expect("deny search");
2409        Unlocked(denied)
2410    }
2411
2412    fn put(root: &Path, path: &str, contents: &[u8]) {
2413        let path = root.join(path);
2414        fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories");
2415        fs::write(path, contents).expect("fixture file");
2416    }
2417
2418    /// Negation, nested files, directory-only and anchored rules, rules below an ignored
2419    /// directory that try to re-include, a re-included directory, a control file that
2420    /// ignores itself, and a listing long enough to split across small batches.
2421    fn rules_tree() -> tempfile::TempDir {
2422        let root = tempfile::tempdir().expect("tempdir");
2423        let root_path = root.path();
2424        put(
2425            root_path,
2426            ".gitignore",
2427            b"*.log\n!keep.log\n/anchored.txt\ncache/\nnode_modules/\nvendor/*\n!vendor/keep/\n",
2428        );
2429        put(root_path, "a.log", b"alog");
2430        put(root_path, "keep.log", b"keeplog");
2431        put(root_path, "anchored.txt", b"anchored");
2432        put(root_path, "cache", b"a file named like a directory rule");
2433        put(root_path, "README.md", b"readme!");
2434        put(root_path, "src/anchored.txt", b"not anchored here");
2435        put(root_path, "src/x.log", b"xlog-");
2436        put(root_path, "src/keep.log", b"kept everywhere");
2437        put(root_path, "src/main.rs", b"fn main() {}");
2438        put(root_path, "sub/.gitignore", b"!*.log\n*.tmp\n");
2439        put(root_path, "sub/y.log", b"re-included");
2440        put(root_path, "sub/z.tmp", b"tmp");
2441        put(root_path, "sub/cache/data.bin", b"cached bytes");
2442        put(root_path, "sub/deep/.gitignore", b"*\n!.gitignore\n");
2443        put(root_path, "sub/deep/f.txt", b"deep");
2444        put(root_path, "sub/deep/inner/g.txt", b"deeper");
2445        put(root_path, "node_modules/pkg/.gitignore", b"!*\n");
2446        put(root_path, "node_modules/pkg/index.js", b"module.exports = 1;");
2447        put(root_path, "node_modules/pkg/lib/a.js", b"a");
2448        put(root_path, "vendor/a.c", b"int a;");
2449        put(root_path, "vendor/keep/k.c", b"int k;");
2450        put(root_path, "vendor/drop/d.c", b"int d;");
2451        put(root_path, "selfish/.gitignore", b".gitignore\n*.bak\n");
2452        put(root_path, "selfish/x.bak", b"backup");
2453        put(root_path, "selfish/y.txt", b"kept");
2454        put(root_path, "many/.gitignore", b"*[02468].dat\n");
2455        for file in 0..40 {
2456            put(root_path, &format!("many/f{file:02}.dat"), &vec![b'.'; file + 1]);
2457        }
2458        root
2459    }
2460
2461    /// The control case whose only control-like file is `.GITIGNORE`.
2462    const CASE_VARIANT: &str = "a case-variant control name";
2463
2464    /// The control case whose directory lists `.gitignore` beside `.GITIGNORE`.
2465    const CASE_BOTH: &str = "both spellings of the control name";
2466
2467    fn control_cases() -> Vec<ControlCase> {
2468        let case = |name, root, scan, share, refused| ControlCase {
2469            name,
2470            root,
2471            scan,
2472            share,
2473            refused,
2474            order_dependent: false,
2475            errors: false,
2476            lookups: crate::test_support::CaseLookups::Host,
2477            variant_governs: None,
2478            #[cfg(unix)]
2479            denied: None,
2480        };
2481        let mut cases = vec![
2482            case("rules", rules_tree(), ScanConfig::default(), true, 0),
2483            case(
2484                "rules with hidden entries pruned",
2485                rules_tree(),
2486                ScanConfig {
2487                    hidden: Some(std::sync::Arc::new(crate::HiddenPolicy::prune_hidden(Vec::<
2488                        std::ffi::OsString,
2489                    >::new(
2490                    )))),
2491                    ..ScanConfig::default()
2492                },
2493                true,
2494                0,
2495            ),
2496            case(
2497                "rules under a depth bound",
2498                rules_tree(),
2499                ScanConfig { max_depth: Some(1), ..ScanConfig::default() },
2500                true,
2501                0,
2502            ),
2503        ];
2504
2505        let empty = tempfile::tempdir().expect("tempdir");
2506        put(empty.path(), "only.txt", b"no rules anywhere");
2507        cases.push(case("no control file", empty, ScanConfig::default(), true, 0));
2508
2509        // A line over the limit refuses its whole file, whatever order it arrives in.
2510        let long = tempfile::tempdir().expect("tempdir");
2511        put(long.path(), ".gitignore", b"*.log\n");
2512        put(long.path(), "x.log", b"ignored");
2513        let mut line = vec![b'x'; crate::control::DEFAULT_CONTROL_LINE_LIMIT + 1];
2514        line.push(b'\n');
2515        put(long.path(), "long/.gitignore", &line);
2516        put(long.path(), "long/kept.txt", b"kept");
2517        put(long.path(), "long/y.log", b"unknown");
2518        cases.push(case("line limit", long, ScanConfig::default(), false, 1));
2519
2520        // Each nested file alone exceeds what the root leaves of the budget, so all seventy
2521        // are refused in any order: more than a note names and more than a report retains.
2522        let over = tempfile::tempdir().expect("tempdir");
2523        put(over.path(), ".gitignore", b"*.log\n");
2524        let mut oversized = vec![b'x'; 200];
2525        oversized.push(b'\n');
2526        for directory in 0..70 {
2527            put(over.path(), &format!("d{directory:02}/.gitignore"), &oversized);
2528            put(over.path(), &format!("d{directory:02}/f.log"), b"log");
2529        }
2530        let limits = crate::control::ControlLimits { budget: Some(256), ..Default::default() };
2531        cases.push(case(
2532            "every nested file over the budget",
2533            over,
2534            ScanConfig { control_limits: limits, ..ScanConfig::default() },
2535            false,
2536            70,
2537        ));
2538
2539        // Any two of four fit the budget and no third does: which two is arrival order.
2540        let competing = tempfile::tempdir().expect("tempdir");
2541        for (position, directory) in ["a", "b", "c", "d"].into_iter().enumerate() {
2542            let mut rules = format!("r{position}").into_bytes();
2543            rules.extend(std::iter::repeat_n(b'y', 100));
2544            rules.push(b'\n');
2545            put(competing.path(), &format!("{directory}/.gitignore"), &rules);
2546            put(competing.path(), &format!("{directory}/file.txt"), b"file");
2547        }
2548        let limits = crate::control::ControlLimits { budget: Some(1000), ..Default::default() };
2549        let mut competing = case(
2550            "competing for the budget",
2551            competing,
2552            ScanConfig { control_limits: limits, ..ScanConfig::default() },
2553            false,
2554            2,
2555        );
2556        competing.order_dependent = true;
2557        cases.push(competing);
2558
2559        // A case variant of the control name governs exactly where a lookup of `.gitignore`
2560        // resolves to it, as the open git makes does (fdu-0w1b): on a case-insensitive
2561        // volume, and through folded lookups on a case-sensitive host. Enough entries fill
2562        // a default batch before the listing reaches `.GITIGNORE` (enumeration order
2563        // permitting), so the transient fold's probe must find what the listed variant's
2564        // read and the index find. Hidden pruning drops the variant's row, not its rules.
2565        let probe = tempfile::tempdir().expect("tempdir");
2566        for (lookups, governs) in crate::test_support::CaseLookups::on_this_host(probe.path()) {
2567            for hidden in [None, Some(Vec::<std::ffi::OsString>::new())] {
2568                let variant = tempfile::tempdir().expect("tempdir");
2569                put(variant.path(), ".GITIGNORE", b"*.log\n");
2570                for file in 0..1_200 {
2571                    put(variant.path(), &format!("f{file:04}.log"), b"log");
2572                }
2573                put(variant.path(), "kept.txt", b"kept");
2574                let scan = ScanConfig {
2575                    hidden: hidden
2576                        .map(|allow| std::sync::Arc::new(crate::HiddenPolicy::prune_hidden(allow))),
2577                    ..ScanConfig::default()
2578                };
2579                let mut variant = case(CASE_VARIANT, variant, scan, true, 0);
2580                variant.lookups = lookups;
2581                variant.variant_governs = Some(governs);
2582                cases.push(variant);
2583            }
2584        }
2585
2586        // A case-sensitive directory can list both spellings, and only the exact name
2587        // governs there, whatever the lookups; a case-insensitive one cannot hold both, and
2588        // writing the second spelling there rewrites the first file.
2589        for lookups in
2590            [crate::test_support::CaseLookups::Host, crate::test_support::CaseLookups::Folded]
2591        {
2592            let both = tempfile::tempdir().expect("tempdir");
2593            put(both.path(), ".gitignore", b"*.log\n");
2594            put(both.path(), ".GITIGNORE", b"*.tmp\n");
2595            if fs::read(both.path().join(".gitignore")).expect("read") != b"*.log\n" {
2596                eprintln!("skipped {CASE_BOTH:?}: the temporary directory is case-insensitive");
2597                break;
2598            }
2599            for file in 0..1_200 {
2600                put(both.path(), &format!("f{file:04}.tmp"), b"tmp");
2601            }
2602            put(both.path(), "x.log", b"log");
2603            let mut both = case(CASE_BOTH, both, ScanConfig::default(), true, 0);
2604            both.lookups = lookups;
2605            cases.push(both);
2606        }
2607
2608        #[cfg(unix)]
2609        {
2610            use std::os::unix::fs::PermissionsExt;
2611
2612            // A control file that is a directory or a symlink applies no rules.
2613            let shapes = tempfile::tempdir().expect("tempdir");
2614            put(shapes.path(), ".gitignore", b"*.tmp\n");
2615            put(shapes.path(), "weird/.gitignore/inner.tmp", b"inner");
2616            put(shapes.path(), "weird/kept.txt", b"kept");
2617            put(shapes.path(), "rules.txt", b"*.txt\n");
2618            fs::create_dir(shapes.path().join("linked")).expect("directory");
2619            std::os::unix::fs::symlink("../rules.txt", shapes.path().join("linked/.gitignore"))
2620                .expect("symlink");
2621            put(shapes.path(), "linked/still.txt", b"still counted");
2622            cases.push(case(
2623                "control files that are not files",
2624                shapes,
2625                ScanConfig::default(),
2626                true,
2627                0,
2628            ));
2629
2630            if crate::test_support::require_permission_bits() {
2631                let unreadable = tempfile::tempdir().expect("tempdir");
2632                put(unreadable.path(), ".gitignore", b"*.log\n");
2633                put(unreadable.path(), "x.log", b"ignored");
2634                put(unreadable.path(), "sub/.gitignore", b"!*.log\n");
2635                put(unreadable.path(), "sub/y.log", b"unknown");
2636                let control = unreadable.path().join("sub/.gitignore");
2637                let mut denied =
2638                    case("unreadable control file", unreadable, ScanConfig::default(), false, 0);
2639                fs::set_permissions(&control, fs::Permissions::from_mode(0o000))
2640                    .expect("deny the control file");
2641                denied.denied = Some(Unlocked(control));
2642                denied.errors = true;
2643                cases.push(denied);
2644
2645                // Rules beside a search-denied directory, whose listing shows no control
2646                // file: the children fail, and the classification of the rest is known.
2647                // At every batch size: a listing that records no entry never fills a
2648                // batch, so the directory's control is never probed
2649                // (`StreamingEmission::settle_listing`), on either route.
2650                let sealed = tempfile::tempdir().expect("tempdir");
2651                put(sealed.path(), ".gitignore", b"*.log\n");
2652                put(sealed.path(), "open/x.log", b"ignored");
2653                put(sealed.path(), "open/y.txt", b"kept");
2654                let denied = seal_search_denied(sealed.path());
2655                let mut unsearchable = case(
2656                    "a directory that lists but refuses search",
2657                    sealed,
2658                    ScanConfig::default(),
2659                    true,
2660                    0,
2661                );
2662                unsearchable.denied = Some(denied);
2663                unsearchable.errors = true;
2664                cases.push(unsearchable);
2665            }
2666        }
2667        for case in &cases {
2668            crate::test_support::settle_allocations(case.root.path());
2669        }
2670        cases
2671    }
2672
2673    /// The default summary of `root` under `scan`, from the transient tier and from the
2674    /// index, with the transient report's provenance, which describes the delivery rather
2675    /// than the tree, set to the index's once the parts both routes share agree.
2676    fn transient_and_indexed(root: &Path, scan: &ScanConfig, label: &str) -> (Report, Report) {
2677        let query = summary_query();
2678        let transient = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
2679        assert_eq!(planned(&transient, &query).retained, RetainedState::Summary, "{label}");
2680        let (mut compact, pending, compact_performance) =
2681            prepared(root, &transient, &query).expect("transient report");
2682        pending.join().expect("the transient tier saves nothing");
2683
2684        // `on` with a snapshot path is a delivery about the snapshot, so the same request
2685        // takes the index: the route every default summary took before fdu-1ovb.
2686        let cache = tempfile::tempdir().expect("cache dir");
2687        let indexed = OpenFixture {
2688            scan: scan.clone(),
2689            ..config(CachePolicy::On, Some(cache.path().join("snapshot.fdu")))
2690        };
2691        assert_eq!(planned(&indexed, &query).retained, RetainedState::FullIndex, "{label}");
2692        let (indexed, pending, indexed_performance) =
2693            prepared(root, &indexed, &query).expect("indexed report");
2694        pending.join().expect("save");
2695
2696        assert_eq!(
2697            (
2698                compact_performance.walked_files,
2699                compact_performance.walked_bytes,
2700                compact_performance.walked_allocated,
2701                compact_performance.source,
2702            ),
2703            (
2704                indexed_performance.walked_files,
2705                indexed_performance.walked_bytes,
2706                indexed_performance.walked_allocated,
2707                indexed_performance.source,
2708            ),
2709            "{label}: walked totals"
2710        );
2711        assert_eq!(compact.provenance.source, indexed.provenance.source, "{label}");
2712        assert_eq!(compact.provenance.freshness, indexed.provenance.freshness, "{label}");
2713        assert_eq!(
2714            (compact.provenance.tiers.entries.source, compact.provenance.tiers.entries.freshness),
2715            (indexed.provenance.tiers.entries.source, indexed.provenance.tiers.entries.freshness),
2716            "{label}"
2717        );
2718        compact.provenance = indexed.provenance.clone();
2719        (compact, indexed)
2720    }
2721
2722    /// The transient summary is the indexed summary, whole, for every control case the
2723    /// index classifies: negations, nested and self-ignoring control files, rules below an
2724    /// ignored directory, control files that are not files, refusals by the line limit
2725    /// and by the budget, an unreadable control file, a case-variant control name where
2726    /// the volume is case-insensitive, and the notes and coverage each produces. Across
2727    /// worker counts, batch sizes small enough to split every listing, and both traversal
2728    /// orders (fdu-1ovb).
2729    #[test]
2730    fn compact_summary_equals_the_indexed_summary_under_every_control_case() {
2731        use crate::report_format::{Format, render};
2732
2733        let default_batch = ScanConfig::default().batch_size;
2734        for case in control_cases() {
2735            let _lookups = case.lookups.install(case.root.path());
2736            let mut compared = 0;
2737            for threads in [Some(1), Some(2), Some(4), None] {
2738                if case.order_dependent && threads != Some(1) {
2739                    continue;
2740                }
2741                for batch_size in [default_batch, 1, 3] {
2742                    for order in [crate::ScanOrder::BreadthFirst, crate::ScanOrder::DepthFirst] {
2743                        let scan = ScanConfig { batch_size, threads, order, ..case.scan.clone() };
2744                        let label = format!(
2745                            "{} ({:?} lookups, {threads:?} workers, batch {batch_size}, {order:?})",
2746                            case.name, case.lookups
2747                        );
2748                        let (compact, indexed) =
2749                            transient_and_indexed(case.root.path(), &scan, &label);
2750
2751                        assert_eq!(format!("{compact:#?}"), format!("{indexed:#?}"), "{label}");
2752                        for format in [Format::Text, Format::Json, Format::Yaml] {
2753                            assert_eq!(
2754                                render(&compact, format, false).expect("render"),
2755                                render(&indexed, format, false).expect("render"),
2756                                "{label}: {format:?}"
2757                            );
2758                        }
2759
2760                        // What each case exists to exercise.
2761                        let Section::Summary(row) = indexed.sections[0] else {
2762                            panic!("{label}: a summary")
2763                        };
2764                        let crate::control::ControlCoverage::Observed(coverage) =
2765                            &indexed.ignore_rules
2766                        else {
2767                            panic!("{label}: the default scope observes .gitignore")
2768                        };
2769                        assert_eq!(coverage.refused, case.refused, "{label}");
2770                        assert_eq!(row.ignored.is_some(), case.share, "{label}");
2771                        assert_eq!(!indexed.status.errors.is_empty(), case.errors, "{label}");
2772                        if case.share && case.name.starts_with("rules") {
2773                            let ignored = row.ignored.expect("a share");
2774                            assert!(
2775                                ignored.files > 0 && ignored.files < row.files && ignored.dirs > 0,
2776                                "{label}: {ignored:?} of {row:?}"
2777                            );
2778                        }
2779                        if let Some(governs) = case.variant_governs {
2780                            assert_eq!(coverage.applied, u64::from(governs), "{label}");
2781                            assert_eq!(
2782                                row.ignored.map(|ignored| ignored.files),
2783                                Some(if governs { 1_200 } else { 0 }),
2784                                "{label}"
2785                            );
2786                        }
2787                        if case.name == CASE_BOTH {
2788                            assert_eq!(coverage.applied, 1, "{label}: the exact name alone");
2789                            assert_eq!(
2790                                row.ignored.map(|ignored| ignored.files),
2791                                Some(1),
2792                                "{label}: only `x.log` is ignored"
2793                            );
2794                        }
2795                        if !case.share {
2796                            assert!(
2797                                indexed
2798                                    .notes
2799                                    .iter()
2800                                    .any(|note| note.contains("could not be verified")),
2801                                "{label}: {:?}",
2802                                indexed.notes
2803                            );
2804                        }
2805                        compared += 1;
2806                    }
2807                }
2808            }
2809            assert!(compared > 0, "{} was compared", case.name);
2810        }
2811    }
2812
2813    #[test]
2814    fn compact_summary_never_creates_the_configured_snapshot() {
2815        let root = tempfile::tempdir().expect("tempdir");
2816        fs::write(root.path().join("payload"), b"payload").expect("file");
2817        let cache = root.path().join("must-not-exist.fdu");
2818
2819        let (report, pending, _) =
2820            prepared(root.path(), &blind(CachePolicy::Off, Some(cache.clone())), &summary_query())
2821                .expect("compact report");
2822        pending.join().expect("no pending compact save");
2823
2824        assert!(report.status.complete);
2825        assert!(!cache.exists());
2826    }
2827
2828    // The folded index (H172): every directory, and only the files a share can show.
2829
2830    /// A tree request at `share` of the `size` metric, every other bound the view's own.
2831    fn tree_query(
2832        views: Vec<ViewSpec>,
2833        format: crate::report_format::Format,
2834        share: &str,
2835        size: crate::query::SizeMetric,
2836    ) -> Query {
2837        let mut query = Query { views, format, ..Query::default() };
2838        query.selection.min_share =
2839            Some(crate::query::ShareThreshold::parse(share).expect("a share"));
2840        query.selection.size = size;
2841        query
2842    }
2843
2844    /// Each request the planner must answer from the full index, and each it may answer
2845    /// from a folded one, with the retention that one keeps.
2846    #[test]
2847    fn a_tree_takes_the_folded_index_only_when_the_request_proves_it_suffices() {
2848        use crate::query::{Bound, SizeMetric, SortKey};
2849        use crate::report_format::Format;
2850
2851        let off = config(CachePolicy::Off, None);
2852        let list = Query { views: vec![ViewSpec::List], ..Query::default() };
2853        let size = list.selection.size;
2854        let kept = |largest_files, size| RetainedState::Tree(TreeRetention { largest_files, size });
2855
2856        // `fdu PATH`: the list rendered as a tree, at the tree's own 1% share.
2857        assert_eq!(planned(&off, &list).retained, kept(100, size));
2858        // Every tree rendering, both tree views together, every sort a tree row carries, and
2859        // every display bound: none reads a file the share cannot show.
2860        let mut folded = vec![
2861            Query { format: Format::Tree, ..list.clone() },
2862            Query { views: vec![ViewSpec::List, ViewSpec::Tree], ..list.clone() },
2863        ];
2864        for format in [Format::Text, Format::Tree, Format::Json, Format::Jsonl, Format::Yaml] {
2865            folded.push(Query { views: vec![ViewSpec::Tree], format, ..Query::default() });
2866        }
2867        for sort in [SortKey::Size, SortKey::Name, SortKey::Mtime, SortKey::Count] {
2868            for reverse in [false, true] {
2869                let mut query = list.clone();
2870                query.selection.sort = Some(sort);
2871                query.selection.reverse = reverse;
2872                folded.push(query);
2873            }
2874        }
2875        for bound in [Bound::Limit(0), Bound::Limit(2), Bound::All] {
2876            let mut query = list.clone();
2877            query.selection.depth = Some(bound);
2878            query.selection.breadth = Some(bound);
2879            query.selection.limit = Some(bound);
2880            folded.push(query);
2881        }
2882        for query in &folded {
2883            assert_eq!(planned(&off, query).retained, kept(100, size), "{query:?}");
2884        }
2885        // The share decides how many files, and the metric which.
2886        for (share, largest_files) in
2887            [("10%", 10), ("0.5%", 200), ("3%", 34), ("0.01%", 10_000), ("100%", 1)]
2888        {
2889            for metric in [SizeMetric::Apparent, SizeMetric::Allocated] {
2890                let query = tree_query(vec![ViewSpec::List], Format::Text, share, metric);
2891                assert_eq!(planned(&off, &query).retained, kept(largest_files, metric), "{share}");
2892            }
2893        }
2894        // The ceiling is exact: `100 / 65,536` percent keeps 65,536 files, and any finer
2895        // share would keep more, so it takes the full index.
2896        let ceiling = tree_query(vec![ViewSpec::Tree], Format::Json, "0.00152587890625%", size);
2897        assert_eq!(planned(&off, &ceiling).retained, kept(TreeRetention::MAX_FILES, size));
2898        let past = tree_query(vec![ViewSpec::Tree], Format::Json, "0.0015258%", size);
2899        assert_eq!(planned(&off, &past).retained, RetainedState::FullIndex);
2900        // Neither the scan's reach nor whether it reads `.gitignore` changes what a tree
2901        // reads, and a delivery that keeps no snapshot keeps no index either.
2902        for fixture in [
2903            blind(CachePolicy::Off, None),
2904            OpenFixture {
2905                scan: ScanConfig { max_depth: Some(2), ..ScanConfig::default() },
2906                ..off.clone()
2907            },
2908            config(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))),
2909            config(CachePolicy::On, None),
2910        ] {
2911            assert_eq!(planned(&fixture, &list).retained, kept(100, size), "{fixture:?}");
2912        }
2913
2914        // What reads more than a tree: a flat inventory, extension or type tallies, or a
2915        // files view, even one rendered as a tree.
2916        let mut full = vec![
2917            Query { format: Format::Json, ..list.clone() },
2918            Query { views: vec![ViewSpec::Tree], format: Format::Paths, ..list.clone() },
2919            Query { views: vec![ViewSpec::Tree], format: Format::Long, ..list.clone() },
2920            Query { views: vec![ViewSpec::Files], format: Format::Tree, ..list.clone() },
2921            Query { views: vec![ViewSpec::List, ViewSpec::Types], ..list.clone() },
2922            Query { views: vec![ViewSpec::Tree, ViewSpec::Extensions], ..list.clone() },
2923            Query { views: vec![ViewSpec::Tree, ViewSpec::Summary], ..list.clone() },
2924            Query { views: vec![ViewSpec::Tree, ViewSpec::Files], ..list.clone() },
2925            Query { views: Vec::new(), ..list.clone() },
2926        ];
2927        // Any filter, which measures rows against what it selects rather than the root.
2928        let mut filtered = list.clone();
2929        filtered.selection.include.push(Pattern::parse("*.rs").expect("pattern"));
2930        full.push(filtered);
2931        let mut excluded = list.clone();
2932        excluded.selection.exclude.push(Pattern::parse("target").expect("pattern"));
2933        full.push(excluded);
2934        let mut sized = list.clone();
2935        sized.selection.min_size = Some(1);
2936        full.push(sized);
2937        let mut kinds = list.clone();
2938        kinds.selection.kinds = vec![EntryKind::Dir];
2939        full.push(kinds);
2940        let mut files = list.clone();
2941        files.selection.kinds = vec![EntryKind::File];
2942        full.push(files);
2943        let mut modified = list.clone();
2944        modified.selection.modified.since = Some(0);
2945        full.push(modified);
2946        let mut by_ignored = list.clone();
2947        by_ignored.selection.ignored = IgnoredEntries::Exclude;
2948        full.push(by_ignored);
2949        // `-d 2 -n 30 --sort size --min-size 300M` (fdu-6o5o): its bounds and sort fold
2950        // alone, and its minimum size does not.
2951        let mut reported = list.clone();
2952        reported.selection.min_size = Some(300 << 20);
2953        reported.selection.sort = Some(SortKey::Size);
2954        reported.selection.depth = Some(Bound::Limit(2));
2955        reported.selection.limit = Some(Bound::Limit(30));
2956        full.push(reported);
2957        // A zero share shows every file.
2958        full.push(tree_query(vec![ViewSpec::List], Format::Text, "0%", size));
2959        full.push(tree_query(vec![ViewSpec::Tree], Format::Json, "0.000%", size));
2960        for query in &full {
2961            assert_eq!(planned(&off, query).retained, RetainedState::FullIndex, "{query:?}");
2962        }
2963
2964        // Content analysis, a narrowed population, and deliveries about the snapshot.
2965        let cache = || Some(PathBuf::from("cache.fdu"));
2966        for fixture in [
2967            analyzing(off.clone()),
2968            stale(config(CachePolicy::Auto, cache())),
2969            config(CachePolicy::On, cache()),
2970        ] {
2971            assert_eq!(planned(&fixture, &list).retained, RetainedState::FullIndex, "{fixture:?}");
2972        }
2973        for population in [IgnoredEntries::Exclude, IgnoredEntries::Only] {
2974            let narrowed = OpenFixture {
2975                scan: ScanConfig { population, ..ScanConfig::default() },
2976                ..off.clone()
2977            };
2978            let mut query = list.clone();
2979            query.selection.ignored = population;
2980            assert_eq!(planned(&narrowed, &query).retained, RetainedState::FullIndex);
2981        }
2982        // Every route but a one-shot report keeps its index.
2983        let (request, delivery) = split(Path::new("."), &off, &list);
2984        for route in [Route::Retained, Route::Refresh, Route::Watch, Route::Opened] {
2985            let plan = plan(&request, &delivery, route).expect("a valid plan");
2986            assert_eq!(plan.retained, RetainedState::FullIndex, "{route:?}");
2987        }
2988    }
2989
2990    /// The request each surface builds for a bare `fdu PATH` (the command line in its flag
2991    /// spellings, Python in its field names) plans the folded tree under the deliveries it
2992    /// runs with. The plan is not observable in any output, so a change to `tree_for`,
2993    /// `is_unfiltered`, or a default that sends the default report back to the full index
2994    /// would otherwise show only in a measurement.
2995    #[test]
2996    fn the_default_request_of_every_surface_plans_the_folded_tree() {
2997        use crate::query::{AxisNames, ReadSpec, RequestSpec, SizeMetric};
2998        let root = Path::new(".");
2999        let default_tree =
3000            RetainedState::Tree(TreeRetention { largest_files: 100, size: SizeMetric::Allocated });
3001        let deliveries = [
3002            Delivery::new(CachePolicy::Auto, Some(PathBuf::from("cache.fdu"))),
3003            Delivery::new(CachePolicy::Off, None),
3004        ];
3005        let planned = |spec: &RequestSpec<'_>, axes, delivery: &Delivery| {
3006            let request =
3007                Request::build(spec, SystemTime::UNIX_EPOCH, axes).expect("a valid request");
3008            plan(&request, delivery, Route::OneShot).expect("a valid plan").retained
3009        };
3010        for axes in [&AxisNames::FLAGS, &AxisNames::FIELDS] {
3011            for delivery in &deliveries {
3012                assert_eq!(planned(&RequestSpec::new(root), axes, delivery), default_tree);
3013                // `--view tree` in a machine format is the same tree; the list in one is flat.
3014                let tree_json = RequestSpec {
3015                    read: ReadSpec { views: Some("tree"), format: Some("json"), ..ReadSpec::new() },
3016                    ..RequestSpec::new(root)
3017                };
3018                assert_eq!(planned(&tree_json, axes, delivery), default_tree);
3019                let list_json = RequestSpec {
3020                    read: ReadSpec { format: Some("json"), ..ReadSpec::new() },
3021                    ..RequestSpec::new(root)
3022                };
3023                assert_eq!(planned(&list_json, axes, delivery), RetainedState::FullIndex);
3024            }
3025        }
3026    }
3027
3028    /// A small deterministic generator (xorshift64*), so a randomized tree is the same
3029    /// tree on every run.
3030    struct Shuffle(u64);
3031
3032    impl Shuffle {
3033        fn below(&mut self, bound: usize) -> usize {
3034            self.0 ^= self.0 >> 12;
3035            self.0 ^= self.0 << 25;
3036            self.0 ^= self.0 >> 27;
3037            let drawn = self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 32;
3038            usize::try_from(drawn).expect("32 bits") % bound
3039        }
3040    }
3041
3042    /// A file of `size` bytes, written or left sparse, modified at `minute`.
3043    fn sized_file(path: &Path, size: u64, sparse: bool, minute: u64) {
3044        use std::io::Write as _;
3045        fs::create_dir_all(path.parent().expect("a parent")).expect("parent directories");
3046        let mut file = fs::File::create(path).expect("fixture file");
3047        if sparse {
3048            file.set_len(size).expect("sparse length");
3049        } else {
3050            file.write_all(&vec![b'.'; usize::try_from(size).expect("small")]).expect("contents");
3051        }
3052        file.set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(minute * 60))
3053            .expect("mtime");
3054    }
3055
3056    /// `files` files under a random hierarchy up to five deep: empty, small, many exactly
3057    /// alike, and large ones sparse so apparent and allocated order them differently; a
3058    /// few modification times, so a newest-first sort meets ties; `.gitignore` rules that
3059    /// ignore, re-include, and ignore a whole directory; and symlinks, which a tree counts
3060    /// in no row.
3061    fn random_tree(seed: u64, files: usize) -> tempfile::TempDir {
3062        let mut draw = Shuffle(seed);
3063        let root = tempfile::tempdir().expect("tempdir");
3064        let mut dirs = vec![PathBuf::new()];
3065        for index in 0..files.div_ceil(12) {
3066            let parent = dirs[draw.below(dirs.len())].clone();
3067            if parent.components().count() < 5 {
3068                let dir = parent.join(format!("d{index:03}"));
3069                fs::create_dir(root.path().join(&dir)).expect("directory");
3070                dirs.push(dir);
3071            }
3072        }
3073        let extensions = ["log", "txt", "bin", "tmp", "rs"];
3074        for index in 0..files {
3075            let dir = &dirs[draw.below(dirs.len())];
3076            let name = format!("f{index:05}.{}", extensions[draw.below(extensions.len())]);
3077            let (size, sparse) = match draw.below(20) {
3078                0..=2 => (0, false),
3079                3..=10 => (draw.below(300) as u64, false),
3080                11..=13 => (1_000, false),
3081                14..=17 => (draw.below(40_000) as u64, false),
3082                _ => (50_000 + draw.below(4_000_000) as u64, true),
3083            };
3084            let minute = FIXTURE_MINUTE + draw.below(4) as u64;
3085            sized_file(&root.path().join(dir).join(name), size, sparse, minute);
3086        }
3087        put(root.path(), ".gitignore", b"*.log\n!f000*.log\nd002/\n");
3088        if let Some(dir) = dirs.get(3) {
3089            put(&root.path().join(dir), ".gitignore", b"!*.log\n*.tmp\n");
3090        }
3091        #[cfg(unix)]
3092        for (index, dir) in dirs.iter().take(4).enumerate() {
3093            std::os::unix::fs::symlink(
3094                "elsewhere",
3095                root.path().join(dir).join(format!("l{index}")),
3096            )
3097            .expect("symlink");
3098        }
3099        crate::test_support::settle_allocations(root.path());
3100        root
3101    }
3102
3103    /// One tree built for the share boundary. Fields drop in order: a sealed directory is
3104    /// restored before the tree is removed.
3105    struct BoundaryTree {
3106        name: &'static str,
3107        #[cfg(unix)]
3108        _sealed: Option<Unlocked>,
3109        tree: tempfile::TempDir,
3110    }
3111
3112    /// Trees built for the share boundary, each with what it holds there.
3113    fn boundary_trees() -> Vec<BoundaryTree> {
3114        let mut trees = Vec::new();
3115
3116        // Nine files at exactly 10% of apparent bytes and two tied for the tenth place at
3117        // 5%, which a 10% tree keeps one of and shows neither of; in allocated bytes all
3118        // eleven tie. Two directories, so the tie spans parents.
3119        let ties = tempfile::tempdir().expect("tempdir");
3120        for index in 0..9 {
3121            sized_file(&ties.path().join(format!("a/at{index}")), 1_000, false, index);
3122        }
3123        sized_file(&ties.path().join("a/half"), 500, false, 20);
3124        sized_file(&ties.path().join("b/half"), 500, false, 20);
3125        trees.push(("ties at the retention bound", ties));
3126
3127        // Exactly as many files at 5% as a 5% tree keeps, and empty files beside them.
3128        let exact = tempfile::tempdir().expect("tempdir");
3129        for index in 0..20 {
3130            sized_file(&exact.path().join(format!("d{}/f{index:02}", index % 3)), 50, false, 1);
3131            sized_file(&exact.path().join(format!("d{}/e{index:02}", index % 4)), 0, false, 2);
3132        }
3133        trees.push(("exactly the files a share shows", exact));
3134
3135        // Nothing but empty files: the root total is zero, and no row reaches any share.
3136        let zero = tempfile::tempdir().expect("tempdir");
3137        for index in 0..60 {
3138            sized_file(&zero.path().join(format!("d{}/z{index:02}", index % 5)), 0, false, 3);
3139        }
3140        trees.push(("all files empty", zero));
3141
3142        let empty = tempfile::tempdir().expect("tempdir");
3143        fs::create_dir_all(empty.path().join("only/directories")).expect("directories");
3144        trees.push(("no files", empty));
3145
3146        let single = tempfile::tempdir().expect("tempdir");
3147        sized_file(&single.path().join("one"), 7, false, 4);
3148        trees.push(("one file", single));
3149
3150        // Ignored files at and just below 1%, an ignored directory of small files, and
3151        // enough unignored small files that most of both populations fold.
3152        let ignored = tempfile::tempdir().expect("tempdir");
3153        put(ignored.path(), ".gitignore", b"*.big\nbuild/\n");
3154        sized_file(&ignored.path().join("src/at.big"), 1_000, false, 5);
3155        sized_file(&ignored.path().join("src/below.big"), 999, false, 5);
3156        for index in 0..150 {
3157            sized_file(&ignored.path().join(format!("build/o{index:03}")), 30, false, 6);
3158            sized_file(&ignored.path().join(format!("src/s{index:03}.rs")), 38, false, 7);
3159            sized_file(&ignored.path().join(format!("src/x{index:03}.big")), 5, false, 8);
3160        }
3161        trees.push(("ignored files at the share boundary", ignored));
3162
3163        // Three paths to one file at exactly 10% each, which every route counts once per
3164        // path, beside seven other files at 10% and empty files a share omits.
3165        let linked = tempfile::tempdir().expect("tempdir");
3166        sized_file(&linked.path().join("a/original"), 1_000, false, 9);
3167        for link in ["b/link", "c/deeper/link"] {
3168            fs::create_dir_all(linked.path().join(link).parent().expect("a parent"))
3169                .expect("directories");
3170            fs::hard_link(linked.path().join("a/original"), linked.path().join(link))
3171                .expect("hard link");
3172        }
3173        for index in 0..7 {
3174            sized_file(&linked.path().join(format!("d/f{index}")), 1_000, false, 10);
3175        }
3176        for index in 0..30 {
3177            sized_file(&linked.path().join(format!("e/z{index:02}")), 0, false, 11);
3178        }
3179        trees.push(("hard links at the share boundary", linked));
3180
3181        // A FIFO and a socket, which are entries of no row, beside files a share omits.
3182        #[cfg(unix)]
3183        {
3184            let special = tempfile::tempdir().expect("tempdir");
3185            for index in 0..120 {
3186                sized_file(&special.path().join(format!("s/f{index:03}")), 10, false, 12);
3187            }
3188            sized_file(&special.path().join("s/large"), 5_000, false, 13);
3189            let fifo = special.path().join("s/pipe");
3190            let status =
3191                std::process::Command::new("mkfifo").arg(&fifo).status().expect("run mkfifo");
3192            assert!(status.success(), "mkfifo exited with {status}");
3193            drop(
3194                std::os::unix::net::UnixListener::bind(special.path().join("s/socket"))
3195                    .expect("bind socket"),
3196            );
3197            trees.push(("special files beside folded files", special));
3198        }
3199
3200        // Two names that differ only in an invalid byte are one name once made readable,
3201        // so rows of equal size and equal lossy name fall back to name order.
3202        #[cfg(target_os = "linux")]
3203        {
3204            use std::os::unix::ffi::OsStrExt as _;
3205            let lossy = tempfile::tempdir().expect("tempdir");
3206            for byte in [0xfe_u8, 0xff] {
3207                let bytes = [b't', b'i', b'e', byte];
3208                let name = std::ffi::OsStr::from_bytes(&bytes);
3209                sized_file(&lossy.path().join("n").join(name), 2_000, false, 14);
3210            }
3211            for index in 0..80 {
3212                sized_file(&lossy.path().join(format!("n/small{index:02}")), 3, false, 15);
3213            }
3214            trees.push(("names equal once made readable", lossy));
3215        }
3216        let trees: Vec<BoundaryTree> = trees
3217            .into_iter()
3218            .map(|(name, tree)| BoundaryTree {
3219                name,
3220                #[cfg(unix)]
3221                _sealed: None,
3222                tree,
3223            })
3224            .chain(search_denied_boundary_tree())
3225            .collect();
3226        for tree in &trees {
3227            crate::test_support::settle_allocations(tree.tree.path());
3228        }
3229        trees
3230    }
3231
3232    /// A directory that lists but refuses search, beside files a share omits: its
3233    /// subdirectory and symlink are named by `d_type` and must not be admitted on it.
3234    /// None where the host cannot deny search (Windows, or a root test process).
3235    #[cfg(unix)]
3236    fn search_denied_boundary_tree() -> Option<BoundaryTree> {
3237        if !crate::test_support::require_permission_bits() {
3238            return None;
3239        }
3240        let unsearchable = tempfile::tempdir().expect("tempdir");
3241        sized_file(&unsearchable.path().join("open/large"), 5_000, false, 18);
3242        for index in 0..60 {
3243            sized_file(&unsearchable.path().join(format!("open/s{index:02}")), 5, false, 19);
3244        }
3245        let sealed = seal_search_denied(unsearchable.path());
3246        Some(BoundaryTree {
3247            name: "a directory that lists but refuses search",
3248            _sealed: Some(sealed),
3249            tree: unsearchable,
3250        })
3251    }
3252
3253    #[cfg(not(unix))]
3254    fn search_denied_boundary_tree() -> Option<BoundaryTree> {
3255        None
3256    }
3257
3258    /// The minute [`random_tree`] and [`crowded_tree`] stamp their oldest files with; each
3259    /// stamps the next three as well.
3260    const FIXTURE_MINUTE: u64 = 28_000_000;
3261
3262    /// A tree on which a minimum size of 1,000 bytes reports neither the roll-ups a folded
3263    /// index keeps nor only the files it admits (fdu-6o5o).
3264    ///
3265    /// `hit/` matches by its subtree, so the selection holds everything in it, `hit/covered`
3266    /// included though that file is 900 bytes; at 1% of what the selection holds it is a
3267    /// row. The root holds a file on either side of the minimum, and each of 110 directories
3268    /// below the minimum holds a 950-byte file the selection never counts: larger than
3269    /// `hit/covered`, and more of them than a 1% tree keeps. `hit/many` holds more files
3270    /// the minimum admits than a 10% tree keeps. Ignored files in the root and below it,
3271    /// empty directories, and directories stamped apart give the other filters something
3272    /// to select.
3273    fn crowded_tree() -> tempfile::TempDir {
3274        let tree = tempfile::tempdir().expect("tempdir");
3275        let root = tree.path();
3276        let minute = |offset| FIXTURE_MINUTE + offset;
3277        put(root, ".gitignore", b"*.skip\n");
3278        sized_file(&root.join("hit/large"), 20_000, false, minute(3));
3279        sized_file(&root.join("hit/covered"), 900, false, minute(3));
3280        sized_file(&root.join("hit/x.skip"), 400, false, minute(2));
3281        for index in 0..30 {
3282            let path = root.join(format!("hit/many/m{index:02}.bin"));
3283            sized_file(&path, 1_000, false, minute(index % 4));
3284        }
3285        for index in 0..110 {
3286            sized_file(&root.join(format!("u{index:03}/f.rs")), 950, false, minute(1));
3287        }
3288        sized_file(&root.join("u000/y.skip"), 30, false, minute(1));
3289        sized_file(&root.join("top-big"), 1_500, false, minute(2));
3290        sized_file(&root.join("top-small"), 990, false, minute(0));
3291        sized_file(&root.join("top.skip"), 20, false, minute(1));
3292        for empty in ["empty", "hit/empty"] {
3293            fs::create_dir_all(root.join(empty)).expect("empty directory");
3294        }
3295        // A directory's own time counts in its subtree's newest activity, so each is
3296        // stamped once nothing more is written in it.
3297        #[cfg(unix)]
3298        {
3299            let mut stamps = vec![
3300                ("hit".to_string(), 3),
3301                ("hit/many".to_string(), 2),
3302                ("hit/empty".to_string(), 0),
3303                ("empty".to_string(), 0),
3304            ];
3305            stamps.extend((0..110).map(|index| (format!("u{index:03}"), 1)));
3306            for (directory, offset) in stamps {
3307                fs::File::open(root.join(directory))
3308                    .expect("open directory")
3309                    .set_modified(
3310                        std::time::UNIX_EPOCH + std::time::Duration::from_secs(minute(offset) * 60),
3311                    )
3312                    .expect("directory mtime");
3313            }
3314        }
3315        crate::test_support::settle_allocations(root);
3316        tree
3317    }
3318
3319    /// Worker counts and traversal orders the differential walks each tree under, and
3320    /// whether it compares every request there or the default ones.
3321    fn folded_walks(one_worker: bool) -> Vec<(usize, crate::ScanOrder, bool)> {
3322        use crate::ScanOrder::{BreadthFirst, DepthFirst};
3323        let mut walks = Vec::new();
3324        for threads in [1, 2, 4, 8] {
3325            if one_worker && threads != 1 {
3326                continue;
3327            }
3328            for order in [BreadthFirst, DepthFirst] {
3329                let every = matches!((threads, order), (1, BreadthFirst) | (8, DepthFirst))
3330                    || (one_worker && order == DepthFirst);
3331                walks.push((threads, order, every));
3332            }
3333        }
3334        walks
3335    }
3336
3337    /// The tree requests compared at one share and metric: the defaults, or every
3338    /// rendering, sort, and display bound a tree has.
3339    fn folded_queries(share: &str, size: crate::query::SizeMetric, every: bool) -> Vec<Query> {
3340        use crate::query::{Bound, SortKey};
3341        use crate::report_format::Format;
3342        let list = tree_query(vec![ViewSpec::List], Format::Text, share, size);
3343        let json = tree_query(vec![ViewSpec::Tree], Format::Json, share, size);
3344        let mut queries = vec![list.clone(), json.clone()];
3345        if !every {
3346            return queries;
3347        }
3348        queries.push(Query { format: Format::Tree, ..list.clone() });
3349        for format in [Format::Jsonl, Format::Yaml, Format::Text] {
3350            queries.push(Query { format, ..json.clone() });
3351        }
3352        queries.push(Query { views: vec![ViewSpec::List, ViewSpec::Tree], ..list.clone() });
3353        for sort in [SortKey::Size, SortKey::Name, SortKey::Mtime, SortKey::Count] {
3354            for reverse in [false, true] {
3355                let mut query = list.clone();
3356                query.selection.sort = Some(sort);
3357                query.selection.reverse = reverse;
3358                queries.push(query);
3359            }
3360        }
3361        let bounds: [(Option<Bound>, Option<Bound>, Option<Bound>); 12] = [
3362            (Some(Bound::Limit(0)), None, None),
3363            (None, Some(Bound::Limit(0)), None),
3364            (Some(Bound::Limit(0)), Some(Bound::Limit(0)), None),
3365            (Some(Bound::Limit(1)), None, None),
3366            (Some(Bound::Limit(2)), None, None),
3367            (Some(Bound::All), None, None),
3368            (None, Some(Bound::Limit(1)), None),
3369            (Some(Bound::All), Some(Bound::Limit(3)), None),
3370            (None, None, Some(Bound::Limit(0))),
3371            (None, None, Some(Bound::Limit(1))),
3372            (None, None, Some(Bound::Limit(5))),
3373            (Some(Bound::All), Some(Bound::Limit(2)), Some(Bound::Limit(7))),
3374        ];
3375        for (depth, breadth, limit) in bounds {
3376            let mut query = json.clone();
3377            query.selection.depth = depth;
3378            query.selection.breadth = breadth;
3379            query.selection.limit = limit;
3380            queries.push(query.clone());
3381            query.selection.sort = Some(SortKey::Name);
3382            queries.push(query);
3383        }
3384        queries
3385    }
3386
3387    /// The shares every tree is compared at: the default, finer and coarser ones, one
3388    /// that divides 100 unevenly, the whole, and the ceiling of what a folded index keeps.
3389    const FOLDED_SHARES: [&str; 8] =
3390        ["1%", "0.5%", "5%", "10%", "0.01%", "33%", "100%", "0.00152587890625%"];
3391
3392    /// For every walk of `root` under `scan`, every share and metric, and every request in
3393    /// [`folded_queries`], the report from the folded index the planner chooses is the
3394    /// report from the full index of the same walk: as a value, and rendered. Returns
3395    /// whether any folded index kept fewer entries than the full one, so a tree that stopped
3396    /// exercising the fold is noticed rather than agreeing vacuously.
3397    fn assert_folded_reports_match(
3398        root: &Path,
3399        scan: &ScanConfig,
3400        shares: &[&str],
3401        one_worker: bool,
3402        label: &str,
3403    ) -> bool {
3404        use crate::query::SizeMetric;
3405        let canonical = root.canonicalize().expect("canonical root");
3406        let mut folded_any = false;
3407        for (threads, order, every) in folded_walks(one_worker) {
3408            let fixture = OpenFixture {
3409                scan: ScanConfig { threads: Some(threads), order, ..scan.clone() },
3410                ..config(CachePolicy::Off, None)
3411            };
3412            let mut full = None;
3413            // Every share where every request is compared; the first four elsewhere.
3414            for share in shares.iter().take(if every { shares.len() } else { 4 }) {
3415                for size in [SizeMetric::Apparent, SizeMetric::Allocated] {
3416                    let mut folded: Option<(TreeRetention, crate::Index)> = None;
3417                    for query in folded_queries(share, size, every) {
3418                        let (request, delivery) = split(root, &fixture, &query);
3419                        let scan_config = request.basis.scope.scan_config(&delivery);
3420                        let RetainedState::Tree(retention) =
3421                            plan(&request, &delivery, Route::OneShot).expect("plan").retained
3422                        else {
3423                            panic!("{label}: {query:?} takes the full index")
3424                        };
3425                        let full = full.get_or_insert_with(|| {
3426                            let (index, _) = crate::scan::scan_into_index(&canonical, &scan_config)
3427                                .expect("full index");
3428                            index
3429                        });
3430                        if folded.as_ref().is_none_or(|(held, _)| *held != retention) {
3431                            let (index, _, _) = crate::scan::scan_into_folded_index(
3432                                &canonical,
3433                                &scan_config,
3434                                retention,
3435                                false,
3436                            )
3437                            .expect("folded index");
3438                            assert!(index.is_folded() && !full.is_folded(), "{label}");
3439                            folded_any |= index.len() < full.len();
3440                            folded = Some((retention, index));
3441                        }
3442                        let (_, index) = folded.as_ref().expect("a folded index");
3443                        let mut from_folded =
3444                            report(index, &request, SystemTime::UNIX_EPOCH).expect("folded report");
3445                        let from_full =
3446                            report(full, &request, SystemTime::UNIX_EPOCH).expect("full report");
3447                        // Provenance describes each walk; everything else describes the tree.
3448                        assert_eq!(
3449                            (from_folded.provenance.source, from_folded.provenance.freshness),
3450                            (from_full.provenance.source, from_full.provenance.freshness),
3451                            "{label}"
3452                        );
3453                        from_folded.provenance = from_full.provenance.clone();
3454                        assert_eq!(
3455                            format!("{from_folded:#?}"),
3456                            format!("{from_full:#?}"),
3457                            "{label} ({threads} workers, {order:?}, {share} of {size:?}): {query:?}"
3458                        );
3459                        assert_eq!(
3460                            crate::report_format::render(&from_folded, query.format, false)
3461                                .expect("render"),
3462                            crate::report_format::render(&from_full, query.format, false)
3463                                .expect("render"),
3464                            "{label} ({threads} workers, {order:?}, {share} of {size:?}): {query:?}"
3465                        );
3466                    }
3467                }
3468            }
3469        }
3470        folded_any
3471    }
3472
3473    /// The one-shot route end to end: `prepare_report` plans the folded index for the
3474    /// default tree, and answers exactly what the full index answers under a delivery that
3475    /// requires one, with the same walked totals.
3476    fn assert_folded_route_matches(root: &Path, scan: &ScanConfig, label: &str) {
3477        use crate::report_format::{Format, render};
3478        let list = Query { views: vec![ViewSpec::List], ..Query::default() };
3479        for threads in [Some(1), Some(4)] {
3480            let scan = ScanConfig { threads, ..scan.clone() };
3481            let transient = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
3482            assert!(matches!(planned(&transient, &list).retained, RetainedState::Tree(_)));
3483            let (mut folded, pending, folded_performance) =
3484                prepared(root, &transient, &list).expect("folded report");
3485            pending.join().expect("the folded route saves nothing");
3486
3487            let cache = tempfile::tempdir().expect("cache dir");
3488            let indexed = OpenFixture {
3489                scan: scan.clone(),
3490                ..config(CachePolicy::On, Some(cache.path().join("snapshot.fdu")))
3491            };
3492            assert_eq!(planned(&indexed, &list).retained, RetainedState::FullIndex);
3493            let (indexed, pending, indexed_performance) =
3494                prepared(root, &indexed, &list).expect("indexed report");
3495            pending.join().expect("save");
3496
3497            assert_eq!(folded_performance, indexed_performance, "{label}");
3498            assert_eq!(folded.provenance.source, indexed.provenance.source, "{label}");
3499            assert_eq!(folded.provenance.freshness, indexed.provenance.freshness, "{label}");
3500            folded.provenance = indexed.provenance.clone();
3501            assert_eq!(format!("{folded:#?}"), format!("{indexed:#?}"), "{label}");
3502            for format in [Format::Text, Format::Json, Format::Yaml] {
3503                assert_eq!(
3504                    render(&folded, format, false).expect("render"),
3505                    render(&indexed, format, false).expect("render"),
3506                    "{label}: {format:?}"
3507                );
3508            }
3509        }
3510    }
3511
3512    /// A tree from the folded index is the tree from the full index, whole, wherever the
3513    /// planner chooses the folded one: randomized and boundary-built trees, and every control
3514    /// case the summary differential uses (refusals by budget and line limit, an unreadable
3515    /// control file where the host enforces permissions, case variants, hidden pruning, a
3516    /// scan-depth bound whose subtrees are incomplete); at one, two, four, and eight workers
3517    /// and both traversal orders; at the default, finer, coarser, uneven, whole, and ceiling
3518    /// shares; in both size metrics; sorted by every key a tree row carries, both ways; at
3519    /// every depth, breadth, and row bound; in every rendering. Ties at the retention bound,
3520    /// empty files, an empty tree, and ignored files at the share are among the trees.
3521    #[test]
3522    fn transient_tree_equals_the_indexed_tree_under_every_bound_case() {
3523        let mut folded_trees = 0;
3524        for (seed, files, scan) in [
3525            (0x9E37_79B9_7F4A_7C15_u64, 480, ScanConfig::default()),
3526            (0xD1B5_4A32_D192_ED03, 700, ScanConfig::default()),
3527            (
3528                0x2545_F491_4F6C_DD1D,
3529                360,
3530                ScanConfig { max_depth: Some(3), ..ScanConfig::default() },
3531            ),
3532            (
3533                0x94D0_49BB_1331_11EB,
3534                420,
3535                ScanConfig { read_controls: false, ..ScanConfig::default() },
3536            ),
3537            // The folded walk takes directory kinds from the listing (H185) except under
3538            // `--one-filesystem`, where descent reads each directory's device. Only Unix has
3539            // that identity: elsewhere the scope is refused before any walk, as
3540            // `ScanConfig::unsupported_axis` states, so there is no walk to compare.
3541            #[cfg(unix)]
3542            (
3543                0x7C15_9E37_79B9_7F4A,
3544                390,
3545                ScanConfig { one_filesystem: true, ..ScanConfig::default() },
3546            ),
3547        ] {
3548            let tree = random_tree(seed, files);
3549            let label = format!("random tree {seed:#x} of {files} files");
3550            assert!(
3551                assert_folded_reports_match(tree.path(), &scan, &FOLDED_SHARES, false, &label),
3552                "{label} folds at some share"
3553            );
3554            assert_folded_route_matches(tree.path(), &scan, &label);
3555            folded_trees += 1;
3556        }
3557
3558        for boundary in boundary_trees() {
3559            let (name, tree) = (boundary.name, boundary.tree.path());
3560            let scan = ScanConfig::default();
3561            let folds = assert_folded_reports_match(tree, &scan, &FOLDED_SHARES, false, name);
3562            assert_eq!(
3563                folds,
3564                !matches!(name, "no files" | "one file"),
3565                "{name}: whether the tree has files a share omits"
3566            );
3567            assert_folded_route_matches(tree, &scan, name);
3568            folded_trees += usize::from(folds);
3569        }
3570
3571        // A directory whose listing failed, beside and above directories with folded files:
3572        // the walk is partial, the subtree measurements run over the folded index, and the
3573        // unlisted directory is shown whatever its share.
3574        #[cfg(unix)]
3575        if crate::test_support::require_permission_bits() {
3576            use std::os::unix::fs::PermissionsExt;
3577            let tree = random_tree(0x1405_7B7E_F767_814F, 300);
3578            let locked = tree.path().join("locked");
3579            sized_file(&locked.join("hidden-large"), 900_000, false, 16);
3580            sized_file(&locked.join("inner/hidden-small"), 4, false, 16);
3581            for index in 0..60 {
3582                sized_file(&tree.path().join(format!("beside/b{index:02}")), 7, false, 17);
3583            }
3584            crate::test_support::settle_allocations(tree.path());
3585            fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock");
3586            let _unlocked = Unlocked(locked);
3587            let (index, _) = crate::scan::scan_into_index(tree.path(), &ScanConfig::default())
3588                .expect("a partial index");
3589            assert_ne!(index.state().coverage, crate::Coverage::Complete, "the listing failed");
3590            let label = "a failed listing beside folded files";
3591            assert!(
3592                assert_folded_reports_match(
3593                    tree.path(),
3594                    &ScanConfig::default(),
3595                    &FOLDED_SHARES,
3596                    false,
3597                    label
3598                ),
3599                "{label}"
3600            );
3601            assert_folded_route_matches(tree.path(), &ScanConfig::default(), label);
3602            folded_trees += 1;
3603        }
3604
3605        for case in control_cases() {
3606            let _lookups = case.lookups.install(case.root.path());
3607            let label = format!("{} ({:?} lookups)", case.name, case.lookups);
3608            let folds = assert_folded_reports_match(
3609                case.root.path(),
3610                &case.scan,
3611                &["1%", "10%", "0.5%"],
3612                case.order_dependent,
3613                &label,
3614            );
3615            if !case.order_dependent {
3616                assert_folded_route_matches(case.root.path(), &case.scan, &label);
3617            }
3618            folded_trees += usize::from(folds);
3619        }
3620
3621        // Enough files that the finest share compared still folds some: ten thousand kept
3622        // at 0.01%, of twelve thousand.
3623        let large = random_tree(0x6C8E_9CF5_7093_2BD5, 12_000);
3624        let label = "a tree larger than 0.01% keeps";
3625        assert!(
3626            assert_folded_reports_match(
3627                large.path(),
3628                &ScanConfig::default(),
3629                &["0.01%"],
3630                true,
3631                label
3632            ),
3633            "{label}"
3634        );
3635        assert!(folded_trees > 10, "the differential folded {folded_trees} trees");
3636    }
3637
3638    /// Why a filtered tree takes the full index (fdu-6o5o): a filter measures its rows, and
3639    /// the share that bounds them, against what it selects, which neither the roll-ups nor
3640    /// the largest files a folded index keeps can state.
3641    ///
3642    /// Under a minimum size the root is the sum of what the minimum selects rather than the
3643    /// root's roll-up, and a directory below the minimum is no row. A directory at or above
3644    /// it selects every file beneath it, so `hit/covered`, below the minimum, is a row: a
3645    /// fold that ranked only the files the minimum admits would drop it, and so would one
3646    /// that ranked every file, since 111 files the selection never counts are larger.
3647    #[test]
3648    fn a_filtered_tree_is_measured_against_what_it_selects() {
3649        use crate::query::{SizeMetric, TreeNode};
3650        use crate::report_format::Format;
3651        let tree = crowded_tree();
3652        let off = config(CachePolicy::Off, None);
3653        let unfiltered = tree_query(vec![ViewSpec::Tree], Format::Json, "1%", SizeMetric::Apparent);
3654        let mut filtered = unfiltered.clone();
3655        filtered.selection.min_size = Some(1_000);
3656        assert_eq!(planned(&off, &filtered).retained, RetainedState::FullIndex);
3657        let root_of = |query: &Query| {
3658            let (report, pending, _) = prepared(tree.path(), &off, query).expect("a report");
3659            pending.join().expect("a one-shot report saves nothing");
3660            match report.sections.into_iter().next() {
3661                Some(Section::Tree { root: Some(root), .. }) => *root,
3662                other => panic!("expected a tree, got {other:?}"),
3663            }
3664        };
3665        let rows = |node: &TreeNode| {
3666            node.children.iter().map(|row| (row.name.clone(), row.bytes)).collect::<Vec<_>>()
3667        };
3668        let named = |rows: &[(&str, u64)]| {
3669            rows.iter().map(|(name, bytes)| ((*name).to_string(), *bytes)).collect::<Vec<_>>()
3670        };
3671
3672        let selected = root_of(&filtered);
3673        // `hit/` and `top-big`; the root's roll-up also holds `top-small`, `top.skip`, the
3674        // 950-byte files, `u000/y.skip`, and the control file.
3675        assert_eq!(selected.bytes, 20_000 + 900 + 400 + 30 * 1_000 + 1_500);
3676        assert_eq!(root_of(&unfiltered).bytes, selected.bytes + 990 + 20 + 110 * 950 + 30 + 7);
3677        assert_eq!(rows(&selected), named(&[("hit", 51_300), ("top-big", 1_500)]));
3678        assert_eq!(
3679            rows(&selected.children[0]),
3680            named(&[("many", 30_000), ("large", 20_000), ("covered", 900)])
3681        );
3682    }
3683
3684    /// One selection by each filter a one-shot tree takes, and compositions of them.
3685    ///
3686    /// A maximum size filters opened-root reads alone ([`crate::query::EntrySelection`]), so
3687    /// no one-shot tree has one to compare.
3688    fn filtered_selections() -> Vec<(&'static str, crate::query::Selection)> {
3689        use crate::query::{ModifiedWindow, Selection};
3690        let at = |offset: u64| {
3691            i64::try_from((FIXTURE_MINUTE + offset) * 60 * 1_000_000_000).expect("nanoseconds")
3692        };
3693        let globs = |sources: &[&str]| -> Vec<Pattern> {
3694            sources.iter().map(|source| Pattern::parse(source).expect("pattern")).collect()
3695        };
3696        let window = |since, before| ModifiedWindow { since, before };
3697        let none = Selection::default;
3698        vec![
3699            ("a minimum size", Selection { min_size: Some(1_000), ..none() }),
3700            ("a minimum every nonempty file passes", Selection { min_size: Some(1), ..none() }),
3701            ("included names", Selection { include: globs(&["*.rs", "*.skip"]), ..none() }),
3702            ("included directories", Selection { include: globs(&["hit", "d00*"]), ..none() }),
3703            ("excluded names", Selection { exclude: globs(&["*.log", "*.skip"]), ..none() }),
3704            (
3705                "excluded directories",
3706                Selection { exclude: globs(&["u0*", "d001", "many"]), ..none() },
3707            ),
3708            (
3709                "included and excluded",
3710                Selection {
3711                    include: globs(&["*.rs", "hit"]),
3712                    exclude: globs(&["many", "*.tmp"]),
3713                    ..none()
3714                },
3715            ),
3716            ("modified since", Selection { modified: window(Some(at(2)), None), ..none() }),
3717            ("modified before", Selection { modified: window(None, Some(at(2))), ..none() }),
3718            (
3719                "a modified window",
3720                Selection { modified: window(Some(at(1)), Some(at(3))), ..none() },
3721            ),
3722            ("files", Selection { kinds: vec![EntryKind::File], ..none() }),
3723            ("directories", Selection { kinds: vec![EntryKind::Dir], ..none() }),
3724            ("symlinks", Selection { kinds: vec![EntryKind::Symlink], ..none() }),
3725            ("unignored entries", Selection { ignored: IgnoredEntries::Exclude, ..none() }),
3726            ("ignored entries", Selection { ignored: IgnoredEntries::Only, ..none() }),
3727            (
3728                "a minimum size among included names",
3729                Selection {
3730                    min_size: Some(1_000),
3731                    include: globs(&["*.bin", "*.rs", "top-*"]),
3732                    ..none()
3733                },
3734            ),
3735        ]
3736    }
3737
3738    /// The tree requests one selection is compared under at one share and metric: the
3739    /// list and the tree view, each in the default and in name order; the reporter's
3740    /// `-d 2 -n 30 --sort size` (fdu-6o5o); and a reversed name order cut by depth and
3741    /// breadth.
3742    fn filtered_queries(
3743        selection: &crate::query::Selection,
3744        share: &str,
3745        size: crate::query::SizeMetric,
3746    ) -> Vec<Query> {
3747        use crate::query::{Bound, SortKey};
3748        use crate::report_format::Format;
3749        let filtered = |views, format| {
3750            let mut query = tree_query(views, format, share, size);
3751            query.selection = crate::query::Selection {
3752                min_share: query.selection.min_share.clone(),
3753                size,
3754                ..selection.clone()
3755            };
3756            query
3757        };
3758        let list = filtered(vec![ViewSpec::List], Format::Text);
3759        let json = filtered(vec![ViewSpec::Tree], Format::Json);
3760        let mut queries = vec![list.clone(), json.clone()];
3761        for query in [&list, &json] {
3762            let mut named = query.clone();
3763            named.selection.sort = Some(SortKey::Name);
3764            queries.push(named);
3765        }
3766        let mut reported = list;
3767        reported.selection.sort = Some(SortKey::Size);
3768        reported.selection.depth = Some(Bound::Limit(2));
3769        reported.selection.limit = Some(Bound::Limit(30));
3770        queries.push(reported);
3771        let mut bounded = json;
3772        bounded.selection.sort = Some(SortKey::Name);
3773        bounded.selection.reverse = true;
3774        bounded.selection.depth = Some(Bound::Limit(1));
3775        bounded.selection.breadth = Some(Bound::Limit(2));
3776        queries.push(bounded);
3777        queries
3778    }
3779
3780    /// `actual` is `expected`, as a value and rendered, once the provenance that describes
3781    /// each walk rather than the tree is set aside.
3782    fn assert_same_answer(
3783        mut actual: Report,
3784        expected: &Report,
3785        format: crate::report_format::Format,
3786        context: &str,
3787    ) {
3788        use crate::report_format::render;
3789        assert_eq!(
3790            (actual.provenance.source, actual.provenance.freshness),
3791            (expected.provenance.source, expected.provenance.freshness),
3792            "{context}"
3793        );
3794        actual.provenance = expected.provenance.clone();
3795        assert_eq!(format!("{actual:#?}"), format!("{expected:#?}"), "{context}");
3796        assert_eq!(
3797            render(&actual, format, false).expect("render"),
3798            render(expected, format, false).expect("render"),
3799            "{context}"
3800        );
3801    }
3802
3803    /// For every selection in [`filtered_selections`], at two shares that omit rows, in
3804    /// both metrics, and for every request in [`filtered_queries`]: the one-shot route the
3805    /// planner chooses answers what the full index of the tree answers, or both refuse; and
3806    /// wherever that route is a folded index, the folded index answers the same. Each
3807    /// selection must change the tree's answer, so that no comparison agrees for want of a
3808    /// filter.
3809    fn assert_filtered_reports_match(root: &Path, scan: &ScanConfig, label: &str) {
3810        use crate::query::{Selection, SizeMetric};
3811        use crate::report_format::Format;
3812        let root = root.canonicalize().expect("canonical root");
3813        let fixture = OpenFixture { scan: scan.clone(), ..config(CachePolicy::Off, None) };
3814        let scan_config = {
3815            let (request, delivery) = split(&root, &fixture, &Query::default());
3816            request.basis.scope.scan_config(&delivery)
3817        };
3818        let (full, _) = crate::scan::scan_into_index(&root, &scan_config).expect("full index");
3819        let answer = |query: &Query| {
3820            let (request, _) = split(&root, &fixture, query);
3821            report(&full, &request, SystemTime::UNIX_EPOCH)
3822        };
3823        let whole = tree_query(vec![ViewSpec::Tree], Format::Json, "0%", SizeMetric::Apparent);
3824        let unfiltered = format!("{:#?}", answer(&whole).expect("unfiltered report").sections);
3825        let mut vacuous = Vec::new();
3826        for (name, selection) in filtered_selections() {
3827            // A selection by ignored state over a tree whose `.gitignore` went unread has no
3828            // answer on any route.
3829            let refused = !scan.read_controls && selection.ignored != IgnoredEntries::Include;
3830            let every_row = Query {
3831                selection: Selection {
3832                    min_share: whole.selection.min_share.clone(),
3833                    size: whole.selection.size,
3834                    ..selection.clone()
3835                },
3836                ..whole.clone()
3837            };
3838            match answer(&every_row) {
3839                Ok(filtered) => {
3840                    assert!(!refused, "{label}: {name} is answered");
3841                    if format!("{:#?}", filtered.sections) == unfiltered {
3842                        vacuous.push(name);
3843                    }
3844                }
3845                Err(error) => assert!(refused, "{label}: {name}: {error}"),
3846            }
3847            for share in ["1%", "10%"] {
3848                for size in [SizeMetric::Apparent, SizeMetric::Allocated] {
3849                    for query in filtered_queries(&selection, share, size) {
3850                        let context = format!("{label}, {name}, {share} of {size:?}: {query:?}");
3851                        let (request, delivery) = split(&root, &fixture, &query);
3852                        let (expected, routed) =
3853                            match (answer(&query), prepare_report(&request, &delivery)) {
3854                                (Ok(expected), Ok((routed, pending, _))) => {
3855                                    pending.join().expect("a one-shot report saves nothing");
3856                                    (expected, routed)
3857                                }
3858                                (Err(_), Err(_)) if refused => continue,
3859                                (expected, routed) => panic!(
3860                                    "{context}: {:?} beside {:?}",
3861                                    expected.err(),
3862                                    routed.err()
3863                                ),
3864                            };
3865                        assert_same_answer(routed, &expected, query.format, &context);
3866                        let RetainedState::Tree(retention) =
3867                            plan(&request, &delivery, Route::OneShot).expect("plan").retained
3868                        else {
3869                            continue;
3870                        };
3871                        let (index, _, _) = crate::scan::scan_into_folded_index(
3872                            &root,
3873                            &scan_config,
3874                            retention,
3875                            false,
3876                        )
3877                        .expect("folded index");
3878                        assert!(index.is_folded(), "{context}");
3879                        let folded =
3880                            report(&index, &request, SystemTime::UNIX_EPOCH).expect("folded");
3881                        assert_same_answer(folded, &expected, query.format, &context);
3882                    }
3883                }
3884            }
3885        }
3886        assert!(vacuous.is_empty(), "{label}: these select the whole tree: {vacuous:?}");
3887    }
3888
3889    /// A filtered tree answers on the one-shot route what the full index answers, under
3890    /// every filter a one-shot tree takes, alone and composed: a minimum size, names and
3891    /// directories included and excluded, a modified window open at either end and closed,
3892    /// entry kinds, and selection by ignored state; in the default and in name order, cut
3893    /// by depth, breadth, and rows, at shares that omit rows, in both metrics. The trees
3894    /// are a randomized one, read with and without `.gitignore` and bounded in scan depth,
3895    /// and [`crowded_tree`], where more files pass a filter than a folded tree keeps and a
3896    /// fold that ranked only those would lose a row. Wherever the planner folds a filtered
3897    /// request, the folded index must answer the same.
3898    #[test]
3899    fn a_filtered_tree_answers_as_the_full_index_on_every_route() {
3900        let random = random_tree(0x9E37_79B9_7F4A_7C15, 480);
3901        let crowded = crowded_tree();
3902        let deep = |max_depth| ScanConfig { max_depth: Some(max_depth), ..ScanConfig::default() };
3903        let blind = ScanConfig { read_controls: false, ..ScanConfig::default() };
3904        for (tree, scan, label) in [
3905            (&random, ScanConfig::default(), "a random tree"),
3906            (&random, deep(3), "a random tree scanned three deep"),
3907            (&random, blind, "a random tree read without .gitignore"),
3908            (&crowded, ScanConfig::default(), "the crowded tree"),
3909            (&crowded, deep(1), "the crowded tree scanned one deep"),
3910        ] {
3911            assert_filtered_reports_match(tree.path(), &scan, label);
3912        }
3913    }
3914
3915    #[test]
3916    fn full_index_report_exposes_scan_diagnostics_when_requested() {
3917        let root = tempfile::tempdir().expect("tempdir");
3918        fs::create_dir(root.path().join("nested")).expect("directory");
3919        fs::write(root.path().join("nested/file.txt"), b"trace me").expect("file");
3920        let query = Query { views: vec![ViewSpec::Tree], ..Query::default() };
3921
3922        let (report, pending, performance, diagnostics) =
3923            prepared_with_diagnostics(root.path(), &config(CachePolicy::Off, None), &query)
3924                .expect("full-index report");
3925        pending.join().expect("no pending save");
3926
3927        assert!(report.status.complete);
3928        assert_eq!(performance.walked_files, 1);
3929        let diagnostics = diagnostics.expect("full-index scan diagnostics");
3930        assert_eq!(diagnostics.schema, crate::scan::SCAN_DIAGNOSTICS_SCHEMA);
3931        assert_eq!(diagnostics.worker_policy.ready_directories_at_finish, 0);
3932        assert_eq!(diagnostics.worker_policy.in_flight_directories_at_finish, 0);
3933    }
3934
3935    /// A tree of `dirs` directories under the root, each holding `files` files of
3936    /// distinct sizes, with its file count and byte total.
3937    fn wide_tree(dirs: usize, files: usize) -> (tempfile::TempDir, u64, u64) {
3938        let root = tempfile::tempdir().expect("tempdir");
3939        let mut bytes = 0;
3940        for directory in 0..dirs {
3941            let path = root.path().join(format!("d{directory:03}"));
3942            fs::create_dir(&path).expect("directory");
3943            for file in 0..files {
3944                let size = directory * files + file + 1;
3945                fs::write(path.join(format!("f{file}.txt")), vec![b'.'; size]).expect("file");
3946                bytes += size as u64;
3947            }
3948        }
3949        (root, (dirs * files) as u64, bytes)
3950    }
3951
3952    /// [`prepared`], reporting through `progress`.
3953    fn prepared_with_progress(
3954        root: &Path,
3955        config: &OpenFixture,
3956        query: &Query,
3957        progress: &Progress,
3958    ) -> Result<(Report, PendingSave, PerformanceSummary)> {
3959        let (request, delivery) = split(root, config, query);
3960        prepare_report_with_progress(&request, &delivery, progress)
3961    }
3962
3963    fn analyzing(fixture: OpenFixture) -> OpenFixture {
3964        OpenFixture {
3965            analysis: crate::content::AnalysisRequest {
3966                profile: crate::content::AnalysisSet::LINES_ONLY,
3967                workers: 0,
3968            },
3969            ..fixture
3970        }
3971    }
3972
3973    /// The invariant the plan makes testable: when a route completes, the handle's
3974    /// files, bytes, and allocated bytes equal the walked totals the route's own
3975    /// performance summary reports, and its directories equal the directories the route
3976    /// read. The allocated figure is also the answer's own total, which is what lets a
3977    /// display put it beside the answer. Every one-shot route: the cold full index, the
3978    /// transient summary fold, a cold run with content analysis, and a warm revalidation
3979    /// of the snapshot that run left.
3980    #[test]
3981    fn progress_ends_at_the_walked_totals_of_every_one_shot_route() {
3982        use crate::ProgressPhase::{Scanning, Summarizing};
3983        let (root, files, bytes) = wide_tree(6, 4);
3984        let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
3985
3986        let progress = Progress::new();
3987        let (_, pending, performance) =
3988            prepared_with_progress(root.path(), &config(CachePolicy::Off, None), &tree, &progress)
3989                .expect("cold full-index report");
3990        pending.join().expect("no save");
3991        let snapshot = progress.snapshot();
3992        assert_eq!(performance.source, ReportSource::ColdScan);
3993        assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
3994        assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "cold full index");
3995        assert_eq!(snapshot.allocated, performance.walked_allocated);
3996        assert!(snapshot.allocated > 0, "files with content occupy blocks");
3997        assert_eq!(snapshot.directories, 7, "the root and its six children");
3998        assert_eq!(
3999            (snapshot.phase, snapshot.analysis),
4000            (Summarizing, None),
4001            "the walk ended, the index was assembled, then the answer was built"
4002        );
4003
4004        let progress = Progress::new();
4005        let (report, pending, performance) = prepared_with_progress(
4006            root.path(),
4007            &blind(CachePolicy::Off, None),
4008            &summary_query(),
4009            &progress,
4010        )
4011        .expect("compact summary report");
4012        pending.join().expect("no save");
4013        let Section::Summary(row) = report.sections[0] else { panic!("summary section") };
4014        let snapshot = progress.snapshot();
4015        assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
4016        assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "summary fold");
4017        assert_eq!(snapshot.allocated, performance.walked_allocated);
4018        assert_eq!(snapshot.allocated, row.allocated, "the progress figure is the answer's");
4019        assert_eq!(snapshot.directories, row.dirs + 1, "the row's directories and the root");
4020        assert_eq!((snapshot.phase, snapshot.analysis), (Scanning, None));
4021
4022        // Outside the tree: a cache inside it is two more files for the warm walk.
4023        let cache_dir = tempfile::tempdir().expect("cache dir");
4024        let cache = cache_dir.path().join("snapshot.fdu");
4025        let progress = Progress::new();
4026        let (_, pending, performance) = prepared_with_progress(
4027            root.path(),
4028            &analyzing(config(CachePolicy::Auto, Some(cache.clone()))),
4029            &tree,
4030            &progress,
4031        )
4032        .expect("cold analyzed report");
4033        let snapshot = progress.snapshot();
4034        assert_eq!(snapshot.phase, Summarizing, "the answer is built while the save runs");
4035        pending.join().expect("save");
4036        assert_eq!(performance.source, ReportSource::ColdScan);
4037        assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "cold with analysis");
4038        assert_eq!(snapshot.allocated, performance.walked_allocated);
4039        assert_eq!(snapshot.directories, 7);
4040        assert_eq!(performance.fresh_files, files, "every file is a lines candidate");
4041        assert_eq!(snapshot.analysis, Some((files, files)));
4042
4043        let progress = Progress::new();
4044        let (_, pending, performance) = prepared_with_progress(
4045            root.path(),
4046            &analyzing(config(CachePolicy::Auto, Some(cache))),
4047            &tree,
4048            &progress,
4049        )
4050        .expect("warm analyzed report");
4051        pending.join().expect("nothing to save");
4052        let snapshot = progress.snapshot();
4053        assert_eq!(performance.source, ReportSource::WarmRevalidate);
4054        assert_eq!((performance.walked_files, performance.walked_bytes), (files, bytes));
4055        assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "warm revalidation");
4056        assert_eq!(snapshot.allocated, performance.walked_allocated);
4057        assert_eq!(snapshot.directories, 7);
4058        assert_eq!(performance.fresh_files, 0, "the sidecar answered every candidate");
4059        assert_eq!((snapshot.phase, snapshot.analysis), (Summarizing, Some((0, 0))));
4060    }
4061
4062    /// A cache-only report walks nothing: it ends building the answer, and its walk
4063    /// counters stay at zero.
4064    #[test]
4065    fn a_cache_only_report_ends_summarizing_and_walks_nothing() {
4066        use crate::ProgressPhase::Summarizing;
4067        let (root, _, _) = wide_tree(3, 2);
4068        let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
4069        let cache_dir = tempfile::tempdir().expect("cache dir");
4070        let cache = cache_dir.path().join("snapshot.fdu");
4071        let (_, pending, _) =
4072            prepared(root.path(), &config(CachePolicy::On, Some(cache.clone())), &tree)
4073                .expect("a report that writes the snapshot");
4074        pending.join().expect("save");
4075
4076        let progress = Progress::new();
4077        let (_, pending, performance) = prepared_with_progress(
4078            root.path(),
4079            &stale(config(CachePolicy::Auto, Some(cache))),
4080            &tree,
4081            &progress,
4082        )
4083        .expect("cache-only report");
4084        pending.join().expect("nothing to save");
4085        let snapshot = progress.snapshot();
4086        assert_eq!(performance.source, ReportSource::CacheOnly);
4087        assert_eq!(snapshot.phase, Summarizing);
4088        assert_eq!((snapshot.directories, snapshot.files, snapshot.bytes), (0, 0, 0));
4089    }
4090
4091    /// The position of `phase` in `order`, so a poller can assert phases never go back.
4092    fn rank(phase: crate::ProgressPhase, order: &[crate::ProgressPhase]) -> usize {
4093        order
4094            .iter()
4095            .position(|expected| *expected == phase)
4096            .unwrap_or_else(|| panic!("{phase:?} is not a phase of this route"))
4097    }
4098
4099    /// What a ticker thread sees: every counter non-decreasing from one snapshot to the
4100    /// next, and the phase moving only forward through the route's order. Deterministic
4101    /// without a timing assumption, because each claim is about consecutive reads of one
4102    /// monotonic cell, whatever the interleaving; the poller just reads until the run is
4103    /// over. The tree spans many worker chunks and many small batches, so the counters
4104    /// are added to from several threads while the poller reads.
4105    #[test]
4106    fn progress_is_monotonic_and_phases_advance_in_order_while_a_report_runs() {
4107        use crate::ProgressPhase::{
4108            Analyzing, Indexing, Loading, Revalidating, Saving, Scanning, Starting, Summarizing,
4109        };
4110        let (root, files, bytes) = wide_tree(48, 6);
4111        let cache_dir = tempfile::tempdir().expect("cache dir");
4112        let cache = cache_dir.path().join("snapshot.fdu");
4113        let fixture = OpenFixture {
4114            scan: ScanConfig { threads: Some(3), batch_size: 4, ..ScanConfig::default() },
4115            ..analyzing(config(CachePolicy::Auto, Some(cache)))
4116        };
4117        let tree = Query { views: vec![ViewSpec::Tree], ..Query::default() };
4118
4119        let routes: [(&str, &[crate::ProgressPhase]); 2] = [
4120            ("cold", &[Starting, Loading, Scanning, Indexing, Analyzing, Saving, Summarizing]),
4121            ("warm", &[Starting, Loading, Revalidating, Analyzing, Saving, Summarizing]),
4122        ];
4123        for (route, order) in routes {
4124            let progress = Progress::new();
4125            // Read before the run can begin, so the first phase seen is the handle's
4126            // initial one whatever the scheduler does with the poller.
4127            let initial = progress.snapshot();
4128            let done = std::sync::atomic::AtomicBool::new(false);
4129            let (performance, seen) = std::thread::scope(|scope| {
4130                let poller = scope.spawn(|| {
4131                    let polled = progress.clone();
4132                    let mut previous = initial;
4133                    let mut seen = vec![previous.phase];
4134                    loop {
4135                        let finished = done.load(std::sync::atomic::Ordering::Acquire);
4136                        let current = polled.snapshot();
4137                        assert!(current.directories >= previous.directories, "{route}");
4138                        assert!(current.files >= previous.files, "{route}");
4139                        assert!(current.bytes >= previous.bytes, "{route}");
4140                        assert!(
4141                            rank(current.phase, order) >= rank(previous.phase, order),
4142                            "{route}: {:?} after {:?}",
4143                            current.phase,
4144                            previous.phase
4145                        );
4146                        if let (Some(before), Some(after)) = (previous.analysis, current.analysis) {
4147                            assert!(after.0 >= before.0 && after.0 <= after.1, "{route}");
4148                            assert_eq!(after.1, before.1, "{route}: the total is fixed");
4149                        }
4150                        if current.phase != previous.phase {
4151                            seen.push(current.phase);
4152                        }
4153                        previous = current;
4154                        // Read once more after the run reports done, so the final state
4155                        // is checked against the last mid-run read.
4156                        if finished {
4157                            break;
4158                        }
4159                        std::thread::yield_now();
4160                    }
4161                    seen
4162                });
4163                let (_, pending, performance) =
4164                    prepared_with_progress(root.path(), &fixture, &tree, &progress)
4165                        .expect("report");
4166                pending.join().expect("save");
4167                done.store(true, std::sync::atomic::Ordering::Release);
4168                (performance, poller.join().expect("poller"))
4169            });
4170            let snapshot = progress.snapshot();
4171            assert_eq!(
4172                (snapshot.files, snapshot.bytes),
4173                (performance.walked_files, performance.walked_bytes),
4174                "{route}"
4175            );
4176            assert_eq!((snapshot.files, snapshot.bytes), (files, bytes), "{route}");
4177            assert_eq!(snapshot.directories, 49, "{route}");
4178            assert_eq!(seen.first(), Some(&Starting), "{route}: {seen:?}");
4179            assert_eq!(
4180                seen.last().copied(),
4181                Some(snapshot.phase),
4182                "{route}: the poller saw the final phase"
4183            );
4184            assert_eq!(snapshot.phase, Summarizing, "{route}: the answer is built last");
4185            if route == "cold" {
4186                assert_eq!(snapshot.analysis, Some((files, files)));
4187            } else {
4188                assert_eq!(snapshot.analysis, Some((0, 0)));
4189                assert!(!seen.contains(&Indexing), "{route}: a warm run assembles no index");
4190            }
4191        }
4192    }
4193
4194    /// The handle observes the run and changes nothing about it: the report prepared
4195    /// with one renders to the bytes of the report prepared without, and the
4196    /// performance summary is the same value, on the indexed and the compact routes.
4197    #[test]
4198    fn a_report_prepared_with_a_handle_is_the_report_prepared_without() {
4199        let (root, _, _) = wide_tree(5, 3);
4200        let tree = Query { views: vec![ViewSpec::Tree, ViewSpec::Files], ..Query::default() };
4201        let cases = [
4202            ("full index", config(CachePolicy::Off, None), tree),
4203            ("compact summary", blind(CachePolicy::Off, None), summary_query()),
4204        ];
4205        for (route, fixture, query) in cases {
4206            let (mut plain, pending, plain_performance) =
4207                prepared(root.path(), &fixture, &query).expect("plain report");
4208            pending.join().expect("no save");
4209            let progress = Progress::new();
4210            let (observed, pending, observed_performance) =
4211                prepared_with_progress(root.path(), &fixture, &query, &progress)
4212                    .expect("observed report");
4213            pending.join().expect("no save");
4214
4215            assert_eq!(plain_performance, observed_performance, "{route}");
4216            plain.provenance = observed.provenance.clone();
4217            let json = |report: &Report| {
4218                crate::report_format::render(report, crate::report_format::Format::Json, false)
4219                    .expect("render")
4220            };
4221            assert_eq!(json(&plain), json(&observed), "{route}");
4222            assert!(progress.snapshot().files > 0, "{route}: the handle did observe the run");
4223        }
4224    }
4225}