stuffr_core/salvage.rs
1//! The salvage engine: format-agnostic scanning, bounding and collection.
2//!
3//! Salvage recovers entries from a damaged archive by scanning for records
4//! directly, rather than trusting a single index (a central directory, an
5//! EOCD) that may be truncated, zeroed, or — the motivating case — may
6//! shadow real records behind a repeated name. The per-format part of that
7//! (recognising a local header, computing a checksum) is a
8//! [`SalvageScan`] implementation; this module is everything that does not
9//! need to know which format it is scanning.
10//!
11//! **This module implements no per-format scanning at all.** It is proven
12//! entirely against mocks — [`SalvageScan`] is the seam Task 2 onward builds
13//! real scanners against (starting with zip).
14//!
15//! ## Verification and shadowing (Task 4)
16//!
17//! Two things this module now does, both format-agnostic:
18//!
19//! - **Deciding [`SalvageStatus`]** is delegated to [`SalvageScan::verify`],
20//! called once per candidate in [`annotate_candidates`]. The default
21//! answers [`SalvageStatus::Unattested`] — "nothing attests that this
22//! candidate is an entry at all" — which is the honest reading of a method
23//! that reads nothing, and the right answer for a mock with no
24//! [`Candidate::verifier`] and for a real format with neither a checksum
25//! nor a self-verifying header (cpio, ar). A format with a self-verifying
26//! header and no content checksum (tar) overrides it to answer
27//! [`SalvageStatus::Complete`]. A format that carries a content checksum
28//! (zip's CRC-32) overrides `verify` to
29//! decode the payload and compare it, answering `Intact` on agreement and
30//! `Partial` otherwise — never `Complete`, which is reserved for "no way
31//! to prove", not "tried and it disagreed". Earlier revisions of this
32//! module recorded every candidate as `Complete` unconditionally; that was
33//! a placeholder, not a claim about zip, and it was wrong for every zip
34//! entry the moment a real scanner carried a CRC-32.
35//! - **Detecting a shadowed record** (setting [`SalvagedEntry::shadows`]) is
36//! generic too: [`annotate_candidates`] remembers the first candidate to
37//! report each distinct `(name, declared_len, verifier)` triple and marks
38//! any LATER candidate reporting the identical triple as shadowing it.
39//! This is a MEASUREMENT, never an inference from [`EntryMeta::name`]
40//! alone repeating: two different files that happen to share a name must
41//! not be linked this way, only two records whose original writer
42//! computed the same checksum, over the same declared length, under the
43//! same name. The checksum alone is not enough either — fixed after this
44//! module's own fix-round review measured it against real archives: every
45//! empty file and every directory entry in a zip has `Crc32(0)` — a
46//! structural certainty for a zero-byte payload, not a collision — so a
47//! checksum-only match reported an ordinary Python package's dozen
48//! `__init__.py` files, and its directories, as duplicates of one another.
49//! The degenerate case is refused outright rather than papered over by the
50//! wider conjunction: a candidate with `declared_len == Some(0)` never
51//! shadows and is never shadowed, because a checksum over zero bytes is
52//! the same value for every empty entry that ever existed and proves
53//! nothing about any of them being copies of each other.
54//! - **Detecting a repeated NAME** ([`SalvagedEntry::collides_with`]) is a
55//! second, deliberately separate annotation, added after the final
56//! whole-branch review found the word "shadow" doing two jobs. The two
57//! facts are different and both worth printing: a shadow is a record
58//! MEASURED to be a copy of an earlier one, a collision is a record whose
59//! name an earlier one already used and whose content this scan could not
60//! prove identical. `stuffr list`'s own `Fidelity::EntryCountMismatch`
61//! warning is about the second, so on an archive whose duplicates differ,
62//! `list` reported two records shadowed while `salvage --list` marked
63//! none — one tool telling a user two contradictory things about the same
64//! eight records. None of the guards on shadow detection above apply to a
65//! collision, and applying them would be wrong: they exist because
66//! claiming two records are COPIES needs proof, and a collision claims
67//! nothing about content at all. The two are mutually exclusive — a
68//! proven copy is reported as a shadow and nothing else, since that is
69//! the strictly more informative claim.
70//!
71//! [`collect_candidates`] (the resync loop: find a candidate, advance) and
72//! [`annotate_candidates`] (bound + verify + shadow, above) are both
73//! exposed separately from [`salvage_all`] — which is just the two of them
74//! run in sequence — so a format that reconciles more than one candidate
75//! source (zip's central-directory fallback, `zip_salvage.rs`) can collect
76//! from each source, merge, and annotate the merged list exactly once
77//! rather than annotating twice and reconciling two already-decided
78//! outcomes. The run's ceiling on a single entry is applied once, in
79//! `annotate_candidates`, to whatever the merge produced — see
80//! [`UnverifiedCause::OverEntryCeiling`] for why it is a status there
81//! rather than a [`crate::Error`] anywhere.
82
83use std::collections::HashMap;
84use std::io::{self, Read, SeekFrom, Write};
85use std::path::Path;
86
87use crate::archive::EntryMeta;
88use crate::error::{Error, Result};
89use crate::format::FormatId;
90use crate::source::SeekRead;
91
92/// 4 GiB. A scanned length is untrustworthy TWICE over — attacker-controlled
93/// AND possibly corrupt — so this is a structural ceiling, not a budget.
94///
95/// Applied per ENTRY, never to the run: an entry over it is reported
96/// [`UnverifiedCause::OverEntryCeiling`] and skipped, and the rest of the
97/// archive is recovered as usual. It used to abort the run outright, which
98/// meant one absurd declaration cost every entry around it.
99pub const MAX_SALVAGE_ENTRY: u64 = 4 * 1024 * 1024 * 1024;
100
101/// What a format can prove about a candidate it found.
102#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
103pub enum Verifier {
104 /// CRC-32, as zip, arj and gzip compute it.
105 Crc32(u32),
106 /// CRC-16/ARC, as lha, arc and zoo compute it.
107 Crc16(u16),
108}
109
110/// A record a scanner believes it found, before anything is decoded.
111///
112/// # Why this is `#[non_exhaustive]` AND has a constructor
113///
114/// Ruling S-I, deferred from Salvage Stage 2 Task 1 to Task 9 on purpose:
115/// this struct is the one shape [`SalvageScan::next_candidate`] *forces* an
116/// external implementor to build, so every field added to it used to break
117/// every out-of-crate scanner at once — and at `0.5.0` all four crates are
118/// published, so "out-of-crate" is not hypothetical. The attribute alone
119/// would have been the wrong fix (`CONTRIBUTING.md`'s `#[non_exhaustive]`
120/// section says why: it forbids `..` construction from another crate, which
121/// is exactly what an implementor must do), which is why it waited for
122/// [`Candidate::new`] — and for all five in-tree scanners to exist, so the
123/// constructor's shape could be read off real callers rather than guessed.
124///
125/// The three arguments [`Candidate::new`] takes are the three facts every
126/// scanner knows the instant it recognises a record. The four `with_*`
127/// setters cover the rest, each taking the field's own type unchanged so
128/// nothing is hidden behind a conversion.
129#[derive(Debug)]
130#[non_exhaustive]
131pub struct Candidate {
132 /// Byte offset of the record's header in the archive.
133 pub offset: u64,
134 /// Byte offset where this record's PAYLOAD begins — past whatever
135 /// fixed and variable-length header fields this format's record
136 /// carries (a zip local header's name/extra fields, ARC's fixed
137 /// 28-byte record, and so on).
138 ///
139 /// Task 3c added this field after `entries.rs`'s write path hardcoded
140 /// a ZIP local header's own 30-byte-plus-name-plus-extra layout as
141 /// the way to locate every format's payload — which crashed
142 /// (`Error::Io`, exit 1, the wildcard this project treats as a
143 /// defect) the moment a non-zip scanner (ARC) reported a candidate
144 /// near the end of the file, because reading 30 bytes from ITS offset
145 /// ran past EOF. Each scanner already computes this value once, at
146 /// discovery, to know where ITS OWN payload begins — this field
147 /// carries that computation forward instead of a consumer re-deriving
148 /// (and mis-deriving) it later. A future scanner (zoo, lha, arj) fills
149 /// this the same way; nothing outside the scanner that discovered a
150 /// candidate needs to know its record's own layout at all.
151 pub payload_start: u64,
152 pub meta: EntryMeta,
153 /// Declared payload length. `None` when the header does not carry one.
154 pub declared_len: Option<u64>,
155 /// The checksum the ORIGINAL writer computed, when the format has one.
156 /// `None` is what produces the intact/complete split honestly, rather
157 /// than by a per-format convention someone has to remember.
158 pub verifier: Option<Verifier>,
159 /// How many of `declared_len`'s bytes the SOURCE actually holds, when
160 /// it holds fewer — the truncated-tail case. `None` means the declared
161 /// payload is entirely present (the ordinary case) or nothing was
162 /// declared at all; `Some(n)` always means `n < declared_len`.
163 ///
164 /// # Why a scanner reports this instead of dropping the candidate
165 ///
166 /// A truncated archive — an interrupted download — is the single most
167 /// common damaged zip there is, and it was the one shape salvage said
168 /// nothing at all about. `zip_salvage.rs`'s validation gate refused any
169 /// candidate whose declared payload ran past the end of the file, which
170 /// is a correct refusal to INVENT the missing bytes (`zip -FF`
171 /// fabricates them and reports success) implemented as SILENCE: no row,
172 /// no note, no non-zero exit. Swept across six truncation points inside
173 /// a three-header archive's last entry, every one reported "2 scanned"
174 /// at exit 0 — the third header was not even counted.
175 ///
176 /// Reporting the candidate with this field set is what turns the
177 /// refusal into a statement. Nothing is invented: the declared length
178 /// stays exactly what the header said, so a reader can see both figures,
179 /// and the bytes that ARE present are still a genuine prefix. What a
180 /// scanner must NOT do is fold the shortfall into `declared_len` — that
181 /// would make a truncated entry indistinguishable from a whole one of
182 /// the smaller size, which is the same mistake in a different place.
183 pub available_len: Option<u64>,
184 /// The archive's own record marks this entry DELETED — the format's
185 /// owner removed it and the writer left the record, and its payload,
186 /// in place.
187 ///
188 /// `false` for every format that has no such flag (zip, tar, ar, cpio,
189 /// arc), which is why it is a plain `bool` rather than an `Option`:
190 /// "this format cannot express deletion" and "this record is not
191 /// deleted" lead to the identical, unannotated row, and a third state
192 /// would be a distinction no consumer could act on.
193 ///
194 /// # Why a scanner reports one rather than dropping it (Ruling S-R)
195 ///
196 /// Salvage Stage 2 Task 4, for ZOO — the first format here with the
197 /// flag. `zoo d` marks an entry `deleted = 1` and leaves it whole in the
198 /// file; `zoolist.c` neither lists nor extracts it, and `legacy::zoo`'s
199 /// reader follows that exactly. The flag is ONE BYTE, so in a damaged
200 /// archive a bit flip turns a live entry into one no ordinary verb will
201 /// ever hand back — which is precisely the archive this verb exists for.
202 /// So the scanner reports it.
203 ///
204 /// **And the report carries a marker, which is the part that took a
205 /// review to get right.** Everywhere else salvage accepts less than an
206 /// ordinary entry it says so on the artifact or in the row: a `Partial`
207 /// lands as `NAME.partial` and never under its real name, a second
208 /// record under a taken name lands as `NAME.salvaged-N`, a proven copy
209 /// prints `[shadowed]`. A deleted record reported `Intact` under its
210 /// real name at exit 0 was the one place that leniency was invisible —
211 /// four verbs call the archive empty and the fifth writes the file, with
212 /// nothing telling a user which it was.
213 ///
214 /// **The exit code deliberately does NOT move.** Recovering a deleted
215 /// record is this verb working as designed, not degraded fidelity — see
216 /// `stuffr::entries`'s own exit-code aggregation, which this field is
217 /// invisible to.
218 pub marked_deleted: bool,
219}
220
221impl Candidate {
222 /// A candidate at `offset`, whose payload begins at `payload_start`,
223 /// carrying `meta`.
224 ///
225 /// Nothing else is claimed. The four fields this does not take default
226 /// to the answers that assert the least:
227 ///
228 /// * `declared_len: None` — the header carries no length, so nothing
229 /// bounds the payload and [`annotate_candidates`] reports
230 /// [`UnverifiedCause::NoDeclaredLength`] rather than reading it.
231 /// * `verifier: None` — the format offers nothing to prove the content
232 /// with, so no shadow is ever claimed for this record (see
233 /// [`SalvagedEntry::shadows`], which needs a verifier to compare).
234 /// * `available_len: None` — every declared byte is present, which is
235 /// the ordinary case; a scanner sets this only when it has MEASURED a
236 /// shortfall.
237 /// * `marked_deleted: false` — the format has no deleted flag, or this
238 /// record is not marked. Four of the five in-tree scanners leave it.
239 ///
240 /// One default is worth naming because it decides a STATUS, not just a
241 /// field: with no `verifier`, [`SalvageScan::verify`]'s own default
242 /// answers [`SalvageStatus::Unattested`] — the tier that asserts the
243 /// least, matching the discipline of the four above. It answered
244 /// [`SalvageStatus::Complete`] until Salvage Stage 3, which was the
245 /// lenient direction: `Complete` claims a header self-check the default
246 /// never performs, and routes to exit 0. A scanner carrying a checksum
247 /// has to write its own `verify` regardless (the default checks
248 /// nothing), and that `verify` reads this very field, so the omission
249 /// fails in the scanner's own first test rather than silently.
250 pub fn new(offset: u64, payload_start: u64, meta: EntryMeta) -> Self {
251 Self {
252 offset,
253 payload_start,
254 meta,
255 declared_len: None,
256 verifier: None,
257 available_len: None,
258 marked_deleted: false,
259 }
260 }
261
262 /// Sets [`Self::declared_len`].
263 #[must_use]
264 pub fn with_declared_len(mut self, declared_len: Option<u64>) -> Self {
265 self.declared_len = declared_len;
266 self
267 }
268
269 /// Sets [`Self::verifier`].
270 #[must_use]
271 pub fn with_verifier(mut self, verifier: Option<Verifier>) -> Self {
272 self.verifier = verifier;
273 self
274 }
275
276 /// Sets [`Self::available_len`] — the truncated-tail case, and only
277 /// when the shortfall was measured. See that field's doc for why a
278 /// scanner must never fold it into [`Self::declared_len`] instead.
279 #[must_use]
280 pub fn with_available_len(mut self, available_len: Option<u64>) -> Self {
281 self.available_len = available_len;
282 self
283 }
284
285 /// Sets [`Self::marked_deleted`] — Ruling S-R. See that field's doc.
286 #[must_use]
287 pub fn with_marked_deleted(mut self, marked_deleted: bool) -> Self {
288 self.marked_deleted = marked_deleted;
289 self
290 }
291}
292
293/// Why an entry is [`SalvageStatus::Unverified`].
294///
295/// Three causes, and the STATUS does not distinguish them because the
296/// decision they lead to is identical (listed, not written, exit 3) — the
297/// same split [`crate::salvage`]'s consumers apply to
298/// [`SalvageStatus::Partial`] via their own cause type (`entries.rs`'s
299/// `PartialCause`, one crate up): **a tier carries a decision, a message
300/// carries a cause.**
301#[derive(Debug, Clone, Copy, PartialEq, Eq)]
302pub enum UnverifiedCause {
303 /// This build recognises the entry's compression method but cannot
304 /// decode it (zip method 99 is AES, 14 is LZMA — recognised, not
305 /// decoded).
306 UndecodableMethod,
307 /// No length was available to bound the payload, so it was never read —
308 /// a data-descriptor entry the raw scan found with no central-directory
309 /// record to reconcile against. A CD-reconciled data-descriptor entry
310 /// gets a real declared length and CRC from the central directory and
311 /// does NOT reach this cause; see `zip_salvage.rs`'s
312 /// `candidate_from_cd_record`.
313 NoDeclaredLength,
314 /// The bytes this entry would have required are past the ceiling this
315 /// run will read for ONE entry — `policy.max_entry` narrowed by the
316 /// scanner's own [`SalvageScan::max_whole_entry`] — so nothing was read
317 /// and nothing was decoded. See [`annotate_candidates`], which is the
318 /// one place that decides this, for the whole rule.
319 ///
320 /// # Why this is a cause and not an `Err`
321 ///
322 /// Salvage Stage 2 Task 3c's fix round 3. This refusal used to be an
323 /// [`crate::Error::ResourceLimit`] raised from two different places — the
324 /// engine's own [`collect_candidates`], and (for ARC) the scanner's own
325 /// `verify` — and an `Err` at either site **aborts the whole run**:
326 /// measured at the CLI on a two-entry archive whose second entry was
327 /// over the ceiling, `stuffr salvage --list` printed no rows at all and
328 /// exited 6, and `salvage -C out` left the destination empty, losing a
329 /// first entry that decodes perfectly. That is the one thing this verb
330 /// exists not to do. Three fix rounds chased the shape from site to
331 /// site; what actually closes it is that an over-ceiling entry has a
332 /// STATUS, so there is no `Err` left to propagate.
333 ///
334 /// Distinct from [`SalvageStatus::Partial`], which is what this used to
335 /// collapse into once the `Err` was folded away at the write layer: a
336 /// truncated entry's genuine prefix IS recovered (as `NAME.partial`),
337 /// while an over-ceiling entry has real bytes nobody read. Reporting it
338 /// as `Partial (truncated)` stated two false things — the entry is not
339 /// truncated, and the empty `NAME.partial` it produced said nothing
340 /// survived.
341 ///
342 /// # Why it carries both figures (fix round 4, NEW-F)
343 ///
344 /// The `Err` this replaced carried a sentence naming the declared size
345 /// and the ceiling; the first version of this variant was a unit, so a
346 /// user meeting a container's own fixed ceiling was told neither number
347 /// and could not tell whether `--max-entry` would help. The cause is
348 /// the only place that knows both, so it carries both and the CLI row
349 /// prints them — the project's own "a message carries a cause", with
350 /// the numbers that make the message actionable.
351 OverEntryCeiling {
352 /// Bytes reading this entry would have required: the BOUNDED
353 /// figure the comparison actually used — `available_len` when the
354 /// payload is truncated, the header's `declared_len` otherwise —
355 /// never the raw declaration when the file holds less than it.
356 needed: u64,
357 /// The ceiling in force for this run: [`SalvagePolicy::max_entry`]
358 /// narrowed by [`SalvageScan::max_whole_entry`]. Equal to
359 /// `max_entry` means `--max-entry` can raise it; below it means a
360 /// whole-decoding container's own fixed ceiling decided, and no
361 /// flag moves that.
362 ceiling: u64,
363 },
364}
365
366/// What was proven about an entry AFTER decoding it.
367#[derive(Debug, Clone, Copy, PartialEq, Eq)]
368pub enum SalvageStatus {
369 /// A checksum the original writer computed agrees.
370 Intact,
371 /// Every declared byte was present and the header self-verified, but the
372 /// format offers NO CHECKSUM AT ALL to prove the content — `tar`, in
373 /// this project's own formats. Never the right answer for a format that
374 /// DOES carry one; see [`SalvageStatus::Unverified`] for that case,
375 /// which this status is not permitted to stand in for.
376 ///
377 /// # `cpio` and `ar` are NOT this tier, and were named here until Stage 3
378 ///
379 /// This doc said "tar, cpio and ar" from Stage 1 until Salvage Stage 3,
380 /// which is where the sentence's own *"and the header self-verified"*
381 /// clause was measured against the three formats it named. `tar` has a
382 /// header checksum and really does self-verify. **`cpio` and `ar` have
383 /// neither a content checksum nor a self-verifying header**, so for
384 /// those two the clause is not merely unproven — there is nothing in the
385 /// format it could ever be true of, and a scanner answering `Complete`
386 /// for one would be asserting a header self-check that does not exist.
387 /// They are [`SalvageStatus::Unattested`].
388 ///
389 /// `stuffr_core::testing::check_salvage_claim` is what holds the line: this
390 /// tier requires a format whose only checksum covers the HEADER
391 /// (`Attestation::HeaderChecksumOnly`) AND that checksum to have been
392 /// checked — so the checksumless pair cannot reach it, and neither can a
393 /// format with a content checksum, whose honest tiers are `Intact`,
394 /// `Partial` and `Unverified`.
395 Complete,
396 /// Nothing attests that this candidate is an entry AT ALL: the format
397 /// carries **no checksum and no self-verifying header**, so a
398 /// coincidental header match and a real entry are indistinguishable in
399 /// the bytes — `cpio` and `ar`, in this project's own formats.
400 ///
401 /// The entry is still WRITTEN, under its real name: `salvage` is the
402 /// recovery-biased verb, and a tier that refused to write would make
403 /// these two formats unsalvageable rather than honestly salvaged. What
404 /// the tier buys is that the run says so — `salvage_exit_code` answers
405 /// **4** for it (degraded fidelity, never 3: nothing was refused and no
406 /// capability was missing), and the `--list` row carries the reason
407 /// beside the entry. There is no flag to opt into or out of this; the
408 /// formats' own shape decides it.
409 ///
410 /// # Not [`Self::Unverified`] — and the near-miss name is the hazard
411 ///
412 /// The variant deliberately is NOT called `Unverifiable`: that is two
413 /// letters from the [`Self::Unverified`] this enum already has, in an
414 /// enum whose variants select exit codes, and a reviewer skimming match
415 /// arms would not reliably see the difference. The distinction, stated
416 /// wherever both appear:
417 ///
418 /// * [`Self::Unverified`] — the format *has* a checksum; this build
419 /// could not use it (an undecodable method), or the header declared
420 /// nothing to check.
421 /// * `Unattested` — the format has **no checksum and no self-verifying
422 /// header**, so nothing attests that this candidate is an entry at
423 /// all.
424 ///
425 /// The consequences differ as much as the claims do: `Unverified` is
426 /// NOT written and exits 3 (this build could not do it — a rebuild may
427 /// fix it), `Unattested` IS written and exits 4 (nothing can fix it;
428 /// the format carries no evidence to find).
429 Unattested,
430 /// Nothing about this entry's content was verified.
431 ///
432 /// Two causes ([`UnverifiedCause`]), and the tier does not distinguish
433 /// them because the decision is identical — the entry is listed, not
434 /// written, and the run exits 3:
435 /// * this build could not decode the method (zip 99 is AES, 14 is
436 /// LZMA);
437 /// * no length was available to bound the payload, so it was never
438 /// read (a data-descriptor entry the raw scan found with no
439 /// central-directory record to reconcile against).
440 ///
441 /// Distinct from [`Self::Complete`], which asserts every DECLARED byte
442 /// was present — a claim that requires something to have been declared
443 /// in the first place. Reporting `Complete` for either cause above was
444 /// this enum's third instance of "nothing to disprove" being read as
445 /// "proven" (after the Task 1 hardcoded placeholder and the Task 4
446 /// undecodable-method case this variant was originally added for) — the
447 /// exit-code table this status exists to keep honest reserves exit 3 for
448 /// both causes, never exit 0, which is what `Complete` would have routed
449 /// either one to.
450 Unverified(UnverifiedCause),
451 /// The payload ran out, the decoder failed mid-stream, or the content
452 /// decoded whole and disagreed with the checksum the original writer
453 /// computed. All three are the same fact from a caller's perspective —
454 /// this build could not confirm the content is what was written — and
455 /// none of them is `Complete` (something WAS checked) or `Intact` (it
456 /// did not agree, or never finished).
457 Partial,
458}
459
460/// Which class of evidence a FORMAT offers for one of its entries — a
461/// per-format constant, and the third argument to the honesty oracle
462/// `stuffr_core::testing::check_salvage_claim`.
463///
464/// **Always compiled, unlike the oracle**, since Salvage Stage 3 Task 5:
465/// `stuffr formats`' SALVAGE column renders it (`yes` only for
466/// [`Attestation::ContentChecksum`]), so it cannot live behind the
467/// `testing` feature with the oracle that consumes it. Which format has
468/// which class is stated once, beside the scanner dispatch
469/// (`stuffr::entries::salvage_attestation`); the fuzz target and the corpus
470/// generator read it from there rather than keeping a table of their own.
471///
472/// **Three classes, which is why this is an enum and not a `bool`.** Salvage
473/// Stage 3 Task 1 first shipped the argument as `format_offers_verifier:
474/// bool`, and its fix round's review showed with a truth table that no
475/// reading of a two-valued flag can hold the boundary: read as "a content
476/// checksum", it let `cpio` claim [`SalvageStatus::Complete`]; read as "a
477/// content checksum OR a self-verifying header" (the shipped reading), it
478/// let `tar` claim [`SalvageStatus::Intact`] and — live against the five
479/// scanners that shipped before this stage — let `zip`, `arc`, `zoo`, `lha`
480/// or `arj` answer `Complete` for an entry whose CRC was compared and
481/// DISAGREED, turning an exit-4 `Partial` into an exit-0 clean recovery.
482/// Both false claims sat in the one cell (`checked`, `offers`) that tar's
483/// honest `Complete` also occupies, so no function of those inputs could
484/// refuse them and permit it. Ruling 3-G.
485///
486/// Two bools (`offers_content_checksum`, `header_self_verifies`) were
487/// considered and refused: four states for three classes, one of them
488/// meaningless, and the split pushed back onto every caller. Here each
489/// scanner NAMES its class.
490///
491/// **Deliberately NOT `#[non_exhaustive]`**, for the reason Ruling 3-D kept
492/// it off [`SalvageStatus`]: a fourth class of evidence must fail to compile
493/// at every `match`, `check_salvage_claim`'s own included, rather than be
494/// absorbed by a wildcard arm that decides what it permits by accident.
495#[derive(Debug, Clone, Copy, PartialEq, Eq)]
496pub enum Attestation {
497 /// A checksum over the entry's CONTENT, computed by the original writer —
498 /// `zip`, `arc`, `zoo`, `lha`, `arj`. The only class that can earn
499 /// [`SalvageStatus::Intact`].
500 ContentChecksum,
501 /// A checksum over the HEADER only; the content has none — `tar`. The
502 /// only class that can earn [`SalvageStatus::Complete`]: "every declared
503 /// byte was present and the header self-verified" is exactly what this
504 /// class can prove and all it can prove.
505 HeaderChecksumOnly,
506 /// Neither — `cpio`, `ar`. A coincidental header match and a real entry
507 /// are indistinguishable in the bytes, so the only honest tier this
508 /// class can reach on a whole entry is [`SalvageStatus::Unattested`].
509 Nothing,
510}
511
512/// One entry the scan recovered.
513///
514/// `#[non_exhaustive]` plus [`SalvagedEntry::new`], for the same reason
515/// [`Candidate`] carries both — see that type's own doc. This one is built
516/// by [`annotate_candidates`] in ordinary use, but every per-format scanner
517/// crate builds one directly to drive its `write_payload` seam, so the
518/// literal it used to need was an out-of-crate literal all the same.
519#[derive(Debug)]
520#[non_exhaustive]
521pub struct SalvagedEntry {
522 /// This entry's position in SCAN order — not an index any container's
523 /// own format assigns, and never renumbered by what `list` would show.
524 /// A later task's CLI surfaces this distinction explicitly (`--index`
525 /// on `salvage` names a scan position, not a list index).
526 pub scan_position: usize,
527 pub offset: u64,
528 /// Carried forward from [`Candidate::payload_start`] — see that
529 /// field's doc for why a consumer must use this rather than
530 /// re-deriving a payload's location from `offset` itself.
531 pub payload_start: u64,
532 pub meta: EntryMeta,
533 pub status: SalvageStatus,
534 /// Set when this record's `(name, declared_len, verifier)` all agree
535 /// with an EARLIER record's — a measurement, never inferred from a
536 /// repeated name alone, and never set for a degenerate zero-length
537 /// checksum (every empty file and every directory entry shares one).
538 /// See [`annotate_candidates`] for where this is computed.
539 ///
540 /// **This is the byte-identical duplicate, and ONLY that.** The other,
541 /// weaker fact — an earlier record under the same name whose content
542 /// this scan could not prove identical — is [`Self::collides_with`], and
543 /// the two are mutually exclusive by construction. They were one word
544 /// ("shadow") until the final whole-branch review measured what that
545 /// cost: `stuffr list`'s own fidelity warning uses "shadowed" for a
546 /// repeated NAME, so on an archive whose duplicates differ, `list` said
547 /// two records were shadowed and `salvage --list` marked none.
548 pub shadows: Option<usize>,
549 /// Set when an EARLIER record carries the same `meta.name` and this one
550 /// is not that record's [`Self::shadows`] — i.e. the name repeats but
551 /// the content could not be proven identical.
552 ///
553 /// This is the fact `stuffr list`'s `Fidelity::EntryCountMismatch`
554 /// warning is about: a later record repeating a name is what makes an
555 /// earlier one unreachable through a central directory's name index,
556 /// whatever the bytes under it. It is also the fact that decides
557 /// anything on the WRITE side, because two records under one name and
558 /// two different payloads cannot both land on one path.
559 ///
560 /// Deliberately NOT gated on a verifier, a declared length or the
561 /// degenerate zero-length case the way [`Self::shadows`] is: those
562 /// guards exist because claiming two records are COPIES of each other
563 /// needs proof, and this annotation claims no such thing. A repeated
564 /// name is directly observed, and `None` here means only that no
565 /// earlier record used this name — never that one did and this could
566 /// not tell.
567 pub collides_with: Option<usize>,
568 /// Carried forward unchanged from [`Candidate::marked_deleted`] — see
569 /// that field for the whole ruling. Unlike [`Self::shadows`] and
570 /// [`Self::collides_with`], this is not something this module MEASURES:
571 /// it is a fact the archive's own record states, which only the scanner
572 /// that read that record can know.
573 pub marked_deleted: bool,
574}
575
576impl SalvagedEntry {
577 /// An entry at scan position `scan_position`, found at `offset`, whose
578 /// payload begins at `payload_start`, carrying `meta` and proven to
579 /// `status`.
580 ///
581 /// The three fields this does not take default to "nothing was
582 /// observed": no shadow, no name collision, not marked deleted. All
583 /// three are annotations [`annotate_candidates`] adds (or, for
584 /// `marked_deleted`, carries forward from [`Candidate`]), so a caller
585 /// building one directly — every per-format `write_payload` test does —
586 /// wants exactly those defaults.
587 pub fn new(
588 scan_position: usize,
589 offset: u64,
590 payload_start: u64,
591 meta: EntryMeta,
592 status: SalvageStatus,
593 ) -> Self {
594 Self {
595 scan_position,
596 offset,
597 payload_start,
598 meta,
599 status,
600 shadows: None,
601 collides_with: None,
602 marked_deleted: false,
603 }
604 }
605
606 /// Sets [`Self::shadows`] — the byte-identical duplicate, and only
607 /// that. See the field's own doc for what separates it from
608 /// [`Self::collides_with`].
609 #[must_use]
610 pub fn with_shadows(mut self, shadows: Option<usize>) -> Self {
611 self.shadows = shadows;
612 self
613 }
614
615 /// Sets [`Self::collides_with`] — the weaker, name-only annotation.
616 #[must_use]
617 pub fn with_collides_with(mut self, collides_with: Option<usize>) -> Self {
618 self.collides_with = collides_with;
619 self
620 }
621
622 /// Sets [`Self::marked_deleted`], carried forward from
623 /// [`Candidate::marked_deleted`] rather than measured here.
624 #[must_use]
625 pub fn with_marked_deleted(mut self, marked_deleted: bool) -> Self {
626 self.marked_deleted = marked_deleted;
627 self
628 }
629}
630
631/// The result of a scan: every entry the scanner recovered, in scan order.
632///
633/// Later tasks may extend this struct; the `entries` field stays public,
634/// and `#[non_exhaustive]` is what makes "extend" honest. No constructor is
635/// paired with it, unlike [`Candidate`] and [`SalvagedEntry`]: this struct
636/// is built by [`salvage_all`] alone, in this crate, and no scanner — in
637/// this workspace or outside it — has any reason to build one.
638#[derive(Debug)]
639#[non_exhaustive]
640pub struct SalvageOutcome {
641 pub entries: Vec<SalvagedEntry>,
642 /// Header shapes the scanner recognised and could not gate, in scan
643 /// order — see [`Sighting`]. Always empty from [`salvage_all`]: the
644 /// engine cannot know what its scanner refused, so a format's own
645 /// `salvage_*` entry point fills it in after the scan. A sighting is
646 /// never an entry; the facade's `salvage_exit_code` counts a
647 /// non-selective run that has one as degraded (exit 4) since the
648 /// Salvage Stage 3 final review's I-1, which reversed Ruling S-X there.
649 pub sightings: Vec<Sighting>,
650 /// Where a SEQUENTIAL-ONLY scanner's walk stopped short of the end of
651 /// the source, if it did — see [`WalkStop`]. Always `None` from
652 /// [`salvage_all`], for the same reason [`Self::sightings`] is always
653 /// empty there: only the scanner knows its walk ended early, so its own
654 /// `salvage_*` entry point fills this in. Like a sighting it is never an
655 /// entry and never touches the run's exit code.
656 pub walk_stop: Option<WalkStop>,
657}
658
659/// A header a scanner SAW and has no gate for (Rulings S-V and 3-J): its
660/// structure is a header's, and something about it — a field this build
661/// cannot check, a shape a shifted copy also takes — leaves the scan unable
662/// to tell it from a phantom. So it is counted here and never becomes a
663/// [`Candidate`], because the engine advances past a candidate by its
664/// declared length and a phantom's length is how a scan jumps a real header.
665///
666/// **Why it has to be reported at all.** A run that recovers nothing while
667/// seeing one refuses with [`Error::Unsupported`] (exit 3) — the scanner's own
668/// job, since only it can name the shape. A run that recovered OTHER entries
669/// reports them at its ordinary exit code (Ruling S-X), and until Salvage
670/// Stage 3's Task 2-N it said nothing else: an entry `stuffr list` shows was
671/// simply absent from `salvage --list`, at exit 0, with an empty stderr. The
672/// sightings recorded here are what the caller prints instead
673/// ([`describe_sightings`]), in a mixed run as in any other — and since the
674/// stage's final review (I-1, reversing S-X for this case) a non-selective
675/// mixed run also exits 4, not 0: a `salvage -C out && rm` script must not
676/// delete an archive holding an entry `stuffr list` reads.
677#[derive(Debug, Clone, PartialEq, Eq)]
678#[non_exhaustive]
679pub struct Sighting {
680 /// The scanner's format, for the sentence [`describe_sightings`] writes.
681 pub format: FormatId,
682 /// Where the header was seen.
683 pub offset: u64,
684 /// What was seen, as a plural noun phrase that follows a count —
685 /// `"header(s) whose magic field … is neither \`ustar\` nor blank"`. One
686 /// phrase per shape, shared with the scanner's own exit-3 refusal, so
687 /// the two can never describe one shape two ways.
688 pub shape: &'static str,
689 /// Why the header was not listed — which sentence
690 /// [`describe_sightings`] reports it in.
691 pub kind: SightingKind,
692}
693
694/// Why a [`Sighting`]'s header was not listed. The two are different claims,
695/// so [`describe_sightings`] words them apart.
696#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
697#[non_exhaustive]
698pub enum SightingKind {
699 /// A header with a header's structure, in a shape this build has no
700 /// gate for — every sighting before 0.10.3.
701 #[default]
702 Ungateable,
703 /// A header the scan could gate, but whose extension chains it did not
704 /// read because a bound on the work it spends on them was reached —
705 /// tar's per-header and scan-wide budgets (0.10.3). The header itself
706 /// may be genuine: the scan-wide budget can be spent by forged chains
707 /// elsewhere in the file. The scan carries on past it.
708 WorkBounded,
709 /// A candidate header whose verdict needed a long field — a name, a
710 /// variable part, an extension chain, the bytes a header CRC covers —
711 /// that the scan did not read, because its [`ScanBudget`] was spent
712 /// (0.10.3). Like [`Self::WorkBounded`] the header itself may be
713 /// genuine, starved by forged candidates elsewhere in the file, and the
714 /// scan carries on past it. Its own kind rather than
715 /// [`Self::WorkBounded`] because that one's sentence names extension
716 /// chains, which is false of a zip name or a CRC.
717 VerdictBounded,
718}
719
720impl Sighting {
721 /// An [`SightingKind::Ungateable`] sighting.
722 pub fn new(format: FormatId, offset: u64, shape: &'static str) -> Self {
723 Self {
724 format,
725 offset,
726 shape,
727 kind: SightingKind::Ungateable,
728 }
729 }
730
731 /// A [`SightingKind::WorkBounded`] sighting.
732 pub fn work_bounded(format: FormatId, offset: u64, shape: &'static str) -> Self {
733 Self {
734 kind: SightingKind::WorkBounded,
735 ..Self::new(format, offset, shape)
736 }
737 }
738
739 /// A [`SightingKind::VerdictBounded`] sighting: the candidate at
740 /// `offset` was refused unexamined because the scan's [`ScanBudget`]
741 /// was spent.
742 pub fn verdict_bounded(format: FormatId, offset: u64, shape: &'static str) -> Self {
743 Self {
744 kind: SightingKind::VerdictBounded,
745 ..Self::new(format, offset, shape)
746 }
747 }
748}
749
750/// The most offsets [`describe_sightings`] lists per shape before it says
751/// how many more there were. The count itself is always the complete figure.
752const SIGHTING_OFFSETS_SHOWN: usize = 8;
753
754/// The one sentence a caller prints for `sightings`, or `None` when there
755/// are none — how many headers of each shape were seen, where, and that the
756/// ordinary verbs may read them or name the variant they are in. It is the
757/// same wording family as each scanner's exit-3 refusal (its shape phrases
758/// are the scanner's own), and it is written here, once, so every scanner
759/// that records a [`Sighting`] gets the same report without a line of its
760/// own.
761///
762/// **"Or name the variant" is cpio's clause (Salvage Stage 3 Task 3).** The
763/// tail said only "may read these headers normally", which is true of a tar
764/// header the scan cannot gate and false of a cpio `odc` or `newc-crc`
765/// header: `stuffr list` refuses both variants at exit 3, naming them. A note
766/// telling a user to expect `list` to read them would be the false "list
767/// reads it" claim Task 2-N's fix round removed from tar.
768///
769/// **[`SightingKind::WorkBounded`] sightings get their own sentence**
770/// (0.10.3): "the structure of a header, but in a shape it has no gate for"
771/// is false of a header the scan gated and then stopped spending work on.
772/// When a run has both kinds, the two sentences are joined; the
773/// [`SightingKind::Ungateable`] one is worded exactly as before.
774///
775/// **So do [`SightingKind::VerdictBounded`] ones** (0.10.3 Task 4b): the
776/// work-bounded sentence names extension chains, which is false of a zip
777/// name, a ZOO variable part or the bytes an ARJ header CRC covers. Each
778/// sentence that applies is joined in the same way, and the two older ones
779/// are worded exactly as before.
780pub fn describe_sightings(sightings: &[Sighting]) -> Option<String> {
781 let of_kind = |kind: SightingKind| -> Vec<&Sighting> {
782 sightings.iter().filter(|s| s.kind == kind).collect()
783 };
784 let ungateable = of_kind(SightingKind::Ungateable);
785 let bounded = of_kind(SightingKind::WorkBounded);
786 let verdicts = of_kind(SightingKind::VerdictBounded);
787 let mut sentences = Vec::new();
788 if !ungateable.is_empty() {
789 sentences.push(format!(
790 "this build's salvage scanner saw {} — each with the structure of a header, but in \
791 a shape it has no gate for, so none is listed or recovered; `stuffr list` and \
792 `stuffr unpack` may read these headers normally, or name the variant they are in, \
793 and it is the SCAN that stops there",
794 sighting_groups(&ungateable)
795 ));
796 }
797 if !bounded.is_empty() {
798 sentences.push(format!(
799 "this build's salvage scanner bounds the work it spends on extension chains, and \
800 did not read the chains in front of {} — so whatever those chains describe is not \
801 listed or recovered there, though the scan went on past them; `stuffr list` and \
802 `stuffr unpack` may read these headers normally",
803 sighting_groups(&bounded)
804 ));
805 }
806 if !verdicts.is_empty() {
807 sentences.push(format!(
808 "this build's salvage scanner bounds the work it spends judging candidate headers \
809 across the whole scan, and did not judge {} — so whatever those headers \
810 describe is not listed or recovered there, though the scan went on past them; \
811 `stuffr list` and `stuffr unpack` may read these headers normally",
812 sighting_groups(&verdicts)
813 ));
814 }
815 (!sentences.is_empty()).then(|| sentences.join("; and separately, "))
816}
817
818/// `"{count} {format} {shape} at offset(s) {offsets}"` per (format, shape),
819/// joined by `"; and "` — the middle of each [`describe_sightings`] sentence.
820fn sighting_groups(sightings: &[&Sighting]) -> String {
821 // Grouped by (format, shape) in first-seen order: a handful of shapes
822 // at most, so a linear search beats a map and keeps the order stable.
823 let mut groups: Vec<((FormatId, &'static str), Vec<u64>)> = Vec::new();
824 for s in sightings {
825 let key = (s.format, s.shape);
826 match groups.iter_mut().find(|(k, _)| *k == key) {
827 Some((_, offsets)) => offsets.push(s.offset),
828 None => groups.push((key, vec![s.offset])),
829 }
830 }
831 let parts: Vec<String> = groups
832 .iter()
833 .map(|((format, shape), offsets)| {
834 let mut shown: Vec<String> = offsets
835 .iter()
836 .take(SIGHTING_OFFSETS_SHOWN)
837 .map(u64::to_string)
838 .collect();
839 if offsets.len() > SIGHTING_OFFSETS_SHOWN {
840 shown.push(format!(
841 "and {} more",
842 offsets.len() - SIGHTING_OFFSETS_SHOWN
843 ));
844 }
845 format!(
846 "{} {format} {shape} at offset(s) {}",
847 offsets.len(),
848 shown.join(", ")
849 )
850 })
851 .collect();
852 parts.join("; and ")
853}
854
855/// Where a scanner that can ONLY walk its format in order — record after
856/// record from the start, the way the ordinary reader does — had to stop
857/// before the end of the source (Salvage Stage 3 Task 4).
858///
859/// # Why a walk stop is not a [`Sighting`]
860///
861/// A sighting is a header the scan SAW, whose structure is a header's and
862/// which it cannot gate. A walk stop is the opposite: a place where the
863/// walk found nothing it could read as a record at all, and — because the
864/// format carries nothing strong enough to search for (an `ar` member's
865/// only marker is two bytes) — cannot look any further. Folding it into
866/// [`Sighting`] would make "saw a real header" and "could see nothing" one
867/// channel, and [`describe_sightings`]'s own sentence ("`stuffr list` may
868/// read these headers normally") would then be false about it.
869///
870/// # What it must say, and why it has to be said at all
871///
872/// The entries a sequential walk cannot reach are **unreachable by
873/// construction, not absent**: a run that recovers three entries from a
874/// ten-entry archive with a hole after the third reports three rows, and
875/// nothing in those rows says seven more may sit behind the hole. So the
876/// caller prints [`describe_walk_stop`] of this on every run that has one,
877/// and the offset it names is where a user with a hex editor would look.
878#[derive(Debug, Clone, PartialEq, Eq)]
879#[non_exhaustive]
880pub struct WalkStop {
881 /// The scanner's format, for the sentence [`describe_walk_stop`] writes.
882 pub format: FormatId,
883 /// Where the record the walk could not read begins — the first byte
884 /// after the last record it did read.
885 pub offset: u64,
886 /// Bytes from [`Self::offset`] to the end of the source.
887 pub remaining: u64,
888 /// Which of the two things this stop is — see [`WalkStopKind`].
889 pub kind: WalkStopKind,
890 /// The ordinary reader's own words for why that record would not read,
891 /// so the note and `stuffr list`'s refusal of the same bytes agree.
892 pub cause: String,
893}
894
895/// The three stops a sequential walk can make, each told apart by what the
896/// scanner measured rather than guessed.
897#[derive(Debug, Clone, Copy, PartialEq, Eq)]
898pub enum WalkStopKind {
899 /// The file ends inside the record at the stop point — the reader ran
900 /// out of bytes mid-record, or fewer bytes remain than the format's
901 /// smallest record header. Everything from the stop point on belongs to
902 /// that incomplete record, so no entry can be hiding after it — the
903 /// common damage, and the one a sequential walk handles fully.
904 CutShort,
905 /// Enough bytes remain to hold further records, and the walk could not
906 /// read the one at the stop point. Whatever those bytes hold is
907 /// unreachable by construction.
908 Unreadable,
909 /// The record at the stop point is one the scanner RECOGNISES — its
910 /// [`WalkStop::cause`] names it — in a shape this build cannot read (Task
911 /// 4 fix round 1: an `ar` symbol table whose header the reader cannot
912 /// parse; GNU `ar`'s Mach-O `__.SYMDEF` with a blank mode was the first,
913 /// and is read since 0.8.1). Not evidence of damage; what follows is
914 /// unreachable to this scanner.
915 UnsupportedShape,
916}
917
918impl WalkStop {
919 pub fn new(
920 format: FormatId,
921 offset: u64,
922 remaining: u64,
923 kind: WalkStopKind,
924 cause: impl Into<String>,
925 ) -> Self {
926 Self {
927 format,
928 offset,
929 remaining,
930 kind,
931 cause: cause.into(),
932 }
933 }
934}
935
936/// The one sentence a caller prints for a [`WalkStop`] — written here, once,
937/// so every sequential-only scanner gets the same report without a line of
938/// its own, and so its kinds can never be described two ways each.
939///
940/// An [`WalkStopKind::Unreadable`] stop says the entries after it are
941/// *unreachable by construction, not absent*, with the offset and the byte
942/// count; a [`WalkStopKind::CutShort`] one says that nothing after it is
943/// missing from the report, which is exactly as true — the file ended inside
944/// that record; an [`WalkStopKind::UnsupportedShape`] one names the shape and
945/// says it is not evidence of damage.
946///
947/// Two of the sentences say the walk goes "as `stuffr list` does". That is a
948/// contract on every scanner that reports a [`WalkStop`]: it walks with its
949/// format's ORDINARY reader, over the file's own bytes, repairing none of
950/// them. `ar`'s walk served its global header as `!<arch>\n` until Stage 3
951/// Task 4's fix round 1, which made the phrase false (a bad magic `list`
952/// refuses was walked past) — a scanner that needs a repair like that must
953/// get its own sentence here rather than borrow this one.
954pub fn describe_walk_stop(stop: &WalkStop) -> String {
955 let WalkStop {
956 format,
957 offset,
958 remaining,
959 kind,
960 cause,
961 } = stop;
962 let bytes = if *remaining == 1 { "byte" } else { "bytes" };
963 match kind {
964 WalkStopKind::Unreadable => format!(
965 "this build's {format} salvage scanner walks the archive in order from its start, \
966 as `stuffr list` does, and stopped at offset {offset} with {remaining} {bytes} still \
967 to go: {cause}. The {format} format marks its records with nothing strong enough \
968 to search for, so the scanner cannot find its way past a record it cannot read; \
969 any entries in those {remaining} {bytes} are unreachable by construction — not \
970 absent, and not counted in this report"
971 ),
972 WalkStopKind::UnsupportedShape => format!(
973 "this build's {format} salvage scanner walks the archive in order from its start, \
974 as `stuffr list` does, and stopped at offset {offset} with {remaining} {bytes} still \
975 to go: {cause}. That is a record shape this build does not read, not evidence of \
976 damage; any entries in those {remaining} {bytes} are unreachable to this scanner \
977 and not counted in this report"
978 ),
979 WalkStopKind::CutShort => format!(
980 "this build's {format} salvage scanner walks the archive in order from its start, \
981 and the file ends {remaining} {bytes} into the record at offset {offset} ({cause}): \
982 too few for any further entry, so the archive is cut short there and nothing \
983 after that point is missing from this report"
984 ),
985 }
986}
987
988#[derive(Debug, Clone, Copy, PartialEq, Eq)]
989pub enum PartialPolicy {
990 Keep,
991 Skip,
992 Ask,
993}
994
995pub struct SalvagePolicy {
996 /// Recovery-biased by default: salvage is the one permissive verb, and
997 /// the strict path is the entire rest of the tool.
998 pub partial: PartialPolicy,
999 /// Structural ceiling on any one entry's length, refused BEFORE
1000 /// allocation — and refused as a per-entry
1001 /// [`UnverifiedCause::OverEntryCeiling`], never as an error that ends
1002 /// the run. [`SalvageScan::max_whole_entry`] narrows it further for a
1003 /// format that decodes an entry whole; the two compose as a minimum and
1004 /// are applied together, once, in [`annotate_candidates`].
1005 pub max_entry: u64,
1006 /// Demand proof: every [`SalvageStatus::Partial`] entry is skipped,
1007 /// whatever [`Self::partial`] says, and so is every
1008 /// [`SalvageStatus::Unattested`] one (Ruling 3-H, Salvage Stage 3).
1009 ///
1010 /// The second clause is this flag's doing, unlike the two refused
1011 /// below: an `Unattested` entry is WRITTEN by default, under its real
1012 /// name, and only `strict` declines it — the format carries nothing
1013 /// that could prove it, which is exactly what "demand proof" asks for.
1014 ///
1015 /// **Those two are the whole of it, and this doc claimed three things
1016 /// until the final whole-branch review's F5.** It read "partial skipped,
1017 /// ceiling fixed, nothing unverifiable", and only the first clause was
1018 /// this flag's doing. The ceiling is NOT fixed under `strict`:
1019 /// [`Self::max_entry`] applies identically with and without it, and
1020 /// `--max-entry` is honoured either way. And an
1021 /// [`SalvageStatus::Unverified`] entry is never written under any
1022 /// policy at all — that is the ops layer's unconditional rule, not
1023 /// something `strict` causes — so crediting it here made a flag look
1024 /// load-bearing for a guarantee that holds without it.
1025 ///
1026 /// The true clauses are enough to justify the flag: `salvage` is this
1027 /// tool's single recovery-biased verb, so `strict` is how a caller asks
1028 /// for the discipline every other verb applies by default.
1029 pub strict: bool,
1030}
1031
1032impl Default for SalvagePolicy {
1033 fn default() -> Self {
1034 Self {
1035 partial: PartialPolicy::Keep,
1036 max_entry: MAX_SALVAGE_ENTRY,
1037 strict: false,
1038 }
1039 }
1040}
1041
1042/// Reusable scratch for a scanner's forward search for its next signature
1043/// (a magic, a checksum-valid block). Reads start at [`Self::FIRST_CHUNK`],
1044/// or at `2 * sig_len - 1` when that is larger, so the first read always
1045/// tests `sig_len` offsets, and double up to [`Self::MAX_CHUNK`], into one
1046/// buffer kept across calls,
1047/// so a search whose answer is a few bytes away reads a few bytes: a scan
1048/// that resumes one byte past a refused candidate costs work proportional to
1049/// the distance it actually searched, never a fixed 64 KiB per call. That is
1050/// the property `testing::check_scan_is_linear` holds every scanner to.
1051/// O(1) memory however far the next signature is.
1052///
1053/// **The bytes a search read past its answer are kept for the next call
1054/// (0.10.3 Task 4).** A call whose `from` lies inside what the previous
1055/// call read tests those bytes again without reading them again, and reads
1056/// only past them. Without that, every call paid at least its first read
1057/// afresh, so candidates a few bytes apart — an accepted header every
1058/// eleven bytes, say — cost a first read EACH: 64 bytes, or `2 * sig_len -
1059/// 1` for a long signature, per candidate. A scan whose `from` only moves
1060/// forward therefore reads each byte about once in its searches, however
1061/// dense its candidates are.
1062///
1063/// **One `ForwardSearch` serves one source**, whose bytes do not change
1064/// while it is in use — a scanner owns one per scan. Pass a different
1065/// source and the kept bytes are that other source's. A call with a
1066/// smaller `file_len` than the last never tests a kept byte at or past it.
1067#[derive(Debug, Default)]
1068pub struct ForwardSearch {
1069 /// Bytes read from the source, starting at offset `window_start`.
1070 window: Vec<u8>,
1071 window_start: u64,
1072}
1073
1074impl ForwardSearch {
1075 /// Size of the first read of every search, unless the signature is long
1076 /// enough that `2 * sig_len - 1` is larger — see [`Self::find`].
1077 pub const FIRST_CHUNK: usize = 64;
1078 /// Ceiling the read size doubles up to.
1079 pub const MAX_CHUNK: usize = 64 * 1024;
1080
1081 /// An empty search; the buffer grows on first use and is then reused.
1082 pub fn new() -> Self {
1083 Self::default()
1084 }
1085
1086 /// The first offset `>= from` at which `matches` accepts the `sig_len`
1087 /// bytes starting there, or `None` when none does before `file_len`.
1088 /// `matches` sees exactly `sig_len` bytes. Every offset is tested once,
1089 /// carrying `sig_len - 1` bytes across a chunk boundary. Never reads at
1090 /// or past `file_len`, and stops at EOF if the source is shorter.
1091 ///
1092 /// The first read is `max(FIRST_CHUNK, 2 * sig_len - 1)` bytes, so it
1093 /// tests at least `sig_len` offsets. A first read of only `sig_len`
1094 /// bytes tests ONE offset: for tar's 512-byte block, a header one block
1095 /// on then cost 512 + 1,024 bytes, three times the distance searched.
1096 pub fn find(
1097 &mut self,
1098 src: &mut dyn SeekRead,
1099 from: u64,
1100 file_len: u64,
1101 sig_len: usize,
1102 matches: impl FnMut(&[u8]) -> bool,
1103 ) -> io::Result<Option<u64>> {
1104 self.find_to_eof(src, from, file_len, sig_len, sig_len, matches)
1105 }
1106
1107 /// [`Self::find`] for a record whose length is not fixed: offsets with
1108 /// at least `sig_len` bytes before `file_len` are offered exactly
1109 /// `sig_len` bytes, and the last offsets — those with fewer, down to
1110 /// `min_len` — are offered every byte left before `file_len`. A record
1111 /// whose longest form is `sig_len` bytes can then be judged whole
1112 /// inside the search, wherever it starts, without a second read near
1113 /// EOF. `min_len` is at least 1 and at most `sig_len`.
1114 pub fn find_to_eof(
1115 &mut self,
1116 src: &mut dyn SeekRead,
1117 from: u64,
1118 file_len: u64,
1119 sig_len: usize,
1120 min_len: usize,
1121 mut matches: impl FnMut(&[u8]) -> bool,
1122 ) -> io::Result<Option<u64>> {
1123 debug_assert!(1 <= min_len && min_len <= sig_len);
1124 if min_len == 0 || file_len.saturating_sub(from) < min_len as u64 {
1125 return Ok(None);
1126 }
1127 // Keep what the last call read from `from` on, if it read `from`;
1128 // otherwise start empty. Never keep a byte at or past `file_len`.
1129 let kept_end = self.window_start + self.window.len() as u64;
1130 if from >= self.window_start && from < kept_end {
1131 self.window.drain(..(from - self.window_start) as usize);
1132 let fits = (file_len - from).min(self.window.len() as u64);
1133 self.window.truncate(fits as usize);
1134 } else {
1135 self.window.clear();
1136 }
1137 self.window_start = from;
1138 let mut chunk = Self::FIRST_CHUNK.max(2 * sig_len - 1);
1139 let mut at_eof = false;
1140 loop {
1141 // Every offset with a full `sig_len` bytes in the window.
1142 if self.window.len() >= sig_len {
1143 let last = self.window.len() - sig_len;
1144 if let Some(at) = (0..=last).find(|&i| matches(&self.window[i..i + sig_len])) {
1145 return Ok(Some(self.window_start + at as u64));
1146 }
1147 let tested = last + 1;
1148 self.window_start += tested as u64;
1149 self.window.drain(..tested);
1150 }
1151 let end = self.window_start + self.window.len() as u64;
1152 if at_eof || end >= file_len {
1153 // Nothing more to read: the offsets left have fewer than
1154 // `sig_len` bytes, and are offered what there is down to
1155 // `min_len`. They stay in the window, so a later call
1156 // re-tests them without a read.
1157 let short = self.window.len().saturating_sub(min_len - 1);
1158 let tail = (0..short).find(|&i| matches(&self.window[i..]));
1159 return Ok(tail.map(|at| self.window_start + at as u64));
1160 }
1161 let old = self.window.len();
1162 let want = (chunk as u64).min(file_len - end) as usize;
1163 src.seek(SeekFrom::Start(end))?;
1164 self.window.resize(old + want, 0);
1165 let n = src.read(&mut self.window[old..])?;
1166 self.window.truncate(old + n);
1167 if n == 0 {
1168 at_eof = true;
1169 } else if n == want {
1170 // Grow only on a full read: a source that trickles short
1171 // reads must not make every call zero a 64 KiB buffer for
1172 // 7 bytes.
1173 chunk = (chunk * 2).min(Self::MAX_CHUNK);
1174 }
1175 }
1176 }
1177}
1178
1179/// The work one salvage scan may spend judging candidates beyond their
1180/// fixed headers: bytes a verdict consumes (a long name, a variable part, an
1181/// extension chain, the bytes a header CRC covers), counted whether they come
1182/// from the source or from a search window. A legitimate archive consumes
1183/// each verdict's bytes once, at most `file_len` in total, so the limit of
1184/// `2 × file_len + 32 KiB` never trips on real input; forged candidates
1185/// that would spend past it are refused unexamined and reported as a
1186/// verdict-bounded sighting ([`SightingKind::VerdictBounded`]), and the
1187/// scan goes on.
1188///
1189/// **Why the slack is only 32 KiB (Rulings T4b-2 and T4b-5).** A
1190/// legitimate scan's verdict bytes all lie inside the file, so they total
1191/// at most `file_len` — and where a scanner pays for a second read too
1192/// (cpio pays for a name in the scan and again when the header before it
1193/// corroborates its size), at most `2 × file_len`. The per-byte term covers
1194/// real input on its own, and the slack only rounds. It was 1 MiB, which
1195/// let a fuzz-scale input of a few KiB spend about 1 MiB: an 8 KiB file of
1196/// overlapping zip names read 536,640 bytes, past
1197/// `stuffr_core::testing::check_scan_is_linear`'s `8 × file_len + 64 KiB`.
1198/// At 32 KiB the same file reads 65,536 bytes, half that bound. 16 KiB
1199/// was tried first and changed the outcome of 27 fuzz-corpus ZOO inputs,
1200/// whose overlapping type-2 records each read most of the file; at 32 KiB,
1201/// with ZOO charging each variable part once, all 27 keep their outcome.
1202///
1203/// **Why a scan needs one (0.10.3).** A scanner resumes one byte past every
1204/// candidate it refuses, and past every candidate whose payload runs off the
1205/// end of the file, so candidates can overlap. Where a candidate's verdict
1206/// needs a field the format lets run to 64 KiB — a zip name, a ZOO type-2
1207/// variable part, an LHA extension chain — packing such candidates every few
1208/// dozen bytes made the scan read that field once per candidate: measured at
1209/// 495x, 962x and 957x the input. No verdict can be reached by reading less,
1210/// so the WORK is bounded instead, across the whole scan: tar's extension-read
1211/// budget (Ruling T3-1), generalised.
1212///
1213/// One per scan, shared by every pass the scan makes. A refused candidate is
1214/// reported through [`Sighting::verdict_bounded`], so a genuine header
1215/// starved by forgeries elsewhere is named, never lost silently.
1216#[derive(Debug)]
1217pub struct ScanBudget {
1218 limit: u64,
1219 spent: u64,
1220}
1221
1222impl ScanBudget {
1223 /// Verdict bytes the scan may spend per byte of input.
1224 pub const PER_INPUT_BYTE: u64 = 2;
1225 /// Verdict bytes the scan may spend on top of [`Self::PER_INPUT_BYTE`]:
1226 /// rounding only, and small enough that it never dominates a small
1227 /// input's `check_scan_is_linear` bound — see the type's doc.
1228 pub const SLACK: u64 = 32 * 1024;
1229
1230 /// The budget for one scan of a `file_len`-byte input:
1231 /// `2 × file_len + 32 KiB`.
1232 pub fn for_input(file_len: u64) -> Self {
1233 Self {
1234 limit: file_len
1235 .saturating_mul(Self::PER_INPUT_BYTE)
1236 .saturating_add(Self::SLACK),
1237 spent: 0,
1238 }
1239 }
1240
1241 /// Charges `bytes` if they fit; `false` (charging nothing) if they would
1242 /// take the scan past its limit: the caller refuses the candidate
1243 /// without examining it.
1244 pub fn try_spend(&mut self, bytes: u64) -> bool {
1245 match self.spent.checked_add(bytes) {
1246 Some(total) if total <= self.limit => {
1247 self.spent = total;
1248 true
1249 }
1250 _ => false,
1251 }
1252 }
1253
1254 /// Bytes charged so far.
1255 pub fn spent(&self) -> u64 {
1256 self.spent
1257 }
1258}
1259
1260/// The per-format seam. A scanner recognises its own format's record shape
1261/// (a local header's magic, a member's checksum field) and reports what it
1262/// finds as a stream of [`Candidate`]s; [`salvage_all`] supplies everything
1263/// that does not depend on which format that is.
1264pub trait SalvageScan {
1265 /// Find the next candidate at or after `from`. `Ok(None)` ends the scan.
1266 fn next_candidate(&mut self, src: &mut dyn SeekRead, from: u64) -> Result<Option<Candidate>>;
1267
1268 /// The largest single entry this scanner is willing to have read for it
1269 /// — the ceiling [`annotate_candidates`] applies, narrowed further by
1270 /// `policy.max_entry`, before it calls [`Self::verify`] at all.
1271 ///
1272 /// `u64::MAX` (the default) means "this scanner imposes none of its
1273 /// own": zip streams every payload it verifies through a `take`, so the
1274 /// only figure that bounds it is the policy's. A format that **decodes
1275 /// an entry whole** — ARC, and the legacy containers still to come —
1276 /// overrides this with its own container-side ceiling
1277 /// (`arc.rs`'s `MAX_ARC_ENTRY_LEN`, 256 MiB), because for those the
1278 /// figure really does become a single allocation.
1279 ///
1280 /// **Declaring a ceiling here is the ONLY way a scanner refuses an
1281 /// entry for its size**, and that is the point: the refusal is then a
1282 /// [`SalvageStatus`] the engine assigns
1283 /// ([`UnverifiedCause::OverEntryCeiling`]), not a [`crate::Error`] the
1284 /// scanner raises, so it can never abort a run over one entry. A
1285 /// scanner that instead checks a size inside [`Self::verify`] and
1286 /// `?`-propagates the failure reintroduces the exact defect three fix
1287 /// rounds of Task 3c chased.
1288 ///
1289 /// **That is documented, not enforced, and saying so is more useful
1290 /// than claiming otherwise.** An earlier wording here called the defect
1291 /// "unrepresentable"; it is not. [`Self::verify`] still returns
1292 /// `Result` — it must, so a scanner CAN report a genuine run-level
1293 /// fault — so the wrong thing is still expressible. What changed is
1294 /// that the right thing now exists, costs one method, and is written
1295 /// into [`Self::verify`]'s contract below.
1296 fn max_whole_entry(&self) -> u64 {
1297 u64::MAX
1298 }
1299
1300 /// Decide what was actually proven about a candidate's payload, after
1301 /// discovery — called once per candidate, in file order, by
1302 /// [`annotate_candidates`].
1303 ///
1304 /// # The contract on `Err`
1305 ///
1306 /// **`Err` here aborts the WHOLE run**, discarding every entry already
1307 /// recovered, so it is reserved for a fault that really is about the
1308 /// run and not about this one entry (the destination is gone, the
1309 /// source handle itself failed). Everything a single candidate can be
1310 /// wrong about — a short read, a bad checksum, a method this build
1311 /// cannot decode, a device error partway through one payload — is a
1312 /// [`SalvageStatus`], never an `Err`. `zip_salvage.rs`'s own
1313 /// `verify_candidate` has stated this in prose since Stage 1 and folds
1314 /// every one of those; ARC's did not, for one case (its size ceiling),
1315 /// and that single `?` cost three fix rounds. The size ceiling now
1316 /// belongs to [`Self::max_whole_entry`], which the engine applies, so
1317 /// there is nothing left for an implementation to `?` on.
1318 ///
1319 /// # What an implementation may assume
1320 ///
1321 /// The candidate's bounded length (`available_len` if the payload was
1322 /// truncated, else `declared_len`) is already **at or below**
1323 /// [`Self::max_whole_entry`] — [`annotate_candidates`] answers
1324 /// [`UnverifiedCause::OverEntryCeiling`] itself, without calling this,
1325 /// for anything above it. So an implementation that decodes whole may
1326 /// size a buffer from that length without re-checking it.
1327 ///
1328 /// The default answers [`SalvageStatus::Unattested`] unconditionally,
1329 /// which is the least this method can claim: it reads nothing, decodes
1330 /// nothing and compares nothing, so "nothing attests that this candidate
1331 /// is an entry at all" is exactly what happened. A format that carries a
1332 /// checksum overrides this to decode the payload — reusing its own
1333 /// existing codec machinery, never inventing a new one here — and
1334 /// compare it against [`Candidate::verifier`]; a format that carries a
1335 /// self-verifying HEADER and no content checksum (tar) overrides it to
1336 /// check that header and answer [`SalvageStatus::Complete`].
1337 ///
1338 /// **It answered `Complete` until Salvage Stage 3**, and the change is
1339 /// not cosmetic. `Complete` asserts that every declared byte was present
1340 /// *and the header self-verified* — two claims this default has never
1341 /// been in a position to make — and it routes to **exit 0**, so a
1342 /// checksumless scanner that simply forgot to override was reporting a
1343 /// clean recovery over candidates nothing had examined. `Unattested`
1344 /// routes to exit 4 and says why on the row. The same argument
1345 /// [`Candidate::new`] makes for its own four defaults: the default is
1346 /// the answer that asserts the LEAST, so forgetting to override is
1347 /// visible rather than flattering.
1348 fn verify(&self, _src: &mut dyn SeekRead, _candidate: &Candidate) -> Result<SalvageStatus> {
1349 Ok(SalvageStatus::Unattested)
1350 }
1351
1352 /// Decode one salvaged entry's payload out of the archive at
1353 /// `archive_path` and into `out`, answering whether the entry's whole
1354 /// declared length was produced.
1355 ///
1356 /// # Why this is on the trait at all (final whole-branch review, F2)
1357 ///
1358 /// `0.6.0` spends a breaking release making this trait implementable
1359 /// from outside. It was implementable and not USABLE: the ops layer's
1360 /// payload dispatch was a private `match` on a [`crate::FormatId`]
1361 /// string, so a scanner written outside this workspace got
1362 /// [`salvage_all`] and none of the behaviour that makes `salvage` what
1363 /// it is. This method is where a scanner's write half now DECLARES
1364 /// itself, beside the scan half, in one place a reader can see both.
1365 ///
1366 /// **It does not by itself make an out-of-tree scanner reachable from
1367 /// `stuffr salvage`, and pretending otherwise would be the same
1368 /// over-claim F2 named.** The ops layer still resolves a format to a
1369 /// scanner by name, with no registration point for a third party to add
1370 /// one; closing that needs a salvage registry and is Stage 3's. What
1371 /// changed is that the seam is now declared rather than scattered — the
1372 /// five in-tree scanners implement this method and the ops layer calls
1373 /// it, so the next one has a signature to fit rather than a convention
1374 /// to guess.
1375 ///
1376 /// # The contract
1377 ///
1378 /// `Ok(true)` means every declared byte was produced; `Ok(false)` means
1379 /// the payload ran short or the decoder stopped early, which the caller
1380 /// renders as a `.partial` write. **`Err` is reserved for the same
1381 /// narrow set [`Self::verify`] reserves it for** — with one addition
1382 /// the ops layer relies on: [`crate::Error::Io`] here is treated as a
1383 /// fact about THIS entry (the archive could not be re-opened or read
1384 /// for it, the destination refused the bytes) and folded into that
1385 /// entry's own outcome, never the run's. A size refusal is never an
1386 /// `Err`: it belongs to [`Self::max_whole_entry`], which the engine
1387 /// applies before `verify` is ever called.
1388 ///
1389 /// The default refuses. A scanner that only ever reports
1390 /// [`SalvageStatus::Unverified`] has nothing to write, and inheriting a
1391 /// refusal is the honest answer for it; the alternative — a required
1392 /// method — would be a breaking change to every implementation for no
1393 /// gain to one that cannot write anything anyway.
1394 fn write_payload(
1395 &self,
1396 _archive_path: &Path,
1397 _entry: &SalvagedEntry,
1398 _compressed_len: u64,
1399 _out: &mut dyn Write,
1400 ) -> Result<bool> {
1401 Err(Error::Unsupported(
1402 "this salvage scanner has no payload writer".to_string(),
1403 ))
1404 }
1405}
1406
1407/// Walk `scan` over `src`, collecting every candidate it reports into a
1408/// [`SalvageOutcome`].
1409///
1410/// **Requires a seekable source.** Scanning searches back and forth over the
1411/// archive rather than reading it once in order, so this takes
1412/// `&mut dyn SeekRead` rather than the ladder's `Box<dyn Source>` (whose
1413/// `as_seek()` may be `None` for a pipe). A caller sitting on a
1414/// non-seekable source must spool it first — exactly as the `arj` and `zoo`
1415/// containers already do, reporting `Rung::Spilled` — which is a later
1416/// task's ops-layer concern, not this function's.
1417///
1418/// Every candidate's length is bounded against the run's entry ceiling
1419/// (`policy.max_entry`, narrowed by [`SalvageScan::max_whole_entry`])
1420/// **before** anything is sized from it — in [`annotate_candidates`], one
1421/// step before the only place that could size anything, and as a per-entry
1422/// [`UnverifiedCause::OverEntryCeiling`] rather than a [`crate::Error`] that ends
1423/// the run.
1424pub fn salvage_all(
1425 scan: &mut dyn SalvageScan,
1426 src: &mut dyn SeekRead,
1427 policy: &SalvagePolicy,
1428) -> Result<SalvageOutcome> {
1429 let candidates = collect_candidates(scan, src)?;
1430 annotate_candidates(&*scan, src, candidates, policy)
1431}
1432
1433/// The resync loop alone: find a candidate, advance strictly past it,
1434/// repeat. Never reads a payload, never sizes anything from a declared
1435/// length, and never calls [`SalvageScan::verify`] — see
1436/// [`annotate_candidates`] for both the entry ceiling and the verification.
1437///
1438/// Exposed separately from [`salvage_all`] so a format that reconciles more
1439/// than one candidate source (zip's central-directory fallback) can collect
1440/// from each source before merging, and annotate the merged list once.
1441///
1442/// # The entry ceiling used to be enforced here, and was an `Err`
1443///
1444/// Salvage Stage 2 Task 3c's fix round 3 moved it to
1445/// [`annotate_candidates`] and turned it into a status. Two things were
1446/// wrong with it here, and only the second is obvious:
1447///
1448/// 1. **It refused a whole archive over one entry.** Measured at the CLI on
1449/// a 78-byte two-entry ARC, `stuffr salvage --list --max-entry 4`
1450/// printed no rows and exited 6 — the healthy first entry, which the
1451/// same binary recovers byte-for-byte at the default ceiling, simply
1452/// lost. That is the same defect the ARC scanner's own ceiling had, in
1453/// the engine rather than in a format.
1454/// 2. **It was one of two ceilings, checked in two places**, so which
1455/// disposition an over-ceiling entry got depended on which check saw it
1456/// first. There is now one rule, applied once, to both figures.
1457///
1458/// Nothing is weakened by the move: this loop sizes nothing from a declared
1459/// length (it only advances `from` by it), so "bounded before anything is
1460/// sized from it" holds exactly as before — the bound simply sits one
1461/// function later, still strictly before the only allocation there is.
1462pub fn collect_candidates(
1463 scan: &mut dyn SalvageScan,
1464 src: &mut dyn SeekRead,
1465) -> Result<Vec<Candidate>> {
1466 let mut candidates = Vec::new();
1467 let mut from = 0u64;
1468
1469 while let Some(candidate) = scan.next_candidate(src, from)? {
1470 // Advance strictly past this candidate so the scan always makes
1471 // progress — but NEVER by a length this candidate merely CLAIMED.
1472 // `available_len.is_some()` means the header's own declared length
1473 // disagreed with the file (there were fewer bytes left than it
1474 // said), and a header that lied about its size once cannot be
1475 // trusted to say how much of the source its "payload" consumed
1476 // either. This used to advance by `available_len` — the bytes
1477 // actually present, not the declared figure — on the reasoning that
1478 // a truncated entry has nothing worth finding after it anyway. That
1479 // is true for a genuinely truncated REAL last entry, but the
1480 // engine cannot tell that case apart from a coincidental
1481 // false-positive header sitting on top of noise with a garbage
1482 // multi-hundred-megabyte declared length: `available_len` is the
1483 // same "everything left in the file" figure either way. Salvage
1484 // Stage 2 Task 3's fix round 1 found exactly this — one
1485 // coincidental marker+name match in an ARC noise corpus, with a
1486 // declared length far past the file, jumped the scan straight to
1487 // EOF in one step and silently dropped every real entry that
1488 // happened to sit between it and the end of the file: no error, no
1489 // `Partial`, no note, in the one verb whose entire job is not
1490 // losing entries silently. So a truncated candidate now advances
1491 // minimally — past its own marker byte only — so the very next
1492 // byte onward is examined for another header. That costs a few
1493 // bytes of re-scanning after a genuinely truncated real entry
1494 // (which has nothing left to find anyway, so the outcome is
1495 // unchanged); it is the entire fix for a phantom, whose lie no
1496 // longer costs the rest of the archive. An UNTRUNCATED candidate's
1497 // declared length was just checked against the file two lines
1498 // above — it fits, so it is not a lie, and advancing by it is
1499 // unchanged.
1500 let advance = if candidate.available_len.is_some() {
1501 1
1502 } else {
1503 candidate.declared_len.unwrap_or(1).max(1)
1504 };
1505 from = candidate.offset.saturating_add(advance);
1506
1507 candidates.push(candidate);
1508 }
1509
1510 Ok(candidates)
1511}
1512
1513/// Turns already-discovered candidates — in file order — into the bounded,
1514/// verified, shadow-annotated entries a caller receives. See this module's
1515/// doc comment for what the passes (bound, verify, shadow) mean and why
1516/// they live here rather than per-format.
1517///
1518/// # The entry ceiling, in one place (fix round 3, Task 3c)
1519///
1520/// `policy.max_entry` narrowed by [`SalvageScan::max_whole_entry`] is the
1521/// run's ceiling on ONE entry, and this is the only place it is applied. A
1522/// candidate whose bounded length (`available_len` if the payload is
1523/// truncated, else `declared_len`) is over it is reported
1524/// [`SalvageStatus::Unverified`]`(`[`UnverifiedCause::OverEntryCeiling`]`)`
1525/// and [`SalvageScan::verify`] is **not called for it at all** — so nothing
1526/// is read and nothing is allocated, which was the whole purpose of the two
1527/// `Err`-raising checks this replaced, without their cost: neither the
1528/// engine's own nor a scanner's could refuse one entry without ending the
1529/// run and discarding every entry already recovered.
1530///
1531/// The bounded length, never the declared one, is what is compared — the
1532/// rule [`Candidate::available_len`]'s own doc argues for and the reason a
1533/// truncated tail's garbage declaration cannot decide anything: a candidate
1534/// whose payload ran out can never cost more than the bytes that exist.
1535pub fn annotate_candidates(
1536 scan: &dyn SalvageScan,
1537 src: &mut dyn SeekRead,
1538 candidates: Vec<Candidate>,
1539 policy: &SalvagePolicy,
1540) -> Result<SalvageOutcome> {
1541 let ceiling = policy.max_entry.min(scan.max_whole_entry());
1542 let mut entries = Vec::with_capacity(candidates.len());
1543 // Earliest scan_position to report each distinct (name, declared_len,
1544 // verifier) triple seen so far — a candidate reporting one already in
1545 // here is shadowing that position. A degenerate zero-length checksum
1546 // (every empty file, every directory entry) is never inserted here and
1547 // never looked up here — see `is_degenerate` below.
1548 let mut seen: HashMap<(String, Option<u64>, Verifier), usize> = HashMap::new();
1549 // Earliest scan_position to report each distinct NAME, regardless of
1550 // anything else about the record — the weaker, directly-observed fact
1551 // `SalvagedEntry::collides_with` reports, and the one `stuffr list`'s
1552 // own fidelity warning is about. Kept separate from `seen` above rather
1553 // than folded into it: that map is deliberately full of guards against
1554 // over-claiming a COPY (a verifier must exist, the declared length must
1555 // agree, a zero-length checksum never counts), and every one of them
1556 // would be wrong here, where nothing about content is being claimed.
1557 let mut names_seen: HashMap<String, usize> = HashMap::new();
1558
1559 for (scan_position, candidate) in candidates.into_iter().enumerate() {
1560 // Bounded length, then the ceiling, then — only if it fits —
1561 // verification. `verify` is what reads and (for a whole-decoding
1562 // format) allocates, so refusing above it is refusing before the
1563 // allocation, which is the property the two `Err`s this replaced
1564 // existed for.
1565 let bounded_len = candidate.available_len.or(candidate.declared_len);
1566 let status = match bounded_len {
1567 Some(needed) if needed > ceiling => {
1568 SalvageStatus::Unverified(UnverifiedCause::OverEntryCeiling { needed, ceiling })
1569 }
1570 _ => scan.verify(src, &candidate)?,
1571 };
1572
1573 let declared_len = candidate.declared_len;
1574 let verifier = candidate.verifier;
1575 // A checksum over zero declared bytes is the same value for every
1576 // empty entry that ever existed (CRC-32 and CRC-16/ARC of an empty
1577 // input are both fixed constants) and proves nothing about any two
1578 // of them being copies of each other — measured against a real
1579 // archive during this task's fix round: every `__init__.py` and
1580 // every directory entry collapsed onto one shadow chain under a
1581 // checksum-only match.
1582 let is_degenerate = declared_len == Some(0);
1583
1584 let shadows = if is_degenerate {
1585 None
1586 } else {
1587 verifier.and_then(|v| {
1588 seen.get(&(candidate.meta.name.clone(), declared_len, v))
1589 .copied()
1590 })
1591 };
1592
1593 if !is_degenerate
1594 && shadows.is_none()
1595 && let Some(v) = verifier
1596 {
1597 seen.entry((candidate.meta.name.clone(), declared_len, v))
1598 .or_insert(scan_position);
1599 }
1600
1601 // The two annotations are mutually exclusive: a record proven to be
1602 // a copy is reported as a shadow and nothing else, because "this is
1603 // a duplicate of #2" is strictly more informative than "something
1604 // earlier used this name". Everything else that repeats a name is a
1605 // collision.
1606 let earliest_with_name = names_seen.get(&candidate.meta.name).copied();
1607 let collides_with = if shadows.is_some() {
1608 None
1609 } else {
1610 earliest_with_name
1611 };
1612 if earliest_with_name.is_none() {
1613 names_seen.insert(candidate.meta.name.clone(), scan_position);
1614 }
1615
1616 let marked_deleted = candidate.marked_deleted;
1617 entries.push(
1618 SalvagedEntry::new(
1619 scan_position,
1620 candidate.offset,
1621 candidate.payload_start,
1622 candidate.meta,
1623 status,
1624 )
1625 .with_shadows(shadows)
1626 .with_collides_with(collides_with)
1627 .with_marked_deleted(marked_deleted),
1628 );
1629 }
1630
1631 Ok(SalvageOutcome {
1632 entries,
1633 sightings: Vec::new(),
1634 walk_stop: None,
1635 })
1636}
1637
1638/// Streams `reader` into `out`, bounded to `expected_len` bytes — the
1639/// entry's own declared uncompressed size — so a small compressed input
1640/// cannot expand arbitrarily far past what its own header claims. Never
1641/// buffers a growable copy of the payload, only a fixed 64 KiB window, and
1642/// never trusts the decoder to stop on its own.
1643///
1644/// Returns `true` when exactly `expected_len` bytes were produced, `false`
1645/// when `reader` ran out or errored first — [`crate`]'s own convention
1646/// throughout this module: "did not fully recover" is a fact a caller
1647/// reports (as [`SalvageStatus::Partial`]'s own two causes already are),
1648/// never a [`crate::Error`] that aborts the whole run over one entry.
1649///
1650/// Lives here, rather than duplicated once per format module, because it
1651/// has no format-specific knowledge at all — only `Read`/`Write` and a byte
1652/// count. Task 3c moved it up from `entries.rs`, where it was zip-shaped
1653/// plumbing bolted onto a zip-only write path; every [`SalvageScan`]
1654/// implementation's own `write_payload` (zip's, ARC's, and any later
1655/// format's) calls this same function once it has a decoded reader in
1656/// hand, so a truncated or overrunning decode is reported identically
1657/// regardless of which format produced it.
1658pub fn stream_bounded_copy(
1659 mut reader: impl Read,
1660 expected_len: u64,
1661 out: &mut dyn Write,
1662) -> Result<bool> {
1663 let mut buf = [0u8; 64 * 1024];
1664 let mut produced = 0u64;
1665 loop {
1666 if produced >= expected_len {
1667 return Ok(true);
1668 }
1669 let want = ((expected_len - produced) as usize).min(buf.len());
1670 let n = match reader.read(&mut buf[..want]) {
1671 Ok(0) => return Ok(false),
1672 Ok(n) => n,
1673 Err(_) => return Ok(false),
1674 };
1675 out.write_all(&buf[..n])?;
1676 produced += n as u64;
1677 }
1678}
1679
1680#[cfg(test)]
1681mod tests {
1682 use std::io::{Cursor, Read, Seek, SeekFrom};
1683
1684 use super::*;
1685
1686 fn bytes(data: &[u8]) -> Cursor<Vec<u8>> {
1687 Cursor::new(data.to_vec())
1688 }
1689
1690 /// A reader that panics if asked to read more bytes in one call than its
1691 /// backing buffer holds. `salvage_all`'s own loop never reads a
1692 /// candidate's payload at all (see the module doc), so a correct
1693 /// implementation never trips this — it exists to catch a regression
1694 /// that reads before bounding, the exact shape this project has shipped
1695 /// six times across `cpio`, `ar` and `zip`.
1696 struct PanickingReader(Cursor<Vec<u8>>);
1697
1698 impl Read for PanickingReader {
1699 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1700 let backing_len = self.0.get_ref().len();
1701 assert!(
1702 buf.len() <= backing_len,
1703 "oversized read requested: {} bytes against a {backing_len}-byte source \
1704 — an allocation or read was sized from a declared length before it was bounded",
1705 buf.len()
1706 );
1707 self.0.read(buf)
1708 }
1709 }
1710
1711 impl Seek for PanickingReader {
1712 fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
1713 self.0.seek(pos)
1714 }
1715 }
1716
1717 fn panicking_reader(len: usize) -> PanickingReader {
1718 PanickingReader(Cursor::new(vec![0u8; len]))
1719 }
1720
1721 /// A scanner that never finds anything, however many times it is asked.
1722 struct NeverFinds;
1723
1724 impl SalvageScan for NeverFinds {
1725 fn next_candidate(
1726 &mut self,
1727 _src: &mut dyn SeekRead,
1728 _from: u64,
1729 ) -> Result<Option<Candidate>> {
1730 Ok(None)
1731 }
1732 }
1733
1734 /// A scanner that reports exactly one candidate, declaring the given
1735 /// length, then ends the scan. It never touches `src` itself — it
1736 /// exists purely to hand `salvage_all` a declared length to bound.
1737 struct DeclaresLength(u64);
1738
1739 impl SalvageScan for DeclaresLength {
1740 fn next_candidate(
1741 &mut self,
1742 _src: &mut dyn SeekRead,
1743 from: u64,
1744 ) -> Result<Option<Candidate>> {
1745 if from > 0 {
1746 return Ok(None);
1747 }
1748 Ok(Some(Candidate {
1749 offset: 0,
1750 payload_start: 0,
1751 meta: EntryMeta::file("absurd"),
1752 declared_len: Some(self.0),
1753 verifier: None,
1754 available_len: None,
1755 marked_deleted: false,
1756 }))
1757 }
1758 }
1759
1760 /// A scanner reporting one candidate whose header DECLARES `declared`
1761 /// bytes while the source holds only `available` of them — the
1762 /// truncated-tail shape, the one a real scanner reports by setting
1763 /// [`Candidate::available_len`].
1764 struct DeclaresMoreThanIsThere {
1765 declared: u64,
1766 available: u64,
1767 }
1768
1769 impl SalvageScan for DeclaresMoreThanIsThere {
1770 fn next_candidate(
1771 &mut self,
1772 _src: &mut dyn SeekRead,
1773 from: u64,
1774 ) -> Result<Option<Candidate>> {
1775 if from > 0 {
1776 return Ok(None);
1777 }
1778 Ok(Some(Candidate {
1779 offset: 0,
1780 payload_start: 0,
1781 meta: EntryMeta::file("cut-short"),
1782 declared_len: Some(self.declared),
1783 verifier: Some(Verifier::Crc32(1)),
1784 available_len: Some(self.available),
1785 marked_deleted: false,
1786 }))
1787 }
1788 }
1789
1790 #[test]
1791 fn a_scanner_that_finds_nothing_yields_no_entries() {
1792 let out = salvage_all(
1793 &mut NeverFinds,
1794 &mut bytes(&[0u8; 4096]),
1795 &SalvagePolicy::default(),
1796 );
1797 assert!(
1798 out.unwrap().entries.is_empty(),
1799 "a scanner finding nothing must yield nothing"
1800 );
1801 }
1802
1803 /// The ceiling test that matters most: a reader that PANICS on an
1804 /// oversized read, so this test fails if the read happens at all — not
1805 /// merely if the wrong status comes back. See this module's
1806 /// falsification note in the task report for what happens when the
1807 /// bound is moved below a read.
1808 ///
1809 /// Fix round 3 (Task 3c): the refusal is a per-entry STATUS now, not an
1810 /// `Err` — so this asserts both halves, that nothing was read AND that
1811 /// the run completed with the entry reported. The old assertion
1812 /// (`unwrap_err()`, exit 6) proved only the first and was satisfied by
1813 /// the behaviour that lost every other entry in the archive.
1814 #[test]
1815 fn an_absurd_declared_length_is_refused_before_the_allocation_it_would_size() {
1816 let mut scan = DeclaresLength(u64::MAX);
1817 let out = salvage_all(
1818 &mut scan,
1819 &mut panicking_reader(4096),
1820 &SalvagePolicy::default(),
1821 )
1822 .expect("an entry over the ceiling must not abort the run");
1823 assert_eq!(out.entries.len(), 1, "the entry is still reported");
1824 assert_eq!(
1825 out.entries[0].status,
1826 SalvageStatus::Unverified(UnverifiedCause::OverEntryCeiling {
1827 needed: u64::MAX,
1828 ceiling: MAX_SALVAGE_ENTRY,
1829 })
1830 );
1831 }
1832
1833 /// The other half of the same rule, and the one the CLI reaches:
1834 /// everything AROUND an over-ceiling entry is still recovered. Measured
1835 /// before this fix as `stuffr salvage --list --max-entry 4` over a
1836 /// 78-byte ARC — no rows at all, exit 6.
1837 #[test]
1838 fn an_entry_over_the_ceiling_does_not_cost_the_entries_around_it() {
1839 struct TwoCandidates;
1840 impl SalvageScan for TwoCandidates {
1841 fn next_candidate(
1842 &mut self,
1843 _src: &mut dyn SeekRead,
1844 from: u64,
1845 ) -> Result<Option<Candidate>> {
1846 let (offset, len, name) = match from {
1847 0 => (0u64, 16u64, "small-first"),
1848 1..=16 => (17u64, 4096u64, "huge-second"),
1849 _ => return Ok(None),
1850 };
1851 Ok(Some(Candidate {
1852 offset,
1853 payload_start: offset,
1854 meta: EntryMeta::file(name),
1855 declared_len: Some(len),
1856 verifier: Some(Verifier::Crc32(len as u32)),
1857 available_len: None,
1858 marked_deleted: false,
1859 }))
1860 }
1861 }
1862
1863 let out = salvage_all(
1864 &mut TwoCandidates,
1865 &mut bytes(&[0u8; 8192]),
1866 &SalvagePolicy {
1867 max_entry: 64,
1868 ..SalvagePolicy::default()
1869 },
1870 )
1871 .expect("one over-ceiling entry must never abort the run");
1872 assert_eq!(out.entries.len(), 2);
1873 // `TwoCandidates` does not override `verify`, so the first entry
1874 // takes the trait's default — `Unattested` since Salvage Stage 3,
1875 // where it was `Complete`. Neither figure is what this test is
1876 // about: the point is that the SECOND entry's ceiling refusal did
1877 // not cost the first entry its row.
1878 assert_eq!(out.entries[0].status, SalvageStatus::Unattested);
1879 assert_eq!(
1880 out.entries[1].status,
1881 SalvageStatus::Unverified(UnverifiedCause::OverEntryCeiling {
1882 needed: 4096,
1883 ceiling: 64,
1884 })
1885 );
1886 }
1887
1888 /// A scanner's own [`SalvageScan::max_whole_entry`] narrows the policy's
1889 /// ceiling — the composition ARC relies on, proven against a mock so it
1890 /// does not depend on ARC's own 256 MiB constant. The candidate here is
1891 /// well under `policy.max_entry` and over the scanner's own figure, so
1892 /// only the `min` can refuse it; the panicking reader proves it is
1893 /// refused without being read.
1894 #[test]
1895 fn a_scanners_own_whole_entry_ceiling_narrows_the_policys() {
1896 struct DecodesWhole;
1897 impl SalvageScan for DecodesWhole {
1898 fn next_candidate(
1899 &mut self,
1900 _src: &mut dyn SeekRead,
1901 from: u64,
1902 ) -> Result<Option<Candidate>> {
1903 if from > 0 {
1904 return Ok(None);
1905 }
1906 Ok(Some(Candidate {
1907 offset: 0,
1908 payload_start: 0,
1909 meta: EntryMeta::file("whole"),
1910 declared_len: Some(2048),
1911 verifier: Some(Verifier::Crc32(7)),
1912 available_len: None,
1913 marked_deleted: false,
1914 }))
1915 }
1916 fn max_whole_entry(&self) -> u64 {
1917 1024
1918 }
1919 fn verify(&self, _src: &mut dyn SeekRead, _c: &Candidate) -> Result<SalvageStatus> {
1920 panic!("verify must not run for a candidate over the ceiling");
1921 }
1922 }
1923
1924 let out = salvage_all(
1925 &mut DecodesWhole,
1926 &mut panicking_reader(4096),
1927 &SalvagePolicy::default(),
1928 )
1929 .expect("a scanner ceiling is a per-entry refusal, not a run failure");
1930 assert_eq!(
1931 out.entries[0].status,
1932 SalvageStatus::Unverified(UnverifiedCause::OverEntryCeiling {
1933 needed: 2048,
1934 // The SCANNER's figure, not the policy's 4 GiB — the `min`
1935 // is what this test is really pinning.
1936 ceiling: 1024,
1937 })
1938 );
1939 }
1940
1941 #[test]
1942 fn a_declared_length_within_the_ceiling_is_collected_rather_than_refused() {
1943 let mut scan = DeclaresLength(4096);
1944 let out = salvage_all(
1945 &mut scan,
1946 &mut panicking_reader(4096),
1947 &SalvagePolicy::default(),
1948 )
1949 .unwrap();
1950 assert_eq!(out.entries.len(), 1);
1951 let entry = &out.entries[0];
1952 assert_eq!(entry.scan_position, 0);
1953 assert_eq!(entry.offset, 0);
1954 // `DeclaresLength` does not override `verify`, so this is the
1955 // trait's default: `Unattested` since Salvage Stage 3, where it was
1956 // `Complete`. The test is about the entry being COLLECTED — its
1957 // declared length sits under the ceiling — not about which tier a
1958 // mock that checks nothing lands in.
1959 assert_eq!(entry.status, SalvageStatus::Unattested);
1960 assert_eq!(entry.shadows, None);
1961 assert_eq!(entry.collides_with, None);
1962 }
1963
1964 /// A truncated candidate is COLLECTED, not dropped — the whole point of
1965 /// [`Candidate::available_len`]. Before it existed, `zip_salvage.rs`
1966 /// refused such a candidate outright and a truncated archive's last
1967 /// entry vanished with no row, no note and exit 0.
1968 #[test]
1969 fn a_candidate_whose_payload_ran_out_is_still_collected() {
1970 let mut scan = DeclaresMoreThanIsThere {
1971 declared: 4000,
1972 available: 1950,
1973 };
1974 let out = salvage_all(
1975 &mut scan,
1976 &mut panicking_reader(4096),
1977 &SalvagePolicy::default(),
1978 )
1979 .unwrap();
1980 assert_eq!(
1981 out.entries.len(),
1982 1,
1983 "a header found but not completable is a fact worth reporting"
1984 );
1985 }
1986
1987 /// The ceiling is checked against what could be READ, not against what
1988 /// a truncated header CLAIMED. Otherwise a garbage length in a tail the
1989 /// scan has already established is not there would abort the entire run
1990 /// at exit 6 — losing every earlier entry over an absent one, in the
1991 /// verb that exists to rescue exactly that archive.
1992 #[test]
1993 fn a_truncated_candidate_is_bounded_by_what_is_there_not_by_what_it_claimed() {
1994 let mut scan = DeclaresMoreThanIsThere {
1995 declared: u64::MAX,
1996 available: 2048,
1997 };
1998 let out = salvage_all(
1999 &mut scan,
2000 &mut panicking_reader(4096),
2001 &SalvagePolicy::default(),
2002 )
2003 .expect("an absent tail must not abort a run that recovered real entries");
2004 assert_eq!(out.entries.len(), 1);
2005 }
2006
2007 /// The complement, so the rule above cannot be read as "truncated
2008 /// candidates are never bounded": what IS there is still bounded, and a
2009 /// truncated candidate holding more available bytes than the ceiling
2010 /// allows is refused exactly as an untruncated one would be — reported
2011 /// [`UnverifiedCause::OverEntryCeiling`], never read (the reader
2012 /// panics if it is), and never allowed to end the run.
2013 #[test]
2014 fn a_truncated_candidate_whose_available_bytes_exceed_the_ceiling_is_refused() {
2015 let mut scan = DeclaresMoreThanIsThere {
2016 declared: u64::MAX,
2017 available: 4096,
2018 };
2019 let out = salvage_all(
2020 &mut scan,
2021 &mut panicking_reader(4096),
2022 &SalvagePolicy {
2023 max_entry: 1024,
2024 ..SalvagePolicy::default()
2025 },
2026 )
2027 .expect("a bounded refusal is per-entry, never a run failure");
2028 assert_eq!(
2029 out.entries[0].status,
2030 // `needed` is 4096 — the bytes that ARE there — never the
2031 // `u64::MAX` this candidate's header declared. Fix round 4
2032 // (NEW-F): the figures a user is shown are the ones the
2033 // comparison used.
2034 SalvageStatus::Unverified(UnverifiedCause::OverEntryCeiling {
2035 needed: 4096,
2036 ceiling: 1024,
2037 }),
2038 "an over-ceiling entry is unverified, not truncated: nobody read it"
2039 );
2040 }
2041
2042 #[test]
2043 fn the_default_policy_is_recovery_biased() {
2044 let policy = SalvagePolicy::default();
2045 assert_eq!(policy.partial, PartialPolicy::Keep);
2046 assert_eq!(policy.max_entry, MAX_SALVAGE_ENTRY);
2047 assert!(!policy.strict);
2048 }
2049
2050 // -----------------------------------------------------------------
2051 // Task 4: `verify` dispatch and shadow detection, proven against a
2052 // mock — same discipline as the rest of this module, since neither
2053 // mechanism needs to know which format it is annotating.
2054 // -----------------------------------------------------------------
2055
2056 /// Reports a fixed, caller-scripted sequence of candidates and answers
2057 /// `verify` from the SAME script rather than decoding anything — this
2058 /// module's engine is proven against mocks, and verification/shadow
2059 /// detection are no exception. Each entry names its own `name` and
2060 /// `declared_len` (rather than a fixed, offset-derived name) so a test
2061 /// can freely vary any one of the three fields shadow detection now
2062 /// requires to agree.
2063 struct ScriptedCandidates {
2064 plan: Vec<(&'static str, Option<u64>, Option<Verifier>, SalvageStatus)>,
2065 next: usize,
2066 }
2067
2068 impl SalvageScan for ScriptedCandidates {
2069 fn next_candidate(
2070 &mut self,
2071 _src: &mut dyn SeekRead,
2072 _from: u64,
2073 ) -> Result<Option<Candidate>> {
2074 if self.next >= self.plan.len() {
2075 return Ok(None);
2076 }
2077 let offset = self.next as u64;
2078 let (name, declared_len, verifier, _) = self.plan[self.next];
2079 self.next += 1;
2080 Ok(Some(Candidate {
2081 offset,
2082 payload_start: offset,
2083 meta: EntryMeta::file(name),
2084 declared_len,
2085 verifier,
2086 available_len: None,
2087 marked_deleted: false,
2088 }))
2089 }
2090
2091 fn verify(&self, _src: &mut dyn SeekRead, candidate: &Candidate) -> Result<SalvageStatus> {
2092 Ok(self.plan[candidate.offset as usize].3)
2093 }
2094 }
2095
2096 /// `verify` decides each entry's status — the whole point of Task 4:
2097 /// this used to be a hardcoded `Complete` for every candidate, which was
2098 /// wrong for any format carrying a real checksum.
2099 #[test]
2100 fn verify_decides_each_entrys_status_not_a_hardcoded_default() {
2101 let mut scan = ScriptedCandidates {
2102 plan: vec![
2103 (
2104 "a",
2105 Some(1),
2106 Some(Verifier::Crc32(1)),
2107 SalvageStatus::Intact,
2108 ),
2109 ("b", Some(1), None, SalvageStatus::Complete),
2110 (
2111 "c",
2112 Some(1),
2113 Some(Verifier::Crc32(2)),
2114 SalvageStatus::Partial,
2115 ),
2116 ],
2117 next: 0,
2118 };
2119 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2120 .expect("scripted candidates must annotate cleanly");
2121 assert_eq!(out.entries[0].status, SalvageStatus::Intact);
2122 assert_eq!(out.entries[1].status, SalvageStatus::Complete);
2123 assert_eq!(out.entries[2].status, SalvageStatus::Partial);
2124 }
2125
2126 /// A later candidate reporting the SAME name, declared_len AND verifier
2127 /// as an earlier one is its shadow — pointing at the earliest position,
2128 /// not merely "some" earlier one.
2129 #[test]
2130 fn a_later_candidate_sharing_an_earlier_verifier_is_marked_as_its_shadow() {
2131 let mut scan = ScriptedCandidates {
2132 plan: vec![
2133 (
2134 "dup",
2135 Some(1),
2136 Some(Verifier::Crc32(7)),
2137 SalvageStatus::Intact,
2138 ),
2139 (
2140 "other",
2141 Some(1),
2142 Some(Verifier::Crc32(9)),
2143 SalvageStatus::Intact,
2144 ),
2145 (
2146 "dup",
2147 Some(1),
2148 Some(Verifier::Crc32(7)),
2149 SalvageStatus::Intact,
2150 ),
2151 ],
2152 next: 0,
2153 };
2154 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2155 .expect("scripted candidates must annotate cleanly");
2156 assert_eq!(out.entries[0].shadows, None);
2157 assert_eq!(out.entries[1].shadows, None);
2158 assert_eq!(
2159 out.entries[2].shadows,
2160 Some(0),
2161 "must point at the earliest matching position"
2162 );
2163 }
2164
2165 fn find_all(data: &[u8], sig: &[u8]) -> Vec<u64> {
2166 let mut search = ForwardSearch::new();
2167 let mut src = Cursor::new(data.to_vec());
2168 let len = data.len() as u64;
2169 let mut from = 0;
2170 let mut hits = Vec::new();
2171 while let Some(at) = search
2172 .find(&mut src, from, len, sig.len(), |w| w == sig)
2173 .unwrap()
2174 {
2175 hits.push(at);
2176 from = at + 1;
2177 }
2178 hits
2179 }
2180
2181 fn naive(data: &[u8], sig: &[u8]) -> Vec<u64> {
2182 (0..=data.len().saturating_sub(sig.len()))
2183 .filter(|&i| data.len() >= sig.len() && &data[i..i + sig.len()] == sig)
2184 .map(|i| i as u64)
2185 .collect()
2186 }
2187
2188 #[test]
2189 fn forward_search_finds_what_a_naive_scan_finds() {
2190 let sig = b"070701";
2191 // Hits at 0, near small-chunk boundaries (each `find` restarts at
2192 // FIRST_CHUNK, so large-chunk straddles are tested separately below),
2193 // and flush against EOF.
2194 let mut data = vec![b'x'; 200_000];
2195 for at in [0usize, 62, 63, 127, 255, 4093, 65_533, 131_070, 199_994] {
2196 data[at..at + sig.len()].copy_from_slice(sig);
2197 }
2198 assert_eq!(find_all(&data, sig), naive(&data, sig));
2199 }
2200
2201 #[test]
2202 fn forward_search_past_eof_and_on_short_input_is_none() {
2203 let mut search = ForwardSearch::new();
2204 let mut src = Cursor::new(b"0707".to_vec());
2205 assert_eq!(search.find(&mut src, 0, 4, 6, |_| true).unwrap(), None);
2206 assert_eq!(search.find(&mut src, 9, 4, 1, |_| true).unwrap(), None);
2207 }
2208
2209 /// A source that returns at most 7 bytes per read.
2210 struct Trickle(Cursor<Vec<u8>>);
2211 impl Read for Trickle {
2212 fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
2213 let n = out.len().min(7);
2214 self.0.read(&mut out[..n])
2215 }
2216 }
2217 impl Seek for Trickle {
2218 fn seek(&mut self, to: SeekFrom) -> std::io::Result<u64> {
2219 self.0.seek(to)
2220 }
2221 }
2222
2223 #[test]
2224 fn forward_search_survives_short_reads() {
2225 let mut data = vec![0u8; 5_000];
2226 data[4_321..4_327].copy_from_slice(b"070701");
2227 let mut search = ForwardSearch::new();
2228 let mut src = Trickle(Cursor::new(data));
2229 assert_eq!(
2230 search
2231 .find(&mut src, 0, 5_000, 6, |w| w == b"070701")
2232 .unwrap(),
2233 Some(4_321)
2234 );
2235 }
2236
2237 #[test]
2238 fn forward_search_no_match_reads_each_byte_once_and_ends_at_eof() {
2239 use crate::testing::CountingSource;
2240 let data = vec![b'x'; 200_000];
2241 let mut src = CountingSource::new(Cursor::new(data));
2242 let mut search = ForwardSearch::new();
2243 assert_eq!(
2244 search
2245 .find(&mut src, 0, 200_000, 6, |w| w == b"070701")
2246 .unwrap(),
2247 None
2248 );
2249 assert_eq!(src.bytes_read(), 200_000);
2250 }
2251
2252 #[test]
2253 fn forward_search_no_match_through_short_reads_ends_at_eof() {
2254 let mut search = ForwardSearch::new();
2255 let mut src = Trickle(Cursor::new(vec![b'x'; 200_000]));
2256 assert_eq!(
2257 search
2258 .find(&mut src, 0, 200_000, 6, |w| w == b"070701")
2259 .unwrap(),
2260 None
2261 );
2262 }
2263
2264 #[test]
2265 fn forward_search_on_a_truncated_source_ends_at_eof() {
2266 let mut search = ForwardSearch::new();
2267 let mut src = Cursor::new(vec![b'x'; 1_000]);
2268 assert_eq!(
2269 search
2270 .find(&mut src, 0, 50_000, 6, |w| w == b"070701")
2271 .unwrap(),
2272 None
2273 );
2274 }
2275
2276 #[test]
2277 fn forward_search_never_looks_past_an_understated_file_len() {
2278 let mut data = vec![b'x'; 10_000];
2279 data[5_000..5_006].copy_from_slice(b"070701");
2280 let mut search = ForwardSearch::new();
2281 let mut src = Cursor::new(data);
2282 assert_eq!(
2283 search
2284 .find(&mut src, 0, 4_000, 6, |w| w == b"070701")
2285 .unwrap(),
2286 None
2287 );
2288 // A hit straddling file_len is not "before file_len" either.
2289 assert_eq!(
2290 search
2291 .find(&mut src, 0, 5_003, 6, |w| w == b"070701")
2292 .unwrap(),
2293 None
2294 );
2295 assert_eq!(
2296 search
2297 .find(&mut src, 0, 5_006, 6, |w| w == b"070701")
2298 .unwrap(),
2299 Some(5_000)
2300 );
2301 }
2302
2303 #[test]
2304 fn forward_search_finds_a_hit_straddling_large_chunk_boundaries() {
2305 let sig = b"070701";
2306 // Cumulative end offset of each read in one `find` from 0.
2307 let mut ends = Vec::new();
2308 let (mut chunk, mut at) = (ForwardSearch::FIRST_CHUNK, 0usize);
2309 while ends.len() < 14 {
2310 at += chunk;
2311 ends.push(at);
2312 chunk = (chunk * 2).min(ForwardSearch::MAX_CHUNK);
2313 }
2314 // The read of MAX_CHUNK / 2 is the last doubling; the next read is
2315 // the first of MAX_CHUNK size.
2316 let (mut sizes, mut chunk) = (Vec::new(), ForwardSearch::FIRST_CHUNK);
2317 for _ in 0..ends.len() {
2318 sizes.push(chunk);
2319 chunk = (chunk * 2).min(ForwardSearch::MAX_CHUNK);
2320 }
2321 let doubling = ends[sizes
2322 .iter()
2323 .position(|&c| c == ForwardSearch::MAX_CHUNK / 2)
2324 .unwrap()];
2325 let first_max = ends[sizes
2326 .iter()
2327 .position(|&c| c == ForwardSearch::MAX_CHUNK)
2328 .unwrap()];
2329 for boundary in [doubling, first_max, first_max + ForwardSearch::MAX_CHUNK] {
2330 for start in [boundary - 5, boundary - 3, boundary - 1] {
2331 let mut data = vec![b'x'; boundary + 100];
2332 data[start..start + sig.len()].copy_from_slice(sig);
2333 let mut search = ForwardSearch::new();
2334 let mut src = Cursor::new(data.clone());
2335 let found = search
2336 .find(&mut src, 0, data.len() as u64, sig.len(), |w| w == sig)
2337 .unwrap();
2338 assert_eq!(found, Some(start as u64), "boundary {boundary}");
2339 }
2340 }
2341 }
2342
2343 #[test]
2344 fn a_nearby_hit_reads_only_a_few_bytes() {
2345 use crate::testing::CountingSource;
2346 let mut data = vec![0u8; 1 << 20];
2347 data[10..16].copy_from_slice(b"070701");
2348 let mut src = CountingSource::new(Cursor::new(data));
2349 let mut search = ForwardSearch::new();
2350 assert_eq!(
2351 search
2352 .find(&mut src, 0, 1 << 20, 6, |w| w == b"070701")
2353 .unwrap(),
2354 Some(10)
2355 );
2356 assert!(
2357 src.bytes_read() <= ForwardSearch::FIRST_CHUNK as u64,
2358 "{}",
2359 src.bytes_read()
2360 );
2361 }
2362
2363 /// A long signature's first read tests `sig_len` offsets, not one: a
2364 /// 512-byte signature 300 bytes on is found by ONE read of `2 * 512 - 1`
2365 /// bytes. With a first read of `sig_len` it took 512 + 1,024.
2366 #[test]
2367 fn a_long_signature_s_first_read_tests_sig_len_offsets() {
2368 use crate::testing::CountingSource;
2369 let sig = [0xA5u8; 512];
2370 let mut data = vec![0u8; 1 << 20];
2371 data[300..812].copy_from_slice(&sig);
2372 let mut src = CountingSource::new(Cursor::new(data));
2373 let mut search = ForwardSearch::new();
2374 assert_eq!(
2375 search
2376 .find(&mut src, 0, 1 << 20, 512, |w| w == sig)
2377 .unwrap(),
2378 Some(300)
2379 );
2380 assert_eq!(src.bytes_read(), 2 * 512 - 1);
2381 }
2382
2383 /// Dense hits cost no first read each: what one call read past its
2384 /// answer is tested by the next without reading it again. A hit every 7
2385 /// bytes through 1 MiB used to cost 64 bytes read per hit (9x); kept,
2386 /// the searches read each byte once.
2387 #[test]
2388 fn dense_hits_read_each_byte_about_once() {
2389 use crate::testing::CountingSource;
2390 let sig = b"070701";
2391 let mut data = vec![b'x'; 1 << 20];
2392 for at in (0..data.len() - sig.len()).step_by(7) {
2393 data[at..at + sig.len()].copy_from_slice(sig);
2394 }
2395 let len = data.len() as u64;
2396 let mut src = CountingSource::new(Cursor::new(data.clone()));
2397 let mut search = ForwardSearch::new();
2398 let (mut from, mut hits) = (0, Vec::new());
2399 while let Some(at) = search
2400 .find(&mut src, from, len, sig.len(), |w| w == sig)
2401 .unwrap()
2402 {
2403 hits.push(at);
2404 // A scanner reads its candidate between searches; the kept
2405 // bytes must survive the source's position moving.
2406 src.seek(SeekFrom::Start(at)).unwrap();
2407 let mut header = [0u8; 6];
2408 src.read_exact(&mut header).unwrap();
2409 from = at + 1;
2410 }
2411 assert_eq!(hits, naive(&data, sig));
2412 let header_reads = 6 * hits.len() as u64;
2413 assert!(
2414 src.bytes_read() - header_reads <= len + ForwardSearch::MAX_CHUNK as u64,
2415 "searches read {} bytes of a {len}-byte input",
2416 src.bytes_read() - header_reads
2417 );
2418 }
2419
2420 /// A call starting before what was kept, or past it, still answers
2421 /// exactly what a fresh search answers.
2422 #[test]
2423 fn kept_bytes_never_change_an_answer() {
2424 let sig = b"070701";
2425 let mut data = vec![b'x'; 300_000];
2426 for at in [5usize, 70, 4_000, 70_000, 250_000, 299_994] {
2427 data[at..at + sig.len()].copy_from_slice(sig);
2428 }
2429 let len = data.len() as u64;
2430 let mut kept = ForwardSearch::new();
2431 let mut src = Cursor::new(data.clone());
2432 for from in [
2433 0u64, 6, 3, 71, 60, 4_001, 200_000, 69_000, 299_995, 250_001, 0,
2434 ] {
2435 for file_len in [len, 250_003, 70_006, 4_005] {
2436 let fresh = ForwardSearch::new()
2437 .find(&mut Cursor::new(data.clone()), from, file_len, 6, |w| {
2438 w == sig
2439 })
2440 .unwrap();
2441 let got = kept
2442 .find(&mut src, from, file_len, 6, |w| w == sig)
2443 .unwrap();
2444 assert_eq!(got, fresh, "from {from}, file_len {file_len}");
2445 }
2446 }
2447 }
2448
2449 /// `find_to_eof` offers the last offsets every byte left, down to
2450 /// `min_len`, and offers the rest exactly `sig_len`.
2451 #[test]
2452 fn find_to_eof_offers_the_tail_what_is_left() {
2453 let data: Vec<u8> = (0..100u8).collect();
2454 let mut offered = Vec::new();
2455 let found = ForwardSearch::new()
2456 .find_to_eof(&mut Cursor::new(data.clone()), 10, 100, 20, 3, |w| {
2457 offered.push((w[0], w.len()));
2458 false
2459 })
2460 .unwrap();
2461 assert_eq!(found, None);
2462 let expected: Vec<(u8, usize)> = (10u8..=97)
2463 .map(|at| (at, 20.min(100 - at as usize)))
2464 .collect();
2465 assert_eq!(offered, expected);
2466 // A hit in the tail answers its offset.
2467 let at = ForwardSearch::new()
2468 .find_to_eof(&mut Cursor::new(data), 0, 100, 20, 1, |w| w[0] == 95)
2469 .unwrap();
2470 assert_eq!(at, Some(95));
2471 }
2472
2473 /// Pins `annotate_candidates`' semantics across the Vec -> HashMap change:
2474 /// a repeat's `collides_with` is the EARLIEST position with its name, and
2475 /// a non-degenerate same-name/length/verifier repeat shadows the earliest.
2476 #[test]
2477 fn annotate_candidates_reports_the_earliest_position_at_scale() {
2478 let names: Vec<&'static str> = (0..3_000)
2479 .map(|i| &*Box::leak(format!("f{i}").into_boxed_str()))
2480 .collect();
2481 let mut plan = Vec::new();
2482 for i in 0..3_000usize {
2483 // Every tenth candidate repeats the name of an original (never a repeat),
2484 // at position (i % 100) / 10 * 10.
2485 let name = if i % 10 == 9 {
2486 names[(i % 100) / 10 * 10]
2487 } else {
2488 names[i]
2489 };
2490 let declared = if i % 20 == 19 { Some(8) } else { Some(4) };
2491 // Some repeats carry a different checksum: same name and length
2492 // but another verifier is a collision, not a shadow.
2493 let crc = if i % 10 == 9 && (i / 10) % 2 == 0 {
2494 8
2495 } else {
2496 7
2497 };
2498 plan.push((
2499 name,
2500 declared,
2501 Some(Verifier::Crc32(crc)),
2502 SalvageStatus::Intact,
2503 ));
2504 }
2505 let keys: Vec<_> = plan.iter().map(|p| (p.0, p.1, p.2)).collect();
2506 let mut scan = ScriptedCandidates { plan, next: 0 };
2507 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2508 .expect("scripted candidates must annotate cleanly");
2509 assert_eq!(out.entries.len(), 3_000);
2510 let mut first_triple = HashMap::new();
2511 let mut first_name = HashMap::new();
2512 for (i, k) in keys.iter().enumerate() {
2513 first_triple.entry(*k).or_insert(i);
2514 first_name.entry(k.0).or_insert(i);
2515 }
2516 for (i, e) in out.entries.iter().enumerate() {
2517 let same_triple = first_triple[&keys[i]];
2518 let same_name = first_name[keys[i].0];
2519 let (want_shadow, want_collide) = if same_triple != i {
2520 (Some(same_triple), None)
2521 } else if same_name != i {
2522 (None, Some(same_name))
2523 } else {
2524 (None, None)
2525 };
2526 assert_eq!(e.shadows, want_shadow, "shadows at {i}");
2527 assert_eq!(e.collides_with, want_collide, "collides_with at {i}");
2528 }
2529 let shadowed = out.entries.iter().filter(|e| e.shadows.is_some()).count();
2530 let collided = out
2531 .entries
2532 .iter()
2533 .filter(|e| e.collides_with.is_some())
2534 .count();
2535 assert!(shadowed > 0 && collided > 0, "{shadowed} {collided}");
2536 }
2537
2538 /// A candidate with no verifier at all (a format with no checksum, or a
2539 /// zip data-descriptor entry) must never be reported as shadowing or
2540 /// shadowed — there is nothing to measure, so nothing is claimed.
2541 #[test]
2542 fn candidates_with_no_verifier_never_shadow_each_other() {
2543 let mut scan = ScriptedCandidates {
2544 plan: vec![
2545 ("a", Some(1), None, SalvageStatus::Complete),
2546 ("a", Some(1), None, SalvageStatus::Complete),
2547 ],
2548 next: 0,
2549 };
2550 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2551 .expect("scripted candidates must annotate cleanly");
2552 assert_eq!(out.entries[0].shadows, None);
2553 assert_eq!(out.entries[1].shadows, None);
2554 }
2555
2556 /// Two candidates whose checksum AND declared_len agree but whose NAME
2557 /// differs must never be linked — the fix-round requirement that a
2558 /// checksum match alone is not enough, only the full
2559 /// `(name, declared_len, verifier)` triple.
2560 #[test]
2561 fn a_matching_verifier_under_a_different_name_is_never_marked_as_a_shadow() {
2562 let mut scan = ScriptedCandidates {
2563 plan: vec![
2564 (
2565 "one.txt",
2566 Some(4),
2567 Some(Verifier::Crc32(42)),
2568 SalvageStatus::Intact,
2569 ),
2570 (
2571 "two.txt",
2572 Some(4),
2573 Some(Verifier::Crc32(42)),
2574 SalvageStatus::Intact,
2575 ),
2576 ],
2577 next: 0,
2578 };
2579 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2580 .expect("scripted candidates must annotate cleanly");
2581 assert_eq!(
2582 out.entries[1].shadows, None,
2583 "same checksum, different name — not a shadow"
2584 );
2585 }
2586
2587 /// The degenerate case: every empty entry (and, in zip, every directory
2588 /// entry) checksums to the same fixed constant over zero declared
2589 /// bytes. Even under the SAME name, a zero-length checksum must never
2590 /// mark a shadow — the value proves nothing, unlike a real collision.
2591 #[test]
2592 fn a_zero_length_declared_verifier_never_shadows_even_under_a_matching_name() {
2593 let mut scan = ScriptedCandidates {
2594 plan: vec![
2595 (
2596 "empty.txt",
2597 Some(0),
2598 Some(Verifier::Crc32(0)),
2599 SalvageStatus::Complete,
2600 ),
2601 (
2602 "empty.txt",
2603 Some(0),
2604 Some(Verifier::Crc32(0)),
2605 SalvageStatus::Complete,
2606 ),
2607 ],
2608 next: 0,
2609 };
2610 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2611 .expect("scripted candidates must annotate cleanly");
2612 assert_eq!(out.entries[0].shadows, None);
2613 assert_eq!(
2614 out.entries[1].shadows, None,
2615 "a zero-byte checksum must never mark a shadow, even under an identical name"
2616 );
2617 assert_eq!(
2618 out.entries[1].collides_with,
2619 Some(0),
2620 "the NAME still repeats, and that is a directly observed fact no checksum \
2621 guard applies to"
2622 );
2623 }
2624
2625 // -----------------------------------------------------------------
2626 // Name collisions: the second annotation, separate from shadowing.
2627 // -----------------------------------------------------------------
2628
2629 /// The finding this annotation closes: two records under one name whose
2630 /// content does NOT agree are what `stuffr list` calls shadowed, and
2631 /// what salvage marked in no way at all — because its own `shadows`
2632 /// means something stricter.
2633 #[test]
2634 fn a_repeated_name_with_different_content_is_reported_as_a_collision() {
2635 let mut scan = ScriptedCandidates {
2636 plan: vec![
2637 (
2638 "dup.txt",
2639 Some(21),
2640 Some(Verifier::Crc32(7)),
2641 SalvageStatus::Intact,
2642 ),
2643 (
2644 "other.txt",
2645 Some(21),
2646 Some(Verifier::Crc32(9)),
2647 SalvageStatus::Intact,
2648 ),
2649 (
2650 "dup.txt",
2651 Some(21),
2652 Some(Verifier::Crc32(11)),
2653 SalvageStatus::Intact,
2654 ),
2655 ],
2656 next: 0,
2657 };
2658 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2659 .expect("scripted candidates must annotate cleanly");
2660 assert_eq!(out.entries[2].shadows, None, "the content does not agree");
2661 assert_eq!(
2662 out.entries[2].collides_with,
2663 Some(0),
2664 "but the name does, and that is the fact `list` warns about"
2665 );
2666 assert_eq!(out.entries[0].collides_with, None);
2667 assert_eq!(out.entries[1].collides_with, None);
2668 }
2669
2670 /// The two annotations are mutually exclusive: a record PROVEN to be a
2671 /// copy says so, and does not also report the weaker fact. Reporting
2672 /// both would make every shadow print two markers for one relationship.
2673 #[test]
2674 fn a_proven_shadow_is_not_also_reported_as_a_name_collision() {
2675 let mut scan = ScriptedCandidates {
2676 plan: vec![
2677 (
2678 "dup.txt",
2679 Some(21),
2680 Some(Verifier::Crc32(7)),
2681 SalvageStatus::Intact,
2682 ),
2683 (
2684 "dup.txt",
2685 Some(21),
2686 Some(Verifier::Crc32(7)),
2687 SalvageStatus::Intact,
2688 ),
2689 ],
2690 next: 0,
2691 };
2692 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2693 .expect("scripted candidates must annotate cleanly");
2694 assert_eq!(out.entries[1].shadows, Some(0));
2695 assert_eq!(out.entries[1].collides_with, None);
2696 }
2697
2698 /// A third record under a name two earlier ones already used points at
2699 /// the EARLIEST of them, the same rule `shadows` follows — so a chain of
2700 /// collisions names one origin rather than each pointing at its
2701 /// predecessor.
2702 #[test]
2703 fn a_third_record_under_one_name_names_the_earliest_position() {
2704 let mut scan = ScriptedCandidates {
2705 plan: vec![
2706 (
2707 "dup",
2708 Some(4),
2709 Some(Verifier::Crc32(1)),
2710 SalvageStatus::Intact,
2711 ),
2712 (
2713 "dup",
2714 Some(4),
2715 Some(Verifier::Crc32(2)),
2716 SalvageStatus::Intact,
2717 ),
2718 (
2719 "dup",
2720 Some(4),
2721 Some(Verifier::Crc32(3)),
2722 SalvageStatus::Intact,
2723 ),
2724 ],
2725 next: 0,
2726 };
2727 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2728 .expect("scripted candidates must annotate cleanly");
2729 assert_eq!(out.entries[1].collides_with, Some(0));
2730 assert_eq!(out.entries[2].collides_with, Some(0));
2731 }
2732
2733 /// The negative double: distinct names must never be marked, however
2734 /// much else they share. Without this, an annotation that fired on
2735 /// everything would pass every test above.
2736 #[test]
2737 fn distinct_names_are_never_reported_as_colliding() {
2738 let mut scan = ScriptedCandidates {
2739 plan: vec![
2740 (
2741 "one.txt",
2742 Some(4),
2743 Some(Verifier::Crc32(42)),
2744 SalvageStatus::Intact,
2745 ),
2746 (
2747 "two.txt",
2748 Some(4),
2749 Some(Verifier::Crc32(42)),
2750 SalvageStatus::Intact,
2751 ),
2752 ],
2753 next: 0,
2754 };
2755 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2756 .expect("scripted candidates must annotate cleanly");
2757 assert_eq!(out.entries[0].collides_with, None);
2758 assert_eq!(out.entries[1].collides_with, None);
2759 }
2760
2761 /// A collision needs no verifier at all, unlike a shadow. A format with
2762 /// no checksum (tar, cpio, ar — and a zip data-descriptor entry) can
2763 /// still repeat a name, and that repetition is exactly as real there.
2764 #[test]
2765 fn a_repeated_name_collides_even_with_no_verifier_to_compare() {
2766 let mut scan = ScriptedCandidates {
2767 plan: vec![
2768 ("same", Some(4), None, SalvageStatus::Complete),
2769 ("same", Some(4), None, SalvageStatus::Complete),
2770 ],
2771 next: 0,
2772 };
2773 let out = salvage_all(&mut scan, &mut bytes(&[0u8; 16]), &SalvagePolicy::default())
2774 .expect("scripted candidates must annotate cleanly");
2775 assert_eq!(
2776 out.entries[1].shadows, None,
2777 "no verifier, so nothing proves they are copies"
2778 );
2779 assert_eq!(
2780 out.entries[1].collides_with,
2781 Some(0),
2782 "but the name repeats, which needs no verifier to observe"
2783 );
2784 }
2785
2786 /// Ruling S-I. `Candidate` is `#[non_exhaustive]`, so an out-of-crate
2787 /// scanner reaches it only through `new` plus the `with_*` setters —
2788 /// which means the DEFAULTS are now public API, not an implementation
2789 /// detail of six struct literals. Each of the four must assert the
2790 /// least: nothing declared, nothing to verify with, nothing missing,
2791 /// nothing deleted.
2792 #[test]
2793 fn a_new_candidate_claims_nothing_it_was_not_told() {
2794 let c = Candidate::new(7, 42, EntryMeta::file("probe"));
2795 assert_eq!(c.offset, 7);
2796 assert_eq!(c.payload_start, 42);
2797 assert_eq!(c.declared_len, None, "no length was declared");
2798 assert_eq!(c.verifier, None, "nothing can prove the content");
2799 assert_eq!(
2800 c.available_len, None,
2801 "a shortfall is MEASURED, never assumed"
2802 );
2803 assert!(!c.marked_deleted, "no record is deleted until one says so");
2804 }
2805
2806 /// The four setters are the only route to those fields from another
2807 /// crate, so each one is pinned rather than trusted to the compiler.
2808 #[test]
2809 fn each_candidate_setter_reaches_its_own_field() {
2810 let c = Candidate::new(0, 0, EntryMeta::file("probe"))
2811 .with_declared_len(Some(9))
2812 .with_verifier(Some(Verifier::Crc16(0xbeef)))
2813 .with_available_len(Some(4))
2814 .with_marked_deleted(true);
2815 assert_eq!(c.declared_len, Some(9));
2816 assert_eq!(c.verifier, Some(Verifier::Crc16(0xbeef)));
2817 assert_eq!(c.available_len, Some(4));
2818 assert!(c.marked_deleted);
2819 }
2820
2821 /// Same ruling, same reason, for the type every per-format
2822 /// `write_payload` test builds directly. Its three defaults are the
2823 /// annotations `annotate_candidates` adds, so an entry built by hand
2824 /// must carry none of them.
2825 #[test]
2826 fn a_new_salvaged_entry_carries_no_annotations() {
2827 let e = SalvagedEntry::new(3, 7, 42, EntryMeta::file("probe"), SalvageStatus::Complete);
2828 assert_eq!(e.scan_position, 3);
2829 assert_eq!(e.offset, 7);
2830 assert_eq!(e.payload_start, 42);
2831 assert_eq!(e.status, SalvageStatus::Complete);
2832 assert_eq!(e.shadows, None);
2833 assert_eq!(e.collides_with, None);
2834 assert!(!e.marked_deleted);
2835
2836 let annotated = SalvagedEntry::new(0, 0, 0, EntryMeta::file("p"), SalvageStatus::Intact)
2837 .with_shadows(Some(1))
2838 .with_collides_with(Some(2))
2839 .with_marked_deleted(true);
2840 assert_eq!(annotated.shadows, Some(1));
2841 assert_eq!(annotated.collides_with, Some(2));
2842 assert!(annotated.marked_deleted);
2843 }
2844
2845 /// Task 2-N: the note a caller prints for its sightings. None for none —
2846 /// a run with nothing refused must print nothing — and otherwise one
2847 /// group per (format, shape), each with its complete count and at most
2848 /// [`SIGHTING_OFFSETS_SHOWN`] offsets, in the order first seen.
2849 #[test]
2850 fn describe_sightings_groups_by_shape_and_bounds_what_it_lists() {
2851 assert_eq!(describe_sightings(&[]), None);
2852
2853 let f = FormatId::new("fmt");
2854 let mut seen: Vec<Sighting> = (0..10u64)
2855 .map(|i| Sighting::new(f, i * 512, "a(s)"))
2856 .collect();
2857 seen.insert(1, Sighting::new(f, 7, "b(s)"));
2858 let note = describe_sightings(&seen).unwrap();
2859 assert!(
2860 note.contains("10 fmt a(s) at offset(s) 0, 512, 1024, 1536, 2048, 2560, 3072, 3584, and 2 more; and 1 fmt b(s) at offset(s) 7"),
2861 "{note}"
2862 );
2863 assert!(note.contains("`stuffr list`"), "{note}");
2864 }
2865
2866 /// 0.10.3: a work-bounded sighting has its own sentence, and an
2867 /// ungateable one keeps its sentence word for word — alone or alongside.
2868 #[test]
2869 fn describe_sightings_words_a_work_bounded_sighting_apart() {
2870 let f = FormatId::new("fmt");
2871 let old = describe_sightings(&[Sighting::new(f, 0, "a(s)")]).unwrap();
2872 assert_eq!(
2873 old,
2874 "this build's salvage scanner saw 1 fmt a(s) at offset(s) 0 — each with the \
2875 structure of a header, but in a shape it has no gate for, so none is listed or \
2876 recovered; `stuffr list` and `stuffr unpack` may read these headers normally, or \
2877 name the variant they are in, and it is the SCAN that stops there"
2878 );
2879 let bounded = describe_sightings(&[Sighting::work_bounded(f, 9, "b(s)")]).unwrap();
2880 assert!(bounded.contains("1 fmt b(s) at offset(s) 9"), "{bounded}");
2881 assert!(!bounded.contains("no gate for"), "{bounded}");
2882 assert!(bounded.contains("bounds the work"), "{bounded}");
2883 // Fix round 1: true for a genuine header starved by forgeries
2884 // elsewhere, and for a scan that goes on past it.
2885 assert!(bounded.contains("the scan went on past them"), "{bounded}");
2886 assert!(!bounded.contains("forged"), "{bounded}");
2887 assert!(!bounded.contains("SCAN that stops"), "{bounded}");
2888 let both = describe_sightings(&[
2889 Sighting::work_bounded(f, 9, "b(s)"),
2890 Sighting::new(f, 0, "a(s)"),
2891 ])
2892 .unwrap();
2893 assert_eq!(both, format!("{old}; and separately, {bounded}"));
2894 }
2895
2896 /// 0.10.3 Task 4b: a verdict-bounded sighting has a third sentence —
2897 /// not the extension-chain one, which would be false of a zip name —
2898 /// and the other two keep their words beside it.
2899 #[test]
2900 fn describe_sightings_words_a_verdict_bounded_sighting_apart() {
2901 let f = FormatId::new("fmt");
2902 let old = describe_sightings(&[Sighting::new(f, 0, "a(s)")]).unwrap();
2903 let bounded = describe_sightings(&[Sighting::work_bounded(f, 9, "b(s)")]).unwrap();
2904 let verdict = describe_sightings(&[Sighting::verdict_bounded(f, 7, "c(s)")]).unwrap();
2905 assert_eq!(
2906 verdict,
2907 "this build's salvage scanner bounds the work it spends judging candidate headers \
2908 across the whole scan, and did not judge 1 fmt c(s) at offset(s) 7 — so \
2909 whatever those headers describe is not listed or recovered there, though the scan \
2910 went on past them; `stuffr list` and `stuffr unpack` may read these headers normally"
2911 );
2912 assert!(!verdict.contains("extension chains"), "{verdict}");
2913 let all = describe_sightings(&[
2914 Sighting::verdict_bounded(f, 7, "c(s)"),
2915 Sighting::work_bounded(f, 9, "b(s)"),
2916 Sighting::new(f, 0, "a(s)"),
2917 ])
2918 .unwrap();
2919 assert_eq!(
2920 all,
2921 format!("{old}; and separately, {bounded}; and separately, {verdict}")
2922 );
2923 }
2924
2925 /// Task 4b: the budget is exactly `2 × file_len + 32 KiB`, a charge
2926 /// that lands on the limit fits, and one past it is refused WHOLE —
2927 /// nothing is charged, so a later, smaller charge can still fit.
2928 #[test]
2929 fn a_scan_budget_spends_to_its_exact_limit_and_refuses_past_it() {
2930 let limit = 2 * 1000 + 32 * 1024;
2931 let mut budget = ScanBudget::for_input(1000);
2932 assert!(budget.try_spend(limit - 10));
2933 assert_eq!(budget.spent(), limit - 10);
2934 assert!(!budget.try_spend(11), "one byte past the limit");
2935 assert_eq!(budget.spent(), limit - 10, "a refusal charges nothing");
2936 assert!(budget.try_spend(10), "exactly to the limit");
2937 assert_eq!(budget.spent(), limit);
2938 assert!(!budget.try_spend(1));
2939 assert_eq!(budget.spent(), limit);
2940 }
2941
2942 /// Spending zero always fits, even on a spent budget, and a charge whose
2943 /// sum overflows `u64` is a refusal, never a wrap.
2944 #[test]
2945 fn a_scan_budget_takes_zero_and_refuses_an_overflowing_charge() {
2946 let mut budget = ScanBudget::for_input(0);
2947 assert!(budget.try_spend(0));
2948 assert_eq!(budget.spent(), 0);
2949 assert!(budget.try_spend(ScanBudget::SLACK));
2950 assert!(budget.try_spend(0), "zero fits a spent budget");
2951 assert!(!budget.try_spend(u64::MAX));
2952 assert_eq!(budget.spent(), ScanBudget::SLACK);
2953
2954 let mut huge = ScanBudget::for_input(u64::MAX);
2955 assert!(huge.try_spend(u64::MAX), "the limit saturates at u64::MAX");
2956 assert!(!huge.try_spend(1), "and a sum past it does not wrap");
2957 assert_eq!(huge.spent(), u64::MAX);
2958 }
2959
2960 /// Task 4: the walk-stop note. A hole says what the brief requires —
2961 /// the entries after it are unreachable by construction, NOT absent,
2962 /// with the offset and the byte count — and a cut-short tail says the
2963 /// opposite, because nothing can be hiding in too few bytes for a record.
2964 #[test]
2965 fn describe_walk_stop_says_unreachable_for_a_hole_and_complete_for_a_cut() {
2966 let f = FormatId::new("fmt");
2967 let hole = describe_walk_stop(&WalkStop::new(
2968 f,
2969 1234,
2970 5000,
2971 WalkStopKind::Unreadable,
2972 "bad field",
2973 ));
2974 for needle in [
2975 "offset 1234",
2976 "5000 bytes",
2977 "bad field",
2978 "unreachable by construction",
2979 "not absent",
2980 "fmt",
2981 ] {
2982 assert!(hole.contains(needle), "missing {needle:?}: {hole}");
2983 }
2984 let cut = describe_walk_stop(&WalkStop::new(f, 90, 1, WalkStopKind::CutShort, "eof"));
2985 assert!(cut.contains("1 byte into the record at offset 90"), "{cut}");
2986 assert!(cut.contains("cut short"), "{cut}");
2987 assert!(!cut.contains("unreachable"), "{cut}");
2988 let shape = describe_walk_stop(&WalkStop::new(
2989 f,
2990 8,
2991 600,
2992 WalkStopKind::UnsupportedShape,
2993 "the symbol table",
2994 ));
2995 assert!(shape.contains("the symbol table"), "{shape}");
2996 assert!(shape.contains("not evidence of damage"), "{shape}");
2997 assert!(!shape.contains("unreachable by construction"), "{shape}");
2998 }
2999}