stuffr_core/fidelity.rs
1use std::fmt;
2
3use crate::display::fmt_name;
4use crate::format::FormatId;
5
6/// Which rung of the adaptive stream ladder a read landed on.
7#[derive(Clone, Copy, PartialEq, Eq, Debug)]
8#[cfg_attr(feature = "serde", derive(serde::Serialize))]
9#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case"))]
10pub enum Rung {
11 /// Input was seekable. Authoritative index, full metadata.
12 Exact,
13 /// Not seekable, parsed forward. Real data, approximate metadata.
14 ForwardOnly,
15 /// Spooled to memory/temp then read exactly. Costs disk, not accuracy.
16 Spilled,
17 /// Best-effort salvage. Partial results by construction.
18 Degraded,
19}
20
21impl Rung {
22 /// Whether the read used the format's authoritative structures.
23 ///
24 /// Note `Spilled` is authoritative: spilling trades disk for accuracy, and
25 /// treating it as lossy would make `--strict-fidelity` reject exact reads.
26 pub fn is_authoritative(&self) -> bool {
27 matches!(self, Rung::Exact | Rung::Spilled)
28 }
29
30 /// Ordering by *quality loss*, worst = highest. Used by `merge`.
31 fn severity(&self) -> u8 {
32 match self {
33 Rung::Exact => 0,
34 Rung::Spilled => 1,
35 Rung::ForwardOnly => 2,
36 Rung::Degraded => 3,
37 }
38 }
39}
40
41impl fmt::Display for Rung {
42 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
43 let s = match self {
44 Rung::Exact => "exact",
45 Rung::ForwardOnly => "forward-only",
46 Rung::Spilled => "spilled",
47 Rung::Degraded => "degraded",
48 };
49 f.write_str(s)
50 }
51}
52
53/// Which metadata fields are **missing**. `true` means absent, not present.
54#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)]
55#[cfg_attr(feature = "serde", derive(serde::Serialize))]
56pub struct MetaFields {
57 pub mtime: bool,
58 pub mode: bool,
59 pub uid_gid: bool,
60 pub comment: bool,
61 pub extra: bool,
62 pub crc: bool,
63}
64
65impl MetaFields {
66 pub fn missing(&self) -> Vec<&'static str> {
67 let mut v = Vec::new();
68 if self.mtime {
69 v.push("mtime")
70 }
71 if self.mode {
72 v.push("mode")
73 }
74 if self.uid_gid {
75 v.push("uid_gid")
76 }
77 if self.comment {
78 v.push("comment")
79 }
80 if self.extra {
81 v.push("extra")
82 }
83 if self.crc {
84 v.push("crc")
85 }
86 v
87 }
88}
89
90/// One specific, structured thing that was lost or approximated.
91#[derive(Clone, PartialEq, Eq, Debug, thiserror::Error)]
92#[cfg_attr(feature = "serde", derive(serde::Serialize))]
93#[cfg_attr(feature = "serde", serde(tag = "kind", rename_all = "kebab-case"))]
94#[non_exhaustive]
95pub enum Fidelity {
96 #[error(
97 "`{format}` central/trailing index was never read; entry metadata came from inline headers"
98 )]
99 TrailingIndexUnread { format: FormatId },
100
101 #[error("size of entry `{}` was only known after its data (data descriptor)", fmt_name(.entry))]
102 SizeFromDataDescriptor { entry: String },
103
104 #[error("total entry count is unknown without the trailing index")]
105 EntryCountUnknown,
106
107 /// The archive's own index declares a different number of records than
108 /// the reader enumerated. Usually FEWER were enumerated, so entries exist
109 /// in the file that no caller can reach; zip can also enumerate MORE, when
110 /// its end-of-central-directory record's two counts contradict each other
111 /// (the v0.7.0 deep-fuzz finding), so the message names both figures
112 /// without saying which way round they are.
113 ///
114 /// Distinct from [`Self::EntryCountUnknown`], which is the forward reader
115 /// admitting it never saw a count at all. Here the count WAS read, from
116 /// the format's authoritative structure, and it disagrees with what came
117 /// back — the strongest evidence available that a read is incomplete, and
118 /// the one a seekable `Rung::Exact` read would otherwise have no way to
119 /// express. Without it a partial enumeration is indistinguishable from a
120 /// complete one, which is the failure this variant exists to prevent:
121 /// listing six of eight records and calling it exact fidelity.
122 ///
123 /// Raised, not recovered. Reaching the shadowed records would mean
124 /// re-parsing the index by hand; naming the shortfall is what the caller
125 /// needs in order to know not to trust the result, and
126 /// `--strict-fidelity` turns it into exit 4.
127 ///
128 /// `reason` is an owned `String` for the same reason
129 /// [`Self::EntrySkipped`]'s is: the cause is specific to the container
130 /// and the reader, and a shortfall that cannot say WHY is one the user
131 /// cannot act on.
132 #[error(
133 "`{format}` index declares {declared} entries but {enumerated} were \
134 enumerated: {}",
135 fmt_name(.reason)
136 )]
137 EntryCountMismatch {
138 format: FormatId,
139 /// What the archive's own index says it holds.
140 declared: u64,
141 /// What the reader was actually able to enumerate.
142 enumerated: u64,
143 /// Why the two differ, in terms the caller can act on.
144 reason: String,
145 },
146
147 #[error("entry `{}` is missing metadata: {}", fmt_name(.entry), fields.missing().join(", "))]
148 MetadataIncomplete { entry: String, fields: MetaFields },
149
150 #[error("decoded a solid block, wasting {wasted_bytes} bytes, to reach the requested entry")]
151 SolidBlockFullyDecoded { wasted_bytes: u64 },
152
153 #[error("skipped encrypted entry `{}`", fmt_name(.entry))]
154 EncryptedEntrySkipped { entry: String },
155
156 /// An entry this build cannot materialise, named along with why.
157 ///
158 /// Distinct from [`Self::MetadataIncomplete`], which is about an entry
159 /// that WAS written and lost some of its metadata: here the entry itself
160 /// is absent from the result. Extraction raises it for a device node,
161 /// fifo or socket — [`crate::EntryKind::Other`], the honest answer a
162 /// container gives for an entry shape `EntryKind` has no variant for yet
163 /// — and for a [`crate::EntryKind::Hardlink`] whose target it did not
164 /// write, or whose target is a directory. Writing one out as a regular
165 /// file carrying its "contents" would be a silent lie about what the
166 /// archive held. The write side
167 /// raises it too, for something met on disk that no container here can
168 /// store — a socket, an undecodable name, a directory being packed into
169 /// `ar`.
170 ///
171 /// `reason` is an owned `String` rather than a `&'static str` because the
172 /// write side's reasons carry runtime detail — the failing errno, the
173 /// container's own name, the undecodable bytes — and a reason that cannot
174 /// name the particular thing that went wrong is a reason the user cannot
175 /// act on.
176 #[error("skipped entry `{}`: {}", fmt_name(.entry), fmt_name(.reason))]
177 EntrySkipped { entry: String, reason: String },
178
179 /// A file that changed length between the moment its size was recorded
180 /// and the moment its bytes were read, stored at the length its header
181 /// already declares.
182 ///
183 /// Distinct from [`Self::EntrySkipped`]: the entry IS in the archive, and
184 /// the archive is structurally valid. What was lost is the tail of a file
185 /// that grew, or the tail of a file that shrank — replaced by zeros,
186 /// because the entry header was written before the payload was read and a
187 /// container cannot un-write it. Refusing instead would throw away the
188 /// whole archive over one file that a live system happened to touch,
189 /// which is what `pack ~ -o backup.tar` over a running home directory does
190 /// routinely; GNU tar prints `file changed as we read it` and continues,
191 /// and this is the same bargain, with the loss named rather than printed
192 /// and forgotten.
193 ///
194 /// `--strict-fidelity` still refuses on it, which is the half of the
195 /// bargain that keeps it honest.
196 #[error(
197 "entry `{}` changed size while it was being read: its header declares \
198 {declared} bytes, {}",
199 fmt_name(.entry),
200 fmt_name(.fixup)
201 )]
202 EntrySizeChanged {
203 entry: String,
204 /// The length written into the entry header, from the walk's `stat`.
205 declared: u64,
206 /// What the file actually supplied and what was done about it.
207 ///
208 /// Prose rather than a second `u64` because the two directions know
209 /// different things: a file that shrank supplies an exact count, while
210 /// a file that grew is only known to have had *more* — measuring how
211 /// much more would mean reading a tail that is deliberately not being
212 /// stored. A field that could name the figure in one case and not the
213 /// other would have to lie in one of them.
214 fixup: String,
215 },
216
217 #[error("stream truncated at offset {at}")]
218 TruncatedStream { at: u64 },
219
220 /// A directory record carries a checksum over ITSELF, and it does not
221 /// match the bytes the record is made of.
222 ///
223 /// Distinct from every other integrity signal in this enum: those are
224 /// about an entry's CONTENT, and a content checksum that fails is
225 /// [`crate::Error::Corrupt`], not a warning, because there is nothing
226 /// honest to hand back. This is about the METADATA record, and the
227 /// verdict is deliberately softer for one measured reason — the format
228 /// that carries such a field may itself treat it as advisory. ZOO is the
229 /// first: `zoo` 2.10's own lister prints a `*` beside an entry whose
230 /// `dir_crc` fails and carries on listing it, so refusing would be
231 /// stricter than the reference implementation, which is how a guard comes
232 /// to fire on input a real tool accepts.
233 ///
234 /// What it must NOT do is stay silent. A record whose own checksum fails
235 /// has been altered since it was written, and every field a caller is
236 /// about to trust — the entry's name, size, offset, and the link to the
237 /// next record — comes out of it. `--strict-fidelity` turns this into
238 /// exit 4, which is the whole point: a verb that says "exact fidelity"
239 /// over an altered record is making a claim the archive itself disputes.
240 ///
241 /// `offset` is the record's own position in the file, because a record
242 /// that fails its checksum may have no usable name to identify it by —
243 /// the reproducer that motivated this variant is a record whose name
244 /// field is intact and whose link field is not.
245 #[error(
246 "`{format}` directory record at offset {offset} fails its own checksum \
247 ({computed:#06x} computed against {recorded:#06x} recorded); the record has been \
248 altered since it was written"
249 )]
250 DirectoryRecordChecksum {
251 format: FormatId,
252 offset: u64,
253 computed: u16,
254 recorded: u16,
255 },
256
257 /// A zip entry's local file header and its central-directory record
258 /// disagree on a field both carry — the "cloaking" shape (GHSA-m665-v7pm-
259 /// 7grj), where one archive shows different names or contents depending
260 /// on which of its two descriptions a reader trusts.
261 ///
262 /// A warning, not [`crate::Error::Corrupt`]: the bytes handed back are
263 /// still a coherent reading (the central directory's on a seekable
264 /// source, the local headers' on a pipe), and the archive is not
265 /// self-contradictory within either description. What cannot be claimed
266 /// is that the reading is the only one, so `--strict-fidelity` turns this
267 /// into exit 4.
268 ///
269 /// `field` is one of `"name"`, `"compression method"`, `"CRC-32"`,
270 /// `"compressed size"`, `"uncompressed size"`, `"encryption flag"`,
271 /// `"data-descriptor flag"`, `"UTF-8 flag"`, `"entry count"` or
272 /// `"order"`. `entry` is the name as
273 /// the authoritative description (the central directory, when it was
274 /// read) spells it, decoded as the reader decodes names.
275 #[error(
276 "`{format}` local header and central directory disagree on {field} for entry `{}`",
277 fmt_name(.entry)
278 )]
279 HeadersDisagree {
280 format: FormatId,
281 entry: String,
282 field: &'static str,
283 },
284}
285
286/// What a read cost in fidelity. Returned alongside data, never logged and
287/// discarded — the caller must be able to discover what was approximated.
288#[derive(Clone, PartialEq, Eq, Debug)]
289#[cfg_attr(feature = "serde", derive(serde::Serialize))]
290pub struct FidelityReport {
291 pub rung: Rung,
292 pub warnings: Vec<Fidelity>,
293}
294
295impl FidelityReport {
296 pub fn exact() -> Self {
297 Self::new(Rung::Exact)
298 }
299
300 pub fn new(rung: Rung) -> Self {
301 Self {
302 rung,
303 warnings: Vec::new(),
304 }
305 }
306
307 pub fn warn(&mut self, w: Fidelity) {
308 self.warnings.push(w);
309 }
310
311 /// Whether anything was approximated. **This is the `--strict-fidelity`
312 /// gate** (exit code 4).
313 ///
314 /// Gating on warnings rather than on the rung is deliberate: a tar read
315 /// from a pipe lands on `ForwardOnly`, but tar has no trailing index, so
316 /// nothing was lost. Failing it would hand a script an error it cannot act
317 /// on. The rung stays diagnostic — reported by `stuffr info`.
318 pub fn has_warnings(&self) -> bool {
319 !self.warnings.is_empty()
320 }
321
322 /// True only if the rung was authoritative *and* nothing was approximated.
323 ///
324 /// Deliberately **not** the `--strict-fidelity` gate — see
325 /// [`Self::has_warnings`]. This is the stronger "the format's own index was
326 /// read and nothing was lost" claim, useful for archival verification.
327 pub fn is_lossless(&self) -> bool {
328 self.rung.is_authoritative() && self.warnings.is_empty()
329 }
330
331 /// Fold another report in: worst rung wins, warnings are unioned.
332 ///
333 /// For combining reports from independently-resolved layers — e.g. a
334 /// codec's own report merged with the inner container's, once a chain has
335 /// more than one — rather than for a single container's own findings,
336 /// which it now records by calling [`Self::warn`] directly on the report
337 /// it inherited from the ladder (see `crate::ladder::seed_report`). A
338 /// warning already present is not repeated, so folding the same loss in
339 /// from two sources still tells the caller about it once.
340 pub fn merge(&mut self, other: &FidelityReport) {
341 if other.rung.severity() > self.rung.severity() {
342 self.rung = other.rung;
343 }
344 for w in &other.warnings {
345 if !self.warnings.contains(w) {
346 self.warnings.push(w.clone());
347 }
348 }
349 }
350}
351
352impl Default for FidelityReport {
353 fn default() -> Self {
354 Self::exact()
355 }
356}
357
358#[cfg(test)]
359mod tests {
360 use super::*;
361 use crate::format::FormatId;
362
363 #[test]
364 fn free_text_in_fidelity_messages_is_escaped() {
365 let r = || String::from("a\x1bb");
366 let all = [
367 Fidelity::EntrySkipped {
368 entry: "e".into(),
369 reason: r(),
370 },
371 Fidelity::EntryCountMismatch {
372 format: FormatId::new("tar"),
373 declared: 1,
374 enumerated: 0,
375 reason: r(),
376 },
377 Fidelity::EntrySizeChanged {
378 entry: "e".into(),
379 declared: 1,
380 fixup: r(),
381 },
382 ];
383 for f in all {
384 let m = f.to_string();
385 assert!(!m.contains('\x1b'), "raw ESC in {m:?}");
386 assert!(m.contains("a\\x1bb"), "{m:?}");
387 }
388 let plain = "a `b` \"c\" naïve 日本";
389 let m = Fidelity::EntrySkipped {
390 entry: "e".into(),
391 reason: plain.into(),
392 }
393 .to_string();
394 assert_eq!(m, format!("skipped entry `e`: {plain}"));
395 }
396
397 #[test]
398 fn entry_names_in_fidelity_messages_are_escaped() {
399 let n = || String::from("a\x1bb");
400 let all = [
401 Fidelity::SizeFromDataDescriptor { entry: n() },
402 Fidelity::MetadataIncomplete {
403 entry: n(),
404 fields: MetaFields::default(),
405 },
406 Fidelity::EncryptedEntrySkipped { entry: n() },
407 Fidelity::EntrySkipped {
408 entry: n(),
409 reason: "r".into(),
410 },
411 Fidelity::EntrySizeChanged {
412 entry: n(),
413 declared: 1,
414 fixup: "f".into(),
415 },
416 Fidelity::HeadersDisagree {
417 format: FormatId::new("zip"),
418 entry: n(),
419 field: "name",
420 },
421 ];
422 for f in all {
423 let m = f.to_string();
424 assert!(m.contains("a\\x1bb"), "{m}");
425 assert!(!m.contains('\x1b'), "{m:?}");
426 }
427 }
428
429 #[test]
430 fn spilled_is_authoritative_even_though_it_is_a_lower_rung() {
431 // Spilling costs disk, not accuracy. Treating it as lossy would make
432 // `--strict-fidelity` reject perfectly exact reads.
433 assert!(Rung::Exact.is_authoritative());
434 assert!(Rung::Spilled.is_authoritative());
435 assert!(!Rung::ForwardOnly.is_authoritative());
436 assert!(!Rung::Degraded.is_authoritative());
437 }
438
439 #[test]
440 fn exact_report_is_lossless() {
441 let r = FidelityReport::exact();
442 assert_eq!(r.rung, Rung::Exact);
443 assert!(r.is_lossless());
444 assert!(r.warnings.is_empty());
445 }
446
447 #[test]
448 fn a_warning_makes_an_otherwise_exact_report_lossy() {
449 let mut r = FidelityReport::exact();
450 r.warn(Fidelity::EntryCountUnknown);
451 assert!(!r.is_lossless());
452 assert_eq!(r.warnings.len(), 1);
453 }
454
455 #[test]
456 fn forward_only_is_lossy_even_with_no_warnings() {
457 assert!(!FidelityReport::new(Rung::ForwardOnly).is_lossless());
458 }
459
460 #[test]
461 fn merge_keeps_the_worst_rung_and_concatenates_warnings() {
462 let mut a = FidelityReport::exact();
463 a.warn(Fidelity::EntryCountUnknown);
464
465 let mut b = FidelityReport::new(Rung::Degraded);
466 b.warn(Fidelity::TruncatedStream { at: 99 });
467
468 a.merge(&b);
469 assert_eq!(a.rung, Rung::Degraded);
470 assert_eq!(a.warnings.len(), 2);
471 }
472
473 #[test]
474 fn merge_does_not_upgrade_a_worse_rung() {
475 let mut a = FidelityReport::new(Rung::Degraded);
476 a.merge(&FidelityReport::exact());
477 assert_eq!(a.rung, Rung::Degraded);
478 }
479
480 #[test]
481 fn merge_does_not_repeat_a_warning_already_present() {
482 // The ladder and the container can each independently observe the same
483 // loss; the caller should be told once.
484 let mut a = FidelityReport::exact();
485 a.warn(Fidelity::EntryCountUnknown);
486
487 let mut b = FidelityReport::new(Rung::ForwardOnly);
488 b.warn(Fidelity::EntryCountUnknown);
489
490 a.merge(&b);
491 assert_eq!(a.warnings.len(), 1);
492 assert_eq!(a.rung, Rung::ForwardOnly);
493 }
494
495 #[test]
496 fn warnings_carry_actionable_detail_not_just_a_kind() {
497 // The whole premise is that the caller can find out WHAT was lost.
498 let f = Fidelity::SizeFromDataDescriptor {
499 entry: "logs/a.txt".into(),
500 };
501 assert!(format!("{f}").contains("logs/a.txt"));
502
503 let f = Fidelity::TrailingIndexUnread {
504 format: FormatId::new("zip"),
505 };
506 assert!(format!("{f}").contains("zip"));
507
508 let f = Fidelity::SolidBlockFullyDecoded {
509 wasted_bytes: 41_943_040,
510 };
511 assert!(format!("{f}").contains("41943040"));
512
513 let f = Fidelity::EntrySkipped {
514 entry: "dev/null".into(),
515 reason: "device nodes, fifos and sockets are not created".into(),
516 };
517 let s = format!("{f}");
518 assert!(
519 s.contains("dev/null") && s.contains("sockets"),
520 "a skip must say both which entry and why: {s}"
521 );
522
523 let f = Fidelity::EntryCountMismatch {
524 format: FormatId::new("zip"),
525 declared: 8,
526 enumerated: 6,
527 reason: "records sharing a name collapse into one".into(),
528 };
529 let s = format!("{f}");
530 assert!(
531 s.contains('8') && s.contains('6') && s.contains("sharing a name"),
532 "a shortfall must name BOTH counts and the cause, or the caller \
533 cannot tell how much is missing or why: {s}"
534 );
535
536 let f = Fidelity::MetadataIncomplete {
537 entry: "b.bin".into(),
538 fields: MetaFields {
539 mode: true,
540 ..Default::default()
541 },
542 };
543 let s = format!("{f}");
544 assert!(s.contains("b.bin") && s.contains("mode"));
545 }
546
547 #[test]
548 fn meta_fields_lists_only_the_missing_ones() {
549 let m = MetaFields {
550 mtime: true,
551 crc: true,
552 ..Default::default()
553 };
554 assert_eq!(m.missing(), vec!["mtime", "crc"]);
555 assert!(MetaFields::default().missing().is_empty());
556 }
557
558 #[test]
559 fn a_forward_only_read_that_lost_nothing_passes_strict_mode() {
560 // tar on a pipe: rung is ForwardOnly, but tar has no trailing index, so
561 // nothing was approximated. Strict mode must not fail this.
562 let r = FidelityReport::new(Rung::ForwardOnly);
563 assert!(!r.has_warnings(), "nothing was approximated");
564 assert!(!r.is_lossless(), "but the rung was not authoritative");
565 }
566
567 #[test]
568 fn strict_mode_fires_on_any_warning_regardless_of_rung() {
569 let mut r = FidelityReport::exact();
570 r.warn(Fidelity::EntryCountUnknown);
571 assert!(r.has_warnings());
572 }
573
574 #[test]
575 fn a_clean_exact_read_passes_both_predicates() {
576 let r = FidelityReport::exact();
577 assert!(!r.has_warnings());
578 assert!(r.is_lossless());
579 }
580}