Skip to main content

rivet/pipeline/
validate_manifest.rs

1//! **Layer: Observability**
2//!
3//! Manifest-aware verification for `--validate` (ADR-0012 §M5 / §M6,
4//! constrained by the ADR-0013 trust-flag contract that says: no new flags;
5//! manifest-aware checks live under the existing `--validate`).
6//!
7//! Although this module reads from a `Destination` (technically L2 surface),
8//! it makes **no execution decisions**: it does not write data, advance
9//! cursors, mutate state, or change the pipeline path.  Its only output is
10//! a structured `ManifestVerification` verdict the run report renders.
11//! Per ADR-0003, that places it firmly in L4 Observability — the
12//! destination read surface is just the carrier.
13//!
14//! Inputs (read-only):
15//! - the destination's `manifest.json` body
16//! - the destination's `_SUCCESS` body, if present
17//! - the listing of every object under the destination prefix
18//!
19//! Outputs:
20//! - [`ManifestVerification`] — a structured verdict the run report renders
21//!   into the operator-facing "Verdicts" section.
22//!
23//! Out of scope here:
24//! - per-file row-count check (that runs *during* the export, against the
25//!   local temp file before upload — see `pipeline::validate::validate_output`).
26//! - source-side reconciliation (lives in [`pipeline::reconcile_cmd`] and
27//!   is what `--reconcile` adds on top of this).
28//! - re-fingerprinting parts (`--validate --deep`, future).
29//!
30//! Failure modes are explicit: each check produces a `Failure` enum variant
31//! that is rendered verbatim in `summary.json` so an Airflow / CI consumer
32//! can branch on the kind, not parse strings.
33
34use serde::{Deserialize, Serialize};
35
36use crate::destination::Destination;
37use crate::error::Result;
38use crate::manifest::{
39    MANIFEST_FILENAME, RunManifest, SUCCESS_FILENAME, join_key, parse_success_marker,
40    success_marker_body,
41};
42use crate::pipeline::manifest_reconcile::{PartPresence, reconcile_manifest_against_listing};
43
44/// Upper bound on a destination control artifact (`manifest.json`) the read
45/// path will materialise into memory.  A `manifest.json` is metadata — a few
46/// KB to low single-digit MB even for very large datasets — so 64 MiB is far
47/// above any legitimate body while still bounding the blast radius.
48///
49/// Security (V21, CWE-400): the manifest readers `head()` an object then read
50/// its full body into a `Vec<u8>`.  An attacker who can write the destination
51/// prefix (a shared bucket prefix, a world-writable export dir) can plant a
52/// multi-GB `manifest.json`; an unbounded read would OOM the next `--resume`,
53/// `--validate`, or `rivet repair`.  [`read_capped`] consults the size the
54/// `head()` already reports and bails before the read when it exceeds this cap.
55pub(crate) const MANIFEST_MAX_BYTES: u64 = 64 * 1024 * 1024;
56
57/// Same V21/CWE-400 defense for the sibling control artifact `_SUCCESS`, read by
58/// the SAME `verify_at_destination` under the SAME destination-writable threat
59/// model but previously UNCAPPED. A legitimate marker is ~22 bytes
60/// (`xxh3:<16-hex>\n`); a body past this small cap is corrupt or planted, so it
61/// is treated as malformed WITHOUT materialising it into memory. 4 KiB is orders
62/// of magnitude above any real marker yet trivially bounded.
63pub(crate) const SUCCESS_MARKER_MAX_BYTES: u64 = 4096;
64
65/// How deep a `rivet validate` pass goes — a graded verify layer over the
66/// same checks, letting an operator trade thoroughness for latency / cost.
67///
68/// The variants are a strict superset chain: `Light ⊂ Sample ⊂ Full`.  Each
69/// level runs every check the level below it does, plus more.  Defined here
70/// (the pipeline layer) and re-exported for the CLI grammar so the **same**
71/// enum gates the checks in [`verify_at_destination`] and parses on the
72/// `--depth` flag — no CLI→pipeline back-dependency.
73///
74/// - **Light**: manifest read + self-consistency + `_SUCCESS` only.  Skips the
75///   `list_prefix` reconcile (no per-part presence/size/checksum) and the
76///   untracked-surplus scan, leaving `parts_verified = 0`.  One `head` + one
77///   `read` of `manifest.json` and `_SUCCESS` — a fast "is this prefix a
78///   complete, marked run?" poll with no prefix listing.
79/// - **Sample**: everything Light does **plus** the part reconcile and
80///   untracked surplus (one `list_prefix`).  This is the pre-graded behaviour
81///   minus the Form B value re-read — full structural verification with no
82///   part downloads.
83/// - **Full** (default): everything Sample does **plus** the Form B value-
84///   checksum re-read (re-reads parts, re-derives per-column checksums).  The
85///   most thorough and the only level that downloads part bodies.  Equivalent
86///   to the pre-graded behaviour, so existing callers are unchanged.
87#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq, Default)]
88pub enum ValidateDepth {
89    /// Manifest read + self-consistency + `_SUCCESS` only (no prefix listing).
90    Light,
91    /// Light + part reconcile + untracked surplus (one `list_prefix`).
92    Sample,
93    /// Sample + the Form B value-checksum re-read (downloads parts).
94    #[default]
95    Full,
96}
97
98impl ValidateDepth {
99    /// True iff this level runs the `list_prefix` reconcile (part presence,
100    /// size, checksum) and the untracked-surplus scan — i.e. anything above
101    /// `Light`.  The single predicate the section-3/5 depth gating keys off.
102    fn runs_part_reconcile(self) -> bool {
103        !matches!(self, ValidateDepth::Light)
104    }
105
106    /// True iff this level downloads part *bodies* — the CDC `__pos` continuity
107    /// check and the Form-B value-checksum re-read, both `Full`-only (the
108    /// `part_reconcile` level above only reads listing metadata). The single
109    /// predicate the section-3/5 part-download gating keys off, parallel to
110    /// [`Self::runs_part_reconcile`] — so adding a depth level edits the enum,
111    /// not the call sites in `validate_cmd`.
112    pub(crate) fn runs_part_download(self) -> bool {
113        matches!(self, ValidateDepth::Full)
114    }
115
116    /// Stable operator-/wire-facing label for the depth a verdict was produced
117    /// at, surfaced in `summary.json` (`depth_level`) and `rivet validate`
118    /// output.
119    pub fn label(self) -> &'static str {
120        match self {
121            ValidateDepth::Light => "light",
122            ValidateDepth::Sample => "sample",
123            ValidateDepth::Full => "full",
124        }
125    }
126}
127
128/// Read `key` into memory only if its `head()`-reported size is within
129/// `max_bytes`; otherwise bail without reading a single byte.
130///
131/// The single enforcement point for the V21 (CWE-400) manifest-read cap shared
132/// by the three control-artifact readers (`--resume` M8 preamble, `--validate`,
133/// `rivet repair`).  Each previously did `head()` then an uncapped `read()`,
134/// discarding the size `head()` already returned; routing through here closes
135/// that gap in one place.
136///
137/// Behaviour:
138/// - object absent (`head` → `None`): `Err` — callers invoke this only after
139///   establishing the object exists, so an absent object here is a hard error,
140///   not the benign "no manifest / legacy prefix" case (which the callers
141///   detect with their own `head()` first).
142/// - oversized (`size_bytes > max_bytes`): `Err` naming the cap, **before** any
143///   body is materialised.
144/// - otherwise: the full body via [`Destination::read`].
145pub(crate) fn read_capped(dest: &dyn Destination, key: &str, max_bytes: u64) -> Result<Vec<u8>> {
146    match dest.head(key)? {
147        None => anyhow::bail!("'{key}' not found at the destination"),
148        Some(meta) => {
149            if meta.size_bytes > max_bytes {
150                anyhow::bail!(
151                    "'{key}' is {} bytes, exceeding the {max_bytes}-byte control-artifact \
152                     read cap — refusing to load it into memory (possible tampering)",
153                    meta.size_bytes
154                );
155            }
156            dest.read(key)
157        }
158    }
159}
160
161/// Outcome of a single `--validate` pass over a destination prefix.
162///
163/// Stable enough to be embedded in `summary.json` directly (see
164/// `pipeline::report::ValidationOutcome`).  Forward-compat: consumers MUST
165/// ignore unknown fields (no `deny_unknown_fields`).
166#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
167pub struct ManifestVerification {
168    /// True iff a `manifest.json` was found at the destination and parsed.
169    /// `false` triggers ADR-0012 M6 fallback (legacy run); higher-level
170    /// check results below are then "skipped" rather than "passed".
171    pub manifest_found: bool,
172    /// Mirrors ADR-0012 M6's required `legacy_run` operator-facing label.
173    pub legacy_run: bool,
174    /// Manifest parts whose presence and recorded `size_bytes` were
175    /// confirmed at the destination.  0 when no manifest was found.
176    pub parts_verified: usize,
177    /// Subset of `parts_verified` whose **content** was confirmed via an MD5
178    /// the store surfaced in its listing (no download) — the rest are size-only.
179    /// Lets `passed: true` say how much of the dataset was content-checked
180    /// rather than implying all of it was.  `#[serde(default)]` for back-compat.
181    #[serde(default)]
182    pub parts_md5_verified: usize,
183    /// Manifest parts that were declared `committed` but not actually
184    /// present, present at a different size, or otherwise mismatched.
185    pub parts_failed: usize,
186    /// True iff `_SUCCESS` exists at the destination AND its body matches
187    /// the fingerprint of the bytes we read for `manifest.json`.  An
188    /// existing `_SUCCESS` whose body diverges from the manifest is itself
189    /// an integrity failure — surfaced via `failures`.
190    pub success_marker_consistent: bool,
191    /// Self-consistency of the manifest (`row_count`, `part_count`,
192    /// duplicate `part_id`s).  Skipped when `manifest_found = false`.
193    pub manifest_self_consistent: bool,
194    /// Final verdict, **derived** (not hand-maintained) — `manifest_found` and
195    /// no *fatal* failure ([`Failure::is_fatal`]).  Stored so it stays in the
196    /// `summary.json` contract, but computed in one place
197    /// ([`ManifestVerification::recompute_passed`]) so a new failure variant is
198    /// fatal by default rather than relying on every site to flip a bool.
199    pub passed: bool,
200    /// Per-failure detail.  May be non-empty with `passed = true` for advisory
201    /// (non-fatal) failures like [`Failure::UntrackedObject`].  Stable variant
202    /// set; new variants land under a new manifest version per ADR-0012.
203    pub failures: Vec<Failure>,
204    /// The graded depth this verdict was produced at: `"light"`, `"sample"`,
205    /// or `"full"` (see [`ValidateDepth`]).  Lets a consumer of `summary.json`
206    /// tell **how much** was actually checked — a `passed: true` at `"light"`
207    /// asserts far less than at `"full"` (no part presence was verified).
208    /// `#[serde(default)]` (→ `"full"`) for back-compat: pre-graded verdicts
209    /// always ran the full pass.
210    #[serde(default = "default_depth_level")]
211    pub depth_level: String,
212}
213
214/// `serde(default)` for [`ManifestVerification::depth_level`]: a verdict that
215/// predates the graded layer always ran the full pass, so an absent field
216/// deserializes to `"full"`.
217fn default_depth_level() -> String {
218    ValidateDepth::Full.label().to_string()
219}
220
221#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
222#[serde(tag = "kind", rename_all = "snake_case")]
223pub enum Failure {
224    /// Manifest declared a part that does not exist at the destination.
225    PartMissing { part_id: u32, path: String },
226    /// Manifest declared a part whose actual size differs from `size_bytes`.
227    PartSizeMismatch {
228        part_id: u32,
229        path: String,
230        expected: u64,
231        actual: u64,
232    },
233    /// Part present at the recorded size but its content MD5 (from the store's
234    /// listing metadata) differs from the manifest's — transit / at-rest
235    /// corruption, caught with no download.
236    PartChecksumMismatch {
237        part_id: u32,
238        path: String,
239        expected: String,
240        actual: String,
241    },
242    /// `--depth full` re-read the parts and a per-column VALUE checksum (Form B)
243    /// disagreed with the manifest — post-write corruption an MD5/size check
244    /// cannot see (a flipped bit inside a data page). Verified-wrong, so it fails
245    /// the verdict and classifies as data-integrity (exit 3), not could-not-verify.
246    ValueChecksumMismatch { detail: String },
247    /// `--depth full` re-read a CDC export's parts and the `__pos` continuity
248    /// check found a gap/duplicate in the change positions — the exported change
249    /// stream is incomplete or reordered. Verified-wrong (same class as a value
250    /// mismatch), so it fails the verdict and classifies as data-integrity (exit
251    /// 3), not could-not-verify (bughunt MED: it was folded into hard_failures →
252    /// exit 1, inconsistent with the value-checksum path).
253    CdcPositionViolation { detail: String },
254    /// `_SUCCESS` exists but its body is malformed (not `xxh3:<16-hex>` after
255    /// trim).  ADR-0012 M2 — orchestrators rely on this format being strict.
256    SuccessMarkerMalformed { body_preview: String },
257    /// `_SUCCESS` body parsed but does not match `xxh3(manifest.json bytes)`.
258    /// Two legitimate sources: (a) someone overwrote `_SUCCESS` after the
259    /// manifest was rewritten — orchestrator bug; (b) the manifest was
260    /// edited in place after the run — operator bug.  Either way the
261    /// manifest is no longer trustworthy.
262    SuccessMarkerStale {
263        marker_fingerprint: String,
264        manifest_fingerprint: String,
265    },
266    /// `RunManifest::validate_self_consistency` rejected the manifest.
267    /// Usually a writer bug (declared row_count != sum of committed parts'
268    /// rows); blocks the rest of the verification because the manifest
269    /// itself is unreliable.
270    ManifestSelfInconsistent { detail: String },
271    /// Reading `manifest.json` returned an I/O error other than "absent".
272    ManifestReadError { detail: String },
273    /// Reading `_SUCCESS` returned an I/O error other than "absent".
274    SuccessMarkerReadError { detail: String },
275    /// Listing the destination prefix returned an I/O error.  Reduces the
276    /// untracked-parts check (M5 surplus) to a no-op for this run.
277    ListPrefixError { detail: String },
278    /// A file is present at the destination prefix but no manifest entry
279    /// references it.  M9-adjacent: `--validate` only flags it; quarantine
280    /// belongs to `--resume`.
281    UntrackedObject { key: String, size_bytes: u64 },
282    /// The export declared `verify: content` but some parts could only be
283    /// size-verified (no comparable content checksum from the store) — the
284    /// declared integrity contract was not met.
285    ContentVerificationUnmet { size_only: usize, total: usize },
286    /// A manifest was *required* at this prefix (the operator pinned a literal
287    /// `--prefix`, asserting a real dataset lives here) but none was found.
288    /// Without this, an absent manifest at an operator-pinned prefix maps to
289    /// the M6 legacy-run label and exits 0 — indistinguishable from a verified
290    /// run, so a CI gate `rivet validate && deploy` sails past a destination
291    /// that was never written.  Fatal: a required-but-missing manifest is a
292    /// refusal reason, not a "cannot certify" advisory.
293    ManifestRequiredButAbsent { prefix: String },
294}
295
296impl Failure {
297    /// Whether this failure invalidates the dataset (flips `passed` to false).
298    ///
299    /// Every variant is fatal **except** [`Failure::UntrackedObject`]: surplus
300    /// objects are an audit signal whose cleanup is `--resume`'s job (ADR-0012
301    /// M9), not a corruption of the manifest-listed parts.  New variants are
302    /// fatal by default — opt out here explicitly, so a forgotten case fails
303    /// closed (safe) rather than silently passing.
304    pub fn is_fatal(&self) -> bool {
305        !matches!(self, Failure::UntrackedObject { .. })
306    }
307
308    /// Whether this failure is COULD-NOT-VERIFY (an operational I/O error that
309    /// prevented the check) rather than VERIFIED-WRONG (the check ran and found
310    /// the data bad). Could-not-verify is the operational class — exit 1, retry —
311    /// NOT the data-integrity stop-the-line class (exit 3). A read/list error
312    /// against a healthy prefix (a chmod-000 manifest, a transient S3 list blip)
313    /// must not page a corruption incident (#7 bughunt: these fell into the
314    /// verdict and drove exit 3). Fatal-by-omission the other way: a NEW variant
315    /// is verified-wrong (exit 3) unless it is explicitly an I/O could-not-verify.
316    pub fn is_could_not_verify(&self) -> bool {
317        matches!(
318            self,
319            Failure::ManifestReadError { .. }
320                | Failure::SuccessMarkerReadError { .. }
321                | Failure::ListPrefixError { .. }
322        )
323    }
324
325    /// Stable `RIVET_VERIFY_*` error code for this failure variant.
326    ///
327    /// One code per variant, intended for orchestrators / CI to branch on
328    /// without parsing the human `Display` string or the per-variant JSON
329    /// fields.  The code is part of the wire contract: it is emitted next to
330    /// `kind` in the JSON report and prefixed in brackets on each pretty line.
331    /// Codes are append-only — never renamed once shipped (a renamed code is a
332    /// silent break for any consumer keying off it).
333    pub fn error_code(&self) -> &'static str {
334        match self {
335            Failure::PartMissing { .. } => "RIVET_VERIFY_PART_MISSING",
336            Failure::PartSizeMismatch { .. } => "RIVET_VERIFY_PART_SIZE_MISMATCH",
337            Failure::PartChecksumMismatch { .. } => "RIVET_VERIFY_PART_CHECKSUM_MISMATCH",
338            Failure::ValueChecksumMismatch { .. } => "RIVET_VERIFY_VALUE_CHECKSUM",
339            Failure::CdcPositionViolation { .. } => "RIVET_VERIFY_CDC_POSITION",
340            Failure::SuccessMarkerMalformed { .. } => "RIVET_VERIFY_SUCCESS_MALFORMED",
341            Failure::SuccessMarkerStale { .. } => "RIVET_VERIFY_SUCCESS_STALE",
342            Failure::ManifestSelfInconsistent { .. } => "RIVET_VERIFY_MANIFEST_INCONSISTENT",
343            Failure::ManifestReadError { .. } => "RIVET_VERIFY_MANIFEST_READ_ERROR",
344            Failure::SuccessMarkerReadError { .. } => "RIVET_VERIFY_SUCCESS_READ_ERROR",
345            Failure::ListPrefixError { .. } => "RIVET_VERIFY_LIST_ERROR",
346            Failure::UntrackedObject { .. } => "RIVET_VERIFY_UNTRACKED_OBJECT",
347            Failure::ContentVerificationUnmet { .. } => "RIVET_VERIFY_CONTENT_UNMET",
348            Failure::ManifestRequiredButAbsent { .. } => "RIVET_VERIFY_MANIFEST_REQUIRED",
349        }
350    }
351}
352
353impl std::fmt::Display for Failure {
354    /// One operator-facing line per failure variant.  Used by:
355    /// - `pipeline::report::render_markdown` (summary.md "failure:" lines)
356    /// - `pipeline::validate_cmd::render_pretty` (`rivet validate` stdout)
357    /// - any future consumer that wants a human-readable failure label
358    ///
359    /// The wire format (`failures[].kind` + per-variant fields) lives in
360    /// the `Serialize` derive above and is the contract Airflow / CI
361    /// consumers branch on.  This `Display` impl is for humans only.
362    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
363        match self {
364            Failure::PartMissing { part_id, path } => {
365                write!(f, "part {} missing at {}", part_id, path)
366            }
367            Failure::PartSizeMismatch {
368                part_id,
369                path,
370                expected,
371                actual,
372            } => write!(
373                f,
374                "part {} size mismatch at {}: manifest {}, dest {}",
375                part_id, path, expected, actual
376            ),
377            Failure::ValueChecksumMismatch { detail } => {
378                write!(
379                    f,
380                    "value checksum mismatch (post-write corruption): {}",
381                    detail
382                )
383            }
384            Failure::CdcPositionViolation { detail } => {
385                write!(f, "cdc __pos continuity violation: {}", detail)
386            }
387            Failure::PartChecksumMismatch {
388                part_id,
389                path,
390                expected,
391                actual,
392            } => write!(
393                f,
394                "part {} content mismatch at {}: manifest md5 {}, dest {}",
395                part_id, path, expected, actual
396            ),
397            Failure::SuccessMarkerMalformed { body_preview } => {
398                write!(f, "_SUCCESS body malformed: {body_preview:?}")
399            }
400            Failure::SuccessMarkerStale {
401                marker_fingerprint,
402                manifest_fingerprint,
403            } => write!(
404                f,
405                "_SUCCESS body {} != manifest fingerprint {} (stale marker)",
406                marker_fingerprint, manifest_fingerprint
407            ),
408            Failure::ManifestSelfInconsistent { detail } => {
409                write!(f, "manifest self-consistency: {detail}")
410            }
411            Failure::ManifestReadError { detail } => {
412                write!(f, "manifest read error: {detail}")
413            }
414            Failure::SuccessMarkerReadError { detail } => {
415                write!(f, "_SUCCESS read error: {detail}")
416            }
417            Failure::ListPrefixError { detail } => {
418                write!(f, "destination listing error: {detail}")
419            }
420            Failure::UntrackedObject { key, size_bytes } => {
421                write!(f, "untracked object: {} ({} bytes)", key, size_bytes)
422            }
423            Failure::ContentVerificationUnmet { size_only, total } => write!(
424                f,
425                "verify: content not met — {size_only} of {total} part(s) only \
426                 size-verified (no store checksum); lower max_file_size so parts \
427                 upload as a single PUT, or the backend exposes no checksum"
428            ),
429            Failure::ManifestRequiredButAbsent { prefix } => write!(
430                f,
431                "no manifest at {prefix}: a manifest was required here (operator \
432                 pinned --prefix) but none was found — this prefix was never \
433                 written, or the data was relocated. Run the export first, or \
434                 drop --prefix to validate the config-resolved destination."
435            ),
436        }
437    }
438}
439
440impl ManifestVerification {
441    /// Base verdict: nothing checked yet (no manifest, all counts zero, all
442    /// sub-checks false, `passed = false`).  Every constructor builds on this
443    /// and overrides only what differs, so a new field lands in **one** place
444    /// rather than several near-identical literals.
445    fn empty() -> Self {
446        Self {
447            manifest_found: false,
448            legacy_run: false,
449            parts_verified: 0,
450            parts_md5_verified: 0,
451            parts_failed: 0,
452            success_marker_consistent: false,
453            manifest_self_consistent: false,
454            passed: false,
455            failures: Vec::new(),
456            // Base level; `verify_at_destination` overwrites this with the
457            // depth it was actually called at before returning any verdict.
458            depth_level: default_depth_level(),
459        }
460    }
461
462    /// Recompute `passed` from the verdict's facts: a manifest was found and no
463    /// **fatal** failure was recorded (advisory failures like `UntrackedObject`
464    /// don't count).  The single source of truth — callers set failures and
465    /// call this once, rather than flipping `passed` by hand at every site.
466    fn recompute_passed(&mut self) {
467        self.passed = self.manifest_found && !self.failures.iter().any(Failure::is_fatal);
468    }
469
470    /// Apply the export's `verify` policy (ADR-0013 / review D).  When content
471    /// verification is required but some parts were only size-verified, record
472    /// a fatal [`Failure::ContentVerificationUnmet`] and re-derive `passed`.
473    /// Policy lives here (one place); the composers — run finalize and the
474    /// `rivet validate` command — just call it with their export's intent.
475    pub fn enforce_content_policy(&mut self, require_content: bool) {
476        if require_content && self.manifest_found {
477            let size_only = self.parts_verified.saturating_sub(self.parts_md5_verified);
478            if size_only > 0 {
479                self.failures.push(Failure::ContentVerificationUnmet {
480                    size_only,
481                    total: self.parts_verified,
482                });
483                self.recompute_passed();
484            }
485        }
486    }
487
488    /// Apply the "a manifest must exist here" policy (finding #20).  When the
489    /// operator pinned a literal `--prefix`, an absent manifest is no longer the
490    /// benign M6 legacy-run case — it almost always means the prefix was never
491    /// written (a misconfigured CI gate). Convert that exact verdict — no
492    /// manifest, no other failure (i.e. the [`ManifestVerification::legacy`]
493    /// shape) — into a fatal [`Failure::ManifestRequiredButAbsent`] so the exit
494    /// gate refuses it loudly instead of silently passing.
495    ///
496    /// Deliberately a no-op for every other shape: a real manifest (passed or
497    /// failed), or an absent manifest that already carries a `ManifestReadError`
498    /// / head failure, is left untouched — those are already classified.  Only
499    /// the "legacy / cannot certify" case is escalated, and only when required.
500    pub fn require_manifest_present(&mut self, prefix: &str) {
501        if !self.manifest_found && !self.has_failures() {
502            self.legacy_run = false;
503            self.failures.push(Failure::ManifestRequiredButAbsent {
504                prefix: prefix.to_string(),
505            });
506            self.recompute_passed();
507        }
508    }
509
510    /// Construct the M6 (legacy run) verdict for a destination that has no
511    /// manifest at all.  Caller composes this with the existing per-file
512    /// row-count check; together they form the legacy `--validate` result.
513    pub fn legacy() -> Self {
514        // `passed = false` is intentional — not "validation failed" but "this
515        // verifier cannot certify"; the caller layers per-file row counts on
516        // top and composes the final verdict.
517        Self {
518            legacy_run: true,
519            ..Self::empty()
520        }
521    }
522
523    /// True iff this verification surfaced any explicit failure (i.e. a
524    /// reason an orchestrator should refuse the run).  Distinct from
525    /// `!passed`, which can also mean "legacy / not applicable".
526    pub fn has_failures(&self) -> bool {
527        !self.failures.is_empty()
528    }
529
530    /// Whether any recorded failure is VERIFIED-WRONG (the check ran and the data
531    /// is bad) as opposed to purely COULD-NOT-VERIFY (an I/O read/list error).
532    /// Drives the exit CLASS: a verdict with a verified-wrong failure is
533    /// data-integrity (exit 3); a verdict whose failures are ALL could-not-verify
534    /// is operational (exit 1). See [`Failure::is_could_not_verify`].
535    pub fn has_verified_wrong_failure(&self) -> bool {
536        self.failures.iter().any(|f| !f.is_could_not_verify())
537    }
538}
539
540/// Run the manifest-aware verification at `manifest_dir` (the destination-
541/// relative directory containing `manifest.json` and `_SUCCESS`).
542///
543/// `manifest_dir` is the same key shape `Destination::write` was called with
544/// for the manifest itself — typically empty (`""`) for prefix-rooted runs,
545/// or the per-export sub-directory.  Trailing `/` is optional.
546///
547/// This function does not panic on any expected I/O outcome — every read
548/// failure becomes a `Failure::*ReadError` so the caller can render a
549/// useful message instead of bailing.
550///
551/// `depth` selects the graded verify layer (see [`ValidateDepth`]):
552/// - [`ValidateDepth::Light`] skips section 3 (the `list_prefix` part
553///   reconcile) and section 5 (untracked surplus), leaving `parts_verified`
554///   at 0 — a fast manifest + `_SUCCESS` poll with no prefix listing.
555/// - [`ValidateDepth::Sample`] and [`ValidateDepth::Full`] run all five
556///   sections here.  The Form B value re-read is **not** in this function;
557///   it is the caller's concern (`run_validate_command`), gated on `Full`.
558///
559/// Regardless of depth, `depth_level` on the returned verdict records the
560/// level this pass ran at.
561pub fn verify_at_destination(
562    dest: &dyn Destination,
563    manifest_dir: &str,
564    depth: ValidateDepth,
565) -> Result<ManifestVerification> {
566    let manifest_key = join_key(manifest_dir, MANIFEST_FILENAME);
567    let success_key = join_key(manifest_dir, SUCCESS_FILENAME);
568
569    // Stamp the depth this pass ran at onto every verdict before it leaves the
570    // function — including the early-return error/legacy shapes — so a consumer
571    // always sees *how much* was checked.  Each `return Ok(v)` below routes
572    // through `with_depth` (or sets `out.depth_level` for the main path).
573    let with_depth = |mut v: ManifestVerification| -> ManifestVerification {
574        v.depth_level = depth.label().to_string();
575        v
576    };
577
578    // ── 1. Manifest read ───────────────────────────────────────────────
579    //
580    // Error-consistency contract: every I/O outcome here surfaces as a
581    // structured `Failure` variant rather than as `Err`.  An operator gets
582    // one verdict shape regardless of whether the destination is missing,
583    // permission-denied, or temporarily unreachable.  The bubbled `Err`
584    // path is reserved for *programmer* errors (caller passes a malformed
585    // `manifest_dir`, a future destination breaks an internal invariant).
586    let manifest_bytes = match dest.head(&manifest_key) {
587        Ok(None) => return Ok(with_depth(ManifestVerification::legacy())),
588        Ok(Some(_)) => match read_capped(dest, &manifest_key, MANIFEST_MAX_BYTES) {
589            Ok(b) => b,
590            Err(e) => {
591                let mut v = ManifestVerification::legacy();
592                v.legacy_run = false;
593                v.failures.push(Failure::ManifestReadError {
594                    detail: format!("{e:#}"),
595                });
596                v.passed = false;
597                return Ok(with_depth(v));
598            }
599        },
600        Err(e) => {
601            // `head` failure is symmetric to a `read` failure — same kind
602            // (`ManifestReadError`) so consumers don't have to branch on
603            // which method tripped.  Distinct from "manifest absent"
604            // (Ok(None) above) which legitimately means "legacy prefix".
605            let mut v = ManifestVerification::legacy();
606            v.legacy_run = false;
607            v.failures.push(Failure::ManifestReadError {
608                detail: format!("manifest head failed: {e:#}"),
609            });
610            v.passed = false;
611            return Ok(with_depth(v));
612        }
613    };
614
615    let manifest: RunManifest = match serde_json::from_slice(&manifest_bytes) {
616        Ok(m) => m,
617        Err(e) => {
618            // A malformed manifest is treated as a self-inconsistency —
619            // semantically equivalent for the operator (the manifest can't
620            // be trusted) but kept distinct in `failures` so the kind is
621            // explicit on the wire.
622            return Ok(with_depth(ManifestVerification {
623                manifest_found: true,
624                failures: vec![Failure::ManifestSelfInconsistent {
625                    detail: format!("manifest.json parse failed: {e}"),
626                }],
627                ..ManifestVerification::empty()
628            }));
629        }
630    };
631
632    // Optimistic base: a found, self-consistent manifest that passes until a
633    // check below flips it.  Overrides only what differs from `empty()`.
634    // Stamp the depth here so the two early `return Ok(out)` paths in section
635    // 4 (success-marker head error, non-utf8 body) carry the right level too.
636    let mut out = ManifestVerification {
637        manifest_found: true,
638        manifest_self_consistent: true,
639        passed: true,
640        depth_level: depth.label().to_string(),
641        ..ManifestVerification::empty()
642    };
643
644    // ── 2. Self-consistency ─────────────────────────────────────────────
645    if let Err(e) = manifest.validate_self_consistency() {
646        out.manifest_self_consistent = false;
647        out.failures.push(Failure::ManifestSelfInconsistent {
648            detail: format!("{e}"),
649        });
650        // Don't short-circuit — we still want to surface part-presence
651        // failures because the operator may want to know both classes at
652        // once rather than fix-then-rerun.
653    }
654
655    // ── 3. Reconcile parts + surplus against ONE prefix listing ────────
656    //
657    // Presence and untracked-surplus both fall out of a single
658    // `reconcile_manifest_against_listing` over one `list_prefix` — the same
659    // pure walk chunked resume uses (`build_resume_plan`).  This replaces the
660    // old per-part `HEAD` loop (N round-trips) and its separate untracked
661    // listing.  Per-part failures are emitted here (step 3); untracked is
662    // emitted at step 5 so the failure ordering an operator reads is stable.
663    //
664    // Trade-off: presence now rides the listing, not per-part `HEAD`.  If the
665    // listing cannot be read, an audit cannot certify the parts — so a list
666    // failure flips `passed = false` (a `ListPrefixError`), rather than the
667    // old behaviour where per-part HEAD still "verified" parts a failed
668    // listing couldn't enumerate.  Every Rivet destination backend offers
669    // strong read-after-write list consistency, so the happy path is one call.
670    //
671    // Graded depth: `Light` skips this `list_prefix` entirely — no part
672    // reconcile, no `ListPrefixError`, `parts_verified` stays 0, and section 5
673    // (untracked) is a no-op since `reconciliation` is `None`.  `Sample` and
674    // `Full` run it.  A `Light` pass therefore certifies only that the
675    // manifest reads, is self-consistent, and `_SUCCESS` matches — never that
676    // the parts are physically present.
677    let reconciliation = if depth.runs_part_reconcile() {
678        match dest.list_prefix(manifest_dir) {
679            Ok(listing) => Some(reconcile_manifest_against_listing(
680                &manifest,
681                &listing,
682                manifest_dir,
683            )),
684            Err(e) => {
685                out.failures.push(Failure::ListPrefixError {
686                    detail: format!("{e:#}"),
687                });
688                None
689            }
690        }
691    } else {
692        None
693    };
694    if let Some(rec) = &reconciliation {
695        for check in &rec.per_part {
696            match &check.presence {
697                PartPresence::Present { md5_verified } => {
698                    out.parts_verified += 1;
699                    if *md5_verified {
700                        out.parts_md5_verified += 1;
701                    }
702                }
703                PartPresence::SizeMismatch { expected, actual } => {
704                    out.parts_failed += 1;
705                    out.failures.push(Failure::PartSizeMismatch {
706                        part_id: check.part_id,
707                        path: check.path.clone(),
708                        expected: *expected,
709                        actual: *actual,
710                    });
711                }
712                PartPresence::Missing => {
713                    out.parts_failed += 1;
714                    out.failures.push(Failure::PartMissing {
715                        part_id: check.part_id,
716                        path: check.path.clone(),
717                    });
718                }
719                PartPresence::ChecksumMismatch { expected, actual } => {
720                    out.parts_failed += 1;
721                    out.failures.push(Failure::PartChecksumMismatch {
722                        part_id: check.part_id,
723                        path: check.path.clone(),
724                        expected: expected.clone(),
725                        actual: actual.clone(),
726                    });
727                }
728            }
729        }
730    }
731
732    // ── 4. _SUCCESS marker consistency ─────────────────────────────────
733    //
734    // Same error-consistency contract as step 1: head/read failures become
735    // `Failure::SuccessMarkerReadError`, not bubbled `Err`.  Absent marker
736    // (Ok(None)) stays informational, not a failure (M2: only successful
737    // runs land _SUCCESS, so its absence on a failed manifest is correct).
738    let success_head = match dest.head(&success_key) {
739        Ok(h) => h,
740        Err(e) => {
741            out.failures.push(Failure::SuccessMarkerReadError {
742                detail: format!("_SUCCESS head failed: {e:#}"),
743            });
744            out.recompute_passed();
745            return Ok(out);
746        }
747    };
748    match success_head {
749        None => {
750            // Absent _SUCCESS is informational, not a failure: per ADR-0012
751            // M2, only successful runs land it.  A failed-then-rewritten
752            // manifest legitimately lacks _SUCCESS.  Leave
753            // `success_marker_consistent = false` (this is a "no signal"
754            // bool, not a "broken" bool) and let the caller decide.
755        }
756        Some(head) if head.size_bytes > SUCCESS_MARKER_MAX_BYTES => {
757            // Refuse to materialise an oversized _SUCCESS: a bare uncapped
758            // dest.read of a multi-GB planted marker OOMs the validate/resume/repair
759            // process (the asymmetry with step 1's manifest.json cap this closes).
760            // success_head already carries the size — no extra round-trip.
761            out.failures.push(Failure::SuccessMarkerMalformed {
762                body_preview: format!(
763                    "(oversized: {} bytes exceeds the {SUCCESS_MARKER_MAX_BYTES}-byte _SUCCESS cap; not read)",
764                    head.size_bytes
765                ),
766            });
767            out.recompute_passed();
768            return Ok(out);
769        }
770        Some(_) => match dest.read(&success_key) {
771            Err(e) => {
772                out.failures.push(Failure::SuccessMarkerReadError {
773                    detail: format!("{e:#}"),
774                });
775            }
776            Ok(body) => {
777                let body_str = match std::str::from_utf8(&body) {
778                    Ok(s) => s,
779                    Err(_) => {
780                        out.failures.push(Failure::SuccessMarkerMalformed {
781                            body_preview: format!("(non-utf8, {} bytes)", body.len()),
782                        });
783                        out.recompute_passed();
784                        return Ok(out);
785                    }
786                };
787                match parse_success_marker(body_str) {
788                    None => {
789                        out.failures.push(Failure::SuccessMarkerMalformed {
790                            body_preview: preview(body_str),
791                        });
792                    }
793                    Some(marker_fp) => {
794                        let manifest_fp = success_marker_body(&manifest_bytes);
795                        // success_marker_body returns the trailing `\n`
796                        // form; trim before comparing to the parsed marker
797                        // (which already trims).
798                        let manifest_fp_trimmed = manifest_fp.trim_end_matches('\n');
799                        if marker_fp == manifest_fp_trimmed {
800                            out.success_marker_consistent = true;
801                        } else {
802                            out.failures.push(Failure::SuccessMarkerStale {
803                                marker_fingerprint: marker_fp.to_string(),
804                                manifest_fingerprint: manifest_fp_trimmed.to_string(),
805                            });
806                        }
807                    }
808                }
809            }
810        },
811    }
812
813    // ── 5. Untracked surplus ───────────────────────────────────────────
814    //
815    // Already computed by the step-3 reconciliation (sidecars, quarantine,
816    // and the doctor probe are filtered there).  Emit it last so the failure
817    // ordering stays parts → marker → untracked.  A list failure left
818    // `reconciliation = None` and already flipped `passed` above.
819    if let Some(rec) = reconciliation {
820        for obj in rec.untracked {
821            out.failures.push(Failure::UntrackedObject {
822                key: obj.key,
823                size_bytes: obj.size_bytes,
824            });
825        }
826    }
827
828    out.recompute_passed();
829    Ok(out)
830}
831
832/// Truncate `s` to a small printable preview for error messages.
833fn preview(s: &str) -> String {
834    let trimmed: String = s.chars().take(40).collect();
835    if s.chars().count() > 40 {
836        format!("{trimmed}…")
837    } else {
838        trimmed
839    }
840}
841
842#[cfg(test)]
843mod tests {
844    use super::*;
845    use crate::config::{DestinationConfig, DestinationType};
846    use crate::destination::local::LocalDestination;
847    use crate::manifest::{
848        MANIFEST_VERSION, ManifestDestination, ManifestPart, ManifestSource, ManifestStatus,
849        PartStatus, RunManifest,
850    };
851    use std::path::Path;
852
853    fn local_dest(base: &Path) -> LocalDestination {
854        LocalDestination::new(&DestinationConfig {
855            destination_type: DestinationType::Local,
856            path: Some(base.to_string_lossy().into_owned()),
857            ..Default::default()
858        })
859        .unwrap()
860    }
861
862    fn part(part_id: u32, rows: i64, size: u64, fp: &str) -> ManifestPart {
863        ManifestPart {
864            part_id,
865            path: format!("part-{part_id:06}.parquet"),
866            rows,
867            size_bytes: size,
868            content_fingerprint: fp.into(),
869            content_md5: String::new(),
870            status: PartStatus::Committed,
871        }
872    }
873
874    fn build_manifest(parts: Vec<ManifestPart>, status: ManifestStatus) -> RunManifest {
875        let row_count: i64 = parts
876            .iter()
877            .filter(|p| p.status == PartStatus::Committed)
878            .map(|p| p.rows)
879            .sum();
880        let part_count = parts
881            .iter()
882            .filter(|p| p.status == PartStatus::Committed)
883            .count() as u32;
884        RunManifest {
885            mode: "batch".to_string(),
886            manifest_version: MANIFEST_VERSION,
887            run_id: "r".into(),
888            export_name: "public.orders".into(),
889            started_at: "2026-05-21T12:00:00Z".into(),
890            finished_at: "2026-05-21T12:01:00Z".into(),
891            status,
892            source: ManifestSource {
893                engine: "postgres".into(),
894                schema: Some("public".into()),
895                table: Some("orders".into()),
896                extraction: None,
897            },
898            destination: ManifestDestination {
899                kind: "local".into(),
900                uri: "file:///tmp/out".into(),
901            },
902            format: "parquet".into(),
903            compression: "zstd".into(),
904            schema_fingerprint: "xxh3:0123456789abcdef".into(),
905            row_count,
906            part_count,
907            parts,
908            column_checksums: None,
909            checksum_key_column: None,
910        }
911    }
912
913    /// Lay out a clean dataset with manifest + _SUCCESS at the root.
914    fn write_dataset(dir: &Path, m: &RunManifest, parts_with_bytes: &[(&str, &[u8])]) {
915        for (name, bytes) in parts_with_bytes {
916            std::fs::write(dir.join(name), bytes).unwrap();
917        }
918        let body = serde_json::to_vec_pretty(m).unwrap();
919        std::fs::write(dir.join(MANIFEST_FILENAME), &body).unwrap();
920        if matches!(m.status, ManifestStatus::Success) {
921            std::fs::write(dir.join(SUCCESS_FILENAME), success_marker_body(&body)).unwrap();
922        }
923    }
924
925    // ── happy path ───────────────────────────────────────────────────────
926
927    #[test]
928    fn happy_path_verifies_all_parts_and_success_marker() {
929        let dir = tempfile::tempdir().unwrap();
930        let m = build_manifest(
931            vec![
932                part(1, 10, 4, "xxh3:1111111111111111"),
933                part(2, 20, 5, "xxh3:2222222222222222"),
934            ],
935            ManifestStatus::Success,
936        );
937        write_dataset(
938            dir.path(),
939            &m,
940            &[
941                ("part-000001.parquet", b"AAAA"),
942                ("part-000002.parquet", b"BBBBB"),
943            ],
944        );
945        let dest = local_dest(dir.path());
946
947        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
948        assert!(v.manifest_found);
949        assert!(!v.legacy_run);
950        assert_eq!(v.parts_verified, 2);
951        assert_eq!(v.parts_failed, 0);
952        assert!(v.success_marker_consistent);
953        assert!(v.manifest_self_consistent);
954        assert!(v.passed);
955        assert!(v.failures.is_empty());
956    }
957
958    // ── M6 legacy run ───────────────────────────────────────────────────
959
960    #[test]
961    fn no_manifest_returns_legacy_run_label() {
962        // Empty prefix — no manifest, no parts.
963        let dir = tempfile::tempdir().unwrap();
964        let dest = local_dest(dir.path());
965        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
966        assert!(!v.manifest_found);
967        assert!(v.legacy_run);
968        assert_eq!(v.parts_verified, 0);
969        assert!(!v.passed);
970        assert!(v.failures.is_empty(), "no failures, just a legacy label");
971    }
972
973    // ── M5 part-presence failures ───────────────────────────────────────
974
975    #[test]
976    fn missing_part_is_flagged_with_part_id_and_path() {
977        let dir = tempfile::tempdir().unwrap();
978        let m = build_manifest(
979            vec![
980                part(1, 10, 4, "xxh3:1111111111111111"),
981                part(2, 20, 5, "xxh3:2222222222222222"),
982            ],
983            ManifestStatus::Success,
984        );
985        write_dataset(
986            dir.path(),
987            &m,
988            &[("part-000001.parquet", b"AAAA")], // part 2 missing
989        );
990        let dest = local_dest(dir.path());
991
992        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
993        assert_eq!(v.parts_verified, 1);
994        assert_eq!(v.parts_failed, 1);
995        assert!(!v.passed);
996        assert!(
997            v.failures
998                .iter()
999                .any(|f| matches!(f, Failure::PartMissing { part_id: 2, .. }))
1000        );
1001    }
1002
1003    #[test]
1004    fn part_size_mismatch_is_flagged_with_expected_and_actual() {
1005        let dir = tempfile::tempdir().unwrap();
1006        let m = build_manifest(
1007            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1008            ManifestStatus::Success,
1009        );
1010        // Manifest claims 4 bytes; we write 6.
1011        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"OOPSIE")]);
1012        let dest = local_dest(dir.path());
1013
1014        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1015        assert!(!v.passed);
1016        let mismatch = v
1017            .failures
1018            .iter()
1019            .find_map(|f| match f {
1020                Failure::PartSizeMismatch {
1021                    part_id,
1022                    expected,
1023                    actual,
1024                    ..
1025                } => Some((*part_id, *expected, *actual)),
1026                _ => None,
1027            })
1028            .expect("must surface the size mismatch");
1029        assert_eq!(mismatch, (1, 4, 6));
1030    }
1031
1032    // ── _SUCCESS marker integrity ───────────────────────────────────────
1033
1034    #[test]
1035    fn stale_success_marker_is_flagged_as_inconsistent() {
1036        // Write a manifest, then overwrite _SUCCESS with the marker for a
1037        // *different* manifest body — simulating an orchestrator that
1038        // mishandled a re-run.
1039        let dir = tempfile::tempdir().unwrap();
1040        let m = build_manifest(
1041            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1042            ManifestStatus::Success,
1043        );
1044        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1045        std::fs::write(
1046            dir.path().join(SUCCESS_FILENAME),
1047            success_marker_body(b"different manifest body"),
1048        )
1049        .unwrap();
1050        let dest = local_dest(dir.path());
1051
1052        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1053        assert!(!v.success_marker_consistent);
1054        assert!(!v.passed);
1055        assert!(
1056            v.failures
1057                .iter()
1058                .any(|f| matches!(f, Failure::SuccessMarkerStale { .. }))
1059        );
1060    }
1061
1062    #[test]
1063    fn malformed_success_marker_body_is_flagged() {
1064        let dir = tempfile::tempdir().unwrap();
1065        let m = build_manifest(
1066            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1067            ManifestStatus::Success,
1068        );
1069        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1070        std::fs::write(dir.path().join(SUCCESS_FILENAME), b"not even xxh3 shaped").unwrap();
1071        let dest = local_dest(dir.path());
1072
1073        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1074        assert!(!v.passed);
1075        assert!(
1076            v.failures
1077                .iter()
1078                .any(|f| matches!(f, Failure::SuccessMarkerMalformed { .. }))
1079        );
1080    }
1081
1082    #[test]
1083    fn absent_success_marker_does_not_fail_validation_alone() {
1084        // ADR-0012 M2: only successful runs land _SUCCESS.  A failed-then-
1085        // rewritten manifest legitimately lacks one — verification must
1086        // not flip `passed` just for that.
1087        let dir = tempfile::tempdir().unwrap();
1088        let m = build_manifest(
1089            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1090            ManifestStatus::Failed,
1091        );
1092        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1093        // Note: write_dataset only writes _SUCCESS for status == Success,
1094        // so no marker exists here.
1095        assert!(!dir.path().join(SUCCESS_FILENAME).exists());
1096        let dest = local_dest(dir.path());
1097
1098        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1099        assert!(v.manifest_found);
1100        assert!(
1101            !v.success_marker_consistent,
1102            "no marker => false (no signal)"
1103        );
1104        // The parts still verified, so passed = true.
1105        assert!(v.passed);
1106        assert!(v.failures.is_empty());
1107    }
1108
1109    // ── self-consistency ────────────────────────────────────────────────
1110
1111    #[test]
1112    fn self_inconsistent_manifest_is_flagged_but_part_check_still_runs() {
1113        let dir = tempfile::tempdir().unwrap();
1114        let mut m = build_manifest(
1115            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1116            ManifestStatus::Success,
1117        );
1118        m.row_count = 9999; // lie
1119
1120        let body = serde_json::to_vec_pretty(&m).unwrap();
1121        std::fs::write(dir.path().join("part-000001.parquet"), b"AAAA").unwrap();
1122        std::fs::write(dir.path().join(MANIFEST_FILENAME), &body).unwrap();
1123        std::fs::write(
1124            dir.path().join(SUCCESS_FILENAME),
1125            success_marker_body(&body),
1126        )
1127        .unwrap();
1128        let dest = local_dest(dir.path());
1129
1130        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1131        assert!(v.manifest_found);
1132        assert!(!v.manifest_self_consistent);
1133        assert!(!v.passed);
1134        // Parts that are physically present still get their `parts_verified`
1135        // counter bumped — both signals are independently useful.
1136        assert_eq!(v.parts_verified, 1);
1137        assert!(
1138            v.failures
1139                .iter()
1140                .any(|f| matches!(f, Failure::ManifestSelfInconsistent { .. }))
1141        );
1142    }
1143
1144    // ── untracked objects ───────────────────────────────────────────────
1145
1146    #[test]
1147    fn verify_counters_track_present_and_failed_parts() {
1148        // Mutation-tier2 gap: the verdict-shaping fields (manifest_found /
1149        // passed / failures) and the per-part counters (parts_verified /
1150        // parts_failed) had no assertion — `+= -> -=` survived. One present
1151        // part + one missing part pin both counters and the verdict.
1152        let dir = tempfile::tempdir().unwrap();
1153        let m = build_manifest(
1154            vec![
1155                part(1, 10, 4, "xxh3:1111111111111111"),
1156                part(2, 10, 4, "xxh3:2222222222222222"),
1157            ],
1158            ManifestStatus::Success,
1159        );
1160        // Only part 1 exists at the destination.
1161        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1162        let dest = local_dest(dir.path());
1163
1164        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1165        assert!(v.manifest_found, "manifest is at the prefix");
1166        assert_eq!(v.parts_verified, 1, "part 1 is present at its size");
1167        assert_eq!(v.parts_failed, 1, "part 2 is missing");
1168        assert!(
1169            v.failures
1170                .iter()
1171                .any(|f| matches!(f, Failure::PartMissing { part_id: 2, .. })),
1172            "the missing part is named in failures: {:?}",
1173            v.failures
1174        );
1175        assert!(!v.passed, "a missing part must fail the verdict");
1176    }
1177
1178    #[test]
1179    fn read_capped_boundary_is_exact() {
1180        // `size > max` (not >=): a body exactly AT the cap must load; one
1181        // byte over must be refused before materialising.
1182        let dir = tempfile::tempdir().unwrap();
1183        std::fs::write(dir.path().join("at-cap.bin"), vec![0u8; 8]).unwrap();
1184        std::fs::write(dir.path().join("over-cap.bin"), vec![0u8; 9]).unwrap();
1185        let dest = local_dest(dir.path());
1186        assert_eq!(
1187            read_capped(&dest, "at-cap.bin", 8)
1188                .expect("exactly at cap loads")
1189                .len(),
1190            8
1191        );
1192        let err = read_capped(&dest, "over-cap.bin", 8).expect_err("over cap refuses");
1193        assert!(
1194            err.to_string().contains("read cap"),
1195            "actionable message: {err:#}"
1196        );
1197    }
1198
1199    #[test]
1200    fn preview_truncates_only_past_forty_chars() {
1201        let s39: String = "x".repeat(39);
1202        let s40: String = "x".repeat(40);
1203        let s41: String = "x".repeat(41);
1204        assert_eq!(preview(&s39), s39, "under the limit: unchanged");
1205        assert_eq!(
1206            preview(&s40),
1207            s40,
1208            "exactly at the limit: unchanged (`>` not `>=`)"
1209        );
1210        assert_eq!(
1211            preview(&s41),
1212            format!("{s40}\u{2026}"),
1213            "past the limit: 40 chars + ellipsis"
1214        );
1215    }
1216
1217    #[test]
1218    fn untracked_object_under_prefix_is_flagged() {
1219        let dir = tempfile::tempdir().unwrap();
1220        let m = build_manifest(
1221            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1222            ManifestStatus::Success,
1223        );
1224        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1225        std::fs::write(dir.path().join("rogue.parquet"), b"XX").unwrap();
1226        let dest = local_dest(dir.path());
1227
1228        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1229        assert!(
1230            v.failures.iter().any(
1231                |f| matches!(f, Failure::UntrackedObject { key, .. } if key == "rogue.parquet")
1232            )
1233        );
1234        // Untracked objects are surfaced but do NOT flip `passed` — that
1235        // is the resume-side decision (M9).  Parts and marker are fine,
1236        // so passed remains true.
1237        assert!(v.passed);
1238    }
1239
1240    #[test]
1241    fn quarantine_prefix_objects_are_silently_ignored() {
1242        let dir = tempfile::tempdir().unwrap();
1243        let m = build_manifest(
1244            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1245            ManifestStatus::Success,
1246        );
1247        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1248        std::fs::create_dir_all(dir.path().join(crate::manifest::QUARANTINE_PREFIX)).unwrap();
1249        std::fs::write(
1250            dir.path()
1251                .join(crate::manifest::QUARANTINE_PREFIX)
1252                .join("old.parquet"),
1253            b"OO",
1254        )
1255        .unwrap();
1256        let dest = local_dest(dir.path());
1257
1258        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1259        assert!(v.passed);
1260        assert!(
1261            !v.failures
1262                .iter()
1263                .any(|f| matches!(f, Failure::UntrackedObject { .. })),
1264            "quarantine_prefix is the legitimate home for these — must not flag"
1265        );
1266    }
1267
1268    #[test]
1269    fn doctor_probe_is_not_flagged_as_untracked() {
1270        // Regression: `rivet doctor` writes `.rivet_doctor_probe` at the
1271        // destination prefix and never removes it.  A subsequent
1272        // `rivet run --validate` against the same prefix must treat it as a
1273        // Rivet sidecar, not foreign data — otherwise `has_failures()` trips
1274        // and the run's `validated` flag is downgraded to FAIL.
1275        let dir = tempfile::tempdir().unwrap();
1276        let m = build_manifest(
1277            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1278            ManifestStatus::Success,
1279        );
1280        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1281        std::fs::write(
1282            dir.path().join(crate::manifest::DOCTOR_PROBE_FILENAME),
1283            b"ok",
1284        )
1285        .unwrap();
1286        let dest = local_dest(dir.path());
1287
1288        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1289        assert!(
1290            !v.has_failures(),
1291            "doctor probe must not surface as a failure: {:?}",
1292            v.failures
1293        );
1294        assert!(v.passed);
1295    }
1296
1297    // ── manifest_dir join semantics ─────────────────────────────────────
1298
1299    #[test]
1300    fn verifies_in_subdirectory_when_manifest_dir_is_non_empty() {
1301        let outer = tempfile::tempdir().unwrap();
1302        std::fs::create_dir_all(outer.path().join("sub/run")).unwrap();
1303        let m = build_manifest(
1304            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1305            ManifestStatus::Success,
1306        );
1307        let body = serde_json::to_vec_pretty(&m).unwrap();
1308        std::fs::write(outer.path().join("sub/run/part-000001.parquet"), b"AAAA").unwrap();
1309        std::fs::write(outer.path().join("sub/run").join(MANIFEST_FILENAME), &body).unwrap();
1310        std::fs::write(
1311            outer.path().join("sub/run").join(SUCCESS_FILENAME),
1312            success_marker_body(&body),
1313        )
1314        .unwrap();
1315        let dest = local_dest(outer.path());
1316
1317        let v = verify_at_destination(&dest, "sub/run", ValidateDepth::Full).unwrap();
1318        assert!(v.passed);
1319        assert_eq!(v.parts_verified, 1);
1320
1321        // Trailing slash is normalised — same outcome.
1322        let v2 = verify_at_destination(&dest, "sub/run/", ValidateDepth::Full).unwrap();
1323        assert!(v2.passed);
1324    }
1325
1326    // ── list-failure semantics (presence now rides the listing) ──────────
1327
1328    /// Wraps a real `LocalDestination` but fails every `list_prefix`. Used to
1329    /// pin the post-refactor contract: presence is derived from the listing,
1330    /// so a listing we cannot read means the audit cannot certify the parts.
1331    struct ListFails(LocalDestination);
1332    impl crate::destination::Destination for ListFails {
1333        fn write(&self, p: &Path, k: &str) -> Result<crate::destination::WriteOutcome> {
1334            self.0.write(p, k)
1335        }
1336        fn capabilities(&self) -> crate::destination::DestinationCapabilities {
1337            self.0.capabilities()
1338        }
1339        fn head(&self, k: &str) -> Result<Option<crate::destination::ObjectMeta>> {
1340            self.0.head(k)
1341        }
1342        fn read(&self, k: &str) -> Result<Vec<u8>> {
1343            self.0.read(k)
1344        }
1345        fn list_prefix(&self, _: &str) -> Result<Vec<crate::destination::ObjectMeta>> {
1346            anyhow::bail!("listing unavailable")
1347        }
1348    }
1349
1350    #[test]
1351    fn list_failure_cannot_certify_parts_and_fails_the_audit() {
1352        let dir = tempfile::tempdir().unwrap();
1353        let m = build_manifest(vec![part(0, 3, 3, "xxh3:0")], ManifestStatus::Success);
1354        write_dataset(dir.path(), &m, &[("part-000000.parquet", b"abc")]);
1355        let dest = ListFails(local_dest(dir.path()));
1356
1357        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1358        // The manifest itself reads + parses fine (HEAD/read still work)…
1359        assert!(v.manifest_found);
1360        assert!(v.manifest_self_consistent);
1361        // …but with no listing we verify zero parts and refuse to pass.
1362        assert!(
1363            !v.passed,
1364            "an audit that cannot list the prefix must not pass"
1365        );
1366        assert_eq!(v.parts_verified, 0);
1367        assert!(
1368            v.failures
1369                .iter()
1370                .any(|f| matches!(f, Failure::ListPrefixError { .. })),
1371            "expected a ListPrefixError, got: {:?}",
1372            v.failures
1373        );
1374    }
1375
1376    // ── manifest read-error semantics (explicit failure, not legacy) ─────
1377
1378    /// Wraps a real `LocalDestination` but fails reading `manifest.json`
1379    /// (head still sees it) — EACCES or a transient store error on a
1380    /// manifest that exists.
1381    struct ManifestReadFails(LocalDestination);
1382    impl crate::destination::Destination for ManifestReadFails {
1383        fn write(&self, p: &Path, k: &str) -> Result<crate::destination::WriteOutcome> {
1384            self.0.write(p, k)
1385        }
1386        fn capabilities(&self) -> crate::destination::DestinationCapabilities {
1387            self.0.capabilities()
1388        }
1389        fn head(&self, k: &str) -> Result<Option<crate::destination::ObjectMeta>> {
1390            self.0.head(k)
1391        }
1392        fn read(&self, k: &str) -> Result<Vec<u8>> {
1393            if k.ends_with(MANIFEST_FILENAME) {
1394                anyhow::bail!("permission denied (simulated)")
1395            }
1396            self.0.read(k)
1397        }
1398        fn list_prefix(&self, p: &str) -> Result<Vec<crate::destination::ObjectMeta>> {
1399            self.0.list_prefix(p)
1400        }
1401    }
1402
1403    #[test]
1404    fn unreadable_manifest_is_explicit_failure_not_legacy() {
1405        // The exit gates (`rivet validate`, run finalize) key off this exact
1406        // shape: `manifest_found: false` but `has_failures()` — distinct
1407        // from M6 legacy (`legacy_run: true`, no failures, exit 0).
1408        let dir = tempfile::tempdir().unwrap();
1409        let m = build_manifest(
1410            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1411            ManifestStatus::Success,
1412        );
1413        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1414        let dest = ManifestReadFails(local_dest(dir.path()));
1415
1416        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1417        assert!(!v.manifest_found);
1418        assert!(!v.legacy_run, "a read error is not the M6 legacy label");
1419        assert!(!v.passed);
1420        assert!(v.has_failures(), "orchestrators need a reason to refuse");
1421        assert!(
1422            matches!(v.failures.as_slice(), [Failure::ManifestReadError { .. }]),
1423            "expected exactly one ManifestReadError, got: {:?}",
1424            v.failures
1425        );
1426    }
1427
1428    /// Same contract when `head` itself errors (cannot even stat the
1429    /// manifest): symmetric `ManifestReadError`, never the legacy label.
1430    struct ManifestHeadFails(LocalDestination);
1431    impl crate::destination::Destination for ManifestHeadFails {
1432        fn write(&self, p: &Path, k: &str) -> Result<crate::destination::WriteOutcome> {
1433            self.0.write(p, k)
1434        }
1435        fn capabilities(&self) -> crate::destination::DestinationCapabilities {
1436            self.0.capabilities()
1437        }
1438        fn head(&self, k: &str) -> Result<Option<crate::destination::ObjectMeta>> {
1439            if k.ends_with(MANIFEST_FILENAME) {
1440                anyhow::bail!("io timeout (simulated)")
1441            }
1442            self.0.head(k)
1443        }
1444        fn read(&self, k: &str) -> Result<Vec<u8>> {
1445            self.0.read(k)
1446        }
1447        fn list_prefix(&self, p: &str) -> Result<Vec<crate::destination::ObjectMeta>> {
1448            self.0.list_prefix(p)
1449        }
1450    }
1451
1452    #[test]
1453    fn manifest_head_error_is_explicit_failure_not_legacy() {
1454        let dir = tempfile::tempdir().unwrap();
1455        let m = build_manifest(
1456            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1457            ManifestStatus::Success,
1458        );
1459        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1460        let dest = ManifestHeadFails(local_dest(dir.path()));
1461
1462        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1463        assert!(!v.manifest_found);
1464        assert!(!v.legacy_run);
1465        assert!(!v.passed);
1466        assert!(
1467            matches!(
1468                v.failures.as_slice(),
1469                [Failure::ManifestReadError { detail }] if detail.contains("manifest head failed")
1470            ),
1471            "expected one ManifestReadError naming the head step, got: {:?}",
1472            v.failures
1473        );
1474    }
1475
1476    // #5: a _SUCCESS whose head-reported size exceeds the cap must be flagged
1477    // WITHOUT being read — a bare uncapped read of a multi-GB planted marker OOMs
1478    // the validate/resume process (asymmetric with the manifest.json cap). The mock
1479    // PANICS if read(_SUCCESS) is called, so the test passing proves the read was
1480    // short-circuited by the size check.
1481    struct SuccessMarkerOversized(LocalDestination);
1482    impl crate::destination::Destination for SuccessMarkerOversized {
1483        fn write(&self, p: &Path, k: &str) -> Result<crate::destination::WriteOutcome> {
1484            self.0.write(p, k)
1485        }
1486        fn capabilities(&self) -> crate::destination::DestinationCapabilities {
1487            self.0.capabilities()
1488        }
1489        fn head(&self, k: &str) -> Result<Option<crate::destination::ObjectMeta>> {
1490            if k.ends_with(crate::manifest::SUCCESS_FILENAME) {
1491                return Ok(Some(crate::destination::ObjectMeta {
1492                    key: k.to_string(),
1493                    size_bytes: SUCCESS_MARKER_MAX_BYTES * 4,
1494                    content_md5: None,
1495                }));
1496            }
1497            self.0.head(k)
1498        }
1499        fn read(&self, k: &str) -> Result<Vec<u8>> {
1500            assert!(
1501                !k.ends_with(crate::manifest::SUCCESS_FILENAME),
1502                "verify must NOT read an oversized _SUCCESS — the size cap must short-circuit \
1503                 before materialising it into memory (the OOM this guards)"
1504            );
1505            self.0.read(k)
1506        }
1507        fn list_prefix(&self, p: &str) -> Result<Vec<crate::destination::ObjectMeta>> {
1508            self.0.list_prefix(p)
1509        }
1510    }
1511
1512    #[test]
1513    fn oversized_success_marker_is_malformed_and_never_read() {
1514        let dir = tempfile::tempdir().unwrap();
1515        let m = build_manifest(
1516            vec![part(1, 10, 4, "xxh3:1111111111111111")],
1517            ManifestStatus::Success,
1518        );
1519        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1520        let dest = SuccessMarkerOversized(local_dest(dir.path()));
1521
1522        let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1523        assert!(
1524            v.failures.iter().any(|f| matches!(
1525                f,
1526                Failure::SuccessMarkerMalformed { body_preview } if body_preview.contains("oversized")
1527            )),
1528            "an oversized _SUCCESS must yield SuccessMarkerMalformed(oversized) without reading it, got: {:?}",
1529            v.failures
1530        );
1531    }
1532
1533    #[test]
1534    fn passed_is_derived_advisory_failures_do_not_fail() {
1535        // An advisory failure (untracked surplus) keeps the verdict passing…
1536        let mut v = ManifestVerification {
1537            manifest_found: true,
1538            ..ManifestVerification::empty()
1539        };
1540        v.failures.push(Failure::UntrackedObject {
1541            key: "stray.parquet".into(),
1542            size_bytes: 9,
1543        });
1544        v.recompute_passed();
1545        assert!(v.passed, "untracked surplus is advisory, not fatal");
1546
1547        // …while any fatal failure flips it.
1548        v.failures.push(Failure::PartMissing {
1549            part_id: 1,
1550            path: "part-000001.parquet".into(),
1551        });
1552        v.recompute_passed();
1553        assert!(!v.passed, "a missing part is fatal");
1554
1555        // No manifest → never passes regardless of failures.
1556        let mut legacy = ManifestVerification::empty();
1557        legacy.recompute_passed();
1558        assert!(!legacy.passed, "no manifest found → cannot certify");
1559    }
1560
1561    #[test]
1562    fn verify_content_policy_fails_only_size_only_parts() {
1563        // 3 parts, 2 content-checked, 1 size-only.
1564        let base = ManifestVerification {
1565            manifest_found: true,
1566            parts_verified: 3,
1567            parts_md5_verified: 2,
1568            ..ManifestVerification::empty()
1569        };
1570        // verify: size → size-only is acceptable, passes.
1571        let mut sz = base.clone();
1572        sz.recompute_passed();
1573        sz.enforce_content_policy(false);
1574        assert!(sz.passed, "size-only OK under verify: size");
1575
1576        // verify: content → the 1 size-only part is a fatal failure.
1577        let mut ct = base.clone();
1578        ct.recompute_passed();
1579        ct.enforce_content_policy(true);
1580        assert!(!ct.passed, "a size-only part fails verify: content");
1581        assert!(
1582            ct.failures.iter().any(|f| matches!(
1583                f,
1584                Failure::ContentVerificationUnmet {
1585                    size_only: 1,
1586                    total: 3
1587                }
1588            )),
1589            "expected ContentVerificationUnmet, got: {:?}",
1590            ct.failures
1591        );
1592
1593        // verify: content with every part md5-checked → satisfied.
1594        let mut all = ManifestVerification {
1595            parts_md5_verified: 3,
1596            ..base
1597        };
1598        all.recompute_passed();
1599        all.enforce_content_policy(true);
1600        assert!(
1601            all.passed && all.failures.is_empty(),
1602            "all md5 meets verify: content"
1603        );
1604    }
1605
1606    // ── require_manifest_present (finding #20: operator-pinned --prefix) ──────
1607
1608    #[test]
1609    fn require_manifest_escalates_legacy_to_fatal_absent() {
1610        // The exact shape `verify_at_destination` returns for an absent manifest
1611        // (`legacy()`): no manifest, no other failure. With a pinned `--prefix`
1612        // this is escalated to a fatal `ManifestRequiredButAbsent` so the exit
1613        // gate refuses it instead of passing as a benign legacy run.
1614        let mut v = ManifestVerification::legacy();
1615        assert!(v.legacy_run && !v.has_failures());
1616
1617        v.require_manifest_present("exports/2026-06-09/orders/");
1618
1619        assert!(!v.legacy_run, "no longer the benign legacy-run label");
1620        assert!(!v.passed, "an absent-but-required manifest cannot pass");
1621        assert!(
1622            matches!(
1623                v.failures.as_slice(),
1624                [Failure::ManifestRequiredButAbsent { prefix }]
1625                    if prefix == "exports/2026-06-09/orders/"
1626            ),
1627            "expected one ManifestRequiredButAbsent naming the prefix, got: {:?}",
1628            v.failures
1629        );
1630    }
1631
1632    #[test]
1633    fn require_manifest_is_noop_on_a_real_passing_manifest() {
1634        // A found, passing verdict is untouched — `--prefix` plus real data is
1635        // the normal "validate this exact prefix" case and must still pass.
1636        let mut v = ManifestVerification {
1637            manifest_found: true,
1638            manifest_self_consistent: true,
1639            parts_verified: 1,
1640            passed: true,
1641            ..ManifestVerification::empty()
1642        };
1643        v.require_manifest_present("exports/orders/");
1644        assert!(
1645            v.passed && v.failures.is_empty(),
1646            "real dataset still passes"
1647        );
1648    }
1649
1650    #[test]
1651    fn require_manifest_does_not_double_flag_a_read_error() {
1652        // An absent manifest that already carries a `ManifestReadError` (head /
1653        // read failed) is already a fatal, classified failure — requiring a
1654        // manifest here must not add a second, redundant failure.
1655        let mut v = ManifestVerification::legacy();
1656        v.legacy_run = false;
1657        v.failures.push(Failure::ManifestReadError {
1658            detail: "permission denied".into(),
1659        });
1660        v.recompute_passed();
1661
1662        v.require_manifest_present("exports/orders/");
1663
1664        assert!(
1665            matches!(v.failures.as_slice(), [Failure::ManifestReadError { .. }]),
1666            "must leave the existing read-error verdict alone, got: {:?}",
1667            v.failures
1668        );
1669    }
1670
1671    // ── graded verify layer (--depth) ───────────────────────────────────
1672
1673    #[test]
1674    fn light_depth_skips_part_reconcile_even_when_a_part_is_missing() {
1675        // A manifest declaring a part that is NOT on disk. At `Full`/`Sample`
1676        // this is a fatal `PartMissing`; at `Light` the `list_prefix` reconcile
1677        // is skipped entirely, so `parts_verified == 0`, no `ListPrefixError`,
1678        // and — with `_SUCCESS` consistent and the manifest self-consistent —
1679        // the verdict still passes. Light certifies the manifest + marker, not
1680        // the parts.
1681        let dir = tempfile::tempdir().unwrap();
1682        let m = build_manifest(
1683            vec![
1684                part(1, 10, 4, "xxh3:1111111111111111"),
1685                part(2, 20, 5, "xxh3:2222222222222222"),
1686            ],
1687            ManifestStatus::Success,
1688        );
1689        // Deliberately write NEITHER part — only manifest.json + _SUCCESS.
1690        write_dataset(dir.path(), &m, &[]);
1691        let dest = local_dest(dir.path());
1692
1693        let v = verify_at_destination(&dest, "", ValidateDepth::Light).unwrap();
1694        assert_eq!(v.depth_level, "light");
1695        assert_eq!(
1696            v.parts_verified, 0,
1697            "light skips the listing — no part is ever verified"
1698        );
1699        assert_eq!(
1700            v.parts_failed, 0,
1701            "no part reconcile means no part failures"
1702        );
1703        assert!(
1704            !v.failures.iter().any(|f| matches!(
1705                f,
1706                Failure::PartMissing { .. } | Failure::ListPrefixError { .. }
1707            )),
1708            "light must not surface part or list failures, got: {:?}",
1709            v.failures
1710        );
1711        assert!(
1712            v.success_marker_consistent,
1713            "_SUCCESS is still checked at light depth"
1714        );
1715        assert!(v.manifest_self_consistent);
1716        assert!(
1717            v.passed,
1718            "manifest + _SUCCESS are consistent, so a light pass certifies it"
1719        );
1720    }
1721
1722    #[test]
1723    fn light_depth_never_lists_so_a_list_failure_cannot_trip() {
1724        // Even with a destination whose `list_prefix` always errors, a light
1725        // pass succeeds: it never calls `list_prefix`, so no `ListPrefixError`.
1726        // This is the direct contrast to `list_failure_cannot_certify_parts…`
1727        // (which runs at Full and *does* fail on the list error).
1728        let dir = tempfile::tempdir().unwrap();
1729        let m = build_manifest(vec![part(0, 3, 3, "xxh3:0")], ManifestStatus::Success);
1730        write_dataset(dir.path(), &m, &[("part-000000.parquet", b"abc")]);
1731        let dest = ListFails(local_dest(dir.path()));
1732
1733        let v = verify_at_destination(&dest, "", ValidateDepth::Light).unwrap();
1734        assert_eq!(v.depth_level, "light");
1735        assert!(
1736            !v.failures
1737                .iter()
1738                .any(|f| matches!(f, Failure::ListPrefixError { .. })),
1739            "light never lists, so a failing list_prefix cannot surface, got: {:?}",
1740            v.failures
1741        );
1742        assert_eq!(v.parts_verified, 0);
1743        assert!(v.passed, "manifest + _SUCCESS consistent → light passes");
1744    }
1745
1746    #[test]
1747    fn sample_depth_runs_part_reconcile_like_full() {
1748        // `Sample` runs every section `verify_at_destination` owns (1-5) — the
1749        // Form B value re-read it skips lives in the *caller*, not here. So a
1750        // missing part is a fatal `PartMissing` at Sample, identical to Full.
1751        let dir = tempfile::tempdir().unwrap();
1752        let m = build_manifest(
1753            vec![
1754                part(1, 10, 4, "xxh3:1111111111111111"),
1755                part(2, 20, 5, "xxh3:2222222222222222"),
1756            ],
1757            ManifestStatus::Success,
1758        );
1759        write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]); // part 2 missing
1760        let dest = local_dest(dir.path());
1761
1762        let v = verify_at_destination(&dest, "", ValidateDepth::Sample).unwrap();
1763        assert_eq!(v.depth_level, "sample");
1764        assert_eq!(v.parts_verified, 1);
1765        assert_eq!(v.parts_failed, 1);
1766        assert!(!v.passed);
1767        assert!(
1768            v.failures
1769                .iter()
1770                .any(|f| matches!(f, Failure::PartMissing { part_id: 2, .. })),
1771            "sample reconciles parts just like full, got: {:?}",
1772            v.failures
1773        );
1774    }
1775
1776    #[test]
1777    fn depth_level_is_stamped_on_every_verdict_shape() {
1778        // The depth label rides the verdict even on the early-return shapes
1779        // (legacy / no manifest), so a consumer always sees how deep the pass
1780        // went.
1781        let dir = tempfile::tempdir().unwrap();
1782        let dest = local_dest(dir.path()); // empty prefix → legacy
1783        for depth in [
1784            ValidateDepth::Light,
1785            ValidateDepth::Sample,
1786            ValidateDepth::Full,
1787        ] {
1788            let v = verify_at_destination(&dest, "", depth).unwrap();
1789            assert!(v.legacy_run, "empty prefix is the legacy shape");
1790            assert_eq!(
1791                v.depth_level,
1792                depth.label(),
1793                "legacy verdict must still carry its depth label"
1794            );
1795        }
1796    }
1797
1798    #[test]
1799    fn error_code_is_stable_and_distinct_per_variant() {
1800        // Each variant maps to its documented `RIVET_VERIFY_*` code. A
1801        // regression guard: renaming a code is a silent break for any CI gate
1802        // keying off it.
1803        let cases: &[(Failure, &str)] = &[
1804            (
1805                Failure::PartMissing {
1806                    part_id: 1,
1807                    path: "p".into(),
1808                },
1809                "RIVET_VERIFY_PART_MISSING",
1810            ),
1811            (
1812                Failure::PartSizeMismatch {
1813                    part_id: 1,
1814                    path: "p".into(),
1815                    expected: 1,
1816                    actual: 2,
1817                },
1818                "RIVET_VERIFY_PART_SIZE_MISMATCH",
1819            ),
1820            (
1821                Failure::PartChecksumMismatch {
1822                    part_id: 1,
1823                    path: "p".into(),
1824                    expected: "a".into(),
1825                    actual: "b".into(),
1826                },
1827                "RIVET_VERIFY_PART_CHECKSUM_MISMATCH",
1828            ),
1829            (
1830                Failure::SuccessMarkerMalformed {
1831                    body_preview: "x".into(),
1832                },
1833                "RIVET_VERIFY_SUCCESS_MALFORMED",
1834            ),
1835            (
1836                Failure::SuccessMarkerStale {
1837                    marker_fingerprint: "a".into(),
1838                    manifest_fingerprint: "b".into(),
1839                },
1840                "RIVET_VERIFY_SUCCESS_STALE",
1841            ),
1842            (
1843                Failure::ManifestSelfInconsistent { detail: "d".into() },
1844                "RIVET_VERIFY_MANIFEST_INCONSISTENT",
1845            ),
1846            (
1847                Failure::ManifestReadError { detail: "d".into() },
1848                "RIVET_VERIFY_MANIFEST_READ_ERROR",
1849            ),
1850            (
1851                Failure::SuccessMarkerReadError { detail: "d".into() },
1852                "RIVET_VERIFY_SUCCESS_READ_ERROR",
1853            ),
1854            (
1855                Failure::ListPrefixError { detail: "d".into() },
1856                "RIVET_VERIFY_LIST_ERROR",
1857            ),
1858            (
1859                Failure::UntrackedObject {
1860                    key: "k".into(),
1861                    size_bytes: 1,
1862                },
1863                "RIVET_VERIFY_UNTRACKED_OBJECT",
1864            ),
1865            (
1866                Failure::ContentVerificationUnmet {
1867                    size_only: 1,
1868                    total: 2,
1869                },
1870                "RIVET_VERIFY_CONTENT_UNMET",
1871            ),
1872            (
1873                Failure::ManifestRequiredButAbsent { prefix: "p".into() },
1874                "RIVET_VERIFY_MANIFEST_REQUIRED",
1875            ),
1876            (
1877                Failure::ValueChecksumMismatch {
1878                    detail: "flip".into(),
1879                },
1880                "RIVET_VERIFY_VALUE_CHECKSUM",
1881            ),
1882            (
1883                Failure::CdcPositionViolation {
1884                    detail: "gap".into(),
1885                },
1886                "RIVET_VERIFY_CDC_POSITION",
1887            ),
1888        ];
1889        for (failure, code) in cases {
1890            assert_eq!(&failure.error_code(), code, "code for {failure:?}");
1891            assert!(
1892                failure.error_code().starts_with("RIVET_VERIFY_"),
1893                "every code shares the RIVET_VERIFY_ prefix"
1894            );
1895        }
1896    }
1897
1898    #[test]
1899    fn cdc_position_violation_is_verified_wrong_not_could_not_verify() {
1900        // #5 / #104 bughunt: a CDC `__pos` continuity violation (a gap/duplicate in
1901        // the exported change stream) is VERIFIED-WRONG — data-integrity, exit 3,
1902        // the same class as a value-checksum mismatch — NOT a could-not-verify I/O
1903        // error (exit 1). This pins the classification so a future edit that made it
1904        // operational (folding it back into hard_failures → exit 1, the pre-fix
1905        // behaviour) goes RED.
1906        let viol = Failure::CdcPositionViolation {
1907            detail: "export 'x': pos gap 5 -> 8".into(),
1908        };
1909        assert!(
1910            !viol.is_could_not_verify(),
1911            "a __pos violation is verified-wrong, not could-not-verify"
1912        );
1913        assert!(viol.is_fatal(), "a __pos violation fails the verdict");
1914
1915        let mut v = ManifestVerification::empty();
1916        v.passed = false;
1917        v.failures.push(viol);
1918        assert!(
1919            v.has_verified_wrong_failure(),
1920            "a verdict carrying a __pos violation must classify as verified-wrong (exit 3)"
1921        );
1922
1923        // The read-error siblings are the OTHER side of the same axis: could-not-
1924        // verify, so a verdict whose ONLY failure is one of them is NOT verified-wrong.
1925        let mut op = ManifestVerification::empty();
1926        op.passed = false;
1927        op.failures.push(Failure::ManifestReadError {
1928            detail: "permission denied".into(),
1929        });
1930        assert!(
1931            !op.has_verified_wrong_failure(),
1932            "a read error alone is could-not-verify (exit 1), not verified-wrong"
1933        );
1934    }
1935}