rivet/pipeline/validate_manifest.rs
1//! **Layer: Observability**
2//!
3//! Manifest-aware verification for `--validate` (ADR-0012 §M5 / §M6,
4//! constrained by the ADR-0013 trust-flag contract that says: no new flags;
5//! manifest-aware checks live under the existing `--validate`).
6//!
7//! Although this module reads from a `Destination` (technically L2 surface),
8//! it makes **no execution decisions**: it does not write data, advance
9//! cursors, mutate state, or change the pipeline path. Its only output is
10//! a structured `ManifestVerification` verdict the run report renders.
11//! Per ADR-0003, that places it firmly in L4 Observability — the
12//! destination read surface is just the carrier.
13//!
14//! Inputs (read-only):
15//! - the destination's `manifest.json` body
16//! - the destination's `_SUCCESS` body, if present
17//! - the listing of every object under the destination prefix
18//!
19//! Outputs:
20//! - [`ManifestVerification`] — a structured verdict the run report renders
21//! into the operator-facing "Verdicts" section.
22//!
23//! Out of scope here:
24//! - per-file row-count check (that runs *during* the export, against the
25//! local temp file before upload — see `pipeline::validate::validate_output`).
26//! - source-side reconciliation (lives in [`pipeline::reconcile_cmd`] and
27//! is what `--reconcile` adds on top of this).
28//! - re-fingerprinting parts (`--validate --deep`, future).
29//!
30//! Failure modes are explicit: each check produces a `Failure` enum variant
31//! that is rendered verbatim in `summary.json` so an Airflow / CI consumer
32//! can branch on the kind, not parse strings.
33
34use serde::{Deserialize, Serialize};
35
36use crate::destination::Destination;
37use crate::error::Result;
38use crate::manifest::{
39 MANIFEST_FILENAME, RunManifest, SUCCESS_FILENAME, join_key, parse_success_marker,
40 success_marker_body,
41};
42use crate::pipeline::manifest_reconcile::{PartPresence, reconcile_manifest_against_listing};
43
44/// Upper bound on a destination control artifact (`manifest.json`) the read
45/// path will materialise into memory. A `manifest.json` is metadata — a few
46/// KB to low single-digit MB even for very large datasets — so 64 MiB is far
47/// above any legitimate body while still bounding the blast radius.
48///
49/// Security (V21, CWE-400): the manifest readers `head()` an object then read
50/// its full body into a `Vec<u8>`. An attacker who can write the destination
51/// prefix (a shared bucket prefix, a world-writable export dir) can plant a
52/// multi-GB `manifest.json`; an unbounded read would OOM the next `--resume`,
53/// `--validate`, or `rivet repair`. [`read_capped`] consults the size the
54/// `head()` already reports and bails before the read when it exceeds this cap.
55pub(crate) const MANIFEST_MAX_BYTES: u64 = 64 * 1024 * 1024;
56
57/// Same V21/CWE-400 defense for the sibling control artifact `_SUCCESS`, read by
58/// the SAME `verify_at_destination` under the SAME destination-writable threat
59/// model but previously UNCAPPED. A legitimate marker is ~22 bytes
60/// (`xxh3:<16-hex>\n`); a body past this small cap is corrupt or planted, so it
61/// is treated as malformed WITHOUT materialising it into memory. 4 KiB is orders
62/// of magnitude above any real marker yet trivially bounded.
63pub(crate) const SUCCESS_MARKER_MAX_BYTES: u64 = 4096;
64
65/// How deep a `rivet validate` pass goes — a graded verify layer over the
66/// same checks, letting an operator trade thoroughness for latency / cost.
67///
68/// The variants are a strict superset chain: `Light ⊂ Sample ⊂ Full`. Each
69/// level runs every check the level below it does, plus more. Defined here
70/// (the pipeline layer) and re-exported for the CLI grammar so the **same**
71/// enum gates the checks in [`verify_at_destination`] and parses on the
72/// `--depth` flag — no CLI→pipeline back-dependency.
73///
74/// - **Light**: manifest read + self-consistency + `_SUCCESS` only. Skips the
75/// `list_prefix` reconcile (no per-part presence/size/checksum) and the
76/// untracked-surplus scan, leaving `parts_verified = 0`. One `head` + one
77/// `read` of `manifest.json` and `_SUCCESS` — a fast "is this prefix a
78/// complete, marked run?" poll with no prefix listing.
79/// - **Sample**: everything Light does **plus** the part reconcile and
80/// untracked surplus (one `list_prefix`). This is the pre-graded behaviour
81/// minus the Form B value re-read — full structural verification with no
82/// part downloads.
83/// - **Full** (default): everything Sample does **plus** the Form B value-
84/// checksum re-read (re-reads parts, re-derives per-column checksums). The
85/// most thorough and the only level that downloads part bodies. Equivalent
86/// to the pre-graded behaviour, so existing callers are unchanged.
87#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq, Default)]
88pub enum ValidateDepth {
89 /// Manifest read + self-consistency + `_SUCCESS` only (no prefix listing).
90 Light,
91 /// Light + part reconcile + untracked surplus (one `list_prefix`).
92 Sample,
93 /// Sample + the Form B value-checksum re-read (downloads parts).
94 #[default]
95 Full,
96}
97
98impl ValidateDepth {
99 /// True iff this level runs the `list_prefix` reconcile (part presence,
100 /// size, checksum) and the untracked-surplus scan — i.e. anything above
101 /// `Light`. The single predicate the section-3/5 depth gating keys off.
102 fn runs_part_reconcile(self) -> bool {
103 !matches!(self, ValidateDepth::Light)
104 }
105
106 /// True iff this level downloads part *bodies* — the CDC `__pos` continuity
107 /// check and the Form-B value-checksum re-read, both `Full`-only (the
108 /// `part_reconcile` level above only reads listing metadata). The single
109 /// predicate the section-3/5 part-download gating keys off, parallel to
110 /// [`Self::runs_part_reconcile`] — so adding a depth level edits the enum,
111 /// not the call sites in `validate_cmd`.
112 pub(crate) fn runs_part_download(self) -> bool {
113 matches!(self, ValidateDepth::Full)
114 }
115
116 /// Stable operator-/wire-facing label for the depth a verdict was produced
117 /// at, surfaced in `summary.json` (`depth_level`) and `rivet validate`
118 /// output.
119 pub fn label(self) -> &'static str {
120 match self {
121 ValidateDepth::Light => "light",
122 ValidateDepth::Sample => "sample",
123 ValidateDepth::Full => "full",
124 }
125 }
126}
127
128/// Read `key` into memory only if its `head()`-reported size is within
129/// `max_bytes`; otherwise bail without reading a single byte.
130///
131/// The single enforcement point for the V21 (CWE-400) manifest-read cap shared
132/// by the three control-artifact readers (`--resume` M8 preamble, `--validate`,
133/// `rivet repair`). Each previously did `head()` then an uncapped `read()`,
134/// discarding the size `head()` already returned; routing through here closes
135/// that gap in one place.
136///
137/// Behaviour:
138/// - object absent (`head` → `None`): `Err` — callers invoke this only after
139/// establishing the object exists, so an absent object here is a hard error,
140/// not the benign "no manifest / legacy prefix" case (which the callers
141/// detect with their own `head()` first).
142/// - oversized (`size_bytes > max_bytes`): `Err` naming the cap, **before** any
143/// body is materialised.
144/// - otherwise: the full body via [`Destination::read`].
145pub(crate) fn read_capped(dest: &dyn Destination, key: &str, max_bytes: u64) -> Result<Vec<u8>> {
146 match dest.head(key)? {
147 None => anyhow::bail!("'{key}' not found at the destination"),
148 Some(meta) => {
149 if meta.size_bytes > max_bytes {
150 anyhow::bail!(
151 "'{key}' is {} bytes, exceeding the {max_bytes}-byte control-artifact \
152 read cap — refusing to load it into memory (possible tampering)",
153 meta.size_bytes
154 );
155 }
156 dest.read(key)
157 }
158 }
159}
160
161/// Outcome of a single `--validate` pass over a destination prefix.
162///
163/// Stable enough to be embedded in `summary.json` directly (see
164/// `pipeline::report::ValidationOutcome`). Forward-compat: consumers MUST
165/// ignore unknown fields (no `deny_unknown_fields`).
166#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
167pub struct ManifestVerification {
168 /// True iff a `manifest.json` was found at the destination and parsed.
169 /// `false` triggers ADR-0012 M6 fallback (legacy run); higher-level
170 /// check results below are then "skipped" rather than "passed".
171 pub manifest_found: bool,
172 /// Mirrors ADR-0012 M6's required `legacy_run` operator-facing label.
173 pub legacy_run: bool,
174 /// Manifest parts whose presence and recorded `size_bytes` were
175 /// confirmed at the destination. 0 when no manifest was found.
176 pub parts_verified: usize,
177 /// Subset of `parts_verified` whose **content** was confirmed via an MD5
178 /// the store surfaced in its listing (no download) — the rest are size-only.
179 /// Lets `passed: true` say how much of the dataset was content-checked
180 /// rather than implying all of it was. `#[serde(default)]` for back-compat.
181 #[serde(default)]
182 pub parts_md5_verified: usize,
183 /// Manifest parts that were declared `committed` but not actually
184 /// present, present at a different size, or otherwise mismatched.
185 pub parts_failed: usize,
186 /// True iff `_SUCCESS` exists at the destination AND its body matches
187 /// the fingerprint of the bytes we read for `manifest.json`. An
188 /// existing `_SUCCESS` whose body diverges from the manifest is itself
189 /// an integrity failure — surfaced via `failures`.
190 pub success_marker_consistent: bool,
191 /// Self-consistency of the manifest (`row_count`, `part_count`,
192 /// duplicate `part_id`s). Skipped when `manifest_found = false`.
193 pub manifest_self_consistent: bool,
194 /// Final verdict, **derived** (not hand-maintained) — `manifest_found` and
195 /// no *fatal* failure ([`Failure::is_fatal`]). Stored so it stays in the
196 /// `summary.json` contract, but computed in one place
197 /// ([`ManifestVerification::recompute_passed`]) so a new failure variant is
198 /// fatal by default rather than relying on every site to flip a bool.
199 pub passed: bool,
200 /// Per-failure detail. May be non-empty with `passed = true` for advisory
201 /// (non-fatal) failures like [`Failure::UntrackedObject`]. Stable variant
202 /// set; new variants land under a new manifest version per ADR-0012.
203 pub failures: Vec<Failure>,
204 /// The graded depth this verdict was produced at: `"light"`, `"sample"`,
205 /// or `"full"` (see [`ValidateDepth`]). Lets a consumer of `summary.json`
206 /// tell **how much** was actually checked — a `passed: true` at `"light"`
207 /// asserts far less than at `"full"` (no part presence was verified).
208 /// `#[serde(default)]` (→ `"full"`) for back-compat: pre-graded verdicts
209 /// always ran the full pass.
210 #[serde(default = "default_depth_level")]
211 pub depth_level: String,
212}
213
214/// `serde(default)` for [`ManifestVerification::depth_level`]: a verdict that
215/// predates the graded layer always ran the full pass, so an absent field
216/// deserializes to `"full"`.
217fn default_depth_level() -> String {
218 ValidateDepth::Full.label().to_string()
219}
220
221#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
222#[serde(tag = "kind", rename_all = "snake_case")]
223pub enum Failure {
224 /// Manifest declared a part that does not exist at the destination.
225 PartMissing { part_id: u32, path: String },
226 /// Manifest declared a part whose actual size differs from `size_bytes`.
227 PartSizeMismatch {
228 part_id: u32,
229 path: String,
230 expected: u64,
231 actual: u64,
232 },
233 /// Part present at the recorded size but its content MD5 (from the store's
234 /// listing metadata) differs from the manifest's — transit / at-rest
235 /// corruption, caught with no download.
236 PartChecksumMismatch {
237 part_id: u32,
238 path: String,
239 expected: String,
240 actual: String,
241 },
242 /// `--depth full` re-read the parts and a per-column VALUE checksum (Form B)
243 /// disagreed with the manifest — post-write corruption an MD5/size check
244 /// cannot see (a flipped bit inside a data page). Verified-wrong, so it fails
245 /// the verdict and classifies as data-integrity (exit 3), not could-not-verify.
246 ValueChecksumMismatch { detail: String },
247 /// `--depth full` re-read a CDC export's parts and the `__pos` continuity
248 /// check found a gap/duplicate in the change positions — the exported change
249 /// stream is incomplete or reordered. Verified-wrong (same class as a value
250 /// mismatch), so it fails the verdict and classifies as data-integrity (exit
251 /// 3), not could-not-verify (bughunt MED: it was folded into hard_failures →
252 /// exit 1, inconsistent with the value-checksum path).
253 CdcPositionViolation { detail: String },
254 /// `_SUCCESS` exists but its body is malformed (not `xxh3:<16-hex>` after
255 /// trim). ADR-0012 M2 — orchestrators rely on this format being strict.
256 SuccessMarkerMalformed { body_preview: String },
257 /// `_SUCCESS` body parsed but does not match `xxh3(manifest.json bytes)`.
258 /// Two legitimate sources: (a) someone overwrote `_SUCCESS` after the
259 /// manifest was rewritten — orchestrator bug; (b) the manifest was
260 /// edited in place after the run — operator bug. Either way the
261 /// manifest is no longer trustworthy.
262 SuccessMarkerStale {
263 marker_fingerprint: String,
264 manifest_fingerprint: String,
265 },
266 /// `RunManifest::validate_self_consistency` rejected the manifest.
267 /// Usually a writer bug (declared row_count != sum of committed parts'
268 /// rows); blocks the rest of the verification because the manifest
269 /// itself is unreliable.
270 ManifestSelfInconsistent { detail: String },
271 /// Reading `manifest.json` returned an I/O error other than "absent".
272 ManifestReadError { detail: String },
273 /// Reading `_SUCCESS` returned an I/O error other than "absent".
274 SuccessMarkerReadError { detail: String },
275 /// Listing the destination prefix returned an I/O error. Reduces the
276 /// untracked-parts check (M5 surplus) to a no-op for this run.
277 ListPrefixError { detail: String },
278 /// A file is present at the destination prefix but no manifest entry
279 /// references it. M9-adjacent: `--validate` only flags it; quarantine
280 /// belongs to `--resume`.
281 UntrackedObject { key: String, size_bytes: u64 },
282 /// The export declared `verify: content` but some parts could only be
283 /// size-verified (no comparable content checksum from the store) — the
284 /// declared integrity contract was not met.
285 ContentVerificationUnmet { size_only: usize, total: usize },
286 /// A manifest was *required* at this prefix (the operator pinned a literal
287 /// `--prefix`, asserting a real dataset lives here) but none was found.
288 /// Without this, an absent manifest at an operator-pinned prefix maps to
289 /// the M6 legacy-run label and exits 0 — indistinguishable from a verified
290 /// run, so a CI gate `rivet validate && deploy` sails past a destination
291 /// that was never written. Fatal: a required-but-missing manifest is a
292 /// refusal reason, not a "cannot certify" advisory.
293 ManifestRequiredButAbsent { prefix: String },
294}
295
296impl Failure {
297 /// Whether this failure invalidates the dataset (flips `passed` to false).
298 ///
299 /// Every variant is fatal **except** [`Failure::UntrackedObject`]: surplus
300 /// objects are an audit signal whose cleanup is `--resume`'s job (ADR-0012
301 /// M9), not a corruption of the manifest-listed parts. New variants are
302 /// fatal by default — opt out here explicitly, so a forgotten case fails
303 /// closed (safe) rather than silently passing.
304 pub fn is_fatal(&self) -> bool {
305 !matches!(self, Failure::UntrackedObject { .. })
306 }
307
308 /// Whether this failure is COULD-NOT-VERIFY (an operational I/O error that
309 /// prevented the check) rather than VERIFIED-WRONG (the check ran and found
310 /// the data bad). Could-not-verify is the operational class — exit 1, retry —
311 /// NOT the data-integrity stop-the-line class (exit 3). A read/list error
312 /// against a healthy prefix (a chmod-000 manifest, a transient S3 list blip)
313 /// must not page a corruption incident (#7 bughunt: these fell into the
314 /// verdict and drove exit 3). Fatal-by-omission the other way: a NEW variant
315 /// is verified-wrong (exit 3) unless it is explicitly an I/O could-not-verify.
316 pub fn is_could_not_verify(&self) -> bool {
317 matches!(
318 self,
319 Failure::ManifestReadError { .. }
320 | Failure::SuccessMarkerReadError { .. }
321 | Failure::ListPrefixError { .. }
322 )
323 }
324
325 /// Stable `RIVET_VERIFY_*` error code for this failure variant.
326 ///
327 /// One code per variant, intended for orchestrators / CI to branch on
328 /// without parsing the human `Display` string or the per-variant JSON
329 /// fields. The code is part of the wire contract: it is emitted next to
330 /// `kind` in the JSON report and prefixed in brackets on each pretty line.
331 /// Codes are append-only — never renamed once shipped (a renamed code is a
332 /// silent break for any consumer keying off it).
333 pub fn error_code(&self) -> &'static str {
334 match self {
335 Failure::PartMissing { .. } => "RIVET_VERIFY_PART_MISSING",
336 Failure::PartSizeMismatch { .. } => "RIVET_VERIFY_PART_SIZE_MISMATCH",
337 Failure::PartChecksumMismatch { .. } => "RIVET_VERIFY_PART_CHECKSUM_MISMATCH",
338 Failure::ValueChecksumMismatch { .. } => "RIVET_VERIFY_VALUE_CHECKSUM",
339 Failure::CdcPositionViolation { .. } => "RIVET_VERIFY_CDC_POSITION",
340 Failure::SuccessMarkerMalformed { .. } => "RIVET_VERIFY_SUCCESS_MALFORMED",
341 Failure::SuccessMarkerStale { .. } => "RIVET_VERIFY_SUCCESS_STALE",
342 Failure::ManifestSelfInconsistent { .. } => "RIVET_VERIFY_MANIFEST_INCONSISTENT",
343 Failure::ManifestReadError { .. } => "RIVET_VERIFY_MANIFEST_READ_ERROR",
344 Failure::SuccessMarkerReadError { .. } => "RIVET_VERIFY_SUCCESS_READ_ERROR",
345 Failure::ListPrefixError { .. } => "RIVET_VERIFY_LIST_ERROR",
346 Failure::UntrackedObject { .. } => "RIVET_VERIFY_UNTRACKED_OBJECT",
347 Failure::ContentVerificationUnmet { .. } => "RIVET_VERIFY_CONTENT_UNMET",
348 Failure::ManifestRequiredButAbsent { .. } => "RIVET_VERIFY_MANIFEST_REQUIRED",
349 }
350 }
351}
352
353impl std::fmt::Display for Failure {
354 /// One operator-facing line per failure variant. Used by:
355 /// - `pipeline::report::render_markdown` (summary.md "failure:" lines)
356 /// - `pipeline::validate_cmd::render_pretty` (`rivet validate` stdout)
357 /// - any future consumer that wants a human-readable failure label
358 ///
359 /// The wire format (`failures[].kind` + per-variant fields) lives in
360 /// the `Serialize` derive above and is the contract Airflow / CI
361 /// consumers branch on. This `Display` impl is for humans only.
362 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
363 match self {
364 Failure::PartMissing { part_id, path } => {
365 write!(f, "part {} missing at {}", part_id, path)
366 }
367 Failure::PartSizeMismatch {
368 part_id,
369 path,
370 expected,
371 actual,
372 } => write!(
373 f,
374 "part {} size mismatch at {}: manifest {}, dest {}",
375 part_id, path, expected, actual
376 ),
377 Failure::ValueChecksumMismatch { detail } => {
378 write!(
379 f,
380 "value checksum mismatch (post-write corruption): {}",
381 detail
382 )
383 }
384 Failure::CdcPositionViolation { detail } => {
385 write!(f, "cdc __pos continuity violation: {}", detail)
386 }
387 Failure::PartChecksumMismatch {
388 part_id,
389 path,
390 expected,
391 actual,
392 } => write!(
393 f,
394 "part {} content mismatch at {}: manifest md5 {}, dest {}",
395 part_id, path, expected, actual
396 ),
397 Failure::SuccessMarkerMalformed { body_preview } => {
398 write!(f, "_SUCCESS body malformed: {body_preview:?}")
399 }
400 Failure::SuccessMarkerStale {
401 marker_fingerprint,
402 manifest_fingerprint,
403 } => write!(
404 f,
405 "_SUCCESS body {} != manifest fingerprint {} (stale marker)",
406 marker_fingerprint, manifest_fingerprint
407 ),
408 Failure::ManifestSelfInconsistent { detail } => {
409 write!(f, "manifest self-consistency: {detail}")
410 }
411 Failure::ManifestReadError { detail } => {
412 write!(f, "manifest read error: {detail}")
413 }
414 Failure::SuccessMarkerReadError { detail } => {
415 write!(f, "_SUCCESS read error: {detail}")
416 }
417 Failure::ListPrefixError { detail } => {
418 write!(f, "destination listing error: {detail}")
419 }
420 Failure::UntrackedObject { key, size_bytes } => {
421 write!(f, "untracked object: {} ({} bytes)", key, size_bytes)
422 }
423 Failure::ContentVerificationUnmet { size_only, total } => write!(
424 f,
425 "verify: content not met — {size_only} of {total} part(s) only \
426 size-verified (no store checksum); lower max_file_size so parts \
427 upload as a single PUT, or the backend exposes no checksum"
428 ),
429 Failure::ManifestRequiredButAbsent { prefix } => write!(
430 f,
431 "no manifest at {prefix}: a manifest was required here (operator \
432 pinned --prefix) but none was found — this prefix was never \
433 written, or the data was relocated. Run the export first, or \
434 drop --prefix to validate the config-resolved destination."
435 ),
436 }
437 }
438}
439
440impl ManifestVerification {
441 /// Base verdict: nothing checked yet (no manifest, all counts zero, all
442 /// sub-checks false, `passed = false`). Every constructor builds on this
443 /// and overrides only what differs, so a new field lands in **one** place
444 /// rather than several near-identical literals.
445 fn empty() -> Self {
446 Self {
447 manifest_found: false,
448 legacy_run: false,
449 parts_verified: 0,
450 parts_md5_verified: 0,
451 parts_failed: 0,
452 success_marker_consistent: false,
453 manifest_self_consistent: false,
454 passed: false,
455 failures: Vec::new(),
456 // Base level; `verify_at_destination` overwrites this with the
457 // depth it was actually called at before returning any verdict.
458 depth_level: default_depth_level(),
459 }
460 }
461
462 /// Recompute `passed` from the verdict's facts: a manifest was found and no
463 /// **fatal** failure was recorded (advisory failures like `UntrackedObject`
464 /// don't count). The single source of truth — callers set failures and
465 /// call this once, rather than flipping `passed` by hand at every site.
466 fn recompute_passed(&mut self) {
467 self.passed = self.manifest_found && !self.failures.iter().any(Failure::is_fatal);
468 }
469
470 /// Apply the export's `verify` policy (ADR-0013 / review D). When content
471 /// verification is required but some parts were only size-verified, record
472 /// a fatal [`Failure::ContentVerificationUnmet`] and re-derive `passed`.
473 /// Policy lives here (one place); the composers — run finalize and the
474 /// `rivet validate` command — just call it with their export's intent.
475 pub fn enforce_content_policy(&mut self, require_content: bool) {
476 if require_content && self.manifest_found {
477 let size_only = self.parts_verified.saturating_sub(self.parts_md5_verified);
478 if size_only > 0 {
479 self.failures.push(Failure::ContentVerificationUnmet {
480 size_only,
481 total: self.parts_verified,
482 });
483 self.recompute_passed();
484 }
485 }
486 }
487
488 /// Apply the "a manifest must exist here" policy (finding #20). When the
489 /// operator pinned a literal `--prefix`, an absent manifest is no longer the
490 /// benign M6 legacy-run case — it almost always means the prefix was never
491 /// written (a misconfigured CI gate). Convert that exact verdict — no
492 /// manifest, no other failure (i.e. the [`ManifestVerification::legacy`]
493 /// shape) — into a fatal [`Failure::ManifestRequiredButAbsent`] so the exit
494 /// gate refuses it loudly instead of silently passing.
495 ///
496 /// Deliberately a no-op for every other shape: a real manifest (passed or
497 /// failed), or an absent manifest that already carries a `ManifestReadError`
498 /// / head failure, is left untouched — those are already classified. Only
499 /// the "legacy / cannot certify" case is escalated, and only when required.
500 pub fn require_manifest_present(&mut self, prefix: &str) {
501 if !self.manifest_found && !self.has_failures() {
502 self.legacy_run = false;
503 self.failures.push(Failure::ManifestRequiredButAbsent {
504 prefix: prefix.to_string(),
505 });
506 self.recompute_passed();
507 }
508 }
509
510 /// Construct the M6 (legacy run) verdict for a destination that has no
511 /// manifest at all. Caller composes this with the existing per-file
512 /// row-count check; together they form the legacy `--validate` result.
513 pub fn legacy() -> Self {
514 // `passed = false` is intentional — not "validation failed" but "this
515 // verifier cannot certify"; the caller layers per-file row counts on
516 // top and composes the final verdict.
517 Self {
518 legacy_run: true,
519 ..Self::empty()
520 }
521 }
522
523 /// True iff this verification surfaced any explicit failure (i.e. a
524 /// reason an orchestrator should refuse the run). Distinct from
525 /// `!passed`, which can also mean "legacy / not applicable".
526 pub fn has_failures(&self) -> bool {
527 !self.failures.is_empty()
528 }
529
530 /// Whether any recorded failure is VERIFIED-WRONG (the check ran and the data
531 /// is bad) as opposed to purely COULD-NOT-VERIFY (an I/O read/list error).
532 /// Drives the exit CLASS: a verdict with a verified-wrong failure is
533 /// data-integrity (exit 3); a verdict whose failures are ALL could-not-verify
534 /// is operational (exit 1). See [`Failure::is_could_not_verify`].
535 pub fn has_verified_wrong_failure(&self) -> bool {
536 self.failures.iter().any(|f| !f.is_could_not_verify())
537 }
538}
539
540/// Run the manifest-aware verification at `manifest_dir` (the destination-
541/// relative directory containing `manifest.json` and `_SUCCESS`).
542///
543/// `manifest_dir` is the same key shape `Destination::write` was called with
544/// for the manifest itself — typically empty (`""`) for prefix-rooted runs,
545/// or the per-export sub-directory. Trailing `/` is optional.
546///
547/// This function does not panic on any expected I/O outcome — every read
548/// failure becomes a `Failure::*ReadError` so the caller can render a
549/// useful message instead of bailing.
550///
551/// `depth` selects the graded verify layer (see [`ValidateDepth`]):
552/// - [`ValidateDepth::Light`] skips section 3 (the `list_prefix` part
553/// reconcile) and section 5 (untracked surplus), leaving `parts_verified`
554/// at 0 — a fast manifest + `_SUCCESS` poll with no prefix listing.
555/// - [`ValidateDepth::Sample`] and [`ValidateDepth::Full`] run all five
556/// sections here. The Form B value re-read is **not** in this function;
557/// it is the caller's concern (`run_validate_command`), gated on `Full`.
558///
559/// Regardless of depth, `depth_level` on the returned verdict records the
560/// level this pass ran at.
561pub fn verify_at_destination(
562 dest: &dyn Destination,
563 manifest_dir: &str,
564 depth: ValidateDepth,
565) -> Result<ManifestVerification> {
566 let manifest_key = join_key(manifest_dir, MANIFEST_FILENAME);
567 let success_key = join_key(manifest_dir, SUCCESS_FILENAME);
568
569 // Stamp the depth this pass ran at onto every verdict before it leaves the
570 // function — including the early-return error/legacy shapes — so a consumer
571 // always sees *how much* was checked. Each `return Ok(v)` below routes
572 // through `with_depth` (or sets `out.depth_level` for the main path).
573 let with_depth = |mut v: ManifestVerification| -> ManifestVerification {
574 v.depth_level = depth.label().to_string();
575 v
576 };
577
578 // ── 1. Manifest read ───────────────────────────────────────────────
579 //
580 // Error-consistency contract: every I/O outcome here surfaces as a
581 // structured `Failure` variant rather than as `Err`. An operator gets
582 // one verdict shape regardless of whether the destination is missing,
583 // permission-denied, or temporarily unreachable. The bubbled `Err`
584 // path is reserved for *programmer* errors (caller passes a malformed
585 // `manifest_dir`, a future destination breaks an internal invariant).
586 let manifest_bytes = match dest.head(&manifest_key) {
587 Ok(None) => return Ok(with_depth(ManifestVerification::legacy())),
588 Ok(Some(_)) => match read_capped(dest, &manifest_key, MANIFEST_MAX_BYTES) {
589 Ok(b) => b,
590 Err(e) => {
591 let mut v = ManifestVerification::legacy();
592 v.legacy_run = false;
593 v.failures.push(Failure::ManifestReadError {
594 detail: format!("{e:#}"),
595 });
596 v.passed = false;
597 return Ok(with_depth(v));
598 }
599 },
600 Err(e) => {
601 // `head` failure is symmetric to a `read` failure — same kind
602 // (`ManifestReadError`) so consumers don't have to branch on
603 // which method tripped. Distinct from "manifest absent"
604 // (Ok(None) above) which legitimately means "legacy prefix".
605 let mut v = ManifestVerification::legacy();
606 v.legacy_run = false;
607 v.failures.push(Failure::ManifestReadError {
608 detail: format!("manifest head failed: {e:#}"),
609 });
610 v.passed = false;
611 return Ok(with_depth(v));
612 }
613 };
614
615 let manifest: RunManifest = match serde_json::from_slice(&manifest_bytes) {
616 Ok(m) => m,
617 Err(e) => {
618 // A malformed manifest is treated as a self-inconsistency —
619 // semantically equivalent for the operator (the manifest can't
620 // be trusted) but kept distinct in `failures` so the kind is
621 // explicit on the wire.
622 return Ok(with_depth(ManifestVerification {
623 manifest_found: true,
624 failures: vec![Failure::ManifestSelfInconsistent {
625 detail: format!("manifest.json parse failed: {e}"),
626 }],
627 ..ManifestVerification::empty()
628 }));
629 }
630 };
631
632 // Optimistic base: a found, self-consistent manifest that passes until a
633 // check below flips it. Overrides only what differs from `empty()`.
634 // Stamp the depth here so the two early `return Ok(out)` paths in section
635 // 4 (success-marker head error, non-utf8 body) carry the right level too.
636 let mut out = ManifestVerification {
637 manifest_found: true,
638 manifest_self_consistent: true,
639 passed: true,
640 depth_level: depth.label().to_string(),
641 ..ManifestVerification::empty()
642 };
643
644 // ── 2. Self-consistency ─────────────────────────────────────────────
645 if let Err(e) = manifest.validate_self_consistency() {
646 out.manifest_self_consistent = false;
647 out.failures.push(Failure::ManifestSelfInconsistent {
648 detail: format!("{e}"),
649 });
650 // Don't short-circuit — we still want to surface part-presence
651 // failures because the operator may want to know both classes at
652 // once rather than fix-then-rerun.
653 }
654
655 // ── 3. Reconcile parts + surplus against ONE prefix listing ────────
656 //
657 // Presence and untracked-surplus both fall out of a single
658 // `reconcile_manifest_against_listing` over one `list_prefix` — the same
659 // pure walk chunked resume uses (`build_resume_plan`). This replaces the
660 // old per-part `HEAD` loop (N round-trips) and its separate untracked
661 // listing. Per-part failures are emitted here (step 3); untracked is
662 // emitted at step 5 so the failure ordering an operator reads is stable.
663 //
664 // Trade-off: presence now rides the listing, not per-part `HEAD`. If the
665 // listing cannot be read, an audit cannot certify the parts — so a list
666 // failure flips `passed = false` (a `ListPrefixError`), rather than the
667 // old behaviour where per-part HEAD still "verified" parts a failed
668 // listing couldn't enumerate. Every Rivet destination backend offers
669 // strong read-after-write list consistency, so the happy path is one call.
670 //
671 // Graded depth: `Light` skips this `list_prefix` entirely — no part
672 // reconcile, no `ListPrefixError`, `parts_verified` stays 0, and section 5
673 // (untracked) is a no-op since `reconciliation` is `None`. `Sample` and
674 // `Full` run it. A `Light` pass therefore certifies only that the
675 // manifest reads, is self-consistent, and `_SUCCESS` matches — never that
676 // the parts are physically present.
677 let reconciliation = if depth.runs_part_reconcile() {
678 match dest.list_prefix(manifest_dir) {
679 Ok(listing) => Some(reconcile_manifest_against_listing(
680 &manifest,
681 &listing,
682 manifest_dir,
683 )),
684 Err(e) => {
685 out.failures.push(Failure::ListPrefixError {
686 detail: format!("{e:#}"),
687 });
688 None
689 }
690 }
691 } else {
692 None
693 };
694 if let Some(rec) = &reconciliation {
695 for check in &rec.per_part {
696 match &check.presence {
697 PartPresence::Present { md5_verified } => {
698 out.parts_verified += 1;
699 if *md5_verified {
700 out.parts_md5_verified += 1;
701 }
702 }
703 PartPresence::SizeMismatch { expected, actual } => {
704 out.parts_failed += 1;
705 out.failures.push(Failure::PartSizeMismatch {
706 part_id: check.part_id,
707 path: check.path.clone(),
708 expected: *expected,
709 actual: *actual,
710 });
711 }
712 PartPresence::Missing => {
713 out.parts_failed += 1;
714 out.failures.push(Failure::PartMissing {
715 part_id: check.part_id,
716 path: check.path.clone(),
717 });
718 }
719 PartPresence::ChecksumMismatch { expected, actual } => {
720 out.parts_failed += 1;
721 out.failures.push(Failure::PartChecksumMismatch {
722 part_id: check.part_id,
723 path: check.path.clone(),
724 expected: expected.clone(),
725 actual: actual.clone(),
726 });
727 }
728 }
729 }
730 }
731
732 // ── 4. _SUCCESS marker consistency ─────────────────────────────────
733 //
734 // Same error-consistency contract as step 1: head/read failures become
735 // `Failure::SuccessMarkerReadError`, not bubbled `Err`. Absent marker
736 // (Ok(None)) stays informational, not a failure (M2: only successful
737 // runs land _SUCCESS, so its absence on a failed manifest is correct).
738 let success_head = match dest.head(&success_key) {
739 Ok(h) => h,
740 Err(e) => {
741 out.failures.push(Failure::SuccessMarkerReadError {
742 detail: format!("_SUCCESS head failed: {e:#}"),
743 });
744 out.recompute_passed();
745 return Ok(out);
746 }
747 };
748 match success_head {
749 None => {
750 // Absent _SUCCESS is informational, not a failure: per ADR-0012
751 // M2, only successful runs land it. A failed-then-rewritten
752 // manifest legitimately lacks _SUCCESS. Leave
753 // `success_marker_consistent = false` (this is a "no signal"
754 // bool, not a "broken" bool) and let the caller decide.
755 }
756 Some(head) if head.size_bytes > SUCCESS_MARKER_MAX_BYTES => {
757 // Refuse to materialise an oversized _SUCCESS: a bare uncapped
758 // dest.read of a multi-GB planted marker OOMs the validate/resume/repair
759 // process (the asymmetry with step 1's manifest.json cap this closes).
760 // success_head already carries the size — no extra round-trip.
761 out.failures.push(Failure::SuccessMarkerMalformed {
762 body_preview: format!(
763 "(oversized: {} bytes exceeds the {SUCCESS_MARKER_MAX_BYTES}-byte _SUCCESS cap; not read)",
764 head.size_bytes
765 ),
766 });
767 out.recompute_passed();
768 return Ok(out);
769 }
770 Some(_) => match dest.read(&success_key) {
771 Err(e) => {
772 out.failures.push(Failure::SuccessMarkerReadError {
773 detail: format!("{e:#}"),
774 });
775 }
776 Ok(body) => {
777 let body_str = match std::str::from_utf8(&body) {
778 Ok(s) => s,
779 Err(_) => {
780 out.failures.push(Failure::SuccessMarkerMalformed {
781 body_preview: format!("(non-utf8, {} bytes)", body.len()),
782 });
783 out.recompute_passed();
784 return Ok(out);
785 }
786 };
787 match parse_success_marker(body_str) {
788 None => {
789 out.failures.push(Failure::SuccessMarkerMalformed {
790 body_preview: preview(body_str),
791 });
792 }
793 Some(marker_fp) => {
794 let manifest_fp = success_marker_body(&manifest_bytes);
795 // success_marker_body returns the trailing `\n`
796 // form; trim before comparing to the parsed marker
797 // (which already trims).
798 let manifest_fp_trimmed = manifest_fp.trim_end_matches('\n');
799 if marker_fp == manifest_fp_trimmed {
800 out.success_marker_consistent = true;
801 } else {
802 out.failures.push(Failure::SuccessMarkerStale {
803 marker_fingerprint: marker_fp.to_string(),
804 manifest_fingerprint: manifest_fp_trimmed.to_string(),
805 });
806 }
807 }
808 }
809 }
810 },
811 }
812
813 // ── 5. Untracked surplus ───────────────────────────────────────────
814 //
815 // Already computed by the step-3 reconciliation (sidecars, quarantine,
816 // and the doctor probe are filtered there). Emit it last so the failure
817 // ordering stays parts → marker → untracked. A list failure left
818 // `reconciliation = None` and already flipped `passed` above.
819 if let Some(rec) = reconciliation {
820 for obj in rec.untracked {
821 out.failures.push(Failure::UntrackedObject {
822 key: obj.key,
823 size_bytes: obj.size_bytes,
824 });
825 }
826 }
827
828 out.recompute_passed();
829 Ok(out)
830}
831
832/// Truncate `s` to a small printable preview for error messages.
833fn preview(s: &str) -> String {
834 let trimmed: String = s.chars().take(40).collect();
835 if s.chars().count() > 40 {
836 format!("{trimmed}…")
837 } else {
838 trimmed
839 }
840}
841
842#[cfg(test)]
843mod tests {
844 use super::*;
845 use crate::config::{DestinationConfig, DestinationType};
846 use crate::destination::local::LocalDestination;
847 use crate::manifest::{
848 MANIFEST_VERSION, ManifestDestination, ManifestPart, ManifestSource, ManifestStatus,
849 PartStatus, RunManifest,
850 };
851 use std::path::Path;
852
853 fn local_dest(base: &Path) -> LocalDestination {
854 LocalDestination::new(&DestinationConfig {
855 destination_type: DestinationType::Local,
856 path: Some(base.to_string_lossy().into_owned()),
857 ..Default::default()
858 })
859 .unwrap()
860 }
861
862 fn part(part_id: u32, rows: i64, size: u64, fp: &str) -> ManifestPart {
863 ManifestPart {
864 part_id,
865 path: format!("part-{part_id:06}.parquet"),
866 rows,
867 size_bytes: size,
868 content_fingerprint: fp.into(),
869 content_md5: String::new(),
870 status: PartStatus::Committed,
871 }
872 }
873
874 fn build_manifest(parts: Vec<ManifestPart>, status: ManifestStatus) -> RunManifest {
875 let row_count: i64 = parts
876 .iter()
877 .filter(|p| p.status == PartStatus::Committed)
878 .map(|p| p.rows)
879 .sum();
880 let part_count = parts
881 .iter()
882 .filter(|p| p.status == PartStatus::Committed)
883 .count() as u32;
884 RunManifest {
885 mode: "batch".to_string(),
886 manifest_version: MANIFEST_VERSION,
887 run_id: "r".into(),
888 export_name: "public.orders".into(),
889 started_at: "2026-05-21T12:00:00Z".into(),
890 finished_at: "2026-05-21T12:01:00Z".into(),
891 status,
892 source: ManifestSource {
893 engine: "postgres".into(),
894 schema: Some("public".into()),
895 table: Some("orders".into()),
896 extraction: None,
897 },
898 destination: ManifestDestination {
899 kind: "local".into(),
900 uri: "file:///tmp/out".into(),
901 },
902 format: "parquet".into(),
903 compression: "zstd".into(),
904 schema_fingerprint: "xxh3:0123456789abcdef".into(),
905 row_count,
906 part_count,
907 parts,
908 column_checksums: None,
909 checksum_key_column: None,
910 }
911 }
912
913 /// Lay out a clean dataset with manifest + _SUCCESS at the root.
914 fn write_dataset(dir: &Path, m: &RunManifest, parts_with_bytes: &[(&str, &[u8])]) {
915 for (name, bytes) in parts_with_bytes {
916 std::fs::write(dir.join(name), bytes).unwrap();
917 }
918 let body = serde_json::to_vec_pretty(m).unwrap();
919 std::fs::write(dir.join(MANIFEST_FILENAME), &body).unwrap();
920 if matches!(m.status, ManifestStatus::Success) {
921 std::fs::write(dir.join(SUCCESS_FILENAME), success_marker_body(&body)).unwrap();
922 }
923 }
924
925 // ── happy path ───────────────────────────────────────────────────────
926
927 #[test]
928 fn happy_path_verifies_all_parts_and_success_marker() {
929 let dir = tempfile::tempdir().unwrap();
930 let m = build_manifest(
931 vec![
932 part(1, 10, 4, "xxh3:1111111111111111"),
933 part(2, 20, 5, "xxh3:2222222222222222"),
934 ],
935 ManifestStatus::Success,
936 );
937 write_dataset(
938 dir.path(),
939 &m,
940 &[
941 ("part-000001.parquet", b"AAAA"),
942 ("part-000002.parquet", b"BBBBB"),
943 ],
944 );
945 let dest = local_dest(dir.path());
946
947 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
948 assert!(v.manifest_found);
949 assert!(!v.legacy_run);
950 assert_eq!(v.parts_verified, 2);
951 assert_eq!(v.parts_failed, 0);
952 assert!(v.success_marker_consistent);
953 assert!(v.manifest_self_consistent);
954 assert!(v.passed);
955 assert!(v.failures.is_empty());
956 }
957
958 // ── M6 legacy run ───────────────────────────────────────────────────
959
960 #[test]
961 fn no_manifest_returns_legacy_run_label() {
962 // Empty prefix — no manifest, no parts.
963 let dir = tempfile::tempdir().unwrap();
964 let dest = local_dest(dir.path());
965 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
966 assert!(!v.manifest_found);
967 assert!(v.legacy_run);
968 assert_eq!(v.parts_verified, 0);
969 assert!(!v.passed);
970 assert!(v.failures.is_empty(), "no failures, just a legacy label");
971 }
972
973 // ── M5 part-presence failures ───────────────────────────────────────
974
975 #[test]
976 fn missing_part_is_flagged_with_part_id_and_path() {
977 let dir = tempfile::tempdir().unwrap();
978 let m = build_manifest(
979 vec![
980 part(1, 10, 4, "xxh3:1111111111111111"),
981 part(2, 20, 5, "xxh3:2222222222222222"),
982 ],
983 ManifestStatus::Success,
984 );
985 write_dataset(
986 dir.path(),
987 &m,
988 &[("part-000001.parquet", b"AAAA")], // part 2 missing
989 );
990 let dest = local_dest(dir.path());
991
992 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
993 assert_eq!(v.parts_verified, 1);
994 assert_eq!(v.parts_failed, 1);
995 assert!(!v.passed);
996 assert!(
997 v.failures
998 .iter()
999 .any(|f| matches!(f, Failure::PartMissing { part_id: 2, .. }))
1000 );
1001 }
1002
1003 #[test]
1004 fn part_size_mismatch_is_flagged_with_expected_and_actual() {
1005 let dir = tempfile::tempdir().unwrap();
1006 let m = build_manifest(
1007 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1008 ManifestStatus::Success,
1009 );
1010 // Manifest claims 4 bytes; we write 6.
1011 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"OOPSIE")]);
1012 let dest = local_dest(dir.path());
1013
1014 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1015 assert!(!v.passed);
1016 let mismatch = v
1017 .failures
1018 .iter()
1019 .find_map(|f| match f {
1020 Failure::PartSizeMismatch {
1021 part_id,
1022 expected,
1023 actual,
1024 ..
1025 } => Some((*part_id, *expected, *actual)),
1026 _ => None,
1027 })
1028 .expect("must surface the size mismatch");
1029 assert_eq!(mismatch, (1, 4, 6));
1030 }
1031
1032 // ── _SUCCESS marker integrity ───────────────────────────────────────
1033
1034 #[test]
1035 fn stale_success_marker_is_flagged_as_inconsistent() {
1036 // Write a manifest, then overwrite _SUCCESS with the marker for a
1037 // *different* manifest body — simulating an orchestrator that
1038 // mishandled a re-run.
1039 let dir = tempfile::tempdir().unwrap();
1040 let m = build_manifest(
1041 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1042 ManifestStatus::Success,
1043 );
1044 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1045 std::fs::write(
1046 dir.path().join(SUCCESS_FILENAME),
1047 success_marker_body(b"different manifest body"),
1048 )
1049 .unwrap();
1050 let dest = local_dest(dir.path());
1051
1052 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1053 assert!(!v.success_marker_consistent);
1054 assert!(!v.passed);
1055 assert!(
1056 v.failures
1057 .iter()
1058 .any(|f| matches!(f, Failure::SuccessMarkerStale { .. }))
1059 );
1060 }
1061
1062 #[test]
1063 fn malformed_success_marker_body_is_flagged() {
1064 let dir = tempfile::tempdir().unwrap();
1065 let m = build_manifest(
1066 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1067 ManifestStatus::Success,
1068 );
1069 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1070 std::fs::write(dir.path().join(SUCCESS_FILENAME), b"not even xxh3 shaped").unwrap();
1071 let dest = local_dest(dir.path());
1072
1073 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1074 assert!(!v.passed);
1075 assert!(
1076 v.failures
1077 .iter()
1078 .any(|f| matches!(f, Failure::SuccessMarkerMalformed { .. }))
1079 );
1080 }
1081
1082 #[test]
1083 fn absent_success_marker_does_not_fail_validation_alone() {
1084 // ADR-0012 M2: only successful runs land _SUCCESS. A failed-then-
1085 // rewritten manifest legitimately lacks one — verification must
1086 // not flip `passed` just for that.
1087 let dir = tempfile::tempdir().unwrap();
1088 let m = build_manifest(
1089 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1090 ManifestStatus::Failed,
1091 );
1092 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1093 // Note: write_dataset only writes _SUCCESS for status == Success,
1094 // so no marker exists here.
1095 assert!(!dir.path().join(SUCCESS_FILENAME).exists());
1096 let dest = local_dest(dir.path());
1097
1098 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1099 assert!(v.manifest_found);
1100 assert!(
1101 !v.success_marker_consistent,
1102 "no marker => false (no signal)"
1103 );
1104 // The parts still verified, so passed = true.
1105 assert!(v.passed);
1106 assert!(v.failures.is_empty());
1107 }
1108
1109 // ── self-consistency ────────────────────────────────────────────────
1110
1111 #[test]
1112 fn self_inconsistent_manifest_is_flagged_but_part_check_still_runs() {
1113 let dir = tempfile::tempdir().unwrap();
1114 let mut m = build_manifest(
1115 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1116 ManifestStatus::Success,
1117 );
1118 m.row_count = 9999; // lie
1119
1120 let body = serde_json::to_vec_pretty(&m).unwrap();
1121 std::fs::write(dir.path().join("part-000001.parquet"), b"AAAA").unwrap();
1122 std::fs::write(dir.path().join(MANIFEST_FILENAME), &body).unwrap();
1123 std::fs::write(
1124 dir.path().join(SUCCESS_FILENAME),
1125 success_marker_body(&body),
1126 )
1127 .unwrap();
1128 let dest = local_dest(dir.path());
1129
1130 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1131 assert!(v.manifest_found);
1132 assert!(!v.manifest_self_consistent);
1133 assert!(!v.passed);
1134 // Parts that are physically present still get their `parts_verified`
1135 // counter bumped — both signals are independently useful.
1136 assert_eq!(v.parts_verified, 1);
1137 assert!(
1138 v.failures
1139 .iter()
1140 .any(|f| matches!(f, Failure::ManifestSelfInconsistent { .. }))
1141 );
1142 }
1143
1144 // ── untracked objects ───────────────────────────────────────────────
1145
1146 #[test]
1147 fn verify_counters_track_present_and_failed_parts() {
1148 // Mutation-tier2 gap: the verdict-shaping fields (manifest_found /
1149 // passed / failures) and the per-part counters (parts_verified /
1150 // parts_failed) had no assertion — `+= -> -=` survived. One present
1151 // part + one missing part pin both counters and the verdict.
1152 let dir = tempfile::tempdir().unwrap();
1153 let m = build_manifest(
1154 vec![
1155 part(1, 10, 4, "xxh3:1111111111111111"),
1156 part(2, 10, 4, "xxh3:2222222222222222"),
1157 ],
1158 ManifestStatus::Success,
1159 );
1160 // Only part 1 exists at the destination.
1161 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1162 let dest = local_dest(dir.path());
1163
1164 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1165 assert!(v.manifest_found, "manifest is at the prefix");
1166 assert_eq!(v.parts_verified, 1, "part 1 is present at its size");
1167 assert_eq!(v.parts_failed, 1, "part 2 is missing");
1168 assert!(
1169 v.failures
1170 .iter()
1171 .any(|f| matches!(f, Failure::PartMissing { part_id: 2, .. })),
1172 "the missing part is named in failures: {:?}",
1173 v.failures
1174 );
1175 assert!(!v.passed, "a missing part must fail the verdict");
1176 }
1177
1178 #[test]
1179 fn read_capped_boundary_is_exact() {
1180 // `size > max` (not >=): a body exactly AT the cap must load; one
1181 // byte over must be refused before materialising.
1182 let dir = tempfile::tempdir().unwrap();
1183 std::fs::write(dir.path().join("at-cap.bin"), vec![0u8; 8]).unwrap();
1184 std::fs::write(dir.path().join("over-cap.bin"), vec![0u8; 9]).unwrap();
1185 let dest = local_dest(dir.path());
1186 assert_eq!(
1187 read_capped(&dest, "at-cap.bin", 8)
1188 .expect("exactly at cap loads")
1189 .len(),
1190 8
1191 );
1192 let err = read_capped(&dest, "over-cap.bin", 8).expect_err("over cap refuses");
1193 assert!(
1194 err.to_string().contains("read cap"),
1195 "actionable message: {err:#}"
1196 );
1197 }
1198
1199 #[test]
1200 fn preview_truncates_only_past_forty_chars() {
1201 let s39: String = "x".repeat(39);
1202 let s40: String = "x".repeat(40);
1203 let s41: String = "x".repeat(41);
1204 assert_eq!(preview(&s39), s39, "under the limit: unchanged");
1205 assert_eq!(
1206 preview(&s40),
1207 s40,
1208 "exactly at the limit: unchanged (`>` not `>=`)"
1209 );
1210 assert_eq!(
1211 preview(&s41),
1212 format!("{s40}\u{2026}"),
1213 "past the limit: 40 chars + ellipsis"
1214 );
1215 }
1216
1217 #[test]
1218 fn untracked_object_under_prefix_is_flagged() {
1219 let dir = tempfile::tempdir().unwrap();
1220 let m = build_manifest(
1221 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1222 ManifestStatus::Success,
1223 );
1224 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1225 std::fs::write(dir.path().join("rogue.parquet"), b"XX").unwrap();
1226 let dest = local_dest(dir.path());
1227
1228 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1229 assert!(
1230 v.failures.iter().any(
1231 |f| matches!(f, Failure::UntrackedObject { key, .. } if key == "rogue.parquet")
1232 )
1233 );
1234 // Untracked objects are surfaced but do NOT flip `passed` — that
1235 // is the resume-side decision (M9). Parts and marker are fine,
1236 // so passed remains true.
1237 assert!(v.passed);
1238 }
1239
1240 #[test]
1241 fn quarantine_prefix_objects_are_silently_ignored() {
1242 let dir = tempfile::tempdir().unwrap();
1243 let m = build_manifest(
1244 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1245 ManifestStatus::Success,
1246 );
1247 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1248 std::fs::create_dir_all(dir.path().join(crate::manifest::QUARANTINE_PREFIX)).unwrap();
1249 std::fs::write(
1250 dir.path()
1251 .join(crate::manifest::QUARANTINE_PREFIX)
1252 .join("old.parquet"),
1253 b"OO",
1254 )
1255 .unwrap();
1256 let dest = local_dest(dir.path());
1257
1258 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1259 assert!(v.passed);
1260 assert!(
1261 !v.failures
1262 .iter()
1263 .any(|f| matches!(f, Failure::UntrackedObject { .. })),
1264 "quarantine_prefix is the legitimate home for these — must not flag"
1265 );
1266 }
1267
1268 #[test]
1269 fn doctor_probe_is_not_flagged_as_untracked() {
1270 // Regression: `rivet doctor` writes `.rivet_doctor_probe` at the
1271 // destination prefix and never removes it. A subsequent
1272 // `rivet run --validate` against the same prefix must treat it as a
1273 // Rivet sidecar, not foreign data — otherwise `has_failures()` trips
1274 // and the run's `validated` flag is downgraded to FAIL.
1275 let dir = tempfile::tempdir().unwrap();
1276 let m = build_manifest(
1277 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1278 ManifestStatus::Success,
1279 );
1280 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1281 std::fs::write(
1282 dir.path().join(crate::manifest::DOCTOR_PROBE_FILENAME),
1283 b"ok",
1284 )
1285 .unwrap();
1286 let dest = local_dest(dir.path());
1287
1288 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1289 assert!(
1290 !v.has_failures(),
1291 "doctor probe must not surface as a failure: {:?}",
1292 v.failures
1293 );
1294 assert!(v.passed);
1295 }
1296
1297 // ── manifest_dir join semantics ─────────────────────────────────────
1298
1299 #[test]
1300 fn verifies_in_subdirectory_when_manifest_dir_is_non_empty() {
1301 let outer = tempfile::tempdir().unwrap();
1302 std::fs::create_dir_all(outer.path().join("sub/run")).unwrap();
1303 let m = build_manifest(
1304 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1305 ManifestStatus::Success,
1306 );
1307 let body = serde_json::to_vec_pretty(&m).unwrap();
1308 std::fs::write(outer.path().join("sub/run/part-000001.parquet"), b"AAAA").unwrap();
1309 std::fs::write(outer.path().join("sub/run").join(MANIFEST_FILENAME), &body).unwrap();
1310 std::fs::write(
1311 outer.path().join("sub/run").join(SUCCESS_FILENAME),
1312 success_marker_body(&body),
1313 )
1314 .unwrap();
1315 let dest = local_dest(outer.path());
1316
1317 let v = verify_at_destination(&dest, "sub/run", ValidateDepth::Full).unwrap();
1318 assert!(v.passed);
1319 assert_eq!(v.parts_verified, 1);
1320
1321 // Trailing slash is normalised — same outcome.
1322 let v2 = verify_at_destination(&dest, "sub/run/", ValidateDepth::Full).unwrap();
1323 assert!(v2.passed);
1324 }
1325
1326 // ── list-failure semantics (presence now rides the listing) ──────────
1327
1328 /// Wraps a real `LocalDestination` but fails every `list_prefix`. Used to
1329 /// pin the post-refactor contract: presence is derived from the listing,
1330 /// so a listing we cannot read means the audit cannot certify the parts.
1331 struct ListFails(LocalDestination);
1332 impl crate::destination::Destination for ListFails {
1333 fn write(&self, p: &Path, k: &str) -> Result<crate::destination::WriteOutcome> {
1334 self.0.write(p, k)
1335 }
1336 fn capabilities(&self) -> crate::destination::DestinationCapabilities {
1337 self.0.capabilities()
1338 }
1339 fn head(&self, k: &str) -> Result<Option<crate::destination::ObjectMeta>> {
1340 self.0.head(k)
1341 }
1342 fn read(&self, k: &str) -> Result<Vec<u8>> {
1343 self.0.read(k)
1344 }
1345 fn list_prefix(&self, _: &str) -> Result<Vec<crate::destination::ObjectMeta>> {
1346 anyhow::bail!("listing unavailable")
1347 }
1348 }
1349
1350 #[test]
1351 fn list_failure_cannot_certify_parts_and_fails_the_audit() {
1352 let dir = tempfile::tempdir().unwrap();
1353 let m = build_manifest(vec![part(0, 3, 3, "xxh3:0")], ManifestStatus::Success);
1354 write_dataset(dir.path(), &m, &[("part-000000.parquet", b"abc")]);
1355 let dest = ListFails(local_dest(dir.path()));
1356
1357 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1358 // The manifest itself reads + parses fine (HEAD/read still work)…
1359 assert!(v.manifest_found);
1360 assert!(v.manifest_self_consistent);
1361 // …but with no listing we verify zero parts and refuse to pass.
1362 assert!(
1363 !v.passed,
1364 "an audit that cannot list the prefix must not pass"
1365 );
1366 assert_eq!(v.parts_verified, 0);
1367 assert!(
1368 v.failures
1369 .iter()
1370 .any(|f| matches!(f, Failure::ListPrefixError { .. })),
1371 "expected a ListPrefixError, got: {:?}",
1372 v.failures
1373 );
1374 }
1375
1376 // ── manifest read-error semantics (explicit failure, not legacy) ─────
1377
1378 /// Wraps a real `LocalDestination` but fails reading `manifest.json`
1379 /// (head still sees it) — EACCES or a transient store error on a
1380 /// manifest that exists.
1381 struct ManifestReadFails(LocalDestination);
1382 impl crate::destination::Destination for ManifestReadFails {
1383 fn write(&self, p: &Path, k: &str) -> Result<crate::destination::WriteOutcome> {
1384 self.0.write(p, k)
1385 }
1386 fn capabilities(&self) -> crate::destination::DestinationCapabilities {
1387 self.0.capabilities()
1388 }
1389 fn head(&self, k: &str) -> Result<Option<crate::destination::ObjectMeta>> {
1390 self.0.head(k)
1391 }
1392 fn read(&self, k: &str) -> Result<Vec<u8>> {
1393 if k.ends_with(MANIFEST_FILENAME) {
1394 anyhow::bail!("permission denied (simulated)")
1395 }
1396 self.0.read(k)
1397 }
1398 fn list_prefix(&self, p: &str) -> Result<Vec<crate::destination::ObjectMeta>> {
1399 self.0.list_prefix(p)
1400 }
1401 }
1402
1403 #[test]
1404 fn unreadable_manifest_is_explicit_failure_not_legacy() {
1405 // The exit gates (`rivet validate`, run finalize) key off this exact
1406 // shape: `manifest_found: false` but `has_failures()` — distinct
1407 // from M6 legacy (`legacy_run: true`, no failures, exit 0).
1408 let dir = tempfile::tempdir().unwrap();
1409 let m = build_manifest(
1410 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1411 ManifestStatus::Success,
1412 );
1413 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1414 let dest = ManifestReadFails(local_dest(dir.path()));
1415
1416 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1417 assert!(!v.manifest_found);
1418 assert!(!v.legacy_run, "a read error is not the M6 legacy label");
1419 assert!(!v.passed);
1420 assert!(v.has_failures(), "orchestrators need a reason to refuse");
1421 assert!(
1422 matches!(v.failures.as_slice(), [Failure::ManifestReadError { .. }]),
1423 "expected exactly one ManifestReadError, got: {:?}",
1424 v.failures
1425 );
1426 }
1427
1428 /// Same contract when `head` itself errors (cannot even stat the
1429 /// manifest): symmetric `ManifestReadError`, never the legacy label.
1430 struct ManifestHeadFails(LocalDestination);
1431 impl crate::destination::Destination for ManifestHeadFails {
1432 fn write(&self, p: &Path, k: &str) -> Result<crate::destination::WriteOutcome> {
1433 self.0.write(p, k)
1434 }
1435 fn capabilities(&self) -> crate::destination::DestinationCapabilities {
1436 self.0.capabilities()
1437 }
1438 fn head(&self, k: &str) -> Result<Option<crate::destination::ObjectMeta>> {
1439 if k.ends_with(MANIFEST_FILENAME) {
1440 anyhow::bail!("io timeout (simulated)")
1441 }
1442 self.0.head(k)
1443 }
1444 fn read(&self, k: &str) -> Result<Vec<u8>> {
1445 self.0.read(k)
1446 }
1447 fn list_prefix(&self, p: &str) -> Result<Vec<crate::destination::ObjectMeta>> {
1448 self.0.list_prefix(p)
1449 }
1450 }
1451
1452 #[test]
1453 fn manifest_head_error_is_explicit_failure_not_legacy() {
1454 let dir = tempfile::tempdir().unwrap();
1455 let m = build_manifest(
1456 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1457 ManifestStatus::Success,
1458 );
1459 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1460 let dest = ManifestHeadFails(local_dest(dir.path()));
1461
1462 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1463 assert!(!v.manifest_found);
1464 assert!(!v.legacy_run);
1465 assert!(!v.passed);
1466 assert!(
1467 matches!(
1468 v.failures.as_slice(),
1469 [Failure::ManifestReadError { detail }] if detail.contains("manifest head failed")
1470 ),
1471 "expected one ManifestReadError naming the head step, got: {:?}",
1472 v.failures
1473 );
1474 }
1475
1476 // #5: a _SUCCESS whose head-reported size exceeds the cap must be flagged
1477 // WITHOUT being read — a bare uncapped read of a multi-GB planted marker OOMs
1478 // the validate/resume process (asymmetric with the manifest.json cap). The mock
1479 // PANICS if read(_SUCCESS) is called, so the test passing proves the read was
1480 // short-circuited by the size check.
1481 struct SuccessMarkerOversized(LocalDestination);
1482 impl crate::destination::Destination for SuccessMarkerOversized {
1483 fn write(&self, p: &Path, k: &str) -> Result<crate::destination::WriteOutcome> {
1484 self.0.write(p, k)
1485 }
1486 fn capabilities(&self) -> crate::destination::DestinationCapabilities {
1487 self.0.capabilities()
1488 }
1489 fn head(&self, k: &str) -> Result<Option<crate::destination::ObjectMeta>> {
1490 if k.ends_with(crate::manifest::SUCCESS_FILENAME) {
1491 return Ok(Some(crate::destination::ObjectMeta {
1492 key: k.to_string(),
1493 size_bytes: SUCCESS_MARKER_MAX_BYTES * 4,
1494 content_md5: None,
1495 }));
1496 }
1497 self.0.head(k)
1498 }
1499 fn read(&self, k: &str) -> Result<Vec<u8>> {
1500 assert!(
1501 !k.ends_with(crate::manifest::SUCCESS_FILENAME),
1502 "verify must NOT read an oversized _SUCCESS — the size cap must short-circuit \
1503 before materialising it into memory (the OOM this guards)"
1504 );
1505 self.0.read(k)
1506 }
1507 fn list_prefix(&self, p: &str) -> Result<Vec<crate::destination::ObjectMeta>> {
1508 self.0.list_prefix(p)
1509 }
1510 }
1511
1512 #[test]
1513 fn oversized_success_marker_is_malformed_and_never_read() {
1514 let dir = tempfile::tempdir().unwrap();
1515 let m = build_manifest(
1516 vec![part(1, 10, 4, "xxh3:1111111111111111")],
1517 ManifestStatus::Success,
1518 );
1519 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]);
1520 let dest = SuccessMarkerOversized(local_dest(dir.path()));
1521
1522 let v = verify_at_destination(&dest, "", ValidateDepth::Full).unwrap();
1523 assert!(
1524 v.failures.iter().any(|f| matches!(
1525 f,
1526 Failure::SuccessMarkerMalformed { body_preview } if body_preview.contains("oversized")
1527 )),
1528 "an oversized _SUCCESS must yield SuccessMarkerMalformed(oversized) without reading it, got: {:?}",
1529 v.failures
1530 );
1531 }
1532
1533 #[test]
1534 fn passed_is_derived_advisory_failures_do_not_fail() {
1535 // An advisory failure (untracked surplus) keeps the verdict passing…
1536 let mut v = ManifestVerification {
1537 manifest_found: true,
1538 ..ManifestVerification::empty()
1539 };
1540 v.failures.push(Failure::UntrackedObject {
1541 key: "stray.parquet".into(),
1542 size_bytes: 9,
1543 });
1544 v.recompute_passed();
1545 assert!(v.passed, "untracked surplus is advisory, not fatal");
1546
1547 // …while any fatal failure flips it.
1548 v.failures.push(Failure::PartMissing {
1549 part_id: 1,
1550 path: "part-000001.parquet".into(),
1551 });
1552 v.recompute_passed();
1553 assert!(!v.passed, "a missing part is fatal");
1554
1555 // No manifest → never passes regardless of failures.
1556 let mut legacy = ManifestVerification::empty();
1557 legacy.recompute_passed();
1558 assert!(!legacy.passed, "no manifest found → cannot certify");
1559 }
1560
1561 #[test]
1562 fn verify_content_policy_fails_only_size_only_parts() {
1563 // 3 parts, 2 content-checked, 1 size-only.
1564 let base = ManifestVerification {
1565 manifest_found: true,
1566 parts_verified: 3,
1567 parts_md5_verified: 2,
1568 ..ManifestVerification::empty()
1569 };
1570 // verify: size → size-only is acceptable, passes.
1571 let mut sz = base.clone();
1572 sz.recompute_passed();
1573 sz.enforce_content_policy(false);
1574 assert!(sz.passed, "size-only OK under verify: size");
1575
1576 // verify: content → the 1 size-only part is a fatal failure.
1577 let mut ct = base.clone();
1578 ct.recompute_passed();
1579 ct.enforce_content_policy(true);
1580 assert!(!ct.passed, "a size-only part fails verify: content");
1581 assert!(
1582 ct.failures.iter().any(|f| matches!(
1583 f,
1584 Failure::ContentVerificationUnmet {
1585 size_only: 1,
1586 total: 3
1587 }
1588 )),
1589 "expected ContentVerificationUnmet, got: {:?}",
1590 ct.failures
1591 );
1592
1593 // verify: content with every part md5-checked → satisfied.
1594 let mut all = ManifestVerification {
1595 parts_md5_verified: 3,
1596 ..base
1597 };
1598 all.recompute_passed();
1599 all.enforce_content_policy(true);
1600 assert!(
1601 all.passed && all.failures.is_empty(),
1602 "all md5 meets verify: content"
1603 );
1604 }
1605
1606 // ── require_manifest_present (finding #20: operator-pinned --prefix) ──────
1607
1608 #[test]
1609 fn require_manifest_escalates_legacy_to_fatal_absent() {
1610 // The exact shape `verify_at_destination` returns for an absent manifest
1611 // (`legacy()`): no manifest, no other failure. With a pinned `--prefix`
1612 // this is escalated to a fatal `ManifestRequiredButAbsent` so the exit
1613 // gate refuses it instead of passing as a benign legacy run.
1614 let mut v = ManifestVerification::legacy();
1615 assert!(v.legacy_run && !v.has_failures());
1616
1617 v.require_manifest_present("exports/2026-06-09/orders/");
1618
1619 assert!(!v.legacy_run, "no longer the benign legacy-run label");
1620 assert!(!v.passed, "an absent-but-required manifest cannot pass");
1621 assert!(
1622 matches!(
1623 v.failures.as_slice(),
1624 [Failure::ManifestRequiredButAbsent { prefix }]
1625 if prefix == "exports/2026-06-09/orders/"
1626 ),
1627 "expected one ManifestRequiredButAbsent naming the prefix, got: {:?}",
1628 v.failures
1629 );
1630 }
1631
1632 #[test]
1633 fn require_manifest_is_noop_on_a_real_passing_manifest() {
1634 // A found, passing verdict is untouched — `--prefix` plus real data is
1635 // the normal "validate this exact prefix" case and must still pass.
1636 let mut v = ManifestVerification {
1637 manifest_found: true,
1638 manifest_self_consistent: true,
1639 parts_verified: 1,
1640 passed: true,
1641 ..ManifestVerification::empty()
1642 };
1643 v.require_manifest_present("exports/orders/");
1644 assert!(
1645 v.passed && v.failures.is_empty(),
1646 "real dataset still passes"
1647 );
1648 }
1649
1650 #[test]
1651 fn require_manifest_does_not_double_flag_a_read_error() {
1652 // An absent manifest that already carries a `ManifestReadError` (head /
1653 // read failed) is already a fatal, classified failure — requiring a
1654 // manifest here must not add a second, redundant failure.
1655 let mut v = ManifestVerification::legacy();
1656 v.legacy_run = false;
1657 v.failures.push(Failure::ManifestReadError {
1658 detail: "permission denied".into(),
1659 });
1660 v.recompute_passed();
1661
1662 v.require_manifest_present("exports/orders/");
1663
1664 assert!(
1665 matches!(v.failures.as_slice(), [Failure::ManifestReadError { .. }]),
1666 "must leave the existing read-error verdict alone, got: {:?}",
1667 v.failures
1668 );
1669 }
1670
1671 // ── graded verify layer (--depth) ───────────────────────────────────
1672
1673 #[test]
1674 fn light_depth_skips_part_reconcile_even_when_a_part_is_missing() {
1675 // A manifest declaring a part that is NOT on disk. At `Full`/`Sample`
1676 // this is a fatal `PartMissing`; at `Light` the `list_prefix` reconcile
1677 // is skipped entirely, so `parts_verified == 0`, no `ListPrefixError`,
1678 // and — with `_SUCCESS` consistent and the manifest self-consistent —
1679 // the verdict still passes. Light certifies the manifest + marker, not
1680 // the parts.
1681 let dir = tempfile::tempdir().unwrap();
1682 let m = build_manifest(
1683 vec![
1684 part(1, 10, 4, "xxh3:1111111111111111"),
1685 part(2, 20, 5, "xxh3:2222222222222222"),
1686 ],
1687 ManifestStatus::Success,
1688 );
1689 // Deliberately write NEITHER part — only manifest.json + _SUCCESS.
1690 write_dataset(dir.path(), &m, &[]);
1691 let dest = local_dest(dir.path());
1692
1693 let v = verify_at_destination(&dest, "", ValidateDepth::Light).unwrap();
1694 assert_eq!(v.depth_level, "light");
1695 assert_eq!(
1696 v.parts_verified, 0,
1697 "light skips the listing — no part is ever verified"
1698 );
1699 assert_eq!(
1700 v.parts_failed, 0,
1701 "no part reconcile means no part failures"
1702 );
1703 assert!(
1704 !v.failures.iter().any(|f| matches!(
1705 f,
1706 Failure::PartMissing { .. } | Failure::ListPrefixError { .. }
1707 )),
1708 "light must not surface part or list failures, got: {:?}",
1709 v.failures
1710 );
1711 assert!(
1712 v.success_marker_consistent,
1713 "_SUCCESS is still checked at light depth"
1714 );
1715 assert!(v.manifest_self_consistent);
1716 assert!(
1717 v.passed,
1718 "manifest + _SUCCESS are consistent, so a light pass certifies it"
1719 );
1720 }
1721
1722 #[test]
1723 fn light_depth_never_lists_so_a_list_failure_cannot_trip() {
1724 // Even with a destination whose `list_prefix` always errors, a light
1725 // pass succeeds: it never calls `list_prefix`, so no `ListPrefixError`.
1726 // This is the direct contrast to `list_failure_cannot_certify_parts…`
1727 // (which runs at Full and *does* fail on the list error).
1728 let dir = tempfile::tempdir().unwrap();
1729 let m = build_manifest(vec![part(0, 3, 3, "xxh3:0")], ManifestStatus::Success);
1730 write_dataset(dir.path(), &m, &[("part-000000.parquet", b"abc")]);
1731 let dest = ListFails(local_dest(dir.path()));
1732
1733 let v = verify_at_destination(&dest, "", ValidateDepth::Light).unwrap();
1734 assert_eq!(v.depth_level, "light");
1735 assert!(
1736 !v.failures
1737 .iter()
1738 .any(|f| matches!(f, Failure::ListPrefixError { .. })),
1739 "light never lists, so a failing list_prefix cannot surface, got: {:?}",
1740 v.failures
1741 );
1742 assert_eq!(v.parts_verified, 0);
1743 assert!(v.passed, "manifest + _SUCCESS consistent → light passes");
1744 }
1745
1746 #[test]
1747 fn sample_depth_runs_part_reconcile_like_full() {
1748 // `Sample` runs every section `verify_at_destination` owns (1-5) — the
1749 // Form B value re-read it skips lives in the *caller*, not here. So a
1750 // missing part is a fatal `PartMissing` at Sample, identical to Full.
1751 let dir = tempfile::tempdir().unwrap();
1752 let m = build_manifest(
1753 vec![
1754 part(1, 10, 4, "xxh3:1111111111111111"),
1755 part(2, 20, 5, "xxh3:2222222222222222"),
1756 ],
1757 ManifestStatus::Success,
1758 );
1759 write_dataset(dir.path(), &m, &[("part-000001.parquet", b"AAAA")]); // part 2 missing
1760 let dest = local_dest(dir.path());
1761
1762 let v = verify_at_destination(&dest, "", ValidateDepth::Sample).unwrap();
1763 assert_eq!(v.depth_level, "sample");
1764 assert_eq!(v.parts_verified, 1);
1765 assert_eq!(v.parts_failed, 1);
1766 assert!(!v.passed);
1767 assert!(
1768 v.failures
1769 .iter()
1770 .any(|f| matches!(f, Failure::PartMissing { part_id: 2, .. })),
1771 "sample reconciles parts just like full, got: {:?}",
1772 v.failures
1773 );
1774 }
1775
1776 #[test]
1777 fn depth_level_is_stamped_on_every_verdict_shape() {
1778 // The depth label rides the verdict even on the early-return shapes
1779 // (legacy / no manifest), so a consumer always sees how deep the pass
1780 // went.
1781 let dir = tempfile::tempdir().unwrap();
1782 let dest = local_dest(dir.path()); // empty prefix → legacy
1783 for depth in [
1784 ValidateDepth::Light,
1785 ValidateDepth::Sample,
1786 ValidateDepth::Full,
1787 ] {
1788 let v = verify_at_destination(&dest, "", depth).unwrap();
1789 assert!(v.legacy_run, "empty prefix is the legacy shape");
1790 assert_eq!(
1791 v.depth_level,
1792 depth.label(),
1793 "legacy verdict must still carry its depth label"
1794 );
1795 }
1796 }
1797
1798 #[test]
1799 fn error_code_is_stable_and_distinct_per_variant() {
1800 // Each variant maps to its documented `RIVET_VERIFY_*` code. A
1801 // regression guard: renaming a code is a silent break for any CI gate
1802 // keying off it.
1803 let cases: &[(Failure, &str)] = &[
1804 (
1805 Failure::PartMissing {
1806 part_id: 1,
1807 path: "p".into(),
1808 },
1809 "RIVET_VERIFY_PART_MISSING",
1810 ),
1811 (
1812 Failure::PartSizeMismatch {
1813 part_id: 1,
1814 path: "p".into(),
1815 expected: 1,
1816 actual: 2,
1817 },
1818 "RIVET_VERIFY_PART_SIZE_MISMATCH",
1819 ),
1820 (
1821 Failure::PartChecksumMismatch {
1822 part_id: 1,
1823 path: "p".into(),
1824 expected: "a".into(),
1825 actual: "b".into(),
1826 },
1827 "RIVET_VERIFY_PART_CHECKSUM_MISMATCH",
1828 ),
1829 (
1830 Failure::SuccessMarkerMalformed {
1831 body_preview: "x".into(),
1832 },
1833 "RIVET_VERIFY_SUCCESS_MALFORMED",
1834 ),
1835 (
1836 Failure::SuccessMarkerStale {
1837 marker_fingerprint: "a".into(),
1838 manifest_fingerprint: "b".into(),
1839 },
1840 "RIVET_VERIFY_SUCCESS_STALE",
1841 ),
1842 (
1843 Failure::ManifestSelfInconsistent { detail: "d".into() },
1844 "RIVET_VERIFY_MANIFEST_INCONSISTENT",
1845 ),
1846 (
1847 Failure::ManifestReadError { detail: "d".into() },
1848 "RIVET_VERIFY_MANIFEST_READ_ERROR",
1849 ),
1850 (
1851 Failure::SuccessMarkerReadError { detail: "d".into() },
1852 "RIVET_VERIFY_SUCCESS_READ_ERROR",
1853 ),
1854 (
1855 Failure::ListPrefixError { detail: "d".into() },
1856 "RIVET_VERIFY_LIST_ERROR",
1857 ),
1858 (
1859 Failure::UntrackedObject {
1860 key: "k".into(),
1861 size_bytes: 1,
1862 },
1863 "RIVET_VERIFY_UNTRACKED_OBJECT",
1864 ),
1865 (
1866 Failure::ContentVerificationUnmet {
1867 size_only: 1,
1868 total: 2,
1869 },
1870 "RIVET_VERIFY_CONTENT_UNMET",
1871 ),
1872 (
1873 Failure::ManifestRequiredButAbsent { prefix: "p".into() },
1874 "RIVET_VERIFY_MANIFEST_REQUIRED",
1875 ),
1876 (
1877 Failure::ValueChecksumMismatch {
1878 detail: "flip".into(),
1879 },
1880 "RIVET_VERIFY_VALUE_CHECKSUM",
1881 ),
1882 (
1883 Failure::CdcPositionViolation {
1884 detail: "gap".into(),
1885 },
1886 "RIVET_VERIFY_CDC_POSITION",
1887 ),
1888 ];
1889 for (failure, code) in cases {
1890 assert_eq!(&failure.error_code(), code, "code for {failure:?}");
1891 assert!(
1892 failure.error_code().starts_with("RIVET_VERIFY_"),
1893 "every code shares the RIVET_VERIFY_ prefix"
1894 );
1895 }
1896 }
1897
1898 #[test]
1899 fn cdc_position_violation_is_verified_wrong_not_could_not_verify() {
1900 // #5 / #104 bughunt: a CDC `__pos` continuity violation (a gap/duplicate in
1901 // the exported change stream) is VERIFIED-WRONG — data-integrity, exit 3,
1902 // the same class as a value-checksum mismatch — NOT a could-not-verify I/O
1903 // error (exit 1). This pins the classification so a future edit that made it
1904 // operational (folding it back into hard_failures → exit 1, the pre-fix
1905 // behaviour) goes RED.
1906 let viol = Failure::CdcPositionViolation {
1907 detail: "export 'x': pos gap 5 -> 8".into(),
1908 };
1909 assert!(
1910 !viol.is_could_not_verify(),
1911 "a __pos violation is verified-wrong, not could-not-verify"
1912 );
1913 assert!(viol.is_fatal(), "a __pos violation fails the verdict");
1914
1915 let mut v = ManifestVerification::empty();
1916 v.passed = false;
1917 v.failures.push(viol);
1918 assert!(
1919 v.has_verified_wrong_failure(),
1920 "a verdict carrying a __pos violation must classify as verified-wrong (exit 3)"
1921 );
1922
1923 // The read-error siblings are the OTHER side of the same axis: could-not-
1924 // verify, so a verdict whose ONLY failure is one of them is NOT verified-wrong.
1925 let mut op = ManifestVerification::empty();
1926 op.passed = false;
1927 op.failures.push(Failure::ManifestReadError {
1928 detail: "permission denied".into(),
1929 });
1930 assert!(
1931 !op.has_verified_wrong_failure(),
1932 "a read error alone is could-not-verify (exit 1), not verified-wrong"
1933 );
1934 }
1935}