rivet-cli 0.21.2

Rivet: PostgreSQL/MySQL/SQL Server/MongoDB → Parquet/CSV (local, S3, GCS, Azure). Crate name rivet-cli; binary rivet.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

# Durability-ordering matrix — the destination artifact is the loader's source of
# truth (ADR-0001), so the delivery position (cursor / slot-ack) must advance ONLY
# AFTER the destination manifest is durable, and the canonical manifest.json +
# _SUCCESS must stay a consistent pair.
#
# This class regressed TWICE and both escaped the existing resilience/cdc ledgers,
# because those crash-recovery cells asserted recovery via a PARQUET GLOB — which
# sees an orphan part that no manifest references, exactly MASKING the bug:
#   * round-2 #11: the CDC slot ack advanced past parts before the run manifest was
#     written → the manifest-authoritative `rivet load` silently dropped them.
#   * round-2 #12: the incremental cursor advanced before the destination manifest
#     → the next run skipped parts no manifest recorded.
#   * round-3 #11-regression: the mid-roll write overwrote the canonical manifest.json
#     leaving _SUCCESS stale → `rivet validate` flagged fatal SuccessMarkerStale on
#     intact data.
#
# The fix in every case is asserted MANIFEST-DRIVEN (only parts a Success manifest
# declares — the loader's view), never a parquet glob. This ledger drift-guards
# that discipline (tests/offline/chunking_matrix_guard.rs).
#
# Columns are PIPELINE SHAPES, not source engines. A cell is EXACTLY one of:
#   test: <fn>  — a test guards it (the guard asserts <fn> exists in src/ or tests/).
#   na:   <why> — the invariant is structurally absent for this shape (e.g. no
#                 mid-run manifest write, or state is re-read from chunk_task).
#   gap:  <why> — a real invariant for this shape that SHOULD be tested, isn't.

engines: [incremental, chunked, keyset, cdc]

scenarios:
  - id: manifest_durable_before_state_advance
    what: "the destination manifest covering a run's parts is durable BEFORE the delivery position (cursor / slot-ack / chunk_task / keyset cursor) advances past them — or, where state advances first (chunked/keyset), a resume RECONSTRUCTS a complete manifest before finalize."
    incremental: { test: crash_after_cursor_commit_is_recoverable_via_a_manifest_driven_read }
    # Round-4 corrected a FALSE na here: chunk_task DOES advance (completed) before
    # any destination manifest, so the advance-past-unmanifested-parts window is real;
    # the fix reconstructs the committed chunks into the finalize manifest on resume.
    chunked:     { test: chunked_crash_resume_writes_a_complete_destination_manifest }
    keyset:      { test: keyset_checkpoint_crash_resume_writes_a_complete_destination_manifest }
    cdc:         { test: roast_manifest_is_durable_before_the_slot_is_acked }

  - id: manifest_driven_recovery
    what: "a crash at the state-advance boundary recovers with ZERO loss when read MANIFEST-DRIVEN (only parts a Success manifest declares, as `rivet load` reads) — the parquet-glob / file_log path masks the orphan class."
    incremental: { test: crash_after_cursor_commit_is_recoverable_via_a_manifest_driven_read }
    # Round-4: the prior na ("no mid-run orphan window") was live-disproven — a
    # chunked crash before the terminal manifest + resume orphaned the pre-crash
    # chunk from the destination manifest (silent ~33% loss). Now fixed + asserted
    # MANIFEST-DRIVEN (reads manifest.json, not file_log/glob).
    chunked:     { test: chunked_crash_resume_writes_a_complete_destination_manifest }
    # Round-5: keyset's checkpoint-resume reconstructs the pre-crash pages from the
    # file_log into the finalize manifest, same class as chunked (both advance a
    # cursor / commit parts before the terminal manifest).
    keyset:      { test: keyset_checkpoint_crash_resume_writes_a_complete_destination_manifest }
    cdc:         { test: cdc_crash_before_manifest_loses_nothing_on_a_manifest_driven_read }

  - id: canonical_success_pair_consistent
    what: "a mid-run/mid-roll manifest write never overwrites the canonical manifest.json while leaving _SUCCESS (which fingerprints it) stale — validate must not flag SuccessMarkerStale on intact data."
    incremental: { na: "the incremental path writes canonical manifest.json + _SUCCESS together exactly once, at finalize_manifest — there is no mid-run canonical write to decouple the pair" }
    chunked:     { na: "same — canonical + _SUCCESS written together only at finalize" }
    keyset:      { na: "same — keyset writes canonical + _SUCCESS together only at finalize; per-page commits write ONLY the run-unique copy" }
    cdc:         { test: mid_roll_manifest_write_does_not_touch_canonical_or_success }

  - id: run_unique_sidecar_survives_shared_prefix
    what: "repeated runs into ONE destination prefix (the until_current scheduler / rapid-incremental model) each keep their immutable run-unique manifest copy — the sidecar is never clobbered by the next run."
    incremental: { test: roast_rapid_incremental_runs_into_same_prefix_must_not_clobber_prior_parts }
    chunked:     { na: "chunked parts carry a random nonce (ADR-0009 RR5) and the run-unique manifest copy is written through the same write_manifest seam the incremental + cdc cells exercise" }
    keyset:      { na: "keyset parts carry a millisecond run-unique stamp (%3f) and the run-unique manifest copy is written through the same write_manifest seam" }
    cdc:         { test: roast_second_run_into_same_prefix_must_not_clobber_prior_manifest }

  # ── Round-7 additions: the sidecar-consistency + manifest-completeness class,
  #    caught on the repair command and the chunked M8 resume preamble ──────────
  - id: repair_updates_the_run_unique_sidecar
    what: "`rivet repair` re-exports missing parts and MUST update the run-unique manifest-<run_id>.json copy the loader reads, not just the canonical manifest.json — else the loader (manifest-authoritative, prefers the run-unique copy) reads the pre-repair part list and silently drops the repaired rows (round-7 HIGH)."
    incremental: { na: "there is no `rivet repair` for an incremental delta — a shortfall re-runs from the persisted cursor, re-exporting the missing window rather than patching a manifest" }
    chunked:     { test: repair_updates_the_run_unique_manifest_copy_not_just_the_canonical }
    keyset:      { na: "repair patches through the SAME shape-agnostic record_repair_parts_in_manifest seam the chunked cell's offline test exercises directly (it asserts the run-unique copy, not a shape-specific flow)" }
    cdc:         { na: "CDC has no repair command — at-least-once replay from the un-acked slot position re-delivers a shortfall" }

  - id: resume_never_quarantines_a_committed_part
    what: "the chunked M8 resume preamble moves 'untracked surplus' objects to _quarantine, but MUST NOT move a part the state DB's file_log records as committed (a resume that committed then crashed before re-finalizing leaves it absent from the stale manifest yet durable) — quarantining it drops the exact rows finalize rehydrates (round-7 MEDIUM silent-loss)."
    incremental: { na: "the incremental path has no M8 quarantine preamble — a single-shot cursor run has no prior-run surplus to reconcile against a stale manifest" }
    chunked:     { test: m8_does_not_quarantine_a_part_recorded_in_the_state_file_log }
    keyset:      { na: "keyset resumes via its own checkpoint (get_resume_run_id + file_log rehydration), not the chunked M8 destination-vs-listing quarantine preamble" }
    cdc:         { na: "CDC resumes by slot/checkpoint position, not a destination-object quarantine sweep" }

  # ── This session: the keyset crash-recovery cursor lifecycle. The single mid-data
  #    cell above (manifest_durable_before_state_advance) did NOT enumerate the
  #    (fault-point × config-variant) sub-cross-product — crash-BEFORE-first-page,
  #    crash-AFTER-data-complete, and the INCREMENTAL variant — and an adversarial
  #    hunt found a HIGH silent-loss in (after-data-complete × incremental). These
  #    three rows raise the keyset column's resolution to that sub-cross-product;
  #    every cell asserts MANIFEST-DRIVEN (the orphaned parquet survives on disk, so
  #    a glob masks the loss). `na` for the other shapes documents structural immunity.
  - id: keyset_fresh_crash_before_first_page_rereads_not_skips
    what: "a FRESH non-incremental keyset run that crashes after open but before its first page commit must re-read from the start on recovery, never resume from a prior completed run's stale high-water mark (whole-table silent skip)."
    incremental: { na: "the stale-cursor skip is a NON-incremental keyset hazard; batch incremental (cursor mode) has no per-run keyset anchor to inherit" }
    chunked:     { na: "chunked resume is --resume-gated over a [min,max] span recomputed each run; a plain re-run ignores the prior checkpoint, so a pre-first-chunk crash cannot inherit a stale cursor" }
    keyset:      { test: keyset_fresh_run_crash_before_first_page_does_not_skip_the_table }
    cdc:         { na: "CDC has no keyset cursor; resume is by slot/checkpoint position" }

  - id: keyset_post_data_failure_non_incremental_clears_anchor
    what: "a NON-incremental keyset run that commits all data then fails a post-data gate must NOT leave a resume anchor — the operator's intended full re-run must re-read, not resume-skip rows updated since."
    incremental: { na: "for incremental a surviving anchor is by-design (the next run continues from the high-water mark) AND must survive to finalize — see the next row" }
    chunked:     { na: "chunked clears completed-chunk state at finalize; a post-data gate failure re-runs the whole span, no cursor to strand" }
    keyset:      { test: keyset_failure_after_data_complete_does_not_resume_and_skip_on_the_next_run }
    cdc:         { na: "no keyset anchor" }

  - id: keyset_incremental_crash_before_finalize_rehydrates_not_orphans
    what: "an INCREMENTAL keyset run that commits all pages then crashes BEFORE finalize_manifest must KEEP its resume anchor so recovery rehydrates the committed pages into the manifest; clearing it early orphans them (the manifest-authoritative loader silently drops the rows — the parquet survives, so ONLY a manifest oracle catches it). Round-2-hunt HIGH regression of the crash-recovery/incremental split."
    incremental: { na: "batch incremental (cursor mode) has no keyset run_id anchor; it re-runs from the persisted cursor" }
    chunked:     { na: "chunked has no keyset_incremental variant; its anchor is per-chunk-task state cleared at finalize" }
    keyset:      { test: keyset_incremental_crash_before_finalize_rehydrates_not_orphans_the_manifest }
    cdc:         { na: "no keyset anchor" }