Skip to main content

keel_cli/
replay.rs

1//! `keel replay <flow>` — journal-driven replay inspection (architecture spec
2//! FR6; dx-spec §5–6).
3//!
4//! A **dry run**: it opens `.keel/journal.db` read-only, walks the recorded
5//! step ledger, and renders exactly what re-entering the flow WOULD do —
6//! which steps substitute from the journal, which re-execute live, where the
7//! replay cursor stands, and what the nondeterminism defense will hold each
8//! step to. No effect fires and nothing is written; actual re-execution stays
9//! with `keel run` / a front-end resume.
10//!
11//! The verdict mirrors the core's semantics (`keel-core/src/flow.rs`,
12//! normative in `conformance/README.md`):
13//!
14//! - `completed` → **pure replay**: every step substitutes, no effect fires; a
15//!   step beyond the ledger is nondeterminism (KEEL-E031).
16//! - `running` / `failed` → **resume**: terminal (`ok`/`error`) records
17//!   substitute; a crashed-mid-step `running` record re-executes live; the
18//!   flow continues live past the ledger. Each resume consumes one flow-level
19//!   attempt, and a still-live lease refuses the resume (KEEL-E030).
20//! - `dead` → **refused**: never auto-resumed (KEEL-E032); inspection only.
21//!
22//! Internal `marker` rows never render as steps, but they are *read*: the
23//! reserved seq-0 counter reports how many resumes the flow has consumed, and
24//! `flow:branch:*` markers surface as recorded nondeterminism divergences with
25//! their expected/observed keys.
26//!
27//! Determinism (dx-spec §5): the `--json` twin carries only values read from
28//! the DB — timestamps are the recorded ms integers, never wall-clock — so
29//! identical journals give byte-identical output.
30
31use std::path::Path;
32
33use rusqlite::Connection;
34use serde::{Deserialize, Serialize};
35
36use crate::render::to_json;
37use crate::{Rendered, evidence, flows};
38
39/// The reserved seq-0 marker key holding the flow-level resume counter
40/// (mirrors `keel-core/src/flow.rs::ATTEMPT_KEY`).
41const ATTEMPT_KEY: &str = "flow:attempt";
42
43/// The prefix of replay-branch divergence markers journaled by the core's
44/// nondeterminism defense (`warn`/`branch` modes).
45const BRANCH_PREFIX: &str = "flow:branch:";
46
47/// The schema tag of the core's step-payload envelope
48/// (mirrors `keel-core/src/flow.rs::STEP_PAYLOAD_SCHEMA`).
49const STEP_PAYLOAD_SCHEMA: &str = "keel.step/v1";
50
51/// What a re-entry would do with one recorded step.
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53enum Action {
54    /// The recorded outcome is returned; the effect is not invoked.
55    Substitute,
56    /// A crashed-mid-step record: the step runs live on resume.
57    ReExecute,
58    /// A non-terminal record inside a completed flow's pure replay — the
59    /// replay would halt with nondeterminism (KEEL-E031) rather than run live.
60    ReplayMiss,
61    /// A dead flow: no resume happens, so no step does anything.
62    None,
63}
64
65impl Action {
66    const fn as_str(self) -> &'static str {
67        match self {
68            Self::Substitute => "substitute",
69            Self::ReExecute => "re-execute",
70            Self::ReplayMiss => "replay-miss",
71            Self::None => "none",
72        }
73    }
74}
75
76/// How a re-entry of this flow behaves as a whole.
77#[derive(Debug, Clone, Copy, PartialEq, Eq)]
78enum Mode {
79    /// Completed flow: every step substitutes; no effect fires.
80    PureReplay,
81    /// Running/failed flow: substitute the ledger, re-execute the crash point,
82    /// continue live.
83    Resume,
84    /// Dead flow: refused (KEEL-E032).
85    Refused,
86}
87
88impl Mode {
89    fn for_status(status: &str) -> Self {
90        match status {
91            "completed" => Self::PureReplay,
92            "dead" => Self::Refused,
93            // 'running' and 'failed' both resume (a failed flow is reset to
94            // running before re-leasing).
95            _ => Self::Resume,
96        }
97    }
98
99    const fn as_str(self) -> &'static str {
100        match self {
101            Self::PureReplay => "pure-replay",
102            Self::Resume => "resume",
103            Self::Refused => "refused",
104        }
105    }
106}
107
108/// One raw step row, markers included (they feed the resume counter and the
109/// divergence report but never render as steps).
110struct StepRow {
111    seq: i64,
112    step_key: String,
113    kind: String,
114    attempt: i64,
115    outcome: String,
116    payload: Option<Vec<u8>>,
117    error_class: Option<String>,
118    started_at: i64,
119    ended_at: Option<i64>,
120}
121
122impl StepRow {
123    /// Whether this record is terminal (substitutable): `ok` or `error`.
124    fn is_terminal(&self) -> bool {
125        self.outcome == "ok" || self.outcome == "error"
126    }
127
128    /// What a re-entry in `mode` does with this record.
129    fn action(&self, mode: Mode) -> Action {
130        match mode {
131            Mode::Refused => Action::None,
132            Mode::PureReplay => {
133                if self.is_terminal() {
134                    Action::Substitute
135                } else {
136                    Action::ReplayMiss
137                }
138            }
139            Mode::Resume => {
140                if self.is_terminal() {
141                    Action::Substitute
142                } else {
143                    Action::ReExecute
144                }
145            }
146        }
147    }
148}
149
150/// One rendered step of the replay plan.
151#[derive(Debug, Serialize)]
152struct PlanStep {
153    action: &'static str,
154    attempt: i64,
155    duration_ms: Option<i64>,
156    ended_at: Option<i64>,
157    error_class: Option<String>,
158    kind: String,
159    outcome: String,
160    seq: i64,
161    started_at: i64,
162    step_key: String,
163}
164
165/// A nondeterminism divergence the defense already recorded (a
166/// `flow:branch:*` marker journaled under `warn`/`branch`).
167#[derive(Debug, Serialize)]
168struct RecordedDivergence {
169    expected: Option<String>,
170    mode: Option<String>,
171    observed: Option<String>,
172    seq: i64,
173}
174
175/// The `keel replay <flow>` report — one struct so the human plan and `--json`
176/// cannot drift.
177#[derive(Debug, Serialize)]
178struct ReplayReport {
179    /// The recorded deploy fence: a resume under a different `code_hash`
180    /// downgrades the nondeterminism response `fail` → `warn` (spec §4.4).
181    code_hash: Option<String>,
182    created_at: i64,
183    divergences: Vec<RecordedDivergence>,
184    entrypoint: String,
185    flow_id: String,
186    lease_expires: Option<i64>,
187    lease_holder: Option<String>,
188    /// Where live execution would begin on a resume: the first re-executed
189    /// seq, or one past the ledger. `null` when nothing runs live
190    /// (pure replay, refused).
191    live_from_seq: Option<i64>,
192    mode: &'static str,
193    /// Resumes this flow has consumed so far (the reserved seq-0 counter;
194    /// 0 when none is recorded).
195    resumes_recorded: i64,
196    status: String,
197    steps: Vec<PlanStep>,
198    steps_reexecute: usize,
199    steps_substitute: usize,
200    updated_at: i64,
201}
202
203/// `keel replay <flow> [--step N]` for `project`. `flow` resolves like
204/// `keel trace` (exact `flow_id`, else a unique substring of id/entrypoint).
205pub fn replay(project: &Path, flow: &str, step: Option<i64>) -> Rendered {
206    let path = evidence::resolved_journal(project).path;
207    if !path.exists() {
208        return flows::soft_error(
209            "no journal yet (.keel/journal.db). Run a flow first with `keel run`.",
210        );
211    }
212    let conn = match flows::open_ro(&path) {
213        Ok(c) => c,
214        Err(e) => return flows::soft_error(&e),
215    };
216    let resolved = match flows::resolve_flow(&conn, flow) {
217        Ok(r) => r,
218        Err(e) => return flows::soft_error(&e),
219    };
220    let extras = match flow_extras(&conn, &resolved.flow_id) {
221        Ok(t) => t,
222        Err(e) => return flows::soft_error(&e),
223    };
224    let rows = match read_step_rows(&conn, &resolved.flow_id) {
225        Ok(r) => r,
226        Err(e) => return flows::soft_error(&e),
227    };
228
229    let mode = Mode::for_status(&resolved.status);
230    if let Some(seq) = step {
231        return step_detail(&resolved, mode, &rows, seq);
232    }
233
234    let report = build_report(&resolved, mode, &rows, extras);
235    let human = plan_human(&report);
236    Rendered::ok(human, to_json(&report))
237}
238
239/// The flow-row columns the trace resolver does not carry.
240struct FlowExtras {
241    code_hash: Option<String>,
242    lease_holder: Option<String>,
243    lease_expires: Option<i64>,
244}
245
246/// Read `code_hash` and the lease columns for one flow.
247fn flow_extras(conn: &Connection, flow_id: &str) -> Result<FlowExtras, String> {
248    conn.query_row(
249        "SELECT code_hash, lease_holder, lease_expires FROM flows WHERE flow_id = ?1",
250        [flow_id],
251        |row| {
252            Ok(FlowExtras {
253                code_hash: row.get(0)?,
254                lease_holder: row.get(1)?,
255                lease_expires: row.get(2)?,
256            })
257        },
258    )
259    .map_err(|e| flows::q(&e))
260}
261
262/// Read every step row (markers included) in seq order.
263fn read_step_rows(conn: &Connection, flow_id: &str) -> Result<Vec<StepRow>, String> {
264    let mut stmt = conn
265        .prepare(
266            "SELECT seq, step_key, kind, attempt, outcome, payload, error_class, \
267             started_at, ended_at FROM steps WHERE flow_id = ?1 ORDER BY seq",
268        )
269        .map_err(|e| flows::q(&e))?;
270    stmt.query_map([flow_id], |row| {
271        Ok(StepRow {
272            seq: row.get(0)?,
273            step_key: row.get(1)?,
274            kind: row.get(2)?,
275            attempt: row.get(3)?,
276            outcome: row.get(4)?,
277            payload: row.get(5)?,
278            error_class: row.get(6)?,
279            started_at: row.get(7)?,
280            ended_at: row.get(8)?,
281        })
282    })
283    .map_err(|e| flows::q(&e))?
284    .collect::<rusqlite::Result<Vec<_>>>()
285    .map_err(|e| flows::q(&e))
286}
287
288/// Assemble the full replay plan from the raw rows.
289fn build_report(
290    resolved: &flows::ResolvedFlow,
291    mode: Mode,
292    rows: &[StepRow],
293    extras: FlowExtras,
294) -> ReplayReport {
295    let mut steps = Vec::new();
296    let mut divergences = Vec::new();
297    let mut resumes_recorded = 0;
298    for row in rows {
299        if row.kind == "marker" {
300            if row.step_key == ATTEMPT_KEY {
301                resumes_recorded = row.attempt;
302            } else if row.step_key.starts_with(BRANCH_PREFIX) {
303                divergences.push(divergence_from(row));
304            }
305            continue;
306        }
307        steps.push(PlanStep {
308            action: row.action(mode).as_str(),
309            attempt: row.attempt,
310            duration_ms: row.ended_at.map(|e| e - row.started_at),
311            ended_at: row.ended_at,
312            error_class: row.error_class.clone(),
313            kind: row.kind.clone(),
314            outcome: row.outcome.clone(),
315            seq: row.seq,
316            started_at: row.started_at,
317            step_key: row.step_key.clone(),
318        });
319    }
320    let steps_substitute = steps
321        .iter()
322        .filter(|s| s.action == Action::Substitute.as_str())
323        .count();
324    let steps_reexecute = steps
325        .iter()
326        .filter(|s| s.action == Action::ReExecute.as_str())
327        .count();
328    let live_from_seq = match mode {
329        Mode::Resume => Some(
330            steps
331                .iter()
332                .find(|s| s.action == Action::ReExecute.as_str())
333                .map_or_else(|| steps.last().map_or(1, |s| s.seq + 1), |s| s.seq),
334        ),
335        Mode::PureReplay | Mode::Refused => None,
336    };
337    ReplayReport {
338        code_hash: extras.code_hash,
339        created_at: resolved.created_at,
340        divergences,
341        entrypoint: resolved.entrypoint.clone(),
342        flow_id: resolved.flow_id.clone(),
343        lease_expires: extras.lease_expires,
344        lease_holder: extras.lease_holder,
345        live_from_seq,
346        mode: mode.as_str(),
347        resumes_recorded,
348        status: resolved.status.clone(),
349        steps,
350        steps_reexecute,
351        steps_substitute,
352        updated_at: resolved.updated_at,
353    }
354}
355
356/// Decode a `flow:branch:*` marker into its expected/observed keys.
357fn divergence_from(row: &StepRow) -> RecordedDivergence {
358    let payload = row.payload.as_deref().and_then(decode_payload);
359    let field = |name: &str| -> Option<String> {
360        payload
361            .as_ref()
362            .and_then(|p| p.get(name))
363            .and_then(|v| v.as_str())
364            .map(str::to_owned)
365    };
366    RecordedDivergence {
367        expected: field("expected"),
368        mode: field("mode"),
369        observed: field("observed"),
370        seq: row.seq,
371    }
372}
373
374/// The human replay plan, derived entirely from [`ReplayReport`].
375fn plan_human(report: &ReplayReport) -> String {
376    let mut lines = vec![
377        format!(
378            "keel \u{25b8} replay {} (dry run \u{2014} no effect fires)\n",
379            report.flow_id
380        ),
381        format!("  entrypoint: {}\n", report.entrypoint),
382        format!("  status:     {}\n", report.status),
383        format!("  verdict:    {}\n", verdict_line(report)),
384    ];
385    // The deploy fence and the lease only gate a live resume; a pure replay
386    // substitutes regardless and a dead flow is refused before either check.
387    if report.mode == Mode::Resume.as_str() {
388        if let Some(hash) = &report.code_hash {
389            lines.push(format!(
390                "  code fence: {hash} \u{2014} a resume under a different deploy downgrades the \
391                 nondeterminism response fail \u{2192} warn\n"
392            ));
393        }
394        if let (Some(holder), Some(expires)) = (&report.lease_holder, report.lease_expires) {
395            lines.push(format!(
396                "  lease:      {holder} (expires at {expires}) \u{2014} a resume while this \
397                 lease is live is refused (KEEL-E030)\n"
398            ));
399        }
400    }
401    if report.resumes_recorded > 0 {
402        lines.push(format!(
403            "  resumes:    {} consumed so far (flow-level attempt counter)\n",
404            report.resumes_recorded
405        ));
406    }
407    lines.push(format!("  steps:      {}\n", report.steps.len()));
408    for s in &report.steps {
409        lines.push(format!(
410            "    {:>3}. {:<7} {:<28} {:<8} \u{2192} {}\n",
411            s.seq, s.kind, s.step_key, s.outcome, s.action,
412        ));
413    }
414    if !report.divergences.is_empty() {
415        lines.push("  recorded divergences (nondeterminism defense, KEEL-E031):\n".to_owned());
416        for d in &report.divergences {
417            lines.push(format!(
418                "    seq {}: {} \u{2014} expected {}, observed {}\n",
419                d.seq,
420                d.mode.as_deref().unwrap_or("?"),
421                d.expected.as_deref().unwrap_or("?"),
422                d.observed.as_deref().unwrap_or("?"),
423            ));
424        }
425    }
426    lines.push(cursor_line(report));
427    lines.concat()
428}
429
430/// One sentence: what a re-entry of this flow does.
431fn verdict_line(report: &ReplayReport) -> String {
432    match report.mode {
433        "pure-replay" => format!(
434            "pure replay \u{2014} all {} steps substitute from the journal; no effect fires",
435            report.steps_substitute
436        ),
437        "refused" => "refused \u{2014} dead flows are never auto-resumed (KEEL-E032)".to_owned(),
438        _ => format!(
439            "resume \u{2014} {} of {} steps substitute; {} re-execute{} live",
440            report.steps_substitute,
441            report.steps.len(),
442            report.steps_reexecute,
443            if report.steps_reexecute == 1 { "s" } else { "" },
444        ),
445    }
446}
447
448/// The closing "where the cursor stands / what to do next" line.
449fn cursor_line(report: &ReplayReport) -> String {
450    match report.mode {
451        "pure-replay" => {
452            let end = report.steps.last().map_or(0, |s| s.seq);
453            format!(
454                "  cursor:     end of ledger \u{2014} the result reconstructs entirely from the \
455                 journal; a step beyond seq {end} would be nondeterminism (KEEL-E031)\n"
456            )
457        }
458        "refused" => {
459            "  next:       inspect the poison step with `keel trace`, fix the cause, and rerun \
460             with a new flow identity; see `keel explain KEEL-E032`\n"
461                .to_owned()
462        }
463        _ => {
464            let seq = report.live_from_seq.unwrap_or(1);
465            format!(
466                "  cursor:     live execution resumes at seq {seq}; steps beyond the ledger run \
467                 live and are journaled\n"
468            )
469        }
470    }
471}
472
473/// The `--step N` detail report: one recorded step in full.
474#[derive(Debug, Serialize)]
475struct StepDetail {
476    action: &'static str,
477    attempt: i64,
478    duration_ms: Option<i64>,
479    ended_at: Option<i64>,
480    error_class: Option<String>,
481    flow_id: String,
482    kind: String,
483    mode: &'static str,
484    outcome: String,
485    /// The decoded MessagePack payload (`null` when absent or undecodable).
486    payload: Option<serde_json::Value>,
487    payload_bytes: Option<usize>,
488    seq: i64,
489    started_at: i64,
490    status: String,
491    step_key: String,
492}
493
494/// Render the `--step N` view, or a precise error when `seq` is not recorded.
495fn step_detail(resolved: &flows::ResolvedFlow, mode: Mode, rows: &[StepRow], seq: i64) -> Rendered {
496    let Some(row) = rows.iter().find(|r| r.seq == seq) else {
497        let real: Vec<i64> = rows
498            .iter()
499            .filter(|r| r.kind != "marker")
500            .map(|r| r.seq)
501            .collect();
502        let range = match (real.first(), real.last()) {
503            (Some(lo), Some(hi)) => format!("recorded steps: seq {lo}\u{2013}{hi}"),
504            _ => "no steps recorded".to_owned(),
505        };
506        return flows::soft_error(&format!(
507            "flow {} has no step at seq {seq} ({range}). Run `keel replay {}` to see the ledger.",
508            resolved.flow_id, resolved.flow_id
509        ));
510    };
511    // Markers never replay; render them with action "none" for inspection.
512    let action = if row.kind == "marker" {
513        Action::None
514    } else {
515        row.action(mode)
516    };
517    let payload = row.payload.as_deref().and_then(decode_payload);
518    let report = StepDetail {
519        action: action.as_str(),
520        attempt: row.attempt,
521        duration_ms: row.ended_at.map(|e| e - row.started_at),
522        ended_at: row.ended_at,
523        error_class: row.error_class.clone(),
524        flow_id: resolved.flow_id.clone(),
525        kind: row.kind.clone(),
526        mode: mode.as_str(),
527        outcome: row.outcome.clone(),
528        payload,
529        payload_bytes: row.payload.as_ref().map(Vec::len),
530        seq: row.seq,
531        started_at: row.started_at,
532        status: resolved.status.clone(),
533        step_key: row.step_key.clone(),
534    };
535    let human = step_human(&report);
536    Rendered::ok(human, to_json(&report))
537}
538
539/// The human `--step` view, derived entirely from [`StepDetail`].
540fn step_human(report: &StepDetail) -> String {
541    let recorded = match report.ended_at {
542        Some(end) => format!(
543            "{} \u{2192} {end} ({}ms)",
544            report.started_at,
545            report.duration_ms.unwrap_or(0)
546        ),
547        None => format!("{} \u{2192} \u{2014} (still running)", report.started_at),
548    };
549    let payload = match (&report.payload, report.payload_bytes) {
550        (Some(v), _) => v.to_string(),
551        (None, Some(n)) => format!("({n} bytes, not decodable as MessagePack)"),
552        (None, None) => "\u{2014}".to_owned(),
553    };
554    let mut lines = vec![
555        format!(
556            "keel \u{25b8} replay {} \u{2014} step {}\n",
557            report.flow_id, report.seq
558        ),
559        format!("  step_key:  {}\n", report.step_key),
560        format!("  kind:      {}\n", report.kind),
561        format!(
562            "  outcome:   {} (attempt {})\n",
563            report.outcome, report.attempt
564        ),
565        format!("  action:    {}\n", action_sentence(report.action)),
566        format!("  recorded:  {recorded}\n"),
567        format!("  payload:   {payload}\n"),
568    ];
569    if let Some(class) = &report.error_class {
570        lines.push(format!("  error:     class {class}\n"));
571    }
572    lines.concat()
573}
574
575/// Expand an action token into its one-line meaning.
576fn action_sentence(action: &str) -> String {
577    let meaning = match action {
578        "substitute" => "the recorded outcome is returned; the effect is not invoked",
579        "re-execute" => "crashed mid-step; a resume runs this step live",
580        "replay-miss" => "not terminal inside a completed flow; replay halts (KEEL-E031)",
581        _ => "internal marker or refused flow; nothing runs",
582    };
583    format!("{action} \u{2014} {meaning}")
584}
585
586/// Decode a step payload: the core's schema-tagged envelope
587/// (`{schema: "keel.step/v1", payload}`), falling back to a bare value so
588/// pre-tag journals — including the golden fixtures — still render.
589fn decode_payload(bytes: &[u8]) -> Option<serde_json::Value> {
590    #[derive(Deserialize)]
591    struct Envelope {
592        schema: String,
593        payload: serde_json::Value,
594    }
595    if let Ok(envelope) = rmp_serde::from_slice::<Envelope>(bytes)
596        && envelope.schema == STEP_PAYLOAD_SCHEMA
597    {
598        return Some(envelope.payload);
599    }
600    rmp_serde::from_slice(bytes).ok()
601}
602
603#[cfg(test)]
604mod tests {
605    use super::*;
606    use crate::{EXIT_FAILURE, EXIT_OK};
607    use std::path::PathBuf;
608
609    /// Build a project dir with `.keel/journal.db` from a golden fixture
610    /// (`conformance/fixtures/journal/<fixture>`), applied over the frozen
611    /// schema — same shape as the `flows` tests.
612    fn project_with_fixture(fixture: &str) -> (tempfile::TempDir, PathBuf) {
613        let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
614        let schema = std::fs::read_to_string(root.join("contracts/journal.sql")).unwrap();
615        let sql = std::fs::read_to_string(root.join("conformance/fixtures/journal").join(fixture))
616            .unwrap();
617        let dir = tempfile::TempDir::new().unwrap();
618        let keel = dir.path().join(".keel");
619        std::fs::create_dir_all(&keel).unwrap();
620        let conn = Connection::open(keel.join("journal.db")).unwrap();
621        conn.execute_batch(&schema).unwrap();
622        conn.execute_batch(&sql).unwrap();
623        let project = dir.path().to_path_buf();
624        (dir, project)
625    }
626
627    /// Open the fixture journal read-write for test-only extra inserts.
628    fn open_rw(project: &Path) -> Connection {
629        Connection::open(project.join(".keel/journal.db")).unwrap()
630    }
631
632    #[test]
633    fn completed_flow_is_pure_replay_all_substitute() {
634        let (_d, project) = project_with_fixture("completed-flow.sql");
635        let r = replay(&project, "01JZWY0A0000000000000001", None);
636        assert_eq!(r.exit, EXIT_OK);
637        assert_eq!(r.json["mode"], "pure-replay");
638        assert_eq!(r.json["steps_substitute"], 5);
639        assert_eq!(r.json["steps_reexecute"], 0);
640        assert!(r.json["live_from_seq"].is_null());
641        assert_eq!(r.json["code_hash"], "ch-9b2e44");
642        for s in r.json["steps"].as_array().unwrap() {
643            assert_eq!(s["action"], "substitute");
644        }
645        assert!(r.human.contains("dry run"));
646        assert!(r.human.contains("pure replay"));
647        assert!(r.human.contains("KEEL-E031"));
648    }
649
650    #[test]
651    fn interrupted_flow_resumes_at_the_crashed_step() {
652        let (_d, project) = project_with_fixture("interrupted-flow.sql");
653        let r = replay(&project, "01JZWY0A0000000000000002", None);
654        assert_eq!(r.exit, EXIT_OK);
655        assert_eq!(r.json["mode"], "resume");
656        assert_eq!(r.json["steps_substitute"], 3);
657        assert_eq!(r.json["steps_reexecute"], 1);
658        assert_eq!(r.json["live_from_seq"], 4);
659        // The crashed step is the one that re-executes.
660        let steps = r.json["steps"].as_array().unwrap();
661        assert_eq!(steps[3]["seq"], 4);
662        assert_eq!(steps[3]["action"], "re-execute");
663        assert_eq!(steps[3]["outcome"], "running");
664        // The lease columns surface verbatim (a live lease refuses a resume).
665        assert_eq!(r.json["lease_holder"], "host-a:pid-4242");
666        assert_eq!(r.json["lease_expires"], 1_783_728_030_000_i64);
667        assert!(r.human.contains("live execution resumes at seq 4"));
668        assert!(r.human.contains("KEEL-E030"));
669    }
670
671    #[test]
672    fn dead_flow_is_refused_inspection_only() {
673        let (_d, project) = project_with_fixture("dead-flow.sql");
674        let r = replay(&project, "01JZWY0A0000000000000003", None);
675        assert_eq!(r.exit, EXIT_OK, "inspection of a dead flow still succeeds");
676        assert_eq!(r.json["mode"], "refused");
677        assert_eq!(r.json["steps_substitute"], 0);
678        assert!(r.json["live_from_seq"].is_null());
679        for s in r.json["steps"].as_array().unwrap() {
680            assert_eq!(s["action"], "none");
681        }
682        assert!(r.human.contains("KEEL-E032"));
683        assert!(r.human.contains("keel explain KEEL-E032"));
684    }
685
686    #[test]
687    fn resume_past_a_complete_ledger_continues_live_after_it() {
688        // A running flow whose recorded steps are all terminal (crash landed
689        // between steps): everything substitutes, live from last seq + 1.
690        let (_d, project) = project_with_fixture("interrupted-flow.sql");
691        open_rw(&project)
692            .execute(
693                "UPDATE steps SET outcome = 'ok', ended_at = started_at + 10 \
694                 WHERE flow_id = '01JZWY0A0000000000000002' AND seq = 4",
695                [],
696            )
697            .unwrap();
698        let r = replay(&project, "01JZWY0A0000000000000002", None);
699        assert_eq!(r.json["steps_substitute"], 4);
700        assert_eq!(r.json["steps_reexecute"], 0);
701        assert_eq!(r.json["live_from_seq"], 5);
702    }
703
704    #[test]
705    fn attempt_marker_and_branch_marker_surface_without_becoming_steps() {
706        let (_d, project) = project_with_fixture("interrupted-flow.sql");
707        let conn = open_rw(&project);
708        // The reserved seq-0 resume counter: 2 resumes consumed.
709        conn.execute(
710            "INSERT INTO steps VALUES ('01JZWY0A0000000000000002', 0, 'flow:attempt', \
711             'marker', 2, 'ok', NULL, NULL, 1783728000000, 1783728000000)",
712            [],
713        )
714        .unwrap();
715        // A recorded warn-mode divergence marker with the core's envelope.
716        let payload = rmp_serde::to_vec_named(&serde_json::json!({
717            "schema": "keel.step/v1",
718            "payload": {"mode": "warn", "expected": "a#1", "observed": "b#1"},
719        }))
720        .unwrap();
721        conn.execute(
722            "INSERT INTO steps VALUES ('01JZWY0A0000000000000002', 5, 'flow:branch:warn', \
723             'marker', 0, 'ok', ?1, NULL, 1783728002000, 1783728002000)",
724            [payload],
725        )
726        .unwrap();
727        let r = replay(&project, "01JZWY0A0000000000000002", None);
728        assert_eq!(r.json["resumes_recorded"], 2);
729        // Markers are not steps: still the 4 real ones.
730        assert_eq!(r.json["steps"].as_array().unwrap().len(), 4);
731        let d = &r.json["divergences"][0];
732        assert_eq!(d["seq"], 5);
733        assert_eq!(d["mode"], "warn");
734        assert_eq!(d["expected"], "a#1");
735        assert_eq!(d["observed"], "b#1");
736        assert!(r.human.contains("recorded divergences"));
737        assert!(r.human.contains("expected a#1, observed b#1"));
738        assert!(r.human.contains("resumes:    2 consumed"));
739    }
740
741    #[test]
742    fn step_detail_decodes_the_recorded_payload() {
743        let (_d, project) = project_with_fixture("completed-flow.sql");
744        let r = replay(&project, "01JZWY0A0000000000000001", Some(1));
745        assert_eq!(r.exit, EXIT_OK);
746        assert_eq!(r.json["step_key"], "api.source.internal#q1");
747        assert_eq!(r.json["action"], "substitute");
748        assert_eq!(r.json["payload"]["rows"], 120);
749        assert_eq!(r.json["duration_ms"], 240);
750        assert!(r.human.contains("{\"rows\":120}"));
751    }
752
753    #[test]
754    fn step_detail_shows_error_class_and_null_payload() {
755        let (_d, project) = project_with_fixture("dead-flow.sql");
756        let r = replay(&project, "01JZWY0A0000000000000003", Some(2));
757        assert_eq!(r.json["outcome"], "error");
758        assert_eq!(r.json["error_class"], "http");
759        assert!(r.json["payload"].is_null());
760        assert!(r.json["payload_bytes"].is_null());
761        assert_eq!(r.json["attempt"], 5);
762        assert!(r.human.contains("class http"));
763    }
764
765    #[test]
766    fn step_detail_unknown_seq_says_what_exists() {
767        let (_d, project) = project_with_fixture("completed-flow.sql");
768        let r = replay(&project, "01JZWY0A0000000000000001", Some(9));
769        assert_eq!(r.exit, EXIT_FAILURE);
770        assert!(r.to_stderr);
771        assert!(r.human.contains("no step at seq 9"));
772        assert!(r.human.contains("seq 1\u{2013}5"));
773        assert!(r.human.contains("keel replay"));
774    }
775
776    #[test]
777    fn unknown_flow_is_a_soft_error_with_a_next_step() {
778        let (_d, project) = project_with_fixture("completed-flow.sql");
779        let r = replay(&project, "does-not-exist", None);
780        assert_eq!(r.exit, EXIT_FAILURE);
781        assert!(r.to_stderr);
782        assert!(r.human.contains("no flow matches"));
783        assert!(r.human.contains("keel flows"));
784    }
785
786    #[test]
787    fn absent_journal_nudges_toward_keel_run() {
788        let dir = tempfile::TempDir::new().unwrap();
789        let r = replay(dir.path(), "anything", None);
790        assert_eq!(r.exit, EXIT_FAILURE);
791        assert!(r.human.contains("no journal yet"));
792    }
793}