mlua-swarm 0.21.0

Swarm engine host built on mlua — long-running stateful runtime with Role/Verb gate, CapToken, 3-stage pipeline, and Middleware overlay.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
//! `RunStore` — persistence for `Run` records (one kick of a `Task`).
//!
//! Part of the issue #13 ID-hierarchy reconciliation: Blueprint -> Task ->
//! Run -> Step -> Attempt. A [`RunId`](crate::types::RunId) is minted
//! server-side each time a [`crate::store::task::TaskRecord`] is kicked; it
//! carries a lightweight trace of the steps dispatched during that kick
//! ([`StepEntry`]) for observability, plus its own outcome status
//! independent of the owning Task's coarser status. A single Task can have
//! N `Run`s over its lifetime (`list_by_task`).
//!
//! Current scope:
//!
//! - [`InMemoryRunStore`] — process-volatile default.
//! - [`SqliteRunStore`] — file-backed persistence via `rusqlite-isle`.
//!   `step_entries` is a JSON column, not normalized into its own table —
//!   this is a trace/observability artifact, not something queried
//!   relationally.
//! - Other persistent backends (Git / mini-app / …) are future carries.

use crate::blueprint::BindingDigest;
use crate::store::replay::{ReplayCursor, ReplayStore};
use crate::types::{RunId, StepId, TaskId};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use thiserror::Error;

pub mod inmemory;
pub mod sqlite;
pub use inmemory::InMemoryRunStore;
pub use sqlite::SqliteRunStore;

// ──────────────────────────────────────────────────────────────────────────
// RunStatus / StepEntry / RunRecord
// ──────────────────────────────────────────────────────────────────────────

/// Lifecycle status of a [`RunRecord`] — the outcome of one specific kick
/// of a Task.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum RunStatus {
    /// Minted, not yet dispatched.
    Pending,
    /// Steps are currently being dispatched for this Run.
    Running,
    /// The Run completed successfully.
    Done,
    /// The Run failed.
    Failed,
    /// The Run was still `Running` when the server process restarted
    /// (issue #35 ST2 boot-time recovery sweep). Terminal — in-flight
    /// `EngineState` is process-local and unrecoverable; this variant
    /// records the fact without attempting to reconstruct or resume it.
    Interrupted,
    /// A cancel request landed on the Run (via `POST /v1/runs/:id/cancel`
    /// / `mse_cancel` / `swarm_cancel`). Terminal — the current wiring
    /// records the intent + trace event; live in-flight abort of the
    /// still-dispatching flow remains a v3 carry, so a Run that reaches
    /// its Ok outcome after this marker keeps its terminal `result_ref`,
    /// but the Cancelled marker itself is observable via
    /// `swarm_status.cancel_requested` and `core.cancel_requested` on
    /// the trace stream.
    Cancelled,
}

/// One worker-reported degradation entry — a worker fell back to a
/// substitute behavior instead of failing outright (e.g. a tool call errored
/// and the worker used a cached/default value). Independent channel from
/// [`StepEntry`]/`result_ref`: degradations never flow through step OUTPUT
/// or the fold path (GH #32; sibling of the GH #34 audit sidecar — both
/// keep observational signal off the BP-chain value). Reported via `POST
/// /v1/worker/degradation`; the server injects `step_ref`/`attempt`/`at`
/// before persisting.
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct DegradationEntry {
    /// The tool (or capability) the worker attempted to use.
    pub tool: String,
    /// The error that triggered the fallback, in the worker's own words.
    pub error: String,
    /// What the worker substituted instead of failing.
    pub fallback: String,
    /// Optional free-form context from the worker.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub note: Option<String>,
    /// The Blueprint step ref (`Step.ref`) this degradation was reported
    /// under, if known. Server-injected metadata, not worker-supplied.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub step_ref: Option<String>,
    /// The attempt number this degradation was reported under, if known.
    /// Server-injected metadata, not worker-supplied.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub attempt: Option<u32>,
    /// Unix epoch seconds — when this entry was recorded. Server-injected.
    pub at: u64,
}

/// One entry in a Run's step trace — appended as the engine dispatches
/// (and finishes) each step. Purely observational: no field here is
/// consulted for flow control.
///
/// The per-step stats extension (started/completed timestamps, duration,
/// token usage, model, worker kind, variant-specific `adapter_data`) is
/// additive: every field is `Option` + `#[serde(default)]` so rows
/// written before the extension deserialize unchanged, and a dispatch
/// where no boundary reported stats still appends a valid entry. The
/// entry stays **write-once** — in-flight visibility belongs to the
/// sibling [`crate::store::trace::TraceEvent`] stream, never to
/// in-place updates here.
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct StepEntry {
    /// The step this entry traces.
    #[schemars(with = "String")]
    pub step_id: StepId,
    /// The Blueprint step ref (`Step.ref`) that was dispatched, if known.
    pub step_ref: Option<String>,
    /// Free-form status label for this step at the time the entry was
    /// recorded (e.g. `"dispatched"`, `"passed"`, `"blocked"`).
    pub status: Option<String>,
    /// Immutable Runner/Agent/Context snapshot digest used for this step.
    /// `None` for rows created before BoundAgent launch wiring.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub binding_digest: Option<BindingDigest>,
    /// Unix epoch seconds — when this entry was recorded.
    pub at: u64,
    /// The attempt number the stats below describe (the LAST attempt the
    /// dispatch ran), when a worker boundary reported one.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub attempt: Option<u32>,
    /// Unix epoch milliseconds — when the dispatcher began this step.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub started_at_ms: Option<i64>,
    /// Unix epoch milliseconds — when the step reached its outcome.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub completed_at_ms: Option<i64>,
    /// Wall-clock dispatch duration in milliseconds (dispatcher-measured,
    /// worker-kind independent).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub duration_ms: Option<u64>,
    /// Worker kind label (`"agent_block"` / `"subprocess"` / `"operator"`
    /// / …) as reported by the worker boundary.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub worker_kind: Option<String>,
    /// The model that served the attempt, when known.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub model: Option<String>,
    /// Normalized token usage, when a worker boundary reported one.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub usage: Option<crate::store::trace::TokenUsage>,
    /// Number of LLM turns the attempt ran, when reported.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub num_turns: Option<u32>,
    /// Worker-kind-specific raw payload (size-capped, engine-opaque).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub adapter_data: Option<serde_json::Value>,
}

impl StepEntry {
    /// Construct an entry with only the pre-stats fields set — the shape
    /// every pre-extension writer produced. Stats fields default to
    /// `None`; the dispatcher's fold fills them when available.
    pub fn basic(
        step_id: StepId,
        step_ref: Option<String>,
        status: Option<String>,
        binding_digest: Option<BindingDigest>,
        at: u64,
    ) -> Self {
        Self {
            step_id,
            step_ref,
            status,
            binding_digest,
            at,
            attempt: None,
            started_at_ms: None,
            completed_at_ms: None,
            duration_ms: None,
            worker_kind: None,
            model: None,
            usage: None,
            num_turns: None,
            adapter_data: None,
        }
    }

    /// Fold a boundary-reported [`crate::store::trace::WorkerStats`]
    /// into this entry (the dispatcher's outcome-time fold). `None`
    /// fields in `stats` leave the entry untouched; `adapter_data` is
    /// size-capped via [`crate::store::trace::cap_payload`].
    pub fn with_worker_stats(mut self, stats: crate::store::trace::WorkerStats) -> Self {
        self.worker_kind = stats.worker_kind.or(self.worker_kind);
        self.model = stats.model.or(self.model);
        self.usage = stats.usage.or(self.usage);
        self.num_turns = stats.num_turns.or(self.num_turns);
        self.adapter_data = stats
            .adapter_data
            .map(crate::store::trace::cap_payload)
            .or(self.adapter_data);
        self
    }
}

/// One persisted `Run` row — one kick of a [`crate::store::task::TaskRecord`].
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct RunRecord {
    /// Run identifier.
    #[schemars(with = "String")]
    pub id: RunId,
    /// The Task this Run was kicked from.
    #[schemars(with = "String")]
    pub task_id: TaskId,
    /// Current lifecycle status.
    pub status: RunStatus,
    /// Trace of dispatched steps, in append order.
    pub step_entries: Vec<StepEntry>,
    /// Worker-reported degradations, in append order (GH #32). Independent
    /// channel from [`Self::step_entries`]/[`Self::result_ref`] — see
    /// [`DegradationEntry`]'s doc for the invariant. `[]` (the default) =
    /// no degradations reported — every pre-#32 Run is unaffected.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub degradations: Vec<DegradationEntry>,
    /// Operator session id bound to this Run, if any (WS operator
    /// correlation).
    pub operator_sid: Option<String>,
    /// The Run's terminal result payload, set once by
    /// [`RunStore::set_result`]. `None` while the Run is in flight.
    #[schemars(with = "Option<serde_json::Value>")]
    pub result_ref: Option<serde_json::Value>,
    /// Opaque JSON snapshot of the launch input this Run was kicked with
    /// (blueprint / init_ctx / operator injection / ttl / …). The server
    /// serializes its own launch-input struct into this string at Run
    /// creation time so an `Interrupted` Run can be resumed under the SAME
    /// `run_id` without re-deriving the input from a since-stale request
    /// body. The store treats it as an opaque blob — the schema is owned by
    /// the caller (the server crate). `None` = no snapshot recorded (older
    /// rows predating resume support, or a caller that never opts in); such
    /// a Run cannot be resumed. Additive with `#[serde(default)]` so
    /// pre-existing serialized rows deserialize unchanged.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub input_json: Option<String>,
    /// Unix epoch seconds — creation time.
    pub created_at: u64,
    /// Unix epoch seconds — last update time.
    pub updated_at: u64,
}

/// Filter/paging parameters for [`RunStore::list`] — the `GET /v1/runs`
/// collection query. All filters AND together; results are newest-first
/// (`created_at` descending, ties broken by insertion order where the
/// backend tracks one).
#[derive(Debug, Clone, Default)]
pub struct RunListFilter {
    /// Only Runs kicked from this Task.
    pub task_id: Option<TaskId>,
    /// Only Runs currently in this status.
    pub status: Option<RunStatus>,
    /// Page size cap. `None` = no cap.
    pub limit: Option<usize>,
    /// Skip the first N matching rows (after ordering).
    pub offset: Option<usize>,
}

/// Errors surfaced by a [`RunStore`] implementation.
#[derive(Debug, Error)]
pub enum RunStoreError {
    /// No Run exists for the given id.
    #[error("run not found: {0}")]
    NotFound(RunId),

    /// `create` was called with an id that is already stored.
    #[error("run already exists: {0}")]
    Duplicate(RunId),

    /// Backend-specific failure not covered by the other variants.
    #[error("other: {0}")]
    Other(String),
}

/// The provenance of a Run snapshot's `bound_agents` array.
///
/// Persisted as the [`BOUND_AGENTS_ORIGIN_KEY`] sibling of `bound_agents`
/// inside the opaque [`RunRecord::input_json`] blob. This is Run-store
/// metadata, **not** a schema-crate Blueprint wire type: it never enters
/// [`crate::blueprint::BoundAgent`], `BoundAgentDigestInput`, or any digest
/// computation. It lives here beside [`RunContext`] — rather than in
/// `crate::service::task_launch` — because both the domain launch service
/// (which writes it) and the server crate's bindings-explain handler (which
/// reads it) consume it, and both already depend on this module; parking it
/// in the service module would force the server crate to reach into a
/// service-private type.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum SnapshotOrigin {
    /// `bound_agents` were resolved and pinned at the Run's initial launch —
    /// the binding identity is the launch-time pin.
    Launch,
    /// `bound_agents` were backfilled from the current Blueprint when a
    /// pre-binding-snapshot Run was resumed or reran. The binding identity
    /// carries no launch-time pin guarantee.
    ResumeBackfill,
}

/// JSON key the [`SnapshotOrigin`] is persisted under, beside `bound_agents`,
/// in [`RunRecord::input_json`].
pub const BOUND_AGENTS_ORIGIN_KEY: &str = "bound_agents_origin";

impl SnapshotOrigin {
    /// Read the origin marker from a decoded launch snapshot. An absent (or
    /// unparseable) [`BOUND_AGENTS_ORIGIN_KEY`] maps to
    /// [`SnapshotOrigin::ResumeBackfill`] — the safe side: a snapshot whose
    /// `bound_agents` were persisted before this marker existed cannot prove
    /// they were pinned at launch, so it must not be reported as a launch pin
    /// and (on the replay axis) must not have binding digests mixed into its
    /// replay keys. Only test artifacts hit this case in practice — the
    /// strict-binding series is unreleased, so no real snapshot predates the
    /// marker.
    pub fn from_snapshot(snapshot: &serde_json::Value) -> Self {
        snapshot
            .get(BOUND_AGENTS_ORIGIN_KEY)
            .and_then(|v| serde_json::from_value::<SnapshotOrigin>(v.clone()).ok())
            .unwrap_or(SnapshotOrigin::ResumeBackfill)
    }
}

/// GH #76 error surface: single-slot breadcrumb the dispatcher writes when a step
/// aborts the flow (currently: [`crate::core::state::DispatchOutcome::Blocked`]),
/// so the surrounding [`crate::service::task_launch::TaskLaunchService::launch`]
/// `map_err` closure can lift `failed_step` + `verdict_value` off the eval
/// boundary into the structured [`crate::service::task_launch::TaskLaunchError::FlowEval`]
/// variant. Sibling to `step_entries` (append-only per-step trace) — this
/// slot is last-write-wins because only ONE aborting step matters for the
/// eval's terminal error envelope, and flow-ir stops dispatching further
/// steps after `EvalError::DispatcherError`.
#[derive(Debug, Clone)]
pub struct LastFailure {
    /// The `StepId` (dispatch-time tid) the dispatcher assigned to the
    /// aborting step.
    pub step_id: StepId,
    /// The Blueprint `Step.ref` that dispatched the aborting step, if
    /// known (dispatcher fills this from its own `ref_` param — never `None`
    /// on the current write path, but modeled `Option` because the
    /// `LastFailure` shape is a public read surface and future breadcrumb
    /// writers may not have a ref in hand).
    pub step_ref: Option<String>,
    /// The verdict value the aborting step carried
    /// (e.g. `DispatchOutcome::Blocked(v)`'s `v`, cloned by the dispatcher
    /// before mapping the outcome to `EvalError::DispatcherError`).
    pub verdict_value: serde_json::Value,
}

/// Pairs a [`RunId`] with the [`RunStore`] used to persist its trace.
///
/// Threaded from the server entry points (`POST /v1/tasks`, `POST
/// /v1/tasks/:id/runs`) down through `TaskApplication::handle_with_run` /
/// `TaskLaunchService::launch` / `EngineDispatcher` (issue #13 run_id
/// propagation) so every step the dispatcher runs can be appended to
/// `RunRecord.step_entries` and the run's id exposed to workers via
/// `Ctx.meta.runtime["run_id"]`. Kept as a distinct type — rather than a
/// new field on `TaskApplicationInput` — so the pre-existing exhaustive
/// struct literal in `mlua-swarm-cli`'s MCP adapter (`TaskApplicationInput
/// { .. }`, no `run_ctx`) keeps compiling unchanged: callers that don't
/// care about run tracing keep calling `TaskApplication::handle` /
/// `TaskLaunchService::launch`, which pass `None` through internally.
#[derive(Clone)]
pub struct RunContext {
    /// The Run this dispatch's steps should be traced into.
    pub run_id: RunId,
    /// Where to append [`StepEntry`] rows as steps are dispatched.
    pub run_store: Arc<dyn RunStore>,
    /// Optional [`ReplayStore`] the engine will append a Ctx-snapshot +
    /// step-output row to after every completed step (see
    /// [`crate::store::replay`] for the primitive). `None` (the default)
    /// disables logging entirely — pre-replay callers keep their behavior
    /// byte-for-byte.
    pub replay_store: Option<Arc<dyn ReplayStore>>,
    /// Optional [`ReplayCursor`] the engine consults BEFORE dispatching
    /// each step. When present and the cursor has a matching row for
    /// `(step_ref, input_hash, occurrence)`, the engine returns the
    /// stored `DispatchOutcome::Pass(value)` verbatim and skips the
    /// Adapter spawn — this is the replay-hit path. `None` (the default)
    /// disables replay entirely.
    pub replay_cursor: Option<Arc<Mutex<ReplayCursor>>>,
    /// Run-pinned replay identity component, keyed by logical agent name.
    pub binding_digests: Arc<HashMap<String, BindingDigest>>,
    /// Whether this dispatch is a resume / rerun-from of an existing Run
    /// rather than an initial launch. `false` (the default) marks an initial
    /// launch. Set to `true` ONLY by the server's resume and rerun-from
    /// handlers — it is the sole, explicit signal that decides a backfilled
    /// snapshot's [`SnapshotOrigin`] (never inferred from replay-cursor or
    /// step-entry state, whose wiring is free to change).
    pub resume: bool,
    /// GH #76 error surface: shared single-slot breadcrumb the dispatcher writes when
    /// a step aborts the flow (`DispatchOutcome::Blocked` → `EvalError`).
    /// Read by the enclosing [`crate::service::task_launch::TaskLaunchService::launch`]
    /// `map_err` closure to populate the structured
    /// [`crate::service::task_launch::TaskLaunchError::FlowEval`] variant's
    /// `failed_step` / `verdict_value` fields. `None` (the default) means
    /// no aborting step was recorded — either the run succeeded, or an
    /// error path fired that does not go through the dispatcher's Blocked
    /// arm (e.g. `EvalError` raised by flow-ir itself before dispatch).
    /// Behind `std::sync::Mutex` to match the `replay_cursor` sibling
    /// (same crate-level convention — dispatcher writes are short critical
    /// sections, no `.await` held across).
    pub last_failure: Arc<Mutex<Option<LastFailure>>>,
    /// Optional [`crate::store::trace::TraceHandle`] bound to this Run —
    /// the write port for the per-Run [`crate::store::trace::TraceEvent`]
    /// stream. When present the dispatcher appends `core.*` events
    /// around every step and registers the handle with the engine
    /// (`Engine::trace_handle`) so middlewares/workers can append their
    /// own kinds. `None` (the default) disables the trace rail entirely
    /// — pre-trace callers keep their behavior byte-for-byte.
    pub trace: Option<crate::store::trace::TraceHandle>,
}

impl RunContext {
    /// Construct a `RunContext` with just the RunStore wired — the same
    /// shape all pre-replay callers use (`replay_store` / `replay_cursor`
    /// both `None`). Preserved as a convenience so a caller that never
    /// opts into replay can keep constructing `RunContext` positionally.
    pub fn new(run_id: RunId, run_store: Arc<dyn RunStore>) -> Self {
        Self {
            run_id,
            run_store,
            replay_store: None,
            replay_cursor: None,
            binding_digests: Arc::new(HashMap::new()),
            resume: false,
            last_failure: Arc::new(Mutex::new(None)),
            trace: None,
        }
    }

    /// Builder-style setter: attach a
    /// [`crate::store::trace::TraceHandle`] so the dispatcher appends
    /// `core.*` trace events around every step and exposes the handle
    /// to middlewares/workers via the engine.
    pub fn with_trace(mut self, trace: crate::store::trace::TraceHandle) -> Self {
        self.trace = Some(trace);
        self
    }

    /// GH #76 error surface: write the aborting-step breadcrumb (last-write-wins).
    /// Called by [`crate::blueprint::EngineDispatcher::dispatch`]'s Blocked
    /// arm BEFORE it maps the outcome to `EvalError::DispatcherError`.
    /// Silently succeeds if the mutex is poisoned — this is an
    /// observability breadcrumb, not a load-bearing invariant, and a
    /// poisoned mutex here must never prevent the primary abort error
    /// from propagating (same fail-open convention as the sibling
    /// `append_step_entry` warn-and-swallow at
    /// `EngineDispatcher::dispatch`).
    pub fn set_last_failure(&self, failure: LastFailure) {
        if let Ok(mut slot) = self.last_failure.lock() {
            *slot = Some(failure);
        }
    }

    /// GH #76 error surface: reconstruct a partial-ctx snapshot from the step-entry
    /// trace persisted so far — the in-tree substitute for a full
    /// `storage.snapshot()` from flow-ir (upstream carry).
    ///
    /// Shape: `{ "steps": { "<step_id>": { "step_ref": ..., "status": ...,
    /// "binding_digest": ..., "at": ... } } }` — a JSON object keyed by
    /// each dispatched `StepId` with its recorded [`StepEntry`] metadata.
    /// This is metadata-level, NOT value-level (no `StepEntry` carries the
    /// step's actual OUTPUT value; that requires upstream mlua-flow-ir
    /// support to expose `storage.snapshot()` on error). Consumers who
    /// need value-level partial ctx must wait for the upstream carry —
    /// see the FlowEval `partial_ctx` field rustdoc.
    ///
    /// Returns `Value::Null` if the store lookup fails (e.g. the row was
    /// deleted between dispatch and error surfacing) — the caller's
    /// `partial_ctx: Option<Value>` field wraps this so `Null` is
    /// distinguishable from "no snapshot attempt at all".
    pub async fn snapshot_partial_ctx(&self) -> serde_json::Value {
        let record = match self.run_store.get(&self.run_id).await {
            Ok(r) => r,
            Err(_) => return serde_json::Value::Null,
        };
        let mut steps = serde_json::Map::new();
        for entry in &record.step_entries {
            let mut fields = serde_json::Map::new();
            if let Some(ref_) = &entry.step_ref {
                fields.insert(
                    "step_ref".to_string(),
                    serde_json::Value::String(ref_.clone()),
                );
            }
            if let Some(status) = &entry.status {
                fields.insert(
                    "status".to_string(),
                    serde_json::Value::String(status.clone()),
                );
            }
            if let Some(digest) = &entry.binding_digest {
                fields.insert(
                    "binding_digest".to_string(),
                    serde_json::Value::String(digest.to_string()),
                );
            }
            fields.insert("at".to_string(), serde_json::Value::Number(entry.at.into()));
            steps.insert(entry.step_id.to_string(), serde_json::Value::Object(fields));
        }
        let mut out = serde_json::Map::new();
        out.insert("steps".to_string(), serde_json::Value::Object(steps));
        serde_json::Value::Object(out)
    }

    /// Builder-style setter: attach a [`ReplayStore`] to log every
    /// completed step's Ctx snapshot + output into.
    pub fn with_replay_store(mut self, store: Arc<dyn ReplayStore>) -> Self {
        self.replay_store = Some(store);
        self
    }

    /// Builder-style setter: attach a [`ReplayCursor`] the dispatcher
    /// consults for a hit before dispatching each step.
    pub fn with_replay_cursor(mut self, cursor: Arc<Mutex<ReplayCursor>>) -> Self {
        self.replay_cursor = Some(cursor);
        self
    }

    /// Attach immutable binding digests so replay keys distinguish the same
    /// step/input executed under different Runner/Agent/Context snapshots.
    pub fn with_binding_digests(mut self, digests: HashMap<String, BindingDigest>) -> Self {
        self.binding_digests = Arc::new(digests);
        self
    }

    /// Builder-style setter: mark this dispatch as a resume / rerun-from of
    /// an existing Run (see [`Self::resume`]). Called only by the server's
    /// resume and rerun-from handlers; every other construction site leaves
    /// the default `false` (initial launch).
    pub fn with_resume(mut self) -> Self {
        self.resume = true;
        self
    }
}

impl std::fmt::Debug for RunContext {
    // `dyn RunStore` carries no `Debug` bound (backend implementations
    // shouldn't be forced to derive it just to satisfy this struct's
    // `Debug`); render `run_store` as its `name()` instead, same idiom as
    // `WorkerInvocation`'s manual `Debug` for its `Arc<dyn OutputSink>`
    // field.
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("RunContext")
            .field("run_id", &self.run_id)
            .field("run_store", &self.run_store.name())
            .field(
                "replay_store",
                &self.replay_store.as_ref().map(|s| s.name()),
            )
            .field("replay_cursor", &self.replay_cursor.is_some())
            .field("binding_digests", &self.binding_digests.len())
            .field("resume", &self.resume)
            .field(
                "last_failure",
                &self.last_failure.lock().ok().and_then(|slot| slot.clone()),
            )
            .field("trace", &self.trace.is_some())
            .finish()
    }
}

// ──────────────────────────────────────────────────────────────────────────
// RunStore trait
// ──────────────────────────────────────────────────────────────────────────

/// Persistence interface for `Run` records — one kick of a Task, in the
/// issue #13 ID hierarchy.
#[async_trait]
pub trait RunStore: Send + Sync {
    /// Backend name — for diagnostics/logging.
    fn name(&self) -> &str;

    /// Create a new Run row. Returns `Duplicate` if `record.id` is already
    /// stored.
    async fn create(&self, record: RunRecord) -> Result<(), RunStoreError>;

    /// Fetch a Run by id.
    async fn get(&self, id: &RunId) -> Result<RunRecord, RunStoreError>;

    /// List every Run kicked from `task_id`, ascending by `created_at`
    /// (oldest kick first).
    async fn list_by_task(&self, task_id: &TaskId) -> Result<Vec<RunRecord>, RunStoreError>;

    /// Append one step-trace entry to a Run's `step_entries`, bumping
    /// `updated_at` to now.
    async fn append_step_entry(&self, id: &RunId, entry: StepEntry) -> Result<(), RunStoreError>;

    /// Append one worker-reported degradation to a Run's `degradations`
    /// (GH #32), bumping `updated_at` to now. Independent of
    /// [`Self::append_step_entry`] — degradations never flow through step
    /// OUTPUT/fold.
    async fn append_degradation(
        &self,
        id: &RunId,
        entry: DegradationEntry,
    ) -> Result<(), RunStoreError>;

    /// Update a Run's status, bumping `updated_at` to now.
    async fn update_status(&self, id: &RunId, status: RunStatus) -> Result<(), RunStoreError>;

    /// Atomically transition a Run's status from `from` to `to`, bumping
    /// `updated_at` to now — the compare-and-set primitive the resume path
    /// (`POST /v1/runs/:id/resume`) uses to guard against a double resume
    /// racing the same `Interrupted` Run into `Running` twice.
    ///
    /// Returns `Ok(true)` when a row with this `id` AND current status
    /// `from` was found and flipped to `to`; `Ok(false)` when the row's
    /// current status was not `from` (a concurrent transition already won,
    /// or the Run is absent). Never a hard error for the status-mismatch /
    /// absent case — the boolean is the caller's race signal.
    async fn try_transition(
        &self,
        id: &RunId,
        from: RunStatus,
        to: RunStatus,
    ) -> Result<bool, RunStoreError>;

    /// Set a Run's terminal `result_ref`, bumping `updated_at` to now.
    async fn set_result(
        &self,
        id: &RunId,
        result_ref: serde_json::Value,
    ) -> Result<(), RunStoreError>;

    /// Replace the opaque launch snapshot after pre-dispatch binding has
    /// enriched it (for example with immutable `bound_agents`).
    async fn set_input_json(&self, id: &RunId, input_json: String) -> Result<(), RunStoreError>;

    /// List every Run currently `Running` (issue #35 ST2 boot sweep +
    /// ST4 occupancy check reuse this). No ordering guarantee.
    async fn list_running(&self) -> Result<Vec<RunRecord>, RunStoreError>;

    /// List Runs matching `filter`, newest-first (`created_at`
    /// descending) — the `GET /v1/runs` collection read.
    async fn list(&self, filter: &RunListFilter) -> Result<Vec<RunRecord>, RunStoreError>;

    /// Delete a Run row (the `DELETE /v1/runs/:id` retention operation).
    /// The caller is responsible for pruning the sibling trace stream
    /// ([`crate::store::trace::RunTraceStore::delete_run`]) — the two
    /// stores are deliberately uncoupled at the trait level.
    async fn delete(&self, id: &RunId) -> Result<(), RunStoreError>;
}

// ──────────────────────────────────────────────────────────────────────────
// Shared inner state used by the InMemory backend.
// ──────────────────────────────────────────────────────────────────────────

#[derive(Default)]
pub(crate) struct Inner {
    /// Insertion order — used as a stable tie-break under `list_by_task()`.
    pub(crate) order: Vec<RunId>,
    pub(crate) records: HashMap<RunId, RunRecord>,
}

pub(crate) type SharedInner = Mutex<Inner>;