Skip to main content

licoup_workflow/
machine.rs

1//! The persisted run snapshot and reducer event vocabulary, reduced by the
2//! generic engine's machine.
3//!
4//! Two kinds of facts stay LicoUp-owned here:
5//!
6//! * [`RunSnapshot`] is a persisted record (`strategy_runs.snapshot_json`):
7//!   it carries the Conversation identity (`conversationId`), the Assistant
8//!   Membership a run answers to (`assistantMembershipId`) and the committed
9//!   route receipt (`routeReceipt`) — identities the generic engine groups
10//!   under its own opaque `contextId`/`runBinding` pair. The engine never
11//!   reads either shape, so the reducer carries them through sideband.
12//! * [`ReducerEvent`] is a persisted record (`strategy_run_events.event_json`)
13//!   plus the durable transition intent (`event` inside it): it keeps the
14//!   Assistant-owned failure events and the three-way `advance`/`return`/
15//!   `terminate` callback decision spelling. Shared variants translate 1:1
16//!   onto the engine's event; the Assistant variants settle locally.
17
18use anyhow::{Result, anyhow, ensure};
19use serde::{Deserialize, Serialize};
20use serde_json::Value;
21use std::collections::{BTreeMap, BTreeSet};
22
23use acativa::workflow::machine as generic;
24
25use crate::{
26    CallbackDecisionKind, CompiledWorkflow, FailureClass, FallbackReceipt, INPUT_ADAPTER_VERSION,
27    PendingCallback, StrategyRunStatus,
28};
29
30// Every shared machine fact is the engine's own type: command spellings,
31// shared-state ledgers, join receipts and the reducer delta are the same
32// bytes on either boundary.
33pub use generic::{
34    CommandKind, CommandStatus, JoinArrival, JoinArrivalReceipt, JoinLedger, JoinSatisfaction,
35    ReducerDelta, RunCommand, SharedEntry, SharedResourceState, SharedWriteReceipt,
36    StaleJoinArrival, VisitFact, binding_session_key, effect_input_for,
37};
38
39/// Whether the run's attempt lineage for one command stays inside the
40/// definition's budget. The engine's check reads shared fields only.
41pub fn lineage_within_budget(
42    snapshot: &RunSnapshot,
43    state_id: &str,
44    visit: u64,
45    slot_id: Option<&str>,
46    item_id: Option<&str>,
47    max_attempts: u8,
48) -> bool {
49    generic::lineage_within_budget(
50        &to_generic_snapshot(snapshot),
51        state_id,
52        visit,
53        slot_id,
54        item_id,
55        max_attempts,
56    )
57}
58
59/// The fallback reason a failed actor or workset command earns, when one is
60/// lawful. The engine's rule reads shared fields only.
61pub fn fallback_reason(
62    workflow: &CompiledWorkflow,
63    snapshot: &RunSnapshot,
64    command: &RunCommand,
65) -> Option<&'static str> {
66    generic::fallback_reason(workflow, &to_generic_snapshot(snapshot), command)
67}
68
69#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
70#[serde(rename_all = "camelCase")]
71pub struct RunSnapshot {
72    pub run_id: String,
73    pub definition_digest: String,
74    pub semantics_digest: String,
75    pub status: StrategyRunStatus,
76    pub sequence: u64,
77    pub input: Value,
78    pub active_states: BTreeSet<String>,
79    pub completed_states: BTreeSet<String>,
80    pub state_visits: BTreeMap<String, u64>,
81    /// Per-visit arrivals, so an old visit of a predecessor can never satisfy a
82    /// new join round.
83    pub join_arrivals: BTreeMap<String, JoinLedger>,
84    /// The run's declared causal input plan.
85    #[serde(default)]
86    pub input_plan: crate::InputPlan,
87    /// Versioned shared state. Reading it without naming a revision is not
88    /// expressible: the only readers are bindings, and a binding records one.
89    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
90    pub shared: BTreeMap<String, SharedResourceState>,
91    /// The exact binding every entered node visit received.
92    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
93    pub bindings: BTreeMap<String, crate::InputBinding>,
94    #[serde(default)]
95    pub actor_sessions: BTreeMap<String, String>,
96    #[serde(default)]
97    pub slot_ordinals: BTreeMap<String, u8>,
98    #[serde(default)]
99    pub slot_candidate_counts: BTreeMap<String, u8>,
100    #[serde(default)]
101    pub attempt_lineage: BTreeMap<String, u8>,
102    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
103    pub merge_sources: BTreeMap<String, String>,
104    #[serde(default, skip_serializing_if = "Vec::is_empty")]
105    pub fallbacks: Vec<FallbackReceipt>,
106    /// The Conversation this run answers to, when it is conversation-bound.
107    #[serde(default, skip_serializing_if = "Option::is_none")]
108    pub conversation_id: Option<String>,
109    /// The Assistant Membership this run answers to.
110    #[serde(default, skip_serializing_if = "Option::is_none")]
111    pub assistant_membership_id: Option<String>,
112    /// The committed route receipt recorded at admission.
113    #[serde(default, skip_serializing_if = "Option::is_none")]
114    pub route_receipt: Option<Value>,
115    #[serde(default, skip_serializing_if = "Option::is_none")]
116    pub cwd: Option<String>,
117    /// Callback-mode edges whose targets wait for the master agent's decision,
118    /// in the deterministic order the waits were entered. The decision event
119    /// always settles the first entry.
120    #[serde(default, skip_serializing_if = "Vec::is_empty")]
121    pub pending_callbacks: Vec<PendingCallback>,
122    pub commands: BTreeMap<String, RunCommand>,
123    #[serde(default, skip_serializing_if = "Option::is_none")]
124    pub diagnostic_code: Option<String>,
125}
126
127impl RunSnapshot {
128    pub fn empty(
129        run_id: impl Into<String>,
130        definition_digest: impl Into<String>,
131        semantics_digest: impl Into<String>,
132    ) -> Self {
133        Self {
134            run_id: run_id.into(),
135            definition_digest: definition_digest.into(),
136            semantics_digest: semantics_digest.into(),
137            status: StrategyRunStatus::Pending,
138            sequence: 0,
139            input: Value::Object(Default::default()),
140            active_states: BTreeSet::new(),
141            completed_states: BTreeSet::new(),
142            state_visits: BTreeMap::new(),
143            join_arrivals: BTreeMap::new(),
144            input_plan: crate::InputPlan::default(),
145            shared: BTreeMap::new(),
146            bindings: BTreeMap::new(),
147            actor_sessions: BTreeMap::new(),
148            slot_ordinals: BTreeMap::new(),
149            slot_candidate_counts: BTreeMap::new(),
150            attempt_lineage: BTreeMap::new(),
151            merge_sources: BTreeMap::new(),
152            fallbacks: Vec::new(),
153            conversation_id: None,
154            assistant_membership_id: None,
155            route_receipt: None,
156            cwd: None,
157            pending_callbacks: Vec::new(),
158            commands: BTreeMap::new(),
159            diagnostic_code: None,
160        }
161    }
162}
163
164#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
165#[serde(tag = "type", rename_all = "kebab-case")]
166pub enum ReducerEvent {
167    /// The run's declared causal input plan, before anything runs. A run that
168    /// never declares one keeps the safe default; a plan that declares an
169    /// adapter version this build does not project is refused, so a stored
170    /// version can never be silently read as a different one.
171    InputPlanDeclared {
172        plan: crate::InputPlan,
173    },
174    Start {
175        input: Value,
176    },
177    AuthorizationGranted {
178        semantics_digest: String,
179    },
180    AuthorizationDenied,
181    AuthorizationRevoked,
182    CommandClaimed {
183        command_id: String,
184        attempt_token: String,
185    },
186    CommandStarted {
187        command_id: String,
188        attempt_token: String,
189    },
190    CommandSucceeded {
191        command_id: String,
192        attempt_token: String,
193        output: Value,
194    },
195    CommandFailed {
196        command_id: String,
197        attempt_token: String,
198        class: FailureClass,
199        code: String,
200    },
201    /// Assistant-owned Graph effects never enter the generic implicit retry,
202    /// fallback, or failure-transition path. The originating Assistant turn
203    /// receives this durable terminal outcome and may choose what to do next.
204    AssistantEffectFailed {
205        command_id: String,
206        attempt_token: String,
207        class: FailureClass,
208        code: String,
209    },
210    /// A drive-level failure has unknown effect position, so the Assistant
211    /// Graph settles in-doubt without duplicating or guessing an effect.
212    AssistantDriveFailed {
213        code: String,
214    },
215    /// The master agent's decision for the oldest pending callback wait. The
216    /// declared callback edge is only taken (`advance`), the completed state
217    /// re-entered (`return`), or the run terminated (`terminate`) after this
218    /// run input arrives; the wait itself is durable.
219    CallbackDecision {
220        state_id: String,
221        state_visit: u64,
222        decision: CallbackDecisionKind,
223    },
224    RetryRequested {
225        command_id: String,
226    },
227    CancelRequested,
228    CancellationAcknowledged {
229        command_id: String,
230        attempt_token: String,
231    },
232    CancellationUnknown {
233        command_id: String,
234        attempt_token: String,
235    },
236    FallbackIssued {
237        failed_command_id: String,
238        next_ordinal: u8,
239        locator: Value,
240        from_value_id: String,
241        to_value_id: String,
242        reason: String,
243        attempts: u8,
244    },
245}
246
247#[derive(Clone, Debug, PartialEq)]
248pub struct ReducerOutput {
249    pub snapshot: RunSnapshot,
250    pub emitted_commands: Vec<RunCommand>,
251    pub applied: bool,
252    /// What this step changed, for a consumer that persists incrementally.
253    pub delta: ReducerDelta,
254}
255
256/// Reduce one event against one snapshot. Assistant-owned failure events
257/// settle under this crate's own rules; every shared event runs through the
258/// engine's reducer over the shared field projection, and the LicoUp-owned
259/// bindings ride the same step untouched.
260pub fn reduce(
261    workflow: &CompiledWorkflow,
262    previous: &RunSnapshot,
263    event: ReducerEvent,
264) -> Result<ReducerOutput> {
265    match event {
266        event @ (ReducerEvent::AssistantEffectFailed { .. }
267        | ReducerEvent::AssistantDriveFailed { .. }) => reduce_assistant(previous, event),
268        event => {
269            let generic = generic::reduce(
270                workflow,
271                &to_generic_snapshot(previous),
272                translate_event(event, previous),
273            )?;
274            Ok(ReducerOutput {
275                snapshot: from_generic_snapshot(generic.snapshot, previous),
276                emitted_commands: generic.emitted_commands,
277                applied: generic.applied,
278                delta: generic.delta,
279            })
280        }
281    }
282}
283
284/// The engine's snapshot view of one LicoUp run: every shared field moves
285/// across; the LicoUp-only bindings have no engine slot and stay sideband —
286/// the engine never reads its own opaque pair either.
287fn to_generic_snapshot(previous: &RunSnapshot) -> generic::RunSnapshot {
288    generic::RunSnapshot {
289        run_id: previous.run_id.clone(),
290        definition_digest: previous.definition_digest.clone(),
291        semantics_digest: previous.semantics_digest.clone(),
292        status: previous.status,
293        sequence: previous.sequence,
294        input: previous.input.clone(),
295        active_states: previous.active_states.clone(),
296        completed_states: previous.completed_states.clone(),
297        state_visits: previous.state_visits.clone(),
298        join_arrivals: previous.join_arrivals.clone(),
299        input_plan: previous.input_plan.clone(),
300        shared: previous.shared.clone(),
301        bindings: previous.bindings.clone(),
302        actor_sessions: previous.actor_sessions.clone(),
303        slot_ordinals: previous.slot_ordinals.clone(),
304        slot_candidate_counts: previous.slot_candidate_counts.clone(),
305        attempt_lineage: previous.attempt_lineage.clone(),
306        merge_sources: previous.merge_sources.clone(),
307        fallbacks: previous.fallbacks.clone(),
308        context_id: previous.conversation_id.clone(),
309        run_binding: previous.route_receipt.clone(),
310        cwd: previous.cwd.clone(),
311        pending_callbacks: previous.pending_callbacks.clone(),
312        commands: previous.commands.clone(),
313        diagnostic_code: previous.diagnostic_code.clone(),
314    }
315}
316
317/// Rebuild the persisted LicoUp snapshot from the engine's result. The engine
318/// never writes the opaque binding pair, so the LicoUp bindings return
319/// exactly as they arrived; every engine-touched field comes back with it.
320fn from_generic_snapshot(generic: generic::RunSnapshot, previous: &RunSnapshot) -> RunSnapshot {
321    RunSnapshot {
322        run_id: generic.run_id,
323        definition_digest: generic.definition_digest,
324        semantics_digest: generic.semantics_digest,
325        status: generic.status,
326        sequence: generic.sequence,
327        input: generic.input,
328        active_states: generic.active_states,
329        completed_states: generic.completed_states,
330        state_visits: generic.state_visits,
331        join_arrivals: generic.join_arrivals,
332        input_plan: generic.input_plan,
333        shared: generic.shared,
334        bindings: generic.bindings,
335        actor_sessions: generic.actor_sessions,
336        slot_ordinals: generic.slot_ordinals,
337        slot_candidate_counts: generic.slot_candidate_counts,
338        attempt_lineage: generic.attempt_lineage,
339        merge_sources: generic.merge_sources,
340        fallbacks: generic.fallbacks,
341        conversation_id: previous.conversation_id.clone(),
342        assistant_membership_id: previous.assistant_membership_id.clone(),
343        route_receipt: previous.route_receipt.clone(),
344        cwd: generic.cwd,
345        pending_callbacks: generic.pending_callbacks,
346        commands: generic.commands,
347        diagnostic_code: generic.diagnostic_code,
348    }
349}
350
351/// Map one shared event onto the engine's event. The only translated variant
352/// is the callback decision: the persisted `advance`/`return`/`terminate`
353/// spelling becomes the engine's positional settlement — `advance` names the
354/// edge's declared target, `return` names the completed state the edge left.
355/// A decision for a wait that is not the oldest pending one is stale; the
356/// engine refuses it on the pending match before reading the decision, so the
357/// positional value it never inspects can be any lawful one.
358fn translate_event(event: ReducerEvent, previous: &RunSnapshot) -> generic::ReducerEvent {
359    match event {
360        ReducerEvent::InputPlanDeclared { plan } => {
361            generic::ReducerEvent::InputPlanDeclared { plan }
362        }
363        ReducerEvent::Start { input } => generic::ReducerEvent::Start { input },
364        ReducerEvent::AuthorizationGranted { semantics_digest } => {
365            generic::ReducerEvent::AuthorizationGranted { semantics_digest }
366        }
367        ReducerEvent::AuthorizationDenied => generic::ReducerEvent::AuthorizationDenied,
368        ReducerEvent::AuthorizationRevoked => generic::ReducerEvent::AuthorizationRevoked,
369        ReducerEvent::CommandClaimed {
370            command_id,
371            attempt_token,
372        } => generic::ReducerEvent::CommandClaimed {
373            command_id,
374            attempt_token,
375        },
376        ReducerEvent::CommandStarted {
377            command_id,
378            attempt_token,
379        } => generic::ReducerEvent::CommandStarted {
380            command_id,
381            attempt_token,
382        },
383        ReducerEvent::CommandSucceeded {
384            command_id,
385            attempt_token,
386            output,
387        } => generic::ReducerEvent::CommandSucceeded {
388            command_id,
389            attempt_token,
390            output,
391        },
392        ReducerEvent::CommandFailed {
393            command_id,
394            attempt_token,
395            class,
396            code,
397        } => generic::ReducerEvent::CommandFailed {
398            command_id,
399            attempt_token,
400            class,
401            code,
402        },
403        ReducerEvent::CallbackDecision {
404            state_id,
405            state_visit,
406            decision,
407        } => {
408            let pending = previous.pending_callbacks.first().filter(|pending| {
409                pending.state_id == state_id && pending.state_visit == state_visit
410            });
411            let decision = match decision {
412                CallbackDecisionKind::Advance => pending
413                    .map(|pending| acativa::CallbackDecision::Enter(pending.target.clone()))
414                    .unwrap_or(acativa::CallbackDecision::Terminate),
415                CallbackDecisionKind::Return => pending
416                    .map(|pending| acativa::CallbackDecision::Enter(pending.state_id.clone()))
417                    .unwrap_or(acativa::CallbackDecision::Terminate),
418                CallbackDecisionKind::Terminate => acativa::CallbackDecision::Terminate,
419            };
420            generic::ReducerEvent::CallbackDecision {
421                state_id,
422                state_visit,
423                decision,
424            }
425        }
426        ReducerEvent::RetryRequested { command_id } => {
427            generic::ReducerEvent::RetryRequested { command_id }
428        }
429        ReducerEvent::CancelRequested => generic::ReducerEvent::CancelRequested,
430        ReducerEvent::CancellationAcknowledged {
431            command_id,
432            attempt_token,
433        } => generic::ReducerEvent::CancellationAcknowledged {
434            command_id,
435            attempt_token,
436        },
437        ReducerEvent::CancellationUnknown {
438            command_id,
439            attempt_token,
440        } => generic::ReducerEvent::CancellationUnknown {
441            command_id,
442            attempt_token,
443        },
444        ReducerEvent::FallbackIssued {
445            failed_command_id,
446            next_ordinal,
447            locator,
448            from_value_id,
449            to_value_id,
450            reason,
451            attempts,
452        } => generic::ReducerEvent::FallbackIssued {
453            failed_command_id,
454            next_ordinal,
455            locator,
456            from_value_id,
457            to_value_id,
458            reason,
459            attempts,
460        },
461        ReducerEvent::AssistantEffectFailed { .. } | ReducerEvent::AssistantDriveFailed { .. } => {
462            unreachable!("assistant events settle under this crate's own rules")
463        }
464    }
465}
466
467/// The reducer step for the Assistant-owned failure events. The bookkeeping —
468/// sequence, adapter gate, delta shape — mirrors the engine's `reduce`
469/// exactly; the settle rules below are this crate's, and neither arm can
470/// enqueue an automatic transition, so no drain step follows them.
471fn reduce_assistant(previous: &RunSnapshot, event: ReducerEvent) -> Result<ReducerOutput> {
472    ensure!(
473        previous.input_plan.input_adapter_version == INPUT_ADAPTER_VERSION,
474        "strategy_input_adapter_mismatch"
475    );
476    let mut snapshot = previous.clone();
477    let mut delta = ReducerDelta {
478        run_id: previous.run_id.clone(),
479        sequence: previous.sequence,
480        status_before: previous.status,
481        status_after: previous.status,
482        ..ReducerDelta::default()
483    };
484    snapshot.sequence = snapshot
485        .sequence
486        .checked_add(1)
487        .ok_or_else(|| anyhow!("strategy_event_sequence_overflow"))?;
488    delta.sequence = snapshot.sequence;
489    let mut applied = true;
490    match event {
491        ReducerEvent::AssistantEffectFailed {
492            command_id,
493            attempt_token,
494            class,
495            code,
496        } => settle_assistant_effect_failure(
497            &mut snapshot,
498            &mut delta,
499            &mut applied,
500            &command_id,
501            &attempt_token,
502            class,
503            &code,
504        )?,
505        ReducerEvent::AssistantDriveFailed { code } => {
506            settle_assistant_drive_failure(&mut snapshot, &mut delta, &mut applied, &code)?
507        }
508        _ => unreachable!("only assistant failure events arrive here"),
509    }
510    if !applied {
511        snapshot.sequence = previous.sequence;
512    }
513    delta.applied = applied;
514    delta.status_after = snapshot.status;
515    if !applied {
516        // A replayed event changes nothing, so its delta claims nothing.
517        delta.entered.clear();
518        delta.bindings.clear();
519        delta.results.clear();
520        delta.joins_satisfied.clear();
521        delta.shared_writes.clear();
522        delta.settled_commands.clear();
523        delta.emitted_commands.clear();
524    }
525    Ok(ReducerOutput {
526        snapshot,
527        emitted_commands: Vec::new(),
528        applied,
529        delta,
530    })
531}
532
533fn set_command_status(
534    snapshot: &mut RunSnapshot,
535    delta: &mut ReducerDelta,
536    command_id: &str,
537    next: CommandStatus,
538) {
539    let Some(command) = snapshot.commands.get_mut(command_id) else {
540        return;
541    };
542    if command.status == next {
543        return;
544    }
545    command.status = next;
546    if !delta
547        .settled_commands
548        .iter()
549        .any(|settled| settled == command_id)
550    {
551        delta.settled_commands.push(command_id.to_owned());
552    }
553}
554
555fn validate_failure_code(code: &str) -> Result<()> {
556    ensure!(
557        !code.is_empty()
558            && code.len() <= 96
559            && code.chars().all(|character| {
560                character.is_ascii_lowercase()
561                    || character.is_ascii_digit()
562                    || matches!(character, '-' | '_')
563            }),
564        "strategy_failure_code_invalid"
565    );
566    Ok(())
567}
568
569fn settle_assistant_effect_failure(
570    snapshot: &mut RunSnapshot,
571    delta: &mut ReducerDelta,
572    applied: &mut bool,
573    command_id: &str,
574    attempt_token: &str,
575    class: FailureClass,
576    code: &str,
577) -> Result<()> {
578    validate_failure_code(code)?;
579    let current = snapshot
580        .commands
581        .get(command_id)
582        .cloned()
583        .ok_or_else(|| anyhow!("strategy_callback_stale"))?;
584    ensure!(
585        current.attempt_token == attempt_token,
586        "strategy_callback_stale"
587    );
588    if matches!(
589        current.status,
590        CommandStatus::Failed | CommandStatus::InDoubt
591    ) && current.failure_class == Some(class)
592        && current.failure_code.as_deref() == Some(code)
593    {
594        *applied = false;
595        return Ok(());
596    }
597    ensure!(
598        matches!(
599            current.status,
600            CommandStatus::Pending | CommandStatus::Claimed | CommandStatus::Running
601        ),
602        "strategy_callback_conflict"
603    );
604    let command = snapshot
605        .commands
606        .get_mut(command_id)
607        .ok_or_else(|| anyhow!("strategy_callback_stale"))?;
608    command.failure_class = Some(class);
609    command.failure_code = Some(code.to_owned());
610    set_command_status(
611        snapshot,
612        delta,
613        command_id,
614        if class == FailureClass::InDoubt {
615            CommandStatus::InDoubt
616        } else {
617            CommandStatus::Failed
618        },
619    );
620    snapshot.pending_callbacks.clear();
621    cancel_unstarted_assistant_commands(snapshot, delta);
622    snapshot.status = if class == FailureClass::InDoubt {
623        StrategyRunStatus::CancelInDoubt
624    } else {
625        StrategyRunStatus::Failed
626    };
627    snapshot.diagnostic_code = Some(code.to_owned());
628    Ok(())
629}
630
631fn settle_assistant_drive_failure(
632    snapshot: &mut RunSnapshot,
633    delta: &mut ReducerDelta,
634    applied: &mut bool,
635    code: &str,
636) -> Result<()> {
637    validate_failure_code(code)?;
638    if matches!(
639        snapshot.status,
640        StrategyRunStatus::Completed
641            | StrategyRunStatus::Failed
642            | StrategyRunStatus::Cancelled
643            | StrategyRunStatus::Blocked
644            | StrategyRunStatus::CancelInDoubt
645            // A run parked on a callback is resting, not in flight. The
646            // drive that parked it has nothing left to prove, so a failure
647            // reported after the park cannot unmake it — and clearing the
648            // pending callbacks here would strand the master's decision.
649            | StrategyRunStatus::Waiting
650    ) {
651        *applied = false;
652        return Ok(());
653    }
654    cancel_unstarted_assistant_commands(snapshot, delta);
655    snapshot.pending_callbacks.clear();
656    let started = snapshot
657        .commands
658        .iter()
659        .filter(|(_, command)| {
660            matches!(
661                command.status,
662                CommandStatus::Claimed | CommandStatus::Running
663            )
664        })
665        .map(|(id, _)| id.clone())
666        .collect::<Vec<_>>();
667    for command_id in started {
668        if let Some(command) = snapshot.commands.get_mut(&command_id) {
669            command.failure_class = Some(FailureClass::InDoubt);
670            command.failure_code = Some(code.to_owned());
671        }
672        set_command_status(snapshot, delta, &command_id, CommandStatus::InDoubt);
673    }
674    snapshot.status = StrategyRunStatus::CancelInDoubt;
675    snapshot.diagnostic_code = Some(code.to_owned());
676    Ok(())
677}
678
679fn cancel_unstarted_assistant_commands(snapshot: &mut RunSnapshot, delta: &mut ReducerDelta) {
680    let unstarted = snapshot
681        .commands
682        .iter()
683        .filter(|(_, command)| {
684            matches!(
685                command.status,
686                CommandStatus::Pending | CommandStatus::Retryable | CommandStatus::CancelRequested
687            )
688        })
689        .map(|(id, _)| id.clone())
690        .collect::<Vec<_>>();
691    for command_id in unstarted {
692        set_command_status(snapshot, delta, &command_id, CommandStatus::Cancelled);
693    }
694}