Skip to main content

agentplane/runtime/
executor.rs

1//! The executor: admission, step dispatch, sealing, and replay.
2//!
3//! M0 executes a single step. The DAG scheduler, plan contract, and topology
4//! checks slot in above this without changing the effect protocol below it —
5//! which is the point of putting the determinism boundary at the effect rather
6//! than at the plan.
7
8#[cfg(feature = "manifest")]
9use std::collections::HashSet;
10use std::collections::{BTreeMap, BTreeSet, HashMap};
11use std::sync::Arc;
12use std::time::Duration;
13
14use serde_json::Value;
15
16use crate::case::{CaseStore, EventStore, TaskStore, TimerStore};
17use crate::core::{
18    ArgSource, Budget, Calendar, Capability, Consumed, CorrelationKey, Delivery, Digest,
19    InboundEvent, Ledger, Outcome, Phase, PlanIR, PlanNode, PolicyBundleIdentity, RunId,
20    RuntimeError, Skill, SkillDescriptor, Spend, StepId, Tainted, WallClock,
21};
22use crate::journal::{Append, JournalStore, Record, RecordKind, ReplayCursor, StepCursor};
23use crate::runtime::BuildError;
24
25use super::ctx::{CaseContext, Mode, Pass, StepCtx};
26use super::metrics;
27use super::telemetry;
28use tracing::Instrument;
29
30#[derive(Debug, Clone)]
31pub(crate) enum CaseBinding {
32    Correlate {
33        kind: String,
34        keys: Vec<CorrelationKey>,
35    },
36    Existing(crate::core::CaseId),
37}
38
39/// What a run is bound to, beyond its target and input: who it acts for,
40/// which matter it belongs to, who asked, which declaration it was reviewed
41/// against. Handed to [`RunRequest::terms`].
42///
43/// The value a front door computes per caller — above all a chain **per run**:
44/// a door that authenticates many callers admits each one's run under the
45/// chain that caller presented. Whether the run happens at most once is not a
46/// term; it is [`RunRequest::once`], because it changes what the answer is.
47///
48/// **Authority is chosen once, by constructor**: [`default`](Self::default)
49/// acts as the plane, [`acting_as`](Self::acting_as) under a caller's chain,
50/// [`served`](Self::served) for a served caller that may have presented none.
51///
52/// ```no_run
53/// # use agentplane::runtime::RunTerms;
54/// # fn f(chain: agentplane::core::Delegation, keys: &[agentplane::core::CorrelationKey]) {
55/// let terms = RunTerms::acting_as(chain).correlated("order", keys);
56/// # }
57/// ```
58///
59/// ```compile_fail
60/// # use agentplane::runtime::RunTerms;
61/// # fn f(chain: agentplane::core::Delegation) {
62/// let terms = RunTerms::served(None).acting_as(chain);
63/// # }
64/// ```
65#[derive(Debug, Clone, Default)]
66pub struct RunTerms {
67    /// Which matter this run belongs to, if any.
68    case: Option<CaseBinding>,
69    /// The admission key to claim. Travels to the `RunAdmitted` record, because
70    /// that is where the store claims it — see
71    /// [`JournalStore::admitted_as`](crate::journal::JournalStore::admitted_as).
72    idempotency_key: Option<String>,
73    /// On whose authority this run acts.
74    acting_as: Acting,
75    /// Who asked for this run, when a person or peer did. Journaled on
76    /// `RunAdmitted`, and barred from deciding the run's own tasks.
77    admitted_by: Option<String>,
78    /// The declaration revision the caller reviewed, when it pinned one.
79    expected_declaration: Option<Digest>,
80    /// Data subjects the embedder names for this run, beside any its
81    /// declaration binds.
82    subjects: Vec<String>,
83}
84
85/// On whose authority a run acts.
86///
87/// Three answers, not two, because "no chain was presented" means different
88/// things at different doors. An in-process embedder that names no chain is
89/// the plane itself, and acts under the plane's
90/// ([`RuntimeBuilder::acting_as`]). A served caller whose credential carried
91/// none is **somebody else** — and falling back to the plane's chain for them
92/// would make the operator's authority an ambient credential every
93/// authenticated peer holds.
94///
95/// Acting under a chain and being bounded by one are separate questions. A
96/// served caller with no chain acts under none, and is still bounded by the
97/// plane's: that chain is the most this plane may do for anybody, so its
98/// scope and its admissibility gate the plan
99/// ([`ceiling`](Self::ceiling)) while nothing of it reaches the policy
100/// context, the journal or a withdrawal lookup.
101#[derive(Debug, Clone, Default)]
102enum Acting {
103    /// The plane's own chain, if it has one: a run the embedder started.
104    #[default]
105    Plane,
106    /// The chain an authenticated caller presented.
107    Caller(crate::core::Delegation),
108    /// A served caller that presented no chain: this run acts under none.
109    Nobody,
110    /// A chain delegated from the plane's own, for a run the plane's run
111    /// commissioned: it acts under that chain, and as the plane, so a peer
112    /// it calls is presented the plane's credential rather than one naming
113    /// the plane's owner as if a person had asked.
114    PlaneDelegate(crate::core::Delegation),
115}
116
117impl Acting {
118    /// The chain this run acts under, given the plane's own.
119    const fn resolve<'a>(
120        &'a self,
121        plane: Option<&'a crate::core::Delegation>,
122    ) -> Option<&'a crate::core::Delegation> {
123        match self {
124            Self::Plane => plane,
125            Self::Caller(chain) | Self::PlaneDelegate(chain) => Some(chain),
126            Self::Nobody => None,
127        }
128    }
129
130    /// The chain that bounds what this run may be admitted for: the one it
131    /// acts under, or — for a served caller that presented none — the
132    /// plane's, which it does not act under.
133    const fn ceiling<'a>(
134        &'a self,
135        plane: Option<&'a crate::core::Delegation>,
136    ) -> Option<&'a crate::core::Delegation> {
137        match self {
138            Self::Plane | Self::Nobody => plane,
139            Self::Caller(chain) | Self::PlaneDelegate(chain) => Some(chain),
140        }
141    }
142}
143
144impl RunTerms {
145    /// Run inside a case that already exists.
146    #[must_use]
147    pub fn in_case(mut self, case: crate::core::CaseId) -> Self {
148        self.case = Some(CaseBinding::Existing(case));
149        self
150    }
151
152    /// Join or open a case by business key before running.
153    #[must_use]
154    pub fn correlated(mut self, kind: &str, keys: &[CorrelationKey]) -> Self {
155        self.case = Some(CaseBinding::Correlate {
156            kind: kind.to_owned(),
157            keys: keys.to_vec(),
158        });
159        self
160    }
161
162    /// Act under this chain rather than the plane's own.
163    ///
164    /// Checked at admission exactly as the plane's chain is: the plan must be
165    /// inside its scope, and [`Delegation::admissible`] must accept this
166    /// plane at this instant. Journaled as the run's `IdentityBound`, so
167    /// "on whose behalf" names the caller and not the plane.
168    ///
169    /// [`Delegation::admissible`]: crate::core::Delegation::admissible
170    #[must_use]
171    pub fn acting_as(chain: crate::core::Delegation) -> Self {
172        Self {
173            acting_as: Acting::Caller(chain),
174            ..Self::default()
175        }
176    }
177
178    /// Act under `chain` as the plane: a commission from a run the plane
179    /// itself admitted, whose sub-run is the plane's work one link down.
180    pub(crate) fn acting_as_plane(mut self, chain: crate::core::Delegation) -> Self {
181        self.acting_as = Acting::PlaneDelegate(chain);
182        self
183    }
184
185    /// Act for a served caller: under the chain its credential carried, or
186    /// under **none** — never the plane's.
187    ///
188    /// The door every surface that admits work for somebody else goes through.
189    /// [`acting_as`](Self::acting_as) alone cannot say "this caller presented
190    /// nothing", and leaving it unset means the plane's own chain.
191    #[must_use]
192    pub fn served(chain: Option<crate::core::Delegation>) -> Self {
193        Self {
194            acting_as: chain.map_or(Acting::Nobody, Acting::Caller),
195            ..Self::default()
196        }
197    }
198
199    /// Record who asked for this run: the authenticated caller a served
200    /// surface admitted it for, or the operator an embedder started it on
201    /// behalf of.
202    ///
203    /// Journaled as `RunAdmitted.admitted_by`, and excluded from deciding
204    /// every task the run opens — the four-eyes control, applied to the one
205    /// actor a declarative agent's approvals could not otherwise name. A
206    /// caller who could approve the call their own request produced would be
207    /// one person on both sides of the review.
208    #[must_use]
209    pub fn admitted_by(mut self, actor: impl Into<String>) -> Self {
210        self.admitted_by = Some(actor.into());
211        self
212    }
213
214    /// Attribute what this run takes in to a data subject.
215    ///
216    /// Recorded in the run's `DataSubjectBound` after any subjects its
217    /// declaration binds, so `agentplane subject` follows the run's input,
218    /// events and case state to the effects they reached. Attribution only:
219    /// no gate reads it.
220    #[must_use]
221    pub fn subject(mut self, subject: impl Into<String>) -> Self {
222        self.subjects.push(subject.into());
223        self
224    }
225
226    /// Admit only if this declaration revision governs the capability.
227    ///
228    /// For a caller that reviewed one revision and schedules runs against a
229    /// plane it does not control: another revision — or no declaration at
230    /// all, including on a build without the `manifest` feature — is refused
231    /// as [`RuntimeError::DeclarationPinMismatch`] before authorization, and
232    /// nothing is recorded.
233    #[must_use]
234    pub const fn expect_declaration(mut self, digest: Digest) -> Self {
235        self.expected_declaration = Some(digest);
236        self
237    }
238
239    fn keyed(mut self, key: &str) -> Self {
240        self.idempotency_key = Some(key.to_owned());
241        self
242    }
243}
244
245/// A commissioned sub-run this process is executing: the run that started it,
246/// and the task running it.
247type Working = (
248    RunId,
249    tokio::task::JoinHandle<Result<RunOutcome, RuntimeError>>,
250);
251
252/// Default lease duration. A crashed owner's runs become claimable this long
253/// after its last heartbeat.
254///
255/// This bounds how long a *dead* owner's runs are stranded, not how long a run
256/// may take: a live run renews while it executes. Set per plane with
257/// [`RuntimeBuilder::lease_ttl`].
258pub const LEASE_TTL: Duration = Duration::from_secs(30);
259
260/// The shortest lease a live run can actually hold.
261///
262/// Both stores keep expiry in whole seconds and lapse on `expires_at <= now`, so
263/// a one-second lease is expired for part of every second it exists. Two is the
264/// smallest value a renewal can stay ahead of.
265pub const MIN_LEASE_TTL: Duration = Duration::from_secs(2);
266
267/// What a run produced.
268#[derive(Debug, Clone)]
269pub struct RunOutcome {
270    pub run_id: RunId,
271    pub status: RunStatus,
272    /// What this run consumed — steps, effects, spend, elapsed, refusals.
273    ///
274    /// Reported rather than left in the ledger because "what did the settlement
275    /// run cost" has to be answerable per item, and a batch sums its items. A
276    /// figure that only exists inside a dropped `Ledger` is a figure nobody can
277    /// bill against.
278    ///
279    /// The whole tally rather than only the money: the counts are what a
280    /// ceiling is sized against, and an operator raising `max_effects` after a
281    /// run stopped short is asking exactly the question this answers.
282    pub consumed: Consumed,
283    /// Terminal hash of the run's chain — what a signature would cover.
284    pub chain_head: Digest,
285    /// What the run produced, **with its label**.
286    ///
287    /// Labelled rather than bare, and the difference is not cosmetic: a
288    /// caller acting on a run's answer needs to know whether a model, a peer
289    /// or a person wrote it. The label was stripped here until an A2A reply
290    /// projection read a marker key out of an untrusted answer and let a
291    /// remote peer choose the envelope its own reply arrived in — a
292    /// confused-deputy reachable because the one fact that would have refused
293    /// it had been dropped at the boundary.
294    pub output: Option<Tainted<Value>>,
295    /// The effect a failed run ended on — its kind, disposition, permanence
296    /// and the peer's code — read from the run's records by
297    /// [`failed_effect`](crate::journal::failed_effect), so a replay answers
298    /// as the live run did. `None` unless the run failed on an effect.
299    pub failed_effect: Option<crate::journal::FailedEffect>,
300}
301
302impl RunOutcome {
303    /// What this run spent, in metered units — the money half of
304    /// [`consumed`](Self::consumed).
305    #[must_use]
306    pub const fn spend(&self) -> Spend {
307        self.consumed.spend
308    }
309
310    /// Why this run ended — [`RunStatus::reason`] without the match.
311    ///
312    /// `None` for a success. The accessor is here as well as on the status
313    /// because this is the type an embedder holds after `run(..)`, and a
314    /// summary field that silently stays empty is the shape this exists to
315    /// prevent.
316    #[must_use]
317    pub fn reason(&self) -> Option<std::borrow::Cow<'_, str>> {
318        self.status.reason()
319    }
320
321    /// The output if the run succeeded, or a [`RunFailure`] saying why not.
322    ///
323    /// Every caller that acts on a run's answer asks the same two questions —
324    /// did it work, and what did it say — and pattern-matching
325    /// [`status`](Self::status) to learn the first is how the second gets read
326    /// off a run that quarantined. This folds the pair into one `?`:
327    ///
328    /// ```ignore
329    /// let answer = runtime.start("greet", input).run().await?.success()?;
330    /// ```
331    ///
332    /// `status` stays public for the callers whose branches genuinely differ —
333    /// a suspended run is not a failed one, and an operator surface treats
334    /// them differently. This is for the caller to whom anything short of an
335    /// answer is an error.
336    ///
337    /// # Errors
338    ///
339    /// [`RunFailure`] for every non-succeeded status, carrying the run id and
340    /// the status itself.
341    pub fn success(self) -> Result<Tainted<Value>, RunFailure> {
342        match self.status {
343            // A succeeded run's answer; `Null` for the degenerate run that
344            // finished without producing one, which is still a success.
345            RunStatus::Succeeded => {
346                Ok(self.output.unwrap_or_else(|| Tainted::trusted(Value::Null)))
347            }
348            status => Err(RunFailure {
349                run_id: self.run_id,
350                status,
351            }),
352        }
353    }
354}
355
356/// What an idempotent admission did.
357///
358/// A duplicate comes back as an answer rather than an error: a caller that
359/// retried wants the original run, not a conflict it has to interpret.
360///
361/// Exhaustive, unlike the error types: a key is unused, or held by a run that
362/// has rested, or held by one that has not. There is no fourth state to leave
363/// room for, and `#[non_exhaustive]` would cost every caller a `_` arm that
364/// means nothing.
365#[derive(Debug, Clone)]
366pub enum Admission {
367    /// The key was unused. This run is the one it admitted.
368    Fresh(RunOutcome),
369    /// The key already admitted a run that has reached a resting point —
370    /// concluded, or suspended waiting for something. Its answer, again.
371    ///
372    /// A suspension counts, and that is the case this exists for: a run parked
373    /// on a four-eyes decision has already opened the task, so a redelivery
374    /// that admitted would put a second identical approval in front of a
375    /// reviewer.
376    ///
377    /// Carries no `output`. A step's result is reconstructed by replay rather
378    /// than stored, so what the journal answers is *what happened* — status,
379    /// reason, chain head. Re-derive the value with [`replay`](Runtime::replay)
380    /// under [`Mode::Strict`], which performs no external effect.
381    Replayed(RunOutcome),
382    /// The key already admitted a run that has not rested yet — still
383    /// executing, or abandoned and awaiting the recovery sweep.
384    ///
385    /// The honest answer to an emitter is *accepted, already in progress* — a
386    /// 200 or 202 naming this run, never a retry-provoking failure.
387    InFlight(RunId),
388}
389
390impl Admission {
391    /// The run this key admitted, whichever of the three this is.
392    #[must_use]
393    pub const fn run_id(&self) -> RunId {
394        match self {
395            Self::Fresh(outcome) | Self::Replayed(outcome) => outcome.run_id,
396            Self::InFlight(run) => *run,
397        }
398    }
399
400    /// Whether this call is the one that admitted the run.
401    #[must_use]
402    pub const fn is_fresh(&self) -> bool {
403        matches!(self, Self::Fresh(_))
404    }
405
406    /// What the run produced, if it has produced anything yet.
407    #[must_use]
408    pub const fn outcome(&self) -> Option<&RunOutcome> {
409        match self {
410            Self::Fresh(outcome) | Self::Replayed(outcome) => Some(outcome),
411            Self::InFlight(_) => None,
412        }
413    }
414}
415
416/// What a keyed [`spawn`](RunRequest::spawn) did.
417///
418/// Separate from [`Admission`] because a spawn returns before the work happens,
419/// so it has no outcome to report.
420#[derive(Debug, Clone, Copy, PartialEq, Eq)]
421pub struct Spawned {
422    /// The run holding the key: the one just started, or the one that already
423    /// held it.
424    pub run: RunId,
425    /// False when the key was already held, so nothing new was started.
426    pub fresh: bool,
427}
428
429/// What a run starts from.
430#[derive(Debug, Clone)]
431pub enum Target {
432    /// A capability a registered skill answers — the degenerate one-node plan.
433    Capability(String),
434    /// A plan, validated and frozen before its first step: one that would
435    /// fail at step seven must not begin at step one.
436    Plan(PlanIR),
437}
438
439impl From<&str> for Target {
440    fn from(capability: &str) -> Self {
441        Self::Capability(capability.to_owned())
442    }
443}
444
445impl From<String> for Target {
446    fn from(capability: String) -> Self {
447        Self::Capability(capability)
448    }
449}
450
451impl From<&String> for Target {
452    fn from(capability: &String) -> Self {
453        Self::Capability(capability.clone())
454    }
455}
456
457impl From<PlanIR> for Target {
458    fn from(plan: PlanIR) -> Self {
459        Self::Plan(plan)
460    }
461}
462
463/// A request with no admission key: answered by what the run did.
464#[derive(Debug, Clone, Copy)]
465pub struct Unkeyed;
466
467/// A request under an admission key: answered by what the key admitted.
468#[derive(Debug, Clone)]
469pub struct Keyed(String);
470
471/// A run about to start — see [`Runtime::start`].
472///
473/// The answer's type follows the key. Unkeyed, a run answers with its
474/// [`RunOutcome`] and a spawn with its [`RunId`]; under
475/// [`once`](Self::once), with an [`Admission`] or a [`Spawned`], because a
476/// key already held is answered by the run that holds it, and a caller must
477/// say what it does then.
478///
479/// ```compile_fail
480/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
481/// let _: agentplane::runtime::RunOutcome = rt.start("c", input).once("k").run().await.unwrap();
482/// # }
483/// ```
484///
485/// ```compile_fail
486/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
487/// rt.start("c", input).once("a").once("b");
488/// # }
489/// ```
490///
491/// ```no_run
492/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
493/// let _: agentplane::runtime::RunOutcome = rt.start("c", input.clone()).run().await.unwrap();
494/// let _: agentplane::runtime::Admission = rt.start("c", input).once("k").run().await.unwrap();
495/// # }
496/// ```
497#[must_use = "a run request does nothing until it is run, spawned or settled"]
498#[derive(Debug)]
499pub struct RunRequest<'a, K> {
500    runtime: &'a Runtime,
501    target: Target,
502    input: Tainted<Value>,
503    terms: RunTerms,
504    key: K,
505}
506
507impl<K> RunRequest<'_, K> {
508    /// Bind the run: a case, a caller's chain, who asked, a pinned
509    /// declaration. A second call replaces the first, as one value.
510    pub fn terms(mut self, terms: RunTerms) -> Self {
511        self.terms = terms;
512        self
513    }
514}
515
516impl<'a> RunRequest<'a, Unkeyed> {
517    /// Admit at most once per tenant under this key.
518    ///
519    /// The door for an at-least-once emitter: a bus that retries until it sees
520    /// a 2xx, a webhook fan-out, a queue with redelivery. The store claims the
521    /// key in the same transaction that writes the run's first record, so a
522    /// second request under it cannot admit a second run — not on a retry, not
523    /// across a restart, not from another instance sharing the store.
524    ///
525    /// **The key must carry its producer.** An id is unique only within one
526    /// emitter, so a bare message id lets two counterparties swallow each
527    /// other's messages as apparent retries.
528    /// [`InboundEvent::dedup_key`](crate::core::InboundEvent::dedup_key) is the
529    /// ready-made spelling.
530    ///
531    /// Not the event buffer: an event nobody subscribes to is dead-lettered by
532    /// [`sweep`](Runtime::sweep), so using the buffer as a ledger would spend
533    /// that signal to buy deduplication.
534    pub fn once(self, key: impl Into<String>) -> RunRequest<'a, Keyed> {
535        RunRequest {
536            runtime: self.runtime,
537            target: self.target,
538            input: self.input,
539            terms: self.terms,
540            key: Keyed(key.into()),
541        }
542    }
543
544    /// Run, and wait until it rests.
545    ///
546    /// # Errors
547    ///
548    /// [`RuntimeError::NoProvider`] when no skill provides the capability, and
549    /// whatever admission refuses — policy, quota, a halted tenant, an
550    /// instance draining, a chain this plane may not act under.
551    pub async fn run(self) -> Result<RunOutcome, RuntimeError> {
552        self.runtime
553            .admit_target(self.target, self.input, self.terms)
554            .await
555    }
556
557    /// Admit, and let the run proceed in the background, returning its id.
558    ///
559    /// **Admission happens before this returns.** The policy gate, the lease
560    /// and the admission records are written first, so a refusal is an error
561    /// here and not a task that never appears, and the id can be read at once.
562    /// The run is durable: a process that dies mid-flight leaves a journal
563    /// another instance resumes.
564    ///
565    /// # Panics
566    ///
567    /// Outside a Tokio runtime, as the rest of this crate's timing does.
568    ///
569    /// # Errors
570    ///
571    /// As [`run`](Self::run).
572    pub async fn spawn(self) -> Result<RunId, RuntimeError> {
573        self.runtime
574            .owned()
575            .spawn_target(self.target, self.input, self.terms)
576            .await
577    }
578}
579
580impl RunRequest<'_, Keyed> {
581    /// Run under the key. A key already held comes back as
582    /// [`Admission::Replayed`] or [`Admission::InFlight`], not an error.
583    ///
584    /// # Errors
585    ///
586    /// As [`RunRequest::run`], and [`RuntimeError::PlanContract`] for an empty
587    /// key or one over [`MAX_ADMISSION_KEY_BYTES`].
588    pub async fn run(self) -> Result<Admission, RuntimeError> {
589        let runtime = self.runtime;
590        let terms = self.terms.keyed(&self.key.0);
591        // Boxed: admission is the largest future in the crate, and the claim
592        // would otherwise carry it inline into every keyed caller's future.
593        let admit = Box::pin(runtime.admit_target(self.target, self.input, terms));
594        match runtime.claimed(&self.key.0, admit).await? {
595            Claimed::Fresh(outcome) => Ok(Admission::Fresh(outcome)),
596            Claimed::Held(run) => runtime.answer_with(run).await,
597        }
598    }
599
600    /// Spawn under the key. A key already held names its run, with `fresh`
601    /// false.
602    ///
603    /// # Panics
604    ///
605    /// Outside a Tokio runtime.
606    ///
607    /// # Errors
608    ///
609    /// As [`run`](Self::run).
610    pub async fn spawn(self) -> Result<Spawned, RuntimeError> {
611        let runtime = self.runtime.owned();
612        let terms = self.terms.keyed(&self.key.0);
613        let admit = Box::pin(runtime.spawn_target(self.target, self.input, terms));
614        Ok(match runtime.claimed(&self.key.0, admit).await? {
615            Claimed::Fresh(run) => Spawned { run, fresh: true },
616            Claimed::Held(run) => Spawned { run, fresh: false },
617        })
618    }
619
620    /// Run under the key until the run it admitted rests, whoever admitted it,
621    /// and answer with its outcome.
622    ///
623    /// The shape a process that may have been killed mid-run wants on restart:
624    /// the same request again. An unused key runs. A key whose run has rested
625    /// answers with that run, its value re-derived by strict replay, which
626    /// performs nothing. A key whose run is still working is resumed when
627    /// nobody holds its lease — its holder has gone — and otherwise followed
628    /// on the journal until it rests or its lease lapses.
629    ///
630    /// # Errors
631    ///
632    /// As [`run`](Self::run), and whatever resuming or replaying the held run
633    /// refuses.
634    pub async fn settle(self) -> Result<RunOutcome, RuntimeError> {
635        let runtime = self.runtime;
636        match self.run().await? {
637            Admission::Fresh(outcome) => Ok(outcome),
638            Admission::Replayed(rested) => runtime.with_output(rested).await,
639            Admission::InFlight(run) => runtime.settled(run).await,
640        }
641    }
642}
643
644/// What a keyed claim found.
645enum Claimed<T> {
646    /// The key was unused, and this admitted.
647    Fresh(T),
648    /// The key already admitted this run.
649    Held(RunId),
650}
651
652/// The stores a full plane runs on, as handles rather than as one type.
653///
654/// [`Runtime::builder_on`] is the shorthand for one backend implementing all
655/// of them, and is the wrong shape for a caller that picks its backend at **run
656/// time** — an operator naming either a file or a connection string holds
657/// `Arc<dyn JournalStore>`, and unsizing a generic parameter needs it `Sized`,
658/// so the wiring is a value that `builder_on` builds.
659///
660/// Here rather than in [`store`](crate::store) because that module is the
661/// backends, gated on `redb` or `postgres`, while this exists whether or not
662/// either is compiled in.
663#[derive(Clone)]
664pub struct Stores {
665    pub journal: std::sync::Arc<dyn crate::journal::JournalStore>,
666    pub cases: std::sync::Arc<dyn crate::case::CaseStore>,
667    pub tasks: std::sync::Arc<dyn crate::case::TaskStore>,
668    pub events: std::sync::Arc<dyn crate::case::EventStore>,
669    pub timers: std::sync::Arc<dyn crate::case::TimerStore>,
670    pub memory: std::sync::Arc<dyn crate::memory::MemoryStore>,
671    /// Where halts, subject withdrawals, rate windows and tenant ceilings live.
672    pub quotas: std::sync::Arc<dyn crate::quota::QuotaStore>,
673    /// Where disclosures of a matter are recorded.
674    pub disclosures: std::sync::Arc<dyn crate::disclosure::DisclosureRegister>,
675}
676
677impl std::fmt::Debug for Stores {
678    /// The handles are trait objects with no identity to print, so what a reader
679    /// wants here is *that the set is complete*, which it is by construction.
680    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
681        f.write_str("Stores { journal, cases, tasks, events, timers, memory, quotas, disclosures }")
682    }
683}
684
685impl Stores {
686    /// Every store from one backend that implements all of them.
687    #[must_use]
688    pub fn on<B: FullBackend + 'static>(store: std::sync::Arc<B>) -> Self {
689        Self {
690            journal: std::sync::Arc::clone(&store) as _,
691            cases: std::sync::Arc::clone(&store) as _,
692            tasks: std::sync::Arc::clone(&store) as _,
693            events: std::sync::Arc::clone(&store) as _,
694            timers: std::sync::Arc::clone(&store) as _,
695            memory: std::sync::Arc::clone(&store) as _,
696            quotas: std::sync::Arc::clone(&store) as _,
697            disclosures: store as _,
698        }
699    }
700}
701
702/// How far into a run's history [`Runtime::running_runs`] reads to attribute it.
703///
704/// `RunAdmitted` is seq 1 and `IdentityBound` follows it when the caller
705/// presented a chain, with the quota pass and the frozen plan between them — so
706/// a handful of records carries both and a long-running agent's history is not
707/// scanned to answer *who is this*.
708const ATTRIBUTION_RECORDS: usize = 8;
709
710/// One run this tenant currently holds an admission slot for.
711///
712/// What [`Runtime::running_runs`] answers with. The ids alone would be a listing
713/// an operator reads out; the attribution is what makes it one they can act on.
714#[derive(Debug, Clone)]
715pub struct LiveRun {
716    pub run: RunId,
717    /// The declared agent governing it, when a declared one does.
718    ///
719    /// `None` for a skill registered directly on the plane — a legitimate shape,
720    /// and one an incident about *a bad revision* can therefore rule out.
721    pub governed_by: Option<crate::journal::AgentIdentity>,
722    /// Who this run acts for: the root of the delegation chain bound at
723    /// admission, when the caller presented one.
724    pub subject: Option<String>,
725    /// The slot is held by a run whose lease has lapsed, so no process is
726    /// executing it.
727    ///
728    /// Not a cancellation candidate: the recovery sweep resumes these, and
729    /// cancelling one unwinds work that was about to continue.
730    pub stranded: bool,
731}
732
733/// A run that stopped short of an answer, as an error a caller can `?`.
734///
735/// Carries the [`RunStatus`] whole rather than a flattened string, so a caller
736/// that started with [`RunOutcome::success`] and later needs to tell a
737/// suspension from a quarantine can match on [`status`](Self::status) without
738/// re-plumbing the call site.
739#[derive(Debug, Clone)]
740pub struct RunFailure {
741    pub run_id: RunId,
742    pub status: RunStatus,
743}
744
745/// **Total, including over the status this type is not supposed to hold.**
746///
747/// `status` is public and `RunStatus` carries `Succeeded`, so a
748/// `RunFailure { status: Succeeded }` is a value any caller can build. Asserting
749/// that away would make *formatting an error* the operation that panics, and a
750/// `Display` impl runs wherever an error is logged — disproportionately during
751/// an incident. So it prints the contradiction instead.
752impl std::fmt::Display for RunFailure {
753    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
754        if matches!(self.status, RunStatus::Succeeded) {
755            return write!(
756                f,
757                "run {} is recorded as a failure carrying a success — \
758                 whatever built this value did so wrongly",
759                self.run_id
760            );
761        }
762        write!(f, "run {} did not succeed: ", self.run_id)?;
763        match &self.status {
764            RunStatus::Succeeded => write!(f, "it succeeded"),
765            RunStatus::Failed(reason) => write!(f, "it failed — {reason}"),
766            RunStatus::Suspended(reason) => write!(f, "it is suspended — {reason:?}"),
767            RunStatus::Exhausted(limit) => write!(f, "a budget stopped it — {limit}"),
768            RunStatus::Quarantined(reason) => write!(f, "it is quarantined — {reason}"),
769            RunStatus::Replanning(reason) => write!(f, "it asked to replan — {reason}"),
770            RunStatus::Observed => write!(
771                f,
772                "it is a session this plane observed rather than executed, so it \
773                 neither succeeded nor failed"
774            ),
775            RunStatus::Cancelled { actor, reason } => {
776                write!(f, "{actor} cancelled it — {reason}")
777            }
778            RunStatus::Swept => write!(f, "the plane recorded a pass over its own state"),
779            RunStatus::BrokeGlass { actor, reason } => {
780                write!(f, "{actor} crossed into this tenant — {reason}")
781            }
782            RunStatus::HaltLifted { actor } => write!(f, "{actor} lifted an emergency stop"),
783            RunStatus::HoldReleased { actor } => write!(f, "{actor} released a legal hold"),
784            RunStatus::Abandoned { actor, reason } => {
785                write!(
786                    f,
787                    "{actor} abandoned it, unresolved and not unwound — {reason}"
788                )
789            }
790            RunStatus::Withheld { subject, reason } => {
791                write!(
792                    f,
793                    "the authority it acts for — '{subject}' — was withdrawn: {reason}"
794                )
795            }
796        }
797    }
798}
799
800impl std::error::Error for RunFailure {}
801
802#[derive(Debug, Clone, PartialEq)]
803pub enum RunStatus {
804    /// All terminal work completed.
805    ///
806    /// Structural, never self-reported: a skill saying "done" is not what makes
807    /// a run succeed. Agents confidently announce success on unmet objectives,
808    /// so completion is determined by the runtime, not claimed by the workload.
809    Succeeded,
810    Failed(String),
811    /// Waiting for something that has not happened. The frame is persisted and
812    /// the task is gone: a suspended run costs disk, not a thread.
813    Suspended(crate::core::SuspendReason),
814    /// A limit stopped it. Not a fault — the run did what it was told, and what
815    /// it was told included a ceiling.
816    Exhausted(crate::core::BudgetExceeded),
817    /// Needs human resolution before anything else may happen to it.
818    Quarantined(String),
819    /// A step asked for a different plan.
820    ///
821    /// Never observed by a caller: the executor either produces a successor and
822    /// keeps going, or turns this into a failure with the reason it refused.
823    /// It exists as a status so a step's request travels the same path every
824    /// other outcome does.
825    Replanning(String),
826    /// An operator stopped it.
827    ///
828    /// Distinct from `Failed` on purpose. A failure is the run discovering it
829    /// cannot proceed; a cancellation is a human deciding it should not. They
830    /// call for different responses — one is investigated, the other was
831    /// intended — and an operator scanning for failures should not have to
832    /// mentally subtract their own interventions.
833    ///
834    /// Completed steps are unwound exactly as they are for a failure: stopping a
835    /// run that has moved money and leaving the movement in place is not
836    /// stopping it.
837    Cancelled {
838        actor: crate::core::Operator,
839        reason: String,
840    },
841    /// An operator closed a run whose outcome could never be established.
842    ///
843    /// The end of the road for a quarantine nobody can answer. Distinct from
844    /// `Cancelled` in the one way that matters to whoever reads it next:
845    /// **nothing was unwound**. A cancellation puts the world back; this leaves
846    /// it exactly as the run left it, because compensating around an unknown
847    /// outcome is the single thing quarantine exists to forbid, and an
848    /// operator's patience running out is not new evidence.
849    ///
850    /// So the doubt does not go away when the run does. It stops being a
851    /// *status* — which is a mutable thing a later action can overwrite — and
852    /// becomes an `agentplane audit` finding, which is derived from the journal
853    /// and outlives every status the run will ever have.
854    Abandoned {
855        actor: crate::core::Operator,
856        reason: String,
857    },
858    /// A pass this plane made over its own state, journaled as a run of its own.
859    ///
860    /// The sweep records what it decided — which obligations breached, which
861    /// tasks it escalated, which runs it took over — rather than logging it, so
862    /// *who decided what* is answerable from the chain six weeks later. It is
863    /// not `Succeeded` because nothing asked for it and there is no answer to
864    /// hand back: a sweep is the plane reporting on itself.
865    Swept,
866    /// An operator crossed into this tenant deliberately, and the reason they
867    /// gave for it.
868    ///
869    /// The one exception to tenant isolation, and the exception's entire value
870    /// is this record. Both halves are carried because an incident review asks
871    /// two questions and neither is worth less than the other: `actor` is read
872    /// back from the crossing record, `reason` from the conclusion, and a
873    /// crossing cannot be recorded without one.
874    BrokeGlass {
875        actor: crate::core::Operator,
876        reason: String,
877    },
878    /// An operator lifted an emergency stop, recorded as a run of its own.
879    ///
880    /// `actor` is read back from the run's
881    /// [`HaltLifted`](crate::journal::RecordKind::HaltLifted) record, which
882    /// also holds the stop it ended.
883    HaltLifted {
884        actor: crate::core::Operator,
885    },
886    /// An operator released a legal hold, recorded as a run of its own.
887    ///
888    /// `actor` is read back from the run's
889    /// [`HoldReleased`](crate::journal::RecordKind::HoldReleased) record.
890    HoldReleased {
891        actor: crate::core::Operator,
892    },
893    /// The authority this run acts under was withdrawn while it was running.
894    ///
895    /// **A pause, not a fault**, for the reason exhaustion is one: the run did
896    /// nothing wrong. Its mutations stand and it stays resumable — lift the halt
897    /// and it continues. Unwinding would reverse correct work for a reason
898    /// unrelated to it; cancelling is the verb for that.
899    ///
900    /// Carries the subject, because *whose* authority is what an operator
901    /// matches against the withdrawal they made.
902    Withheld {
903        subject: String,
904        reason: String,
905    },
906    /// A session this plane **observed** rather than executed, closed.
907    ///
908    /// The one status that is not about work this runtime did. A seat beside
909    /// somebody else's agent produces records at the *asserted* rung
910    /// ([`RecordKind::Observed`](crate::journal::RecordKind::Observed)), in a
911    /// run of their own with no admission — the shape a sweep's run already
912    /// takes — so that they share the chain, the Merkle log and the witness
913    /// while sharing no record kind with a dispatched effect.
914    ///
915    /// **It carries nothing, and the absence is the design.** There is no
916    /// reason to give: the session ended because the party running it ended
917    /// it, and this plane is in no position to say why. The session's own
918    /// identifier is on every observation record rather than here, so a plane
919    /// that stopped mid-session still leaves evidence that says which session
920    /// it describes.
921    ///
922    /// Sealed like any other ending, so an external verifier checks it without
923    /// being taught what an observation is. Never resumable: there is nothing
924    /// to resume onto, because nothing here was executing.
925    Observed,
926}
927
928impl RunStatus {
929    /// Why the run ended, whatever kind of ending it was.
930    ///
931    /// `None` only for [`Succeeded`](Self::Succeeded), which has no reason to
932    /// give. Every other conclusion carries one, and the point of gathering
933    /// them here is that an embedder mapping outcomes onto its own wire type
934    /// should not have to match every variant to find the sentence, and an
935    /// embedder that reads only the status ships an empty summary on failure.
936    ///
937    /// Borrowed where the variant already holds a string and formatted where
938    /// it holds a typed value: a suspension names what it waits for and an
939    /// exhaustion names the ceiling it hit, and both are worth more than the
940    /// word "suspended". `Cancelled` yields the operator's reason; the actor
941    /// who gave it stays on the variant, because *why* and *who* are two
942    /// questions and only one of them is the reason.
943    #[must_use]
944    pub fn reason(&self) -> Option<std::borrow::Cow<'_, str>> {
945        use std::borrow::Cow;
946        match self {
947            // Nothing to say, for three different reasons and one answer: a
948            // success has no why, a sweep's records carry theirs, and an
949            // observed session ended because somebody else ended it.
950            // A lift's record carries what it ended; the lifter gave no reason.
951            Self::Succeeded
952            | Self::Swept
953            | Self::Observed
954            | Self::HaltLifted { .. }
955            | Self::HoldReleased { .. } => None,
956            Self::Failed(reason)
957            | Self::Quarantined(reason)
958            | Self::Replanning(reason)
959            | Self::Cancelled { reason, .. }
960            | Self::Abandoned { reason, .. }
961            | Self::BrokeGlass { reason, .. }
962            | Self::Withheld { reason, .. } => Some(Cow::Borrowed(reason.as_str())),
963            Self::Suspended(reason) => Some(Cow::Owned(reason.to_string())),
964            Self::Exhausted(exceeded) => Some(Cow::Owned(exceeded.to_string())),
965        }
966    }
967
968    /// The run's **own** account of why it ended, for the journal.
969    ///
970    /// [`reason`](Self::reason) answers *what should a person be shown*, which
971    /// includes the words an operator typed when they stopped the run. This
972    /// answers the narrower question the conclusion record asks: *is there a
973    /// sentence here that nothing else on the chain already records?*
974    ///
975    /// `None` for every ending a person caused. Those write their own record —
976    /// [`RunCancelled`](crate::journal::RecordKind::RunCancelled),
977    /// [`QuarantineDecided`](crate::journal::RecordKind::QuarantineDecided),
978    /// [`BreakGlass`](crate::journal::RecordKind::BreakGlass),
979    /// [`AuthorityWithheld`](crate::journal::RecordKind::AuthorityWithheld) —
980    /// carrying the actor and the reason together, in the clear. Repeating the
981    /// reason on the conclusion would make one sentence two facts, and seal the
982    /// copy that an erasure then destroys.
983    ///
984    /// `None` for an exhaustion too: the typed ceiling verdict is the record,
985    /// and the prose was a rendering of it.
986    #[must_use]
987    pub fn own_account(&self) -> Option<std::borrow::Cow<'_, str>> {
988        use std::borrow::Cow;
989        match self {
990            // The runtime's own endings: a skill's words, a provider's words,
991            // a divergence this build found. Nothing else on the chain says it.
992            Self::Failed(reason) | Self::Quarantined(reason) | Self::Replanning(reason) => {
993                Some(Cow::Borrowed(reason.as_str()))
994            }
995            // Nothing to add, for two different reasons and one answer. The
996            // first four are somebody's doing and their own record already
997            // carries both halves. The rest either write no conclusion with a
998            // why — a success, a sweep, a suspension — or carry it typed
999            // instead, which is what `exhaustion` is.
1000            //
1001            // Enumerated rather than left to a wildcard: a status added later
1002            // must be made to answer this question, and `_ => None` would
1003            // answer it silently and wrongly for anything a person causes.
1004            Self::Cancelled { .. }
1005            | Self::Abandoned { .. }
1006            | Self::BrokeGlass { .. }
1007            | Self::HaltLifted { .. }
1008            | Self::HoldReleased { .. }
1009            | Self::Withheld { .. }
1010            | Self::Succeeded
1011            | Self::Swept
1012            // The account is the observation records themselves, and this plane
1013            // has none of its own to add: it did not run the session.
1014            | Self::Observed
1015            | Self::Suspended(_)
1016            | Self::Exhausted(_) => None,
1017        }
1018    }
1019
1020    /// Whether the run stopped without reaching a conclusion.
1021    #[must_use]
1022    pub fn is_suspended(&self) -> bool {
1023        matches!(self, Self::Suspended(_))
1024    }
1025
1026    /// Whether the recorded history can no longer be trusted to describe this
1027    /// code, so a human must look before anything else happens.
1028    #[must_use]
1029    pub fn is_quarantined(&self) -> bool {
1030        matches!(self, Self::Quarantined(_))
1031    }
1032
1033    #[must_use]
1034    pub fn as_str(&self) -> &'static str {
1035        match self {
1036            Self::Succeeded => "succeeded",
1037            Self::Failed(_) => "failed",
1038            Self::Suspended(_) => "suspended",
1039            Self::Exhausted(_) => "exhausted",
1040            Self::Quarantined(_) => "quarantined",
1041            Self::Replanning(_) => "replanning",
1042            Self::Cancelled { .. } => "cancelled",
1043            Self::Abandoned { .. } => "abandoned",
1044            Self::Swept => super::sweeper::SWEEP_OUTCOME,
1045            Self::BrokeGlass { .. } => BREAK_GLASS_OUTCOME,
1046            Self::HaltLifted { .. } => HALT_LIFTED_OUTCOME,
1047            Self::HoldReleased { .. } => HOLD_RELEASED_OUTCOME,
1048            Self::Withheld { .. } => WITHHELD_OUTCOME,
1049            Self::Observed => OBSERVED_OUTCOME,
1050        }
1051    }
1052
1053    /// Who brought this run to this state, where a person did.
1054    ///
1055    /// `None` for every ending the runtime reached on its own, which is most of
1056    /// them. One accessor rather than three `matches!` at each surface, because
1057    /// *who* is the question an operator asks immediately after *what* — and a
1058    /// surface that reads the reason and drops the actor answers half of it.
1059    #[must_use]
1060    pub fn actor(&self) -> Option<&crate::core::Operator> {
1061        match self {
1062            Self::Cancelled { actor, .. }
1063            | Self::Abandoned { actor, .. }
1064            | Self::BrokeGlass { actor, .. }
1065            | Self::HaltLifted { actor }
1066            | Self::HoldReleased { actor } => Some(actor),
1067            // A withdrawal names a *subject*, not an actor: the operator who
1068            // threw the halt and the credential it covers are different
1069            // parties, and answering "who" with the subject would report the
1070            // person whose authority was taken as the one who took it.
1071            Self::Succeeded
1072            | Self::Failed(_)
1073            | Self::Suspended(_)
1074            | Self::Exhausted(_)
1075            | Self::Quarantined(_)
1076            | Self::Replanning(_)
1077            | Self::Withheld { .. }
1078            | Self::Swept
1079            // Somebody was at the keyboard, and it was not this plane's
1080            // operator. Naming the observed party's user here would report a
1081            // person this runtime never authenticated as one of its own.
1082            | Self::Observed => None,
1083        }
1084    }
1085
1086    /// Whether an operator stopped this run.
1087    #[must_use]
1088    pub fn is_cancelled(&self) -> bool {
1089        matches!(self, Self::Cancelled { .. })
1090    }
1091
1092    /// Whether this conclusion freezes the journal and enters the Merkle log.
1093    ///
1094    /// Only conclusions that nothing may resume seal. A failed run may be
1095    /// resumed — its completed effects are read back from history rather than
1096    /// performed again, which is the point of having a journal — and an
1097    /// exhausted run may continue once somebody raises the ceiling. Sealing
1098    /// either would commit the Merkle log to a leaf its own resume is then
1099    /// permitted to grow past: the checkpoint would attest a prefix of a
1100    /// history that kept moving, and the store's refusal of appends after a
1101    /// seal would turn every legitimate resume into an error.
1102    ///
1103    /// **A quarantine does not seal**, and that follows from the same rule
1104    /// rather than being an exception to it. A quarantine is the runtime saying
1105    /// it does not know, which is a story that is not over; a Merkle leaf is a
1106    /// claim that a history is complete. Sealing one published a proof of
1107    /// completeness over an admittedly incomplete record, and — the practical
1108    /// half — froze the journal against the one record that answers it, so
1109    /// "a human must resolve it before it can run again" named a resolution the
1110    /// format made impossible. It enters the log when it reaches a real ending,
1111    /// including [`Abandoned`](Self::Abandoned), which is a person deciding
1112    /// there will never be one.
1113    ///
1114    /// This is the **one** implementation of that rule. `resume_is_closed`
1115    /// refuses exactly the outcomes this method seals — a test pins the
1116    /// agreement — because two copies of one rule agree everywhere except the
1117    /// boundary nobody probed.
1118    #[must_use]
1119    pub fn seals(&self) -> bool {
1120        matches!(
1121            self,
1122            Self::Succeeded
1123                | Self::Cancelled { .. }
1124                | Self::Abandoned { .. }
1125                | Self::Swept
1126                | Self::BrokeGlass { .. }
1127                | Self::HaltLifted { .. }
1128                | Self::HoldReleased { .. }
1129                | Self::Observed
1130        )
1131    }
1132}
1133
1134/// Every outcome a sealed run can carry, as the outcome index spells them.
1135///
1136/// The store indexes runs *by* outcome and deliberately has no "all runs"
1137/// query — a backlog listing is a range read rather than a table scan — so any
1138/// caller that wants every sealed run has to name the outcomes one by one. This
1139/// is the list to name them from, and it lives beside [`RunStatus::seals`]
1140/// because it is the same rule in its other spelling: exactly the statuses that
1141/// seal, with no exception clause. A sweep (`swept`) and a break-glass crossing
1142/// (`broke-glass`) are sealed at birth with no goal to have succeeded or failed
1143/// at, and they are statuses all the same — an ending this plane writes and its
1144/// own reader cannot name is an ending it reports as unrecognised.
1145///
1146/// It lived in the CLI first, as string literals — two implementations of one
1147/// rule, where a new sealing outcome would have been exported by nobody and
1148/// the export that silently omitted it would have dropped exactly the runs an
1149/// auditor came for. A test holds this list to `RunStatus::seals` variant by
1150/// variant, which the literals in a binary could never be held to.
1151pub const SEALED_OUTCOMES: &[&str] = &[
1152    "succeeded",
1153    "cancelled",
1154    "abandoned",
1155    super::sweeper::SWEEP_OUTCOME,
1156    BREAK_GLASS_OUTCOME,
1157    HALT_LIFTED_OUTCOME,
1158    HOLD_RELEASED_OUTCOME,
1159    OBSERVED_OUTCOME,
1160];
1161
1162/// What an offline verb sweeps when the operator names no outcome.
1163///
1164/// Every ending — [`SEALED_OUTCOMES`] — **plus `quarantined`**, which is not an
1165/// ending and is exactly why it belongs: a run set aside for a person is the
1166/// plane's outstanding unresolved risk, and an artifact holding everything that
1167/// finished and nothing that did not is not the one an auditor came for. It is
1168/// also the outcome most likely to be the reason they came.
1169///
1170/// Failed and exhausted runs are deliberately absent, and the distinction is
1171/// not sealing but *whether anything is waiting on a person*. Those two are
1172/// work in progress that a resume moves on its own; a quarantine moves only
1173/// when somebody decides, so it can sit for weeks and still be current.
1174///
1175/// A test holds this to [`SEALED_OUTCOMES`] rather than restating it, so a new
1176/// ending is swept here the day it exists.
1177pub const OUTCOMES_OF_RECORD: &[&str] = &[
1178    "succeeded",
1179    "cancelled",
1180    "abandoned",
1181    "quarantined",
1182    super::sweeper::SWEEP_OUTCOME,
1183    BREAK_GLASS_OUTCOME,
1184    HALT_LIFTED_OUTCOME,
1185    HOLD_RELEASED_OUTCOME,
1186    OBSERVED_OUTCOME,
1187];
1188
1189/// How a resume came to hold its lease, which decides what a failure does
1190/// with it.
1191///
1192/// A lease that lapses while still naming an owner is what the recovery sweep
1193/// lists, so keeping one on failure is how a run stays findable; releasing it
1194/// is right only where the run is findable without it.
1195#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1196enum Claim {
1197    /// Claimed by the resume itself, over a run whose journal already lists
1198    /// it — failed, exhausted, withheld, quarantined, waiting.
1199    Standing,
1200    /// Handed over by a wake or a delivery that recorded the awaited result
1201    /// under it, which took the run off the waiting listing.
1202    Unlisted,
1203    /// Claimed by the recovery sweep over a lease its owner died holding.
1204    Recovering,
1205}
1206
1207/// A run that exists but has not run.
1208///
1209/// Produced by admission and consumed by execution, so the two can happen in
1210/// different places — the same request for a blocking call, and a background
1211/// task for a non-blocking one.
1212struct Admitted {
1213    run: RunId,
1214    epoch: crate::core::Epoch,
1215    quota: QuotaPass,
1216    budget: Budget,
1217    agent: String,
1218    plan: PlanIR,
1219    input: Tainted<Value>,
1220    case: Option<CaseContext>,
1221    /// The chain admission bound — the caller's where one was presented,
1222    /// the plane's otherwise — and what every step of this run acts under.
1223    identity: Option<crate::core::Delegation>,
1224    /// The references its `DataSubjectBound` record makes available.
1225    subjects: BTreeSet<crate::core::SubjectRef>,
1226}
1227
1228/// The quota identity of one live execution pass.
1229///
1230/// Captured once at admission or resume and carried to settlement. Computing
1231/// the period again after the work finishes can cross a day or month boundary,
1232/// authorizing against one ledger and charging another.
1233#[derive(Debug, Clone, Default)]
1234struct QuotaPass {
1235    period: Option<String>,
1236    enabled: bool,
1237    release_slot: bool,
1238    /// How many steps this pass may dispatch at once, when the tenant bounds
1239    /// spend: the width the run's reservation was sized for. `None` leaves
1240    /// the run's own budget as the only bound.
1241    width: Option<usize>,
1242}
1243
1244impl QuotaPass {
1245    fn at(
1246        quota: &crate::quota::TenantQuota,
1247        at: crate::core::Timestamp,
1248        release_slot: bool,
1249    ) -> Self {
1250        Self {
1251            period: quota.bounds_spend().then(|| quota.period.key_for(at)),
1252            enabled: true,
1253            release_slot,
1254            width: None,
1255        }
1256    }
1257
1258    const fn disabled() -> Self {
1259        Self {
1260            period: None,
1261            enabled: false,
1262            release_slot: false,
1263            width: None,
1264        }
1265    }
1266
1267    /// Held to the width a spend reservation was sized for, whenever the
1268    /// tenant bounds spend.
1269    fn sized_for(mut self, width: usize) -> Self {
1270        if self.period.is_some() {
1271            self.width = Some(width);
1272        }
1273        self
1274    }
1275
1276    /// How wide this pass may dispatch: the budget's own width, narrowed to
1277    /// the one its reservation was sized for. A replan may widen the plan; it
1278    /// does not widen what the period holds for it.
1279    fn parallelism(&self, budget: &Budget) -> usize {
1280        self.width
1281            .map_or_else(|| budget.parallelism(), |w| budget.parallelism().min(w))
1282    }
1283
1284    fn period(&self) -> Option<&str> {
1285        self.period.as_deref()
1286    }
1287
1288    fn started(&self) -> Option<RecordKind> {
1289        self.enabled.then(|| RecordKind::QuotaPassStarted {
1290            period: self.period.clone(),
1291            release_slot: self.release_slot,
1292        })
1293    }
1294
1295    fn settlement(
1296        &self,
1297        run: RunId,
1298        epoch: crate::core::Epoch,
1299        spend: Spend,
1300        concludes: bool,
1301    ) -> Option<crate::quota::QuotaSettlement> {
1302        self.enabled.then(|| crate::quota::QuotaSettlement {
1303            run,
1304            epoch,
1305            period: self.period.clone(),
1306            spend,
1307            release_slot: self.release_slot,
1308            concludes,
1309        })
1310    }
1311}
1312
1313/// How many steps a run may have in flight: its budget's width, or its
1314/// admitted plan's node count when that is narrower or nothing narrows it.
1315///
1316/// What a spend reservation multiplies the per-call bound by, since each step
1317/// in flight may hold one operation the ledger admitted and has not billed.
1318fn reserved_width(budget: &Budget, plan: &PlanIR) -> usize {
1319    budget.parallelism().min(plan.nodes.len()).max(1)
1320}
1321
1322/// The peers a plane may call, and how it reaches them.
1323///
1324/// One value because the two are one decision: a registry says what each
1325/// peer is granted and which credential reaches it, and the client is the
1326/// transport those grants are spent over. Held by the plane and handed to
1327/// every step, so a skill calling a peer reaches it through the same checked
1328/// registry a declarative agent does — never one it carries itself.
1329#[derive(Debug)]
1330pub(crate) struct PeerWiring {
1331    pub(crate) registry: crate::peers::PeerRegistry,
1332    pub(crate) client: Arc<dyn crate::peers::PeerClient>,
1333}
1334
1335/// Keeps a run's lease alive for as long as it is executing.
1336///
1337/// A lease answers one question — *is this owner dead?* — and it answers by
1338/// expiry. Without renewal it also answers a question it was never asked: a
1339/// healthy run that outlives its TTL looks exactly like a crashed one, and agent
1340/// runs routinely outlive a lease because a single model call can. Another
1341/// instance then takes the run over, bumps the epoch, and the original is fenced
1342/// on its next append: killed mid-flight, having already done real work.
1343///
1344/// Renewing while executing separates the two. The TTL then bounds how long a
1345/// *crashed* owner strands its runs, which is what it is for, and stops bounding
1346/// how long a run may take, which it should never have bounded.
1347///
1348/// Aborted on drop, so the renewal stops the moment execution returns — by any
1349/// path, including a panic unwinding through it. A heartbeat that outlived its
1350/// run would hold a lease nobody is using and strand it for a full TTL after a
1351/// crash, which is the failure this exists to prevent, arriving late.
1352struct Heartbeat(tokio::task::JoinHandle<()>);
1353
1354impl Drop for Heartbeat {
1355    fn drop(&mut self) {
1356        self.0.abort();
1357    }
1358}
1359
1360/// The runtime.
1361#[derive(Debug, Clone)]
1362pub struct Runtime {
1363    store: Arc<dyn JournalStore>,
1364    skills: HashMap<String, Arc<dyn Skill>>,
1365    by_capability: HashMap<Capability, String>,
1366    /// A handle to this plane, for steps that commission other agents on it.
1367    ///
1368    /// Weak, and that is not an optimisation: a strong self-reference would
1369    /// leak every runtime ever built. It exists because commissioning belongs to
1370    /// the *runtime* — a skill holding an `Arc<Runtime>` cannot work, since the
1371    /// runtime needs the skill before the skill can have the runtime.
1372    self_ref: std::sync::Weak<Self>,
1373    /// The declaration governing each skill, by skill name.
1374    ///
1375    /// Per skill rather than per runtime, because a plane runs several agents
1376    /// and a step must be judged against *its own* agent's manifest.
1377    #[cfg(feature = "manifest")]
1378    governed_by: HashMap<String, Arc<crate::manifest::Manifest>>,
1379    /// Which tenant this plane runs as.
1380    ///
1381    /// One plane serves one tenant; a **process** may run several and serve
1382    /// them all, because `api::Planes` resolves a plane from the authenticated
1383    /// caller's tenant. That is what the composite keys on both backends bought:
1384    /// the tenant leads every key, lease, correlation and blob path, so a query
1385    /// that forgets it misses rather than returning somebody else's rows.
1386    tenant: crate::core::TenantId,
1387    /// Who vouched for each declaration, by agent name.
1388    ///
1389    /// Separate from the manifest because it is *not* in it: a document cannot
1390    /// state who signed it. It arrives beside the manifest from a verified
1391    /// registry resolution, and an absent entry means nobody vouched.
1392    #[cfg(feature = "manifest")]
1393    published_by: HashMap<String, crate::core::KeyId>,
1394    /// Model drivers by the name a manifest calls them.
1395    ///
1396    /// Kept after build — not only consumed by it — so a *hand-written* skill
1397    /// can reach the driver its manifest's model role names through
1398    /// [`StepCtx::complete`], instead of smuggling an `Arc<dyn ModelProvider>`
1399    /// field past the declaration.
1400    #[cfg(feature = "manifest")]
1401    providers: HashMap<String, Arc<dyn crate::model::ModelProvider>>,
1402    owner: String,
1403    /// How long a run's lease lasts, and how long a crashed owner's runs stay
1404    /// unclaimable.
1405    lease_ttl: Duration,
1406    /// Where this plane's agents remember things, when a deployment wires one.
1407    memories: Option<Arc<dyn crate::memory::MemoryStore>>,
1408    semantic: Option<Arc<super::SemanticMemory>>,
1409    authorities: Option<Arc<dyn crate::authority::AuthorityStore>>,
1410    /// The peers this plane may call, and the transport that reaches them.
1411    peers: Option<Arc<PeerWiring>>,
1412    /// The catalogue and transport a **skill** reaches tools through.
1413    ///
1414    /// Held by the plane rather than handed to each skill, and that is the
1415    /// point. A skill that built its own [`ToolCatalog`] could declare a tool
1416    /// read-only that its manifest calls mutating — and `ToolSafety::read_only`
1417    /// carries `Recovery::Retry`, so a timed-out money-moving call gets sent
1418    /// again. `check_catalogue_not_laxer_than_grants` refuses exactly that for
1419    /// the plane's catalogue at build; a hand-built one never passed under it.
1420    /// `StepCtx::call_tool` is the path that cannot.
1421    #[cfg(feature = "manifest")]
1422    tools: Option<(
1423        Arc<crate::tools::ToolCatalog>,
1424        Arc<dyn crate::tools::ToolClient>,
1425    )>,
1426    /// The highest sensitivity this plane will write into an append-only
1427    /// record, when it says.
1428    journal_ceiling: Option<crate::core::Sensitivity>,
1429    /// Where this plane's tool transports may connect.
1430    ///
1431    /// Behind `manifest` because the tool path is.
1432    ///
1433    /// Here rather than on each `ToolClient` because it is a **plane**
1434    /// decision: the client owns the connection, the plane owns the
1435    /// destination, exactly as it does for a model driver's base URL. Absent
1436    /// means no egress control on the tool path, spelled the same way an
1437    /// absent policy engine is.
1438    #[cfg(feature = "manifest")]
1439    egress: Option<Arc<crate::core::Egress>>,
1440    #[cfg(feature = "manifest")]
1441    checkers: crate::content::Checkers,
1442    /// Where live model output is forwarded, when anything listens.
1443    streams: Option<Arc<dyn RunStreamObserver>>,
1444    /// How this plane attributes its metrics.
1445    meter: super::metrics::Meter,
1446    /// Durable per-tenant ceilings, when a deployment wires them.
1447    quotas: Option<Arc<dyn crate::quota::QuotaStore>>,
1448    /// The rate ceilings this plane's declarations state, and the quota store
1449    /// that counts them. `None` when no grant declares one.
1450    #[cfg(feature = "manifest")]
1451    rates: Option<Arc<super::ctx::Rates>>,
1452    quota: crate::quota::TenantQuota,
1453    budget: Budget,
1454    /// Whether every plan this runtime admits must carry a verifier.
1455    require_verifier: bool,
1456
1457    cases: Option<Arc<dyn CaseStore>>,
1458    events: Option<Arc<dyn EventStore>>,
1459    tasks: Option<Arc<dyn TaskStore>>,
1460    timers: Option<Arc<dyn TimerStore>>,
1461    blobs: Option<Arc<dyn crate::blob::BlobStore>>,
1462    /// Where disclosures are recorded, read by a retention pass so it names
1463    /// the copies of what it erased.
1464    disclosures: Option<Arc<dyn crate::disclosure::DisclosureRegister>>,
1465    /// Where webhook registrations and their cursors live.
1466    ///
1467    /// Held by the plane rather than only by whoever delivers, because the
1468    /// registrations that stop being delivered are a **backlog**, and a backlog
1469    /// is answered by an operator on the plane's own surface — not by the
1470    /// worker that gave up on it.
1471    #[cfg(feature = "push")]
1472    push: Option<Arc<dyn crate::push::PushStore>>,
1473    /// Where data keys live, when payload bytes are sealed.
1474    #[cfg(feature = "keyring")]
1475    keyring: Option<Arc<dyn crate::keyring::KeyRing>>,
1476    batches: Option<Arc<dyn crate::batch::BatchStore>>,
1477    policy: Option<Arc<dyn crate::core::PolicyEngine>>,
1478    identity: Option<crate::core::Delegation>,
1479    replanner: Option<Arc<dyn crate::plan::Replanner>>,
1480    calendar: Arc<dyn Calendar>,
1481    signer: Option<Arc<dyn crate::core::Signer>>,
1482    /// Where this plane sends its own events, when a deployment configures it.
1483    ///
1484    /// Registered per run **at admission**, and again by every resume that
1485    /// executes: a destination attached after the first record would silently
1486    /// miss it, and "the journal is the outbox" only holds if the cursor starts
1487    /// at sequence one.
1488    #[cfg(feature = "push")]
1489    outbox: Option<Arc<crate::push::Outbox>>,
1490    /// The parties this plane submits its checkpoints to, and how many must
1491    /// answer.
1492    ///
1493    /// The one control here whose whole value is that it is **not** this
1494    /// plane: the chain, the signatures and the Merkle log all verify against
1495    /// inputs the operator supplies, so none of them detects a run removed or
1496    /// two histories shown to two auditors. Absent means the plane's history
1497    /// is anchored nowhere outside itself, which is a deployment decision and
1498    /// is reported as one by `audit`.
1499    witnesses: Option<Arc<Witnessing>>,
1500    /// The runs this instance put on a background task, and the gate that
1501    /// closes admission when it is asked to stop.
1502    ///
1503    /// Process-local on purpose: it answers *what is this instance executing
1504    /// right now*, which no other instance can be told and no store can hold.
1505    /// The durable cousin is [`Runtime::set_halt`], which stops a tenant
1506    /// everywhere and survives a restart — a different question with a
1507    /// different audience.
1508    inflight: Arc<super::drain::InFlight>,
1509    /// Sub-runs that started commissions set working in this process, by run: the
1510    /// awaiting run takes the outcome from the task that executed it, rather
1511    /// than replaying a run that already did its work here.
1512    commissions: Arc<std::sync::Mutex<HashMap<RunId, Working>>>,
1513}
1514
1515/// The witness set and the bar a submission round is held to.
1516#[derive(Debug)]
1517pub(crate) struct Witnessing {
1518    pub(crate) witnesses: Vec<Arc<dyn crate::journal::Witness>>,
1519    pub(crate) quorum: crate::journal::WitnessQuorum,
1520    /// The size last submitted under a **met** quorum.
1521    ///
1522    /// In memory, deliberately. It exists only to keep a periodic pass from
1523    /// re-submitting a log that has not grown, and the authority on where a
1524    /// witness actually is has always been the witness — which answers `409`
1525    /// with its own size and gets a proof built from there. So a restart
1526    /// re-submits once and the protocol absorbs it; a durable copy would be a
1527    /// second answer to a question the counterparty already answers.
1528    pub(crate) submitted: std::sync::atomic::AtomicU64,
1529    /// The declared maximum time between submissions of an unchanged log;
1530    /// `None` never re-submits one.
1531    pub(crate) interval: Option<std::time::Duration>,
1532    /// The sweep instant of the last round that met quorum, in memory for the
1533    /// reason `submitted` is.
1534    pub(crate) last_met: std::sync::Mutex<Option<crate::core::Timestamp>>,
1535}
1536
1537/// Where a run is entering this plane from.
1538///
1539/// The one thing the drain gate needs to know, and it cannot be derived at the
1540/// gate: by the time admission runs, an agent commissioning another one and a
1541/// peer sending a message look identical. The difference is whose work it is —
1542/// a commission is a step of a run this process is *already* executing and has
1543/// promised to finish, so refusing it would make the drain manufacture the
1544/// undecided effects it exists to prevent.
1545#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1546pub(crate) enum Entry {
1547    /// A caller outside this process: an embedder, an operator, a peer, a batch
1548    /// pass. Refused while draining.
1549    Outside,
1550    /// A run already executing here, ordering work it needs in order to
1551    /// conclude. Never refused for draining — the parent is what the drain is
1552    /// waiting for.
1553    Nested,
1554}
1555
1556/// A backend that implements every store a full plane runs on.
1557///
1558/// Blanket-implemented, so it is a *description* rather than an obligation:
1559/// any type providing the store traits — journal, cases, tasks, events,
1560/// timers, memory, quotas, disclosures — is a `FullBackend` without saying so, and both
1561/// shipped backends are. The quota store is in the set because halts live
1562/// there: a plane without it would admit past every emergency stop. What it buys is [`Runtime::builder_on`], where the
1563/// alternative was a cast of one `Arc` per store that every deployment spelled out
1564/// and none could get differently.
1565pub trait FullBackend:
1566    JournalStore
1567    + CaseStore
1568    + TaskStore
1569    + EventStore
1570    + TimerStore
1571    + crate::memory::MemoryStore
1572    + crate::quota::QuotaStore
1573    + crate::disclosure::DisclosureRegister
1574{
1575}
1576
1577impl<B> FullBackend for B where
1578    B: JournalStore
1579        + CaseStore
1580        + TaskStore
1581        + EventStore
1582        + TimerStore
1583        + crate::memory::MemoryStore
1584        + crate::quota::QuotaStore
1585        + crate::disclosure::DisclosureRegister
1586{
1587}
1588
1589impl Runtime {
1590    /// A builder with the whole case layer wired to one backend.
1591    ///
1592    /// The one-call form of the seven-cast litany: journal, cases, tasks,
1593    /// events, timers, memory and quotas all backed by `store`, which is what a
1594    /// deployment on a single `RedbStore` or `PostgresStore` means anyway. The
1595    /// à-la-carte methods remain — [`cases`](RuntimeBuilder::cases) and
1596    /// friends override an individual store afterwards, and
1597    /// [`builder`](Self::builder) still starts from the journal alone for a
1598    /// plane that wants nothing else.
1599    ///
1600    /// Blob storage is deliberately not included: bytes routinely live in a
1601    /// different system than rows, so [`blobs`](RuntimeBuilder::blobs) stays
1602    /// an explicit decision.
1603    #[must_use]
1604    pub fn builder_on<B: FullBackend + 'static>(store: Arc<B>) -> RuntimeBuilder {
1605        Self::builder_with(Stores::on(store))
1606    }
1607
1608    /// A builder with the whole case layer wired from handles.
1609    ///
1610    /// What [`builder_on`](Self::builder_on) is in terms of: the seven casts live
1611    /// in one place, so a deployment that chose its backend at run time — an
1612    /// operator naming either a file or a connection string — wires exactly what
1613    /// a deployment on one concrete type wires, rather than a hand-written
1614    /// approximation of it that drifts the day an eighth store is added.
1615    #[must_use]
1616    pub fn builder_with(stores: Stores) -> RuntimeBuilder {
1617        Self::builder(stores.journal)
1618            .cases(stores.cases)
1619            .tasks(stores.tasks)
1620            .events(stores.events)
1621            .timers(stores.timers)
1622            .memory(stores.memory)
1623            .quota(stores.quotas, crate::quota::TenantQuota::default())
1624            .disclosures(stores.disclosures)
1625    }
1626
1627    #[must_use]
1628    pub fn builder(store: Arc<dyn JournalStore>) -> RuntimeBuilder {
1629        RuntimeBuilder {
1630            store,
1631            signer: None,
1632            skills: Vec::new(),
1633            owner: None,
1634            lease_ttl: LEASE_TTL,
1635            memories: None,
1636            semantic: None,
1637            authorities: None,
1638            peers: None,
1639            tenant_label: super::telemetry::TenantLabel::default(),
1640            quotas: None,
1641            quota: crate::quota::TenantQuota::default(),
1642            budget: Budget::unlimited(),
1643            require_verifier: false,
1644            cases: None,
1645            events: None,
1646            tasks: None,
1647            timers: None,
1648            blobs: None,
1649            disclosures: None,
1650            #[cfg(feature = "push")]
1651            push: None,
1652            #[cfg(feature = "keyring")]
1653            keyring: None,
1654            #[cfg(feature = "manifest")]
1655            tools: None,
1656            journal_ceiling: None,
1657            #[cfg(feature = "manifest")]
1658            egress: None,
1659            #[cfg(feature = "manifest")]
1660            checkers: crate::content::Checkers::default(),
1661            streams: None,
1662            tenant: crate::core::TenantId::default(),
1663            batches: None,
1664            policy: None,
1665            identity: None,
1666            replanner: None,
1667            calendar: None,
1668            #[cfg(feature = "manifest")]
1669            toolbox: None,
1670            #[cfg(feature = "manifest")]
1671            tool_servers: Vec::new(),
1672            #[cfg(feature = "manifest")]
1673            agents: Vec::new(),
1674            #[cfg(feature = "manifest")]
1675            providers: HashMap::new(),
1676            #[cfg(feature = "push")]
1677            outbox: None,
1678            witnesses: Vec::new(),
1679            quorum: None,
1680            witness_interval: None,
1681        }
1682    }
1683
1684    /// The driver registered under a manifest's name for it, if any.
1685    #[cfg(feature = "manifest")]
1686    pub(crate) fn model_provider(
1687        &self,
1688        name: &str,
1689    ) -> Option<Arc<dyn crate::model::ModelProvider>> {
1690        self.providers.get(name).map(Arc::clone)
1691    }
1692
1693    /// The case store, if this runtime has one.
1694    #[must_use]
1695    pub fn cases(&self) -> Option<&Arc<dyn CaseStore>> {
1696        self.cases.as_ref()
1697    }
1698
1699    /// The worklist, if this runtime has one.
1700    #[must_use]
1701    pub fn tasks(&self) -> Option<&Arc<dyn TaskStore>> {
1702        self.tasks.as_ref()
1703    }
1704
1705    /// The inbound-event store, if this runtime has one.
1706    #[must_use]
1707    pub fn events(&self) -> Option<&Arc<dyn EventStore>> {
1708        self.events.as_ref()
1709    }
1710
1711    /// The ceilings every run under this runtime starts with.
1712    ///
1713    /// Readable because "what is this plane allowed to spend" is a question an
1714    /// operator asks of a running system, and answering it by re-reading the
1715    /// config that *should* have been applied is how a misapplied budget stays
1716    /// invisible.
1717    #[must_use]
1718    pub const fn budget(&self) -> &Budget {
1719        &self.budget
1720    }
1721
1722    /// The blob store, if this runtime has one.
1723    #[must_use]
1724    pub fn blobs(&self) -> Option<&Arc<dyn crate::blob::BlobStore>> {
1725        self.blobs.as_ref()
1726    }
1727
1728    /// Run the live half of the case-layer drill with this plane's own stores.
1729    ///
1730    /// The wiring is the point of this method existing: the drill's value is
1731    /// holding each case's references against the stores the plane *actually
1732    /// runs with*, and an embedder assembling `drill::Stores` by hand could
1733    /// hand it a different blob store than the one their runs write to —
1734    /// a drill that passes over the wrong bucket. Stores this runtime does
1735    /// not have are reported as unchecked by the drill itself, which is a
1736    /// different and better answer than an error: a plane with no blob store
1737    /// has no bytes to lose.
1738    ///
1739    /// **The clock is the caller's**, the same decision
1740    /// [`attention`](Self::attention) and the push sweep make. A rehearsal's
1741    /// instant is a fact about the outside world rather than a journaled
1742    /// observation, and taking it here would be a fourth ambient-clock escape
1743    /// inside the deterministic zone for a value the caller already has. A
1744    /// fixed instant also makes the recorded verdict testable.
1745    ///
1746    /// # Errors
1747    ///
1748    /// If this runtime has no case store — the drill walks cases, so there is
1749    /// nothing to drill — or if the case layer cannot be enumerated.
1750    pub async fn drill(
1751        &self,
1752        at: crate::core::Timestamp,
1753    ) -> Result<crate::drill::DrillReport, RuntimeError> {
1754        let cases = self.cases().ok_or_else(|| {
1755            RuntimeError::PlanContract(
1756                "this runtime has no case store — the drill walks cases, so there is \
1757                 nothing to drill; build it with `.cases(store)`"
1758                    .into(),
1759            )
1760        })?;
1761        let stores = crate::drill::Stores {
1762            cases,
1763            blobs: self.blobs(),
1764            #[cfg(feature = "keyring")]
1765            keys: self.keyring.as_ref(),
1766            tenant: &self.tenant,
1767        };
1768        let report = crate::drill::drill(&stores)
1769            .await
1770            .map_err(RuntimeError::from_store)?;
1771
1772        // **The verdict is recorded, not only returned.** What an audit asks
1773        // is *when did you last rehearse, and did it pass* — and left to the
1774        // caller that answer lives in whatever ran the verb, which is a CI log
1775        // or a wiki page. The checkpoint pins which store it ran against, so a
1776        // drill over a restored copy cannot be read as one over production.
1777        let checkpoint = self
1778            .store
1779            .checkpoint()
1780            .await
1781            .map_err(RuntimeError::from_store)?;
1782        cases
1783            .record_drill(&crate::case::DrillRecord {
1784                at,
1785                sound: report.is_sound(),
1786                cases: report.cases as u64,
1787                findings: report.findings.len() as u64,
1788                not_checked: report.not_checked.len() as u64,
1789                origin: checkpoint.origin.clone(),
1790                size: checkpoint.size,
1791            })
1792            .await
1793            .map_err(RuntimeError::from_store)?;
1794        Ok(report)
1795    }
1796
1797    /// The last recovery rehearsal this plane ran, or `None` if it never has.
1798    ///
1799    /// The half that makes `drill` evidence rather than a command: an audit
1800    /// asks *when did you last rehearse and did it pass*, and that answer has
1801    /// to come from the plane rather than from the thing that scheduled it.
1802    ///
1803    /// # Errors
1804    ///
1805    /// [`RuntimeError::PlanContract`] when this plane has no case store — the
1806    /// same condition that makes a drill impossible in the first place — and
1807    /// [`RuntimeError`] for a store failure.
1808    pub async fn last_drill(&self) -> Result<Option<crate::case::DrillRecord>, RuntimeError> {
1809        self.cases()
1810            .ok_or_else(|| {
1811                RuntimeError::PlanContract(
1812                    "this runtime has no case store, so it has never drilled one".into(),
1813                )
1814            })?
1815            .last_drill()
1816            .await
1817            .map_err(RuntimeError::from_store)
1818    }
1819
1820    /// Erase every closed case opened before `older_than`. The retention verb.
1821    ///
1822    /// The wiring is the point, exactly as it is for [`drill`](Self::drill):
1823    /// erasure derives blob addresses and key scopes from the tenant and the
1824    /// case, so a caller assembling the stores by hand could tombstone
1825    /// addresses no run ever wrote and report a clean pass over nothing.
1826    ///
1827    /// `reason` lands on every tombstone and on each key destruction, so a
1828    /// later read says *expired, on this date, for this reason* rather than
1829    /// *missing* — the distinction the drill's three-way verdict turns on.
1830    ///
1831    /// `at` is a parameter rather than a clock read, for the reason the
1832    /// sweeper's is: a pass that read the clock could not be tested against a
1833    /// year of ageing cases.
1834    ///
1835    /// Read [`RetentionReport::not_erasable`](crate::retention::RetentionReport::not_erasable)
1836    /// before concluding anything. Without a key ring this expires blob
1837    /// tombstones in the live store and **journal payloads stay verbatim**,
1838    /// which is the residual the erasure page names and the one an erasure
1839    /// request actually turns on.
1840    ///
1841    /// # Errors
1842    ///
1843    /// If this runtime has no case store, or the case layer cannot be walked.
1844    /// A store that fails on one case is a report entry, not an error.
1845    pub async fn retain(
1846        &self,
1847        older_than: crate::core::Timestamp,
1848        at: crate::core::Timestamp,
1849        reason: &str,
1850    ) -> Result<crate::retention::RetentionReport, RuntimeError> {
1851        let cases = self.cases().ok_or_else(|| {
1852            RuntimeError::PlanContract(
1853                "this runtime has no case store — retention walks cases, so there is \
1854                 nothing to retain; build it with `.cases(store)`"
1855                    .into(),
1856            )
1857        })?;
1858        let stores = crate::retention::Stores {
1859            cases,
1860            blobs: self.blobs(),
1861            #[cfg(feature = "keyring")]
1862            keys: self.keyring.as_ref(),
1863            tenant: &self.tenant,
1864            disclosures: self.disclosures.as_ref(),
1865        };
1866        crate::retention::retain(&stores, self.store.as_ref(), older_than, at, reason)
1867            .await
1868            .map_err(RuntimeError::from_store)
1869    }
1870
1871    /// The timer store, if this runtime has one.
1872    #[must_use]
1873    pub fn timers(&self) -> Option<&Arc<dyn TimerStore>> {
1874        self.timers.as_ref()
1875    }
1876
1877    #[must_use]
1878    pub fn batches(&self) -> Option<&Arc<dyn crate::batch::BatchStore>> {
1879        self.batches.as_ref()
1880    }
1881
1882    #[cfg(feature = "push")]
1883    #[must_use]
1884    pub fn push(&self) -> Option<&Arc<dyn crate::push::PushStore>> {
1885        self.push.as_ref()
1886    }
1887
1888    /// Re-arm a webhook registration a delivery worker gave up on.
1889    ///
1890    /// A parked registration is not delivered and not deleted: it keeps its
1891    /// cursor, so re-arming resumes at the first record the receiver never
1892    /// acknowledged rather than at the head of the run. That is the whole
1893    /// reason parking exists, and the reason this verb is on the plane — a
1894    /// backlog nobody can empty is a warning line in yesterday's logs.
1895    ///
1896    /// The clock is read here rather than taken from the caller, unlike the
1897    /// delivery worker's: a worker's schedule has to be assertable in a test,
1898    /// and an operator asking for *now* means now.
1899    ///
1900    /// Returns whether a parked registration was found. `false` is an answer —
1901    /// already live, or never existed — and is told rather than swallowed,
1902    /// because an operator left waiting for a sweep with nothing to do has no
1903    /// way to tell the difference.
1904    ///
1905    /// # Errors
1906    ///
1907    /// [`RuntimeError::PlanContract`] if this plane has no push store, and
1908    /// [`RuntimeError::Store`] if it is unreachable.
1909    #[cfg(feature = "push")]
1910    pub async fn rearm_push(&self, task: RunId, id: &str) -> Result<bool, RuntimeError> {
1911        let push = self
1912            .push
1913            .as_ref()
1914            .ok_or_else(|| RuntimeError::PlanContract("this plane has no push store".into()))?;
1915        let at = now_for_admission().unix_timestamp().max(0).unsigned_abs();
1916        push.unpark(task, id, at)
1917            .await
1918            .map_err(RuntimeError::from_store)
1919    }
1920
1921    /// Ask a run to stop, and drive the stop if nothing else will.
1922    ///
1923    /// The request is durable before this returns, so an operator who gets an
1924    /// acknowledgement has one whether or not the run was reachable. What
1925    /// happens next depends on where the run is:
1926    ///
1927    /// * **Suspended** — nothing is executing, so this resumes the run itself.
1928    ///   It observes the request at its first step boundary and unwinds.
1929    /// * **Running here or elsewhere** — the owner observes the request at its
1930    ///   next step boundary. Nothing is interrupted mid-effect, deliberately:
1931    ///   stopping between "announced" and "recorded" manufactures the in-doubt
1932    ///   case the effect protocol exists to avoid.
1933    /// * **Quarantined** — refused with [`RuntimeError::CannotUnwind`]: the
1934    ///   unwind a stop promises is what a quarantine forbids, and reopening or
1935    ///   abandoning it are the verbs that apply.
1936    /// * **Concluded and closed** (succeeded, abandoned, cancelled) — refused
1937    ///   with [`RuntimeError::AlreadyConcluded`] and nothing is recorded; a run
1938    ///   somebody already stopped answers `false`. A failed or exhausted run is
1939    ///   resumable, and stopping it unwinds its work.
1940    /// * **Held by no provider on this plane** — an operator's terminal: the
1941    ///   request is recorded and not driven, and the plane that holds the run's
1942    ///   agent observes it on its next resume.
1943    ///
1944    /// Returns whether *this* call recorded the request. A second caller gets
1945    /// `false`: the first asker stays on the record, because "who intervened"
1946    /// must not be rewritten by a retry.
1947    ///
1948    /// # Errors
1949    ///
1950    /// [`RuntimeError::PlanContract`] for a blank reason,
1951    /// [`RuntimeError::CannotUnwind`] for a quarantined run,
1952    /// [`RuntimeError::AlreadyConcluded`] for a closed one, and
1953    /// [`RuntimeError`] if the store is unreachable or resuming a suspended run
1954    /// fails.
1955    pub async fn request_cancel(
1956        &self,
1957        run: RunId,
1958        actor: &crate::core::Operator,
1959        reason: &str,
1960    ) -> Result<bool, RuntimeError> {
1961        // An intervention with no stated reason documents nobody's judgement,
1962        // and it is the one field of the record the requester writes.
1963        if reason.trim().is_empty() {
1964            return Err(RuntimeError::PlanContract(
1965                "stopping a run needs a reason — what the person stopping it saw".to_owned(),
1966            ));
1967        }
1968        // Checked before recording. Writing first and failing afterwards leaves a
1969        // request standing against an id that does not exist, and the operator's
1970        // retry then comes back "somebody else already asked" — which is a
1971        // confusing way to say "you mistyped".
1972        let records = self
1973            .store
1974            .read(run, 1)
1975            .await
1976            .map_err(RuntimeError::from_store)?;
1977        if records.is_empty() {
1978            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
1979                run.to_string(),
1980            )));
1981        }
1982
1983        // A quarantine is refused here rather than recorded and ignored. The
1984        // stop this verb promises is *unwind and put the world back*, and that
1985        // is the one thing a quarantined run may not do — so the request landed
1986        // durably, answered the operator `recorded: true`, and the resume then
1987        // walked straight past it. An operator who believes they stopped a run
1988        // holding an unresolved payment is worse off than one who was told no.
1989        //
1990        // Read from the recorded conclusion, not from a live status: a run that
1991        // quarantines *after* this call is a genuine race the standing request
1992        // loses, and that is correct — the request was made about a run nobody
1993        // had given up on.
1994        if recorded_conclusion(&records).as_deref() == Some("quarantined") {
1995            return Err(RuntimeError::CannotUnwind {
1996                run: run.to_string(),
1997            });
1998        }
1999        // Refused only where the conclusion closes the run. A failed or an
2000        // exhausted run is resumable and its work stands, so stopping it is the
2001        // operator's remedy — the unwind it asks for is exactly what those
2002        // conclusions left undone. Read from the latest conclusion, not the
2003        // last record: an answer to a doubt may follow a conclusion, and the
2004        // run is no less closed for it. A run already stopped answers `false`
2005        // rather than an error, so a second asker is told the intervention of
2006        // record is somebody else's.
2007        if let Some(status) = resume_is_closed(&records) {
2008            if self.cancellation(run).await?.is_some() {
2009                return Ok(false);
2010            }
2011            return Err(RuntimeError::AlreadyConcluded {
2012                run: run.to_string(),
2013                outcome: recorded_conclusion(&records)
2014                    .unwrap_or_else(|| status.as_str().to_owned()),
2015            });
2016        }
2017
2018        let fresh = self
2019            .store
2020            .request_cancel(run, actor, reason)
2021            .await
2022            .map_err(RuntimeError::from_store)?;
2023
2024        // Drive it. A suspended run has no thread to notice anything, so an
2025        // operator's stop would sit unobserved until the deadline swept it —
2026        // which for a run waiting on a six-week obligation is not a stop.
2027        //
2028        // Resuming a run that has already concluded is a no-op inside `replay`,
2029        // which reads the recorded status back rather than re-executing — so
2030        // there is no "is it finished?" check to race with here.
2031        //
2032        // Driven **only when the run is genuinely idle**: `replay` claims the
2033        // lease before it reads, and a `LeaseHeld` refusal means somebody is
2034        // executing the run right now — here or on another instance. That is
2035        // not a failure of this call. The request is already durable, and the
2036        // live owner checks for it at every step boundary (the cancellation
2037        // read at the top of the executor's ready-set loop), so the stop
2038        // arrives without a second executor ever touching the run. Resuming
2039        // anyway would, on the owner's own instance, hand the resume the *same
2040        // epoch* the live execution is writing under — two executors on one
2041        // chain that fencing, by construction, cannot tell apart.
2042        //
2043        // A plane holding no provider, case store or key ring for the run — an
2044        // operator's terminal, a library plane — records the request and
2045        // leaves the driving to the plane that does: that plane's next resume
2046        // finds the request standing.
2047        if fresh {
2048            match self.replay(run, Mode::Resume).await {
2049                Ok(_)
2050                | Err(
2051                    RuntimeError::LeaseHeld { .. }
2052                    | RuntimeError::NoProvider { .. }
2053                    | RuntimeError::NoCaseStore { .. }
2054                    | RuntimeError::PayloadsSealed { .. },
2055                ) => {}
2056                Err(e) => return Err(e),
2057            }
2058        }
2059        Ok(fresh)
2060    }
2061
2062    /// Answer a quarantine, and let the runtime decide the run again.
2063    ///
2064    /// A quarantine is this runtime saying it does not know, and the one
2065    /// conclusion it cannot reach past on its own. Everything else that stops a
2066    /// run has a party who can move it — a resume for a crash, a raised ceiling
2067    /// for an exhaustion, a reviewer for a suspension. This is the party for
2068    /// this one.
2069    ///
2070    /// # What this does and does not decide
2071    ///
2072    /// The person decides *whether the run may be judged again*, or that it
2073    /// never will be. They do not decide the outcome. A
2074    /// [`Reopen`](crate::core::QuarantineDecision::Reopen) hands the run back
2075    /// to the executor, which re-derives its verdict from a history that now
2076    /// holds whatever the person established with
2077    /// [`reconcile_effect`](Self::reconcile_effect) — and quarantines it again,
2078    /// on the record, if the answer was wrong or incomplete. An
2079    /// [`Abandon`](crate::core::QuarantineDecision::Abandon) closes it where it
2080    /// stands, unwinding nothing.
2081    ///
2082    /// The decision is journaled **before** it is acted on, so a crash between
2083    /// the two leaves the instruction standing rather than lost: the next
2084    /// resume finds it and finishes the job.
2085    ///
2086    /// Returns the outcome of the pass this drove — so an operator who reopens
2087    /// a run learns immediately whether it got any further, rather than having
2088    /// to go and look.
2089    ///
2090    /// # Errors
2091    ///
2092    /// [`RuntimeError::NotQuarantined`] if the run's last conclusion is
2093    /// anything else, [`RuntimeError::LeaseHeld`] if somebody is executing it,
2094    /// and [`RuntimeError`] for a store failure or a refused resume.
2095    ///
2096    /// # Panics
2097    ///
2098    /// Never: the `expect` covers a lease this function has just acquired.
2099    pub async fn decide_quarantine(
2100        &self,
2101        run: RunId,
2102        decider: &crate::core::Operator,
2103        reason: &str,
2104        decision: crate::core::QuarantineDecision,
2105    ) -> Result<RunOutcome, RuntimeError> {
2106        // The whole weight of this record is that a named person took
2107        // responsibility for a fact the runtime could not establish, and
2108        // "unknown decided this for no stated reason" documents nothing while
2109        // looking like process. Only the reason is checked here: `Operator` has
2110        // no empty value, so the decider is guaranteed by the type on every
2111        // path that builds one rather than by a check on this one.
2112        if reason.trim().is_empty() {
2113            return Err(RuntimeError::PlanContract(
2114                "answering a quarantine needs a decider and a reason — who looked, and what they \
2115                 found"
2116                    .to_owned(),
2117            ));
2118        }
2119
2120        let lease = self
2121            .store
2122            .acquire(run, &self.owner, self.lease_ttl)
2123            .await
2124            .map_err(RuntimeError::from_store)?;
2125        let appended = self
2126            .record_decision(run, &lease, decider, reason, decision)
2127            .await;
2128        if let Err(e) = appended {
2129            // The lease was claimed by this call and never handed to a resume,
2130            // so it is this call's to give back. Leaving it would strand the
2131            // run for a TTL and hand it to the recovery sweep, which would
2132            // "recover" a run nobody was running.
2133            let _ = self.store.release_lease(run, lease.epoch).await;
2134            return Err(e);
2135        }
2136        // Handed over rather than released and re-taken: the same window a wake
2137        // path closes. Both decisions travel this path — an abandonment is
2138        // concluded inside the resume, from the record just written. The run
2139        // still stands quarantined on the record, so it keeps its listing
2140        // whatever this pass does.
2141        self.replay_releasing(run, Pass::Resume, Some(lease), Claim::Standing)
2142            .await
2143    }
2144
2145    /// Answer a quarantine **without driving the run**.
2146    ///
2147    /// [`decide_quarantine`](Self::decide_quarantine) records the decision and
2148    /// then hands the run to the executor, which is what an operator on a
2149    /// running plane wants: they learn immediately whether a reopened run got
2150    /// any further.
2151    ///
2152    /// A terminal holding a store cannot do the second half. It has the
2153    /// journal and not the agent — a declarative run's program is a manifest
2154    /// this process was not given — so driving the run would mean refusing the
2155    /// verb for want of an argument the operator may not have during an
2156    /// incident.
2157    ///
2158    /// **That is a supported state rather than a lesser one.** The decision is
2159    /// journaled before it is acted on precisely so a crash between the two
2160    /// leaves the instruction standing, and this is the same position reached
2161    /// deliberately: the next resume finds the record and finishes the job —
2162    /// so what the caller must not do is report the run as moved.
2163    ///
2164    /// # Errors
2165    ///
2166    /// As [`decide_quarantine`](Self::decide_quarantine), minus everything the
2167    /// resume could have raised.
2168    pub async fn record_quarantine_decision(
2169        &self,
2170        run: RunId,
2171        decider: &crate::core::Operator,
2172        reason: &str,
2173        decision: crate::core::QuarantineDecision,
2174    ) -> Result<(), RuntimeError> {
2175        if reason.trim().is_empty() {
2176            return Err(RuntimeError::PlanContract(
2177                "answering a quarantine needs a decider and a reason — who looked, and what they \
2178                 found"
2179                    .to_owned(),
2180            ));
2181        }
2182        let lease = self
2183            .store
2184            .acquire(run, &self.owner, self.lease_ttl)
2185            .await
2186            .map_err(RuntimeError::from_store)?;
2187        let appended = self
2188            .record_decision(run, &lease, decider, reason, decision)
2189            .await;
2190        // Always given back: this verb never executes the run, so the lease is
2191        // held only for the width of one append — the rule
2192        // `reconcile_effect` follows.
2193        let _ = self.store.release_lease(run, lease.epoch).await;
2194        appended
2195    }
2196
2197    /// Append the decision, once the run is known to be asking for one.
2198    async fn record_decision(
2199        &self,
2200        run: RunId,
2201        lease: &crate::journal::Lease,
2202        decider: &crate::core::Operator,
2203        reason: &str,
2204        decision: crate::core::QuarantineDecision,
2205    ) -> Result<(), RuntimeError> {
2206        let records = self
2207            .store
2208            .read(run, 1)
2209            .await
2210            .map_err(RuntimeError::from_store)?;
2211        if records.is_empty() {
2212            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2213                run.to_string(),
2214            )));
2215        }
2216        let status = recorded_conclusion(&records);
2217        // A run whose payloads were erased is asking the same question a
2218        // quarantine asks — *the runtime cannot go on, and a person must
2219        // decide* — so it is answered by the same verb whatever outcome it was
2220        // left under. Without this it has no verb at all: it cannot be
2221        // resumed, and a cancellation promises an unwind that erased arguments
2222        // make impossible. Abandonment is the honest ending and this is the
2223        // door to it.
2224        if status.as_deref() != Some("quarantined") && !payloads_erased(&records) {
2225            return Err(RuntimeError::NotQuarantined {
2226                run: run.to_string(),
2227                status: status.unwrap_or_else(|| "still running".to_owned()),
2228            });
2229        }
2230        let mut append = Append::new(
2231            run,
2232            RecordKind::QuarantineDecided {
2233                decider: decider.clone(),
2234                reason: reason.to_owned(),
2235                decision,
2236            },
2237        );
2238        if let Some(case) = records.iter().find_map(|r| r.body.case) {
2239            append = append.case(case);
2240        }
2241        self.store
2242            .append(lease.epoch, vec![append])
2243            .await
2244            .map_err(RuntimeError::from_store)?;
2245        Ok(())
2246    }
2247
2248    /// Supply, out of band, the outcome of an effect the journal cannot decide.
2249    ///
2250    /// The manual twin of [`Effect::reconcile`](crate::core::Effect::reconcile),
2251    /// and it produces the same record for the same reason: an operator who
2252    /// looks the charge up in the provider's console is answering exactly the
2253    /// question a probe asks, so the answer belongs in the place a probe's
2254    /// answer goes, where replay reads it back and nothing re-derives it.
2255    ///
2256    /// What separates the two is on the record too. A probe's verdict is
2257    /// attributed to the run; this one names the person, because "the provider
2258    /// told us" and "somebody asserted it" are different evidence.
2259    ///
2260    /// **It supplies a missing fact and never replaces a recorded one.** Only
2261    /// an effect [`undecided_effects`](crate::journal::undecided_effects) names
2262    /// may be answered. Without that rule an operator could talk a run out of
2263    /// compensating work that is standing in the world, and the journal would
2264    /// show an orderly reconciliation while it happened.
2265    ///
2266    /// This changes no status by itself: the run stays quarantined until
2267    /// [`decide_quarantine`](Self::decide_quarantine) hands it back. Several
2268    /// doubts can be answered before the run is judged again, and answering one
2269    /// of three is not a decision.
2270    ///
2271    /// # Errors
2272    ///
2273    /// [`RuntimeError::NotUndecided`] if that effect's outcome is already on
2274    /// the record, [`RuntimeError::LeaseHeld`] if somebody is executing the
2275    /// run, and [`RuntimeError`] for a store failure.
2276    pub async fn reconcile_effect(
2277        &self,
2278        run: RunId,
2279        effect: crate::core::EffectKey,
2280        assertion: crate::core::Assertion,
2281        asserted_by: &crate::core::Operator,
2282        note: &str,
2283    ) -> Result<(), RuntimeError> {
2284        // `asserted_by` needs no emptiness check: `Operator` has none to fail.
2285        if note.trim().is_empty() {
2286            return Err(RuntimeError::PlanContract(
2287                "an assertion about an effect needs a name and a note — who established this, and \
2288                 how"
2289                .to_owned(),
2290            ));
2291        }
2292        let lease = self
2293            .store
2294            .acquire(run, &self.owner, self.lease_ttl)
2295            .await
2296            .map_err(RuntimeError::from_store)?;
2297        let result = self
2298            .record_assertion(run, &lease, effect, assertion, asserted_by, note)
2299            .await;
2300        // Always given back: this verb never executes the run, so the lease is
2301        // held only for the width of one append.
2302        let _ = self.store.release_lease(run, lease.epoch).await;
2303        result
2304    }
2305
2306    async fn record_assertion(
2307        &self,
2308        run: RunId,
2309        lease: &crate::journal::Lease,
2310        effect: crate::core::EffectKey,
2311        assertion: crate::core::Assertion,
2312        asserted_by: &crate::core::Operator,
2313        note: &str,
2314    ) -> Result<(), RuntimeError> {
2315        let records = self
2316            .store
2317            .read(run, 1)
2318            .await
2319            .map_err(RuntimeError::from_store)?;
2320        if records.is_empty() {
2321            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2322                run.to_string(),
2323            )));
2324        }
2325        let Some(undecided) = crate::journal::undecided_effects(&records)
2326            .into_iter()
2327            .find(|u| u.effect == effect)
2328        else {
2329            return Err(RuntimeError::NotUndecided {
2330                run: run.to_string(),
2331                effect: effect.to_string(),
2332            });
2333        };
2334
2335        let output = match &assertion {
2336            crate::core::Assertion::Landed(value) => Some(value.clone()),
2337            crate::core::Assertion::DidNotHappen => None,
2338        };
2339        let mut append = Append::new(
2340            run,
2341            RecordKind::EffectReconciled {
2342                disposition: assertion.disposition(),
2343                // Stated on the record rather than left to a reader's default.
2344                // A durable format that does not say what it means is one a
2345                // later build guesses about, and the guess here is the label on
2346                // a value the run is about to act on.
2347                declared: output
2348                    .is_some()
2349                    .then(crate::core::DeclaredOutput::untrusted),
2350                output,
2351                // A person's assertion costs nothing. What the call itself
2352                // consumed, if it consumed anything, is already on the record
2353                // of the attempt this answers.
2354                spend: Spend::ZERO,
2355                // No probe ran: this is a person's assertion, so the
2356                // provider-facing field stays empty and their account goes in
2357                // the field that is read without a key.
2358                detail: None,
2359                note: Some(note.to_owned()),
2360                asserted_by: Some(asserted_by.clone()),
2361            },
2362        )
2363        .step(undecided.step)
2364        .phase(undecided.phase)
2365        .effect(effect);
2366        if let Some(case) = records.iter().find_map(|r| r.body.case) {
2367            append = append.case(case);
2368        }
2369        self.store
2370            .append(lease.epoch, vec![append])
2371            .await
2372            .map_err(RuntimeError::from_store)?;
2373        tracing::info!(
2374            target: telemetry::RECONCILED,
2375            %run,
2376            step = %undecided.step,
2377            verdict = ?assertion.disposition(),
2378            actor = %asserted_by,
2379        );
2380        self.meter
2381            .count(metrics::RECONCILIATIONS, assertion.disposition().as_str());
2382        Ok(())
2383    }
2384
2385    /// Every mutating effect this run's journal cannot decide the outcome of.
2386    ///
2387    /// What an operator holding a quarantine has to go and look up, and the
2388    /// keys they quote back to [`reconcile_effect`](Self::reconcile_effect).
2389    ///
2390    /// # Errors
2391    ///
2392    /// If the store is unreachable.
2393    pub async fn undecided(&self, run: RunId) -> Result<Vec<crate::core::Undecided>, RuntimeError> {
2394        let records = self
2395            .store
2396            .read(run, 1)
2397            .await
2398            .map_err(RuntimeError::from_store)?;
2399        Ok(crate::journal::undecided_effects(&records))
2400    }
2401
2402    /// Stop this tenant from starting new work, or let it start again.
2403    ///
2404    /// The emergency stop. `Some(reason)` halts, `None` lifts, and the reason is
2405    /// required because the next person to look will be somebody else, possibly
2406    /// at three in the morning, and *why* is the whole question.
2407    ///
2408    /// **What it stops, precisely.** New admissions, across every instance,
2409    /// because the flag is in the store rather than in this process — a switch
2410    /// that stops only the instance it was thrown on is the in-process-counter
2411    /// failure arriving during an incident. Refusals are their own error, not a
2412    /// ceiling: a ceiling means *not right now* and invites a retry, which is
2413    /// exactly what somebody pulling this switch is trying to stop.
2414    ///
2415    /// **What a workload-scoped halt does not stop, deliberately.** Runs
2416    /// already executing, and suspended runs resuming. Those are existing work,
2417    /// and refusing to let them continue would strand them mid-saga with
2418    /// reversals unrun — turning an incident into a second one. To stop work in
2419    /// flight, cancel it: that unwinds what it did and records who asked. This
2420    /// is the front door, not a power cut, and saying so is the difference
2421    /// between a control an operator can reason about and one they discover the
2422    /// shape of during an outage.
2423    ///
2424    /// **How wide it reaches** is the [`HaltScope`](crate::quota::HaltScope),
2425    /// and the scopes divide on *which axis they name* rather than on how
2426    /// broad they are. Three name a workload — the tenant, every revision of
2427    /// one declared agent, or one exact reviewed digest — and stop admission.
2428    /// The fourth names an **authority**, any principal on the delegation
2429    /// chain a run was admitted under, and is the one that reaches work
2430    /// already running:
2431    /// there the incident *is* the credential, so a run carrying on under it is
2432    /// the harm rather than a side effect, and it pauses at its next step
2433    /// boundary with its completed work standing
2434    /// ([`withdraws_from`](crate::quota::HaltScope::withdraws_from) is the
2435    /// predicate, so the distinction is checked rather than remembered).
2436    /// Scopes are independent rows — lifting a narrow halt leaves a broad one
2437    /// standing. An ungoverned run, with no manifest, is stopped only by a
2438    /// tenant halt; there is nothing narrower to key it on.
2439    ///
2440    /// `at` is the caller's, never a clock read here: every lifecycle instant in
2441    /// this crate is supplied for the reason a hold's is — a pass that reads its
2442    /// own clock cannot be tested against an ageing plane, and [`I1`] forbids the
2443    /// runtime an ambient clock outside the effect protocol.
2444    ///
2445    /// Requires a quota store; without one there is nowhere durable to keep the
2446    /// flag, and an emergency stop that a restart forgets is not one.
2447    ///
2448    /// [`I1`]: crate::journal
2449    ///
2450    /// # Errors
2451    ///
2452    /// If no quota store is wired, or the store is unreachable.
2453    pub async fn set_halt(
2454        &self,
2455        scope: &crate::quota::HaltScope,
2456        by: &crate::core::Operator,
2457        at: crate::core::Timestamp,
2458        reason: &str,
2459    ) -> Result<(), RuntimeError> {
2460        let quotas = self.quota_store()?;
2461        quotas
2462            .set_halt(scope, by, at, reason)
2463            .await
2464            .map_err(RuntimeError::Store)?;
2465        self.forget_credentials(scope);
2466        Ok(())
2467    }
2468
2469    /// Drop the peer credentials held for a subject a halt names, when it is
2470    /// thrown and when it is lifted — not only when a hop next meets it, which
2471    /// may never happen while the halt stands.
2472    fn forget_credentials(&self, scope: &crate::quota::HaltScope) {
2473        if let (crate::quota::HaltScope::Subject { id }, Some(wiring)) =
2474            (scope, self.peers.as_ref())
2475        {
2476            wiring.registry.forget(id);
2477        }
2478    }
2479
2480    /// Lift one, and let work at that scope start again — recording who did.
2481    ///
2482    /// Answers the run holding the lift record and whether this call removed
2483    /// the row, or `None` when nothing was standing at `scope`: an operator who
2484    /// lifts a scope nobody halted is told so rather than told *done*, and
2485    /// nothing is written.
2486    ///
2487    /// **The record comes first, then the row goes.** The lift is written as
2488    /// the sole [`HaltLifted`](crate::journal::RecordKind::HaltLifted) record
2489    /// of a sealed run of its own, carrying `by`, `at` and the whole stop it
2490    /// ended; only then is the register's row removed. The journal and the
2491    /// register share no transaction, so a lift that cannot be recorded does
2492    /// not happen, and a removal that fails after the record is written is an
2493    /// error naming the record's run — the halt still stands, the register is
2494    /// the answer to what is in force, and lifting again writes a second
2495    /// record. The record attests the instruction and the row as read, not
2496    /// that the removal succeeded.
2497    ///
2498    /// **Only the row the record names goes.** The removal is
2499    /// [`QuotaStore::lift_halt_if`](crate::quota::QuotaStore::lift_halt_if)
2500    /// with that row, so a halt re-thrown between the read and the removal is
2501    /// left standing and the call fails naming the record's run. A row another
2502    /// lift already removed answers `removed: false`.
2503    ///
2504    /// [`QuotaStore::lift_halt`](crate::quota::QuotaStore::lift_halt) is the
2505    /// raw register verb; a direct call to it is unrecorded.
2506    ///
2507    /// # Errors
2508    ///
2509    /// If no quota store is wired, the store is unreachable, or the record
2510    /// cannot be written; [`RuntimeError::ControlStands`] if the row cannot be
2511    /// removed after the record was written, or was re-thrown in between.
2512    pub async fn lift_halt(
2513        &self,
2514        scope: &crate::quota::HaltScope,
2515        by: &crate::core::Operator,
2516        at: crate::core::Timestamp,
2517    ) -> Result<Option<ControlLifted>, RuntimeError> {
2518        let quotas = self.quota_store()?;
2519        let standing = quotas.halts().await.map_err(RuntimeError::Store)?;
2520        let Some(halt) = standing.into_iter().find(|h| &h.scope == scope) else {
2521            return Ok(None);
2522        };
2523        let run = self
2524            .seal_operator_record(
2525                RecordKind::HaltLifted {
2526                    scope: scope.key(),
2527                    by: by.clone(),
2528                    at,
2529                    reason: halt.reason.clone(),
2530                    thrown_by: halt.by.clone(),
2531                    thrown_at: halt.at,
2532                },
2533                None,
2534                HALT_LIFTED_OUTCOME,
2535                None,
2536            )
2537            .await?;
2538        let removed = quotas
2539            .lift_halt_if(&halt)
2540            .await
2541            .map_err(|e| removal_failed(run, &e))?;
2542        self.forget_credentials(scope);
2543        if !removed {
2544            let now = quotas.halts().await.map_err(|e| removal_failed(run, &e))?;
2545            if now.iter().any(|h| &h.scope == scope) {
2546                return Err(RuntimeError::ControlStands {
2547                    run: run.to_string(),
2548                    detail: format!(
2549                        "the halt on '{}' was re-thrown after this lift read it, and the new \
2550                         halt still stands",
2551                        scope.key()
2552                    ),
2553                });
2554            }
2555        }
2556        Ok(Some(ControlLifted {
2557            record: run,
2558            removed,
2559        }))
2560    }
2561
2562    /// Release the legal hold on `case` — recording who did.
2563    ///
2564    /// The hold's counterpart of [`lift_halt`](Self::lift_halt), in the same
2565    /// order for the same reason: the
2566    /// [`HoldReleased`](crate::journal::RecordKind::HoldReleased) record is
2567    /// written, stamped with `case`, before the row goes. It names the releaser
2568    /// and the hold's placing operator and instant, and not the hold's reason,
2569    /// so the erasure the release permits leaves its authorization readable.
2570    ///
2571    /// `None` when the matter is not held, with nothing written; `removed:
2572    /// false` when another release removed the row first.
2573    ///
2574    /// **Only the hold the record names goes.** The removal is
2575    /// [`CaseStore::release_hold_if`](crate::case::CaseStore::release_hold_if)
2576    /// with that hold, so a hold placed again between the read and the removal
2577    /// is left standing and the call fails naming the record's run.
2578    /// [`CaseStore::release_hold`](crate::case::CaseStore::release_hold) is the
2579    /// raw register verb; a direct call to it is unrecorded.
2580    ///
2581    /// # Errors
2582    ///
2583    /// [`StoreError::NotFound`](crate::core::StoreError::NotFound) if the case
2584    /// does not exist; [`RuntimeError::ControlStands`] if the row cannot be
2585    /// removed after the record was written, or was placed again in between;
2586    /// otherwise if no case store is
2587    /// wired, the store is unreachable, or the record cannot be written.
2588    pub async fn release_hold(
2589        &self,
2590        case: crate::core::CaseId,
2591        by: &crate::core::Operator,
2592        at: crate::core::Timestamp,
2593    ) -> Result<Option<ControlLifted>, RuntimeError> {
2594        let cases = self.cases.as_ref().ok_or_else(|| {
2595            RuntimeError::PlanContract("releasing a hold needs a case store".to_owned())
2596        })?;
2597        let Some(standing) = cases.hold(case).await.map_err(RuntimeError::Store)? else {
2598            return match cases.case(case).await.map_err(RuntimeError::Store)? {
2599                Some(_) => Ok(None),
2600                None => Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2601                    format!("case {case}"),
2602                ))),
2603            };
2604        };
2605        let run = self
2606            .seal_operator_record(
2607                RecordKind::HoldReleased {
2608                    by: by.clone(),
2609                    at,
2610                    placed_by: standing.by.clone(),
2611                    placed_at: standing.placed_at,
2612                },
2613                Some(case),
2614                HOLD_RELEASED_OUTCOME,
2615                None,
2616            )
2617            .await?;
2618        let removed = cases
2619            .release_hold_if(case, &standing)
2620            .await
2621            .map_err(|e| removal_failed(run, &e))?;
2622        if !removed
2623            && cases
2624                .hold(case)
2625                .await
2626                .map_err(|e| removal_failed(run, &e))?
2627                .is_some()
2628        {
2629            return Err(RuntimeError::ControlStands {
2630                run: run.to_string(),
2631                detail: format!(
2632                    "the hold on case {case} was placed again after this release read it, \
2633                     and the new hold still stands"
2634                ),
2635            });
2636        }
2637        Ok(Some(ControlLifted {
2638            record: run,
2639            removed,
2640        }))
2641    }
2642
2643    /// Every recorded halt lift, newest first: the one record of each run.
2644    ///
2645    /// # Errors
2646    ///
2647    /// If the journal is unreachable, or a lift run does not hold a lift record.
2648    pub async fn lifted_halts(&self, limit: usize) -> Result<Vec<Record>, RuntimeError> {
2649        self.operator_records(HALT_LIFTED_OUTCOME, limit).await
2650    }
2651
2652    /// Every recorded hold release, newest first: the one record of each run.
2653    ///
2654    /// # Errors
2655    ///
2656    /// If the journal is unreachable, or a release run does not hold a release
2657    /// record.
2658    pub async fn released_holds(&self, limit: usize) -> Result<Vec<Record>, RuntimeError> {
2659        self.operator_records(HOLD_RELEASED_OUTCOME, limit).await
2660    }
2661
2662    /// The first record of each run sealed under `outcome`, in the index's
2663    /// newest-first order. A run whose first record is not the act its outcome
2664    /// names is a store error, never skipped: a history that silently thins is
2665    /// one that reads as complete.
2666    async fn operator_records(
2667        &self,
2668        outcome: &str,
2669        limit: usize,
2670    ) -> Result<Vec<Record>, RuntimeError> {
2671        let runs = self
2672            .store
2673            .runs_by_outcome(outcome, limit)
2674            .await
2675            .map_err(RuntimeError::from_store)?;
2676        let mut found = Vec::with_capacity(runs.len());
2677        for run in runs {
2678            let first = self
2679                .store
2680                .read_page(run, 1, 1)
2681                .await
2682                .map_err(RuntimeError::from_store)?
2683                .into_iter()
2684                .next();
2685            match first {
2686                Some(record)
2687                    if matches!(
2688                        (outcome, record.kind()),
2689                        (HALT_LIFTED_OUTCOME, RecordKind::HaltLifted { .. })
2690                            | (HOLD_RELEASED_OUTCOME, RecordKind::HoldReleased { .. })
2691                    ) =>
2692                {
2693                    found.push(record);
2694                }
2695                _ => {
2696                    return Err(RuntimeError::Store(crate::core::StoreError::Corrupt {
2697                        seq: 1,
2698                        detail: format!(
2699                            "run {run} is sealed as '{outcome}' but holds no such record"
2700                        ),
2701                    }));
2702                }
2703            }
2704        }
2705        Ok(found)
2706    }
2707
2708    /// The quota store, when one is wired.
2709    pub(crate) fn quota_store_if_wired(&self) -> Option<&Arc<dyn crate::quota::QuotaStore>> {
2710        self.quotas.as_ref()
2711    }
2712
2713    /// Every run holding spend against this tenant's periods, and how much.
2714    ///
2715    /// The runs a [`QuotaError::SpentOut`](crate::quota::QuotaError::SpentOut)
2716    /// refusal counts as *reserved*. A run holds its reservation until it
2717    /// concludes, so the ones that stopped waiting on a person — quarantined,
2718    /// exhausted, withheld, failed — hold it until that person acts;
2719    /// [`attention`](Self::attention) names those.
2720    ///
2721    /// # Errors
2722    ///
2723    /// If no quota store is wired, or it is unreachable.
2724    pub async fn reservations(
2725        &self,
2726        limit: usize,
2727    ) -> Result<Vec<crate::quota::Held>, RuntimeError> {
2728        let quotas = self.quotas.as_ref().ok_or_else(|| {
2729            RuntimeError::Store(crate::core::StoreError::Backend(
2730                "no quota store is wired, so no run holds spend against a period".to_owned(),
2731            ))
2732        })?;
2733        quotas
2734            .reservations(limit)
2735            .await
2736            .map_err(RuntimeError::Store)
2737    }
2738
2739    /// The quota store, or the refusal that says why an emergency stop needs one.
2740    fn quota_store(&self) -> Result<&std::sync::Arc<dyn crate::quota::QuotaStore>, RuntimeError> {
2741        self.quotas.as_ref().ok_or_else(|| {
2742            RuntimeError::Store(crate::core::StoreError::Backend(
2743                "an emergency stop needs a quota store to keep the flag in — an \
2744                 in-process one is forgotten by a restart and never seen by a \
2745                 second instance"
2746                    .to_owned(),
2747            ))
2748        })
2749    }
2750
2751    /// Every standing emergency stop on this tenant, and why.
2752    ///
2753    /// The operator's question — *what is stopped right now?* — which a
2754    /// per-scope lookup cannot answer without already knowing what to ask
2755    /// about.
2756    ///
2757    /// # Errors
2758    ///
2759    /// If no quota store is wired, or the store is unreachable.
2760    pub async fn halts(&self) -> Result<Vec<crate::quota::Halt>, RuntimeError> {
2761        let quotas = self.quotas.as_ref().ok_or_else(|| {
2762            RuntimeError::Store(crate::core::StoreError::Backend(
2763                "no quota store is wired, so no emergency stop can be set or read".to_owned(),
2764            ))
2765        })?;
2766        quotas.halts().await.map_err(RuntimeError::Store)
2767    }
2768
2769    /// Which runs are executing **right now**, and under whose authority.
2770    ///
2771    /// [`runs_by_outcome`](crate::journal::JournalStore::runs_by_outcome)
2772    /// answers from the journal's index, which only *concluded* runs are in; the
2773    /// live set is the tenant's admission slots.
2774    ///
2775    /// **The join is the point, not the ids.** An incident is rarely *cancel
2776    /// this run* but a revoked credential or a bad revision, so each entry
2777    /// carries the declared agent and the delegation subject it was admitted
2778    /// under.
2779    ///
2780    /// A slot whose lease has lapsed is marked `stranded` rather than omitted:
2781    /// the recovery sweep resumes those, and cancelling one undoes work that was
2782    /// about to continue.
2783    ///
2784    /// Bounded by `limit`, and reads a page of each run's header records to
2785    /// attribute it — an operator question rather than a hot path.
2786    ///
2787    /// # Errors
2788    ///
2789    /// If no quota store is wired, or either store is unreachable.
2790    pub async fn running_runs(&self, limit: usize) -> Result<Vec<LiveRun>, RuntimeError> {
2791        let quotas = self.quotas.as_ref().ok_or_else(|| {
2792            RuntimeError::Store(crate::core::StoreError::Backend(
2793                "no quota store is wired, so nothing tracks which runs hold this \
2794                 tenant's admission slots"
2795                    .to_owned(),
2796            ))
2797        })?;
2798        let live = quotas
2799            .running_runs(limit)
2800            .await
2801            .map_err(RuntimeError::Store)?;
2802        // One read, not one per run: the stranded set is small and the whole of
2803        // it is needed to answer any entry.
2804        let stranded: std::collections::BTreeSet<RunId> = self
2805            .store
2806            .abandoned_runs(limit)
2807            .await
2808            .map_err(RuntimeError::Store)?
2809            .into_iter()
2810            .collect();
2811
2812        let mut out = Vec::with_capacity(live.len());
2813        for run in live {
2814            // The header records only. `RunAdmitted` is the first and
2815            // `IdentityBound` follows it when the caller presented a chain, so a
2816            // short page carries both without reading a run's whole history —
2817            // which on a long-running agent is the difference between an
2818            // operator listing and a scan.
2819            let head = self
2820                .store
2821                .read_page(run, 1, ATTRIBUTION_RECORDS)
2822                .await
2823                .map_err(RuntimeError::Store)?;
2824            let governed_by = head.iter().find_map(|r| match r.kind() {
2825                crate::journal::RecordKind::RunAdmitted { governed_by, .. } => {
2826                    governed_by.as_ref().map(|id| (**id).clone())
2827                }
2828                _ => None,
2829            });
2830            let subject = head.iter().find_map(|r| match r.kind() {
2831                crate::journal::RecordKind::IdentityBound { chain } => {
2832                    chain.first().map(|p| p.id.clone())
2833                }
2834                _ => None,
2835            });
2836            out.push(LiveRun {
2837                stranded: stranded.contains(&run),
2838                run,
2839                governed_by,
2840                subject,
2841            });
2842        }
2843        Ok(out)
2844    }
2845
2846    /// The runs that are waiting, and what each one waits for.
2847    ///
2848    /// The third listing, beside concluded runs and running ones: a suspended
2849    /// run has no outcome and holds no admission slot, so neither of the others
2850    /// names it. Re-arming these is the recovery runbook's last step, and until
2851    /// this existed that step named a verb with no argument an operator could
2852    /// obtain.
2853    ///
2854    /// Soonest due first, bounded by `limit`. Straight through to the store's
2855    /// maintained index — no page of records is read, because what a run waits
2856    /// for is on the suspension itself.
2857    ///
2858    /// # Errors
2859    ///
2860    /// If the store is unreachable.
2861    pub async fn waiting_runs(
2862        &self,
2863        limit: usize,
2864    ) -> Result<Vec<crate::journal::WaitingRun>, RuntimeError> {
2865        self.store
2866            .waiting_runs(limit)
2867            .await
2868            .map_err(RuntimeError::from_store)
2869    }
2870
2871    /// Stop this instance taking on work, then wait for what it is already
2872    /// doing.
2873    ///
2874    /// A process killed mid-step leaves an announced effect with no terminal
2875    /// record, so its declared [`Recovery`](crate::core::Recovery) has to
2876    /// decide whether it happened — the right answer to an accident, and an
2877    /// expensive one to schedule on every deploy.
2878    ///
2879    /// Admission closes **first**: every later [`start`](Self::start) is
2880    /// refused
2881    /// [`RuntimeError::Draining`](crate::core::RuntimeError::Draining), leaving
2882    /// no lease, no slot and no journal. Then this waits up to `grace` for the
2883    /// runs it put on a background task to reach a journaled resting point.
2884    /// Closing first is what makes the answer worth having — a wait alone
2885    /// reports about an instant — and it is also the only signal a long-lived
2886    /// read gets, so an open A2A subscription ends rather than holding a
2887    /// graceful shutdown for as long as the run it watches.
2888    ///
2889    /// **Run it beside a server's own graceful shutdown, not after it** —
2890    /// `tokio::join!` of the two. Serialising deadlocks one on the other in
2891    /// whichever order you pick: a run awaited inside a request handler is
2892    /// finished by the server, and an open subscription is ended by this.
2893    ///
2894    /// Three things it deliberately does not do. It does not wait for runs on
2895    /// somebody else's task; those belong to that caller's own shutdown. It
2896    /// does not gate resumes or an agent commissioning another one — both
2897    /// continue work this plane already owns, and a commission is a step of a
2898    /// run this is waiting for. And it does not hand back the leases of runs it
2899    /// did not finish: a release says takeover is safe, and for a run still
2900    /// inside `perform` the next owner would re-perform the call beside the one
2901    /// in flight. Left to expire, the lease is the signal the recovery sweep
2902    /// reads as *an instance died holding this run* — which is what a
2903    /// drained-out run is.
2904    ///
2905    /// Idempotent: draining an instance that is already draining waits again.
2906    pub async fn drain(&self, grace: std::time::Duration) -> super::DrainReport {
2907        self.inflight.drain(grace).await
2908    }
2909
2910    /// Whether this instance has stopped admitting runs.
2911    ///
2912    /// For an embedder whose server keeps its listener open while it drains and
2913    /// therefore needs something to answer a readiness probe with. The shipped
2914    /// server needs none: it closes its listeners first, so a probe that
2915    /// connects already fails — which is why this crate's HTTP surface has no
2916    /// `/health` route to flip. See [`crate::api`] on why it has none at all.
2917    #[must_use]
2918    pub fn is_draining(&self) -> bool {
2919        self.inflight.is_draining()
2920    }
2921
2922    /// The cancellation recorded against a run, if one was.
2923    ///
2924    /// Reads the journal rather than a flag: a cancellation is a record, so
2925    /// this answers the same thing after a restart and on another instance.
2926    ///
2927    /// # Errors
2928    ///
2929    /// If the store is unreachable.
2930    pub async fn cancellation(
2931        &self,
2932        run: RunId,
2933    ) -> Result<Option<crate::journal::Cancellation>, RuntimeError> {
2934        self.store
2935            .cancellation(run)
2936            .await
2937            .map_err(RuntimeError::from_store)
2938    }
2939
2940    /// The policy engine, if this runtime has one.
2941    ///
2942    /// Exposed because a surface that faces strangers has to be able to ask
2943    /// whether one exists at all. Inside the process the caller is the
2944    /// embedder's own code and an absent engine is a choice; on a socket it is
2945    /// a hole, and the HTTP surface refuses to start without one.
2946    #[must_use]
2947    pub fn policy(&self) -> Option<&Arc<dyn crate::core::PolicyEngine>> {
2948        self.policy.as_ref()
2949    }
2950
2951    /// Which tenant this plane runs as.
2952    #[must_use]
2953    pub fn tenant(&self) -> &crate::core::TenantId {
2954        &self.tenant
2955    }
2956
2957    /// What this plane's policy engine cannot evaluate for a served caller
2958    /// that presented no chain.
2959    ///
2960    /// The build probes the chain shapes this plane's own runs take: its own
2961    /// chain where it has one, and both shapes where it has none. A served
2962    /// surface adds the one the build cannot know about — a caller acting
2963    /// under no chain on a plane that has one — so every surface that admits
2964    /// work for a stranger asks this before it starts.
2965    #[cfg(any(feature = "a2a-server", feature = "mcp-server"))]
2966    pub(crate) fn served_policy_problems(&self) -> Vec<String> {
2967        match (&self.policy, &self.identity) {
2968            (Some(engine), Some(_)) => runtime_policy_problems(engine.as_ref(), None),
2969            _ => Vec::new(),
2970        }
2971    }
2972
2973    /// How far from its owner the plane's own chain is, or zero without one.
2974    ///
2975    /// The depth a run bound by the plane's chain but acting under none starts
2976    /// from: acting under no chain does not put a served caller closer to the
2977    /// owner than the plane that serves it.
2978    pub(crate) fn own_depth(&self) -> usize {
2979        self.identity
2980            .as_ref()
2981            .map_or(0, crate::core::Delegation::depth)
2982    }
2983
2984    /// The journal, under the name the batch driver reads it by.
2985    #[must_use]
2986    pub(crate) fn meter(&self) -> &super::metrics::Meter {
2987        &self.meter
2988    }
2989
2990    /// Write an operator act as the sole record of a sealed run of its own.
2991    ///
2992    /// The act, then its conclusion, then the seal — so it carries the chain,
2993    /// the signature, Merkle inclusion and the witness, and an offline reader
2994    /// checks it without being taught what the act was. `case` stamps the
2995    /// record onto a matter's history; `reason` is the conclusion's, for the
2996    /// one act whose reason lives there.
2997    async fn seal_operator_record(
2998        &self,
2999        kind: RecordKind,
3000        case: Option<crate::core::CaseId>,
3001        outcome: &str,
3002        reason: Option<&str>,
3003    ) -> Result<RunId, RuntimeError> {
3004        let run = RunId::generate();
3005        let epoch = BREAK_GLASS_EPOCH;
3006        let mut act = Append::new(run, kind);
3007        if let Some(case) = case {
3008            act = act.case(case);
3009        }
3010        self.store
3011            .append(epoch, vec![act])
3012            .await
3013            .map_err(RuntimeError::from_store)?;
3014        let head = self
3015            .store
3016            .head(run)
3017            .await
3018            .map_err(RuntimeError::from_store)?;
3019        self.store
3020            .append(
3021                epoch,
3022                vec![Append::new(
3023                    run,
3024                    RecordKind::RunConcluded {
3025                        outcome: outcome.to_owned(),
3026                        reason: reason.map(str::to_owned),
3027                        exhaustion: None,
3028                        live_spend: Spend::default(),
3029                        chain_head: head.hash,
3030                    },
3031                )],
3032            )
3033            .await
3034            .map_err(RuntimeError::from_store)?;
3035        self.store
3036            .seal(run, epoch, outcome)
3037            .await
3038            .map_err(RuntimeError::from_store)?;
3039        Ok(run)
3040    }
3041
3042    /// Record an operator deliberately crossing into this tenant, and return
3043    /// the run that holds the record.
3044    ///
3045    /// Every other tenancy control keeps a cross-tenant read from being reached
3046    /// by accident. This is the designed exception, and the rule for it is that an
3047    /// exception without a record is indistinguishable from the breach it is
3048    /// meant to be. So the access is written into **this** tenant's journal —
3049    /// the one whose data is about to be reached — in a sealed run of its own,
3050    /// exactly as a sweep writes its decisions. It therefore inherits the hash
3051    /// chain, the per-record signature and the Merkle inclusion, and the
3052    /// offline audit tool reports it without being taught that break-glass
3053    /// exists.
3054    ///
3055    /// **The record is written before any data is served, and a failure to
3056    /// write it is a failure to access.** That direction is the whole control:
3057    /// the alternative — serve first, record best-effort — is a break-glass
3058    /// that works exactly as well when its own evidence is lost.
3059    ///
3060    /// # Errors
3061    ///
3062    /// If `reason` is blank, or if the record cannot be written. An
3063    /// unexplained exception is the thing this record exists to prevent, so it
3064    /// is refused rather than stored empty.
3065    pub async fn record_break_glass(
3066        &self,
3067        actor: &crate::core::Operator,
3068        roles: &[String],
3069        reason: &str,
3070    ) -> Result<RunId, RuntimeError> {
3071        if reason.trim().is_empty() {
3072            return Err(RuntimeError::PlanContract(
3073                "break-glass needs a reason: an unexplained crossing of the tenant \
3074                 boundary is what this record exists to prevent"
3075                    .to_owned(),
3076            ));
3077        }
3078        let run = self
3079            .seal_operator_record(
3080                RecordKind::BreakGlass {
3081                    actor: actor.clone(),
3082                    roles: roles.to_vec(),
3083                    reason: reason.to_owned(),
3084                },
3085                None,
3086                BREAK_GLASS_OUTCOME,
3087                Some(reason),
3088            )
3089            .await?;
3090        tracing::warn!(
3091            tenant = %self.tenant(),
3092            %actor,
3093            %run,
3094            reason,
3095            "break-glass: an operator crossed the tenant boundary"
3096        );
3097        Ok(run)
3098    }
3099
3100    /// The journal, for reads the runtime does not mediate.
3101    ///
3102    /// This is also the embedder's **resumable output stream**:
3103    /// [`JournalStore::read`] is a seq-cursored read over a run's records, so
3104    /// polling from the last sequence seen is a durable, reconnect-safe
3105    /// progress feed that any instance can serve — the exact mechanism the
3106    /// A2A streaming surface is built on. There is deliberately no curated
3107    /// event type between an embedder and the records: a third vocabulary
3108    /// beside the journal's and the wire's would be a second truth wearing
3109    /// ergonomics, and the records are already the history the stream must
3110    /// agree with.
3111    pub fn journal(&self) -> &Arc<dyn JournalStore> {
3112        &self.store
3113    }
3114
3115    /// Which case a run belongs to, or `None` if it belongs to none.
3116    ///
3117    /// Read from the **journal**, not from a column beside it. The binding is
3118    /// stamped on the run's own records at admission, so answering from there
3119    /// is answering from the plan of record — a case-store column saying the
3120    /// same thing would be a second copy of one fact, and the two could
3121    /// disagree about a run the case layer never saw.
3122    ///
3123    /// It is the first question an operator surface asks, which is why it is a
3124    /// method rather than a documented one-liner: every caller was otherwise
3125    /// going to reach for the first record and read `body.case` off it, and a
3126    /// caller who reached for the *last* one instead would still be right today
3127    /// and wrong the moment a run is admitted before its case is known.
3128    ///
3129    /// # Errors
3130    ///
3131    /// If the journal cannot be read. An unknown run is `Ok(None)` rather than
3132    /// an error: *no such run* and *a run in no case* are both honest answers
3133    /// to this question, and neither is a fault.
3134    pub async fn case_of(&self, run: RunId) -> Result<Option<crate::core::CaseId>, RuntimeError> {
3135        Ok(self
3136            .store
3137            .read(run, 1)
3138            .await
3139            .map_err(RuntimeError::from_store)?
3140            .first()
3141            .and_then(|record| record.body.case))
3142    }
3143
3144    #[must_use]
3145    /// The witness set and quorum this plane was built with, if any.
3146    pub(crate) fn witnessing(&self) -> Option<&Arc<Witnessing>> {
3147        self.witnesses.as_ref()
3148    }
3149
3150    pub fn store(&self) -> &Arc<dyn JournalStore> {
3151        &self.store
3152    }
3153
3154    /// This **process instance's** identity, as it appears in run leases.
3155    ///
3156    /// Not the agent's name: several instances of one agent are normal. See
3157    /// [`RuntimeBuilder::owner`].
3158    ///
3159    /// Public because "which instance holds this run" is a question an operator
3160    /// asks of a stuck system, and the answer is otherwise only in a store row.
3161    #[must_use]
3162    pub fn owner_id(&self) -> &str {
3163        &self.owner
3164    }
3165
3166    /// The input as the declared admission rules leave it: refused with no
3167    /// record, as an admission policy denial is, or with its sensitivity
3168    /// raised to every matched classification — which `RunAdmitted` then
3169    /// records as the input's label.
3170    #[cfg(feature = "manifest")]
3171    fn admit_content(
3172        &self,
3173        agent: &str,
3174        input: Tainted<Value>,
3175    ) -> Result<Tainted<Value>, RuntimeError> {
3176        let Some(manifest) = self
3177            .resolve(agent)
3178            .ok()
3179            .and_then(|skill| self.governing(skill.as_ref()))
3180        else {
3181            return Ok(input);
3182        };
3183        let outcome = super::ctx::judged(
3184            manifest.spec.security.content.as_ref(),
3185            crate::content::At::Admission,
3186            input.peek(),
3187        )
3188        .map_err(RuntimeError::PolicyDenied)?;
3189        if let Some(hit) = outcome.refused.first() {
3190            return Err(RuntimeError::PolicyDenied(
3191                crate::core::PolicyError::Content {
3192                    rule: hit.rule.clone(),
3193                    pointer: hit.pointer.clone(),
3194                },
3195            ));
3196        }
3197        let raised = outcome.raise(input.label().sensitivity);
3198        if raised == input.label().sensitivity {
3199            return Ok(input);
3200        }
3201        Ok(input.raised_to(raised))
3202    }
3203
3204    #[cfg(not(feature = "manifest"))]
3205    #[allow(clippy::unused_self, clippy::unnecessary_wraps)]
3206    fn admit_content(
3207        &self,
3208        _agent: &str,
3209        input: Tainted<Value>,
3210    ) -> Result<Tainted<Value>, RuntimeError> {
3211        Ok(input)
3212    }
3213
3214    /// The content rules' verdict on an event delivered to a suspended run,
3215    /// under the declaration governing the run's skill; `None` where no rule
3216    /// matched or none is declared.
3217    #[cfg(feature = "manifest")]
3218    fn delivered_verdict(
3219        &self,
3220        history: &[crate::journal::Record],
3221        payload: &serde_json::Value,
3222    ) -> Option<crate::core::ContentVerdict> {
3223        let capability = history.iter().find_map(|r| match r.kind() {
3224            RecordKind::RunAdmitted { capability, .. } => Some(capability.clone()),
3225            _ => None,
3226        })?;
3227        let manifest = self.governing(self.resolve(&capability).ok()?.as_ref())?;
3228        super::ctx::verdict(super::ctx::judged(
3229            manifest.spec.security.content.as_ref(),
3230            crate::content::At::Source(super::ctx::AWAIT_KIND),
3231            payload,
3232        ))
3233    }
3234
3235    #[cfg(not(feature = "manifest"))]
3236    #[allow(clippy::unused_self)]
3237    fn delivered_verdict(
3238        &self,
3239        _history: &[crate::journal::Record],
3240        _payload: &serde_json::Value,
3241    ) -> Option<crate::core::ContentVerdict> {
3242        None
3243    }
3244
3245    /// The declaration governing a skill, if its agent has one.
3246    ///
3247    /// Per skill, not per plane: a runtime runs several agents, and a step must
3248    /// be judged against the manifest of the agent whose skill it is. Looking up
3249    /// one plane-wide manifest would apply another agent's ceilings.
3250    #[cfg(feature = "manifest")]
3251    fn governing(&self, skill: &dyn Skill) -> Option<Arc<crate::manifest::Manifest>> {
3252        self.governed_by.get(&skill.descriptor().name).cloned()
3253    }
3254
3255    /// Renew this run's lease until the returned guard is dropped.
3256    ///
3257    /// Requires a Tokio runtime, as the rest of this crate's timing does.
3258    fn heartbeat(&self, run: RunId, epoch: crate::core::Epoch) -> Heartbeat {
3259        let store = Arc::clone(&self.store);
3260        let owner = self.owner.clone();
3261        let ttl = self.lease_ttl;
3262        // A third of the TTL, so two renewals can be lost to a slow store before
3263        // the lease lapses. Renewing *at* the TTL would mean any hesitation
3264        // leaves it expired, and an expired lease is one anybody may take —
3265        // including, per `acquire`, this caller, which would fence the run with
3266        // its own heartbeat.
3267        let period = ttl / 3;
3268        Heartbeat(tokio::spawn(async move {
3269            loop {
3270                tokio::time::sleep(period).await;
3271                // `renew`, never `acquire`. A renewal extends only a lease
3272                // still held by exactly this `(owner, epoch)`; it cannot
3273                // claim. That distinction is load-bearing here: this task
3274                // races the run's own conclusion, and an acquire arriving
3275                // just after `conclude` released the lease would re-take it —
3276                // a live, never-released lease over a concluded run, which
3277                // the recovery sweep then "recovers" forever.
3278                //
3279                // Stop only when the store says the lease is gone — released
3280                // by the conclusion, lapsed and reclaimed, or taken over: the
3281                // run's next append will be fenced, which is the correct
3282                // outcome, and claiming now would only resurrect ownership
3283                // this instance no longer has. Any other failure is the store
3284                // not answering, and giving up then would let a run that is
3285                // still executing lapse into somebody else's recovery.
3286                match store.renew(run, &owner, epoch, ttl).await {
3287                    Ok(_) => {}
3288                    Err(crate::core::StoreError::LeaseNotHeld { .. }) => return,
3289                    Err(error) => tracing::warn!(
3290                        target: "agentplane::lease",
3291                        run = %run,
3292                        error = %error,
3293                        "a lease renewal failed; the next one retries"
3294                    ),
3295                }
3296            }
3297        }))
3298    }
3299
3300    /// The standing withdrawal that reaches `identity`, if one does.
3301    ///
3302    /// The one predicate the step boundary and a subject-bound hop both ask,
3303    /// so the two cannot disagree about whose authority is withdrawn. Read on
3304    /// a live pass only — a halt thrown today must not decide what a run did
3305    /// last year — and an unreachable store propagates, because carrying on
3306    /// would leave a withdrawal unenforced exactly while accounting is down.
3307    pub(crate) async fn withdrawal_against(
3308        &self,
3309        identity: Option<&crate::core::Delegation>,
3310    ) -> Result<Option<Withdrawal>, crate::core::StoreError> {
3311        let (Some(quotas), Some(chain)) = (self.quotas.as_ref(), identity) else {
3312            return Ok(None);
3313        };
3314        let halts = quotas.halts().await?;
3315        Ok(halts.into_iter().find_map(|halt| {
3316            let subject = halt.scope.withdraws_from(chain)?.to_owned();
3317            Some(Withdrawal {
3318                subject,
3319                reason: halt.reason,
3320                by: halt.by,
3321            })
3322        }))
3323    }
3324
3325    /// Stop a run because the authority it acts under was withdrawn.
3326    ///
3327    /// A **pause**: `stop` is reached with a status `maybe_unwind` passes
3328    /// straight through, so the run's mutations stand and it stays resumable.
3329    /// That is the treatment exhaustion gets and for the same reason — the run
3330    /// did nothing wrong, and reversing correct work because a credential lapsed
3331    /// would be a second incident.
3332    ///
3333    /// `record` is false on a resume that finds the withdrawal still standing:
3334    /// the withholding already on the chain says so, and a second one per resume
3335    /// attempt would turn a queue of retries into a queue of records.
3336    #[allow(clippy::too_many_arguments)]
3337    async fn withhold(
3338        &self,
3339        cx: Unwind<'_>,
3340        withdrawal: Withdrawal,
3341        record: bool,
3342        completed: &[(StepId, Capability)],
3343        outputs: &BTreeMap<StepId, Tainted<Value>>,
3344        cursor: &mut ReplayCursor,
3345        case_id: Option<crate::core::CaseId>,
3346        quota: &QuotaPass,
3347    ) -> Result<RunOutcome, RuntimeError> {
3348        let Withdrawal {
3349            subject,
3350            reason,
3351            by,
3352        } = withdrawal;
3353        if record {
3354            let stamped = (cx.stamp)(Append::new(
3355                cx.run,
3356                RecordKind::AuthorityWithheld {
3357                    subject: subject.clone(),
3358                    reason: reason.clone(),
3359                    by: by.clone(),
3360                },
3361            ));
3362            self.store
3363                .append(cx.epoch, vec![stamped])
3364                .await
3365                .map_err(RuntimeError::from_store)?;
3366        }
3367        self.stop(
3368            cx,
3369            RunStatus::Withheld { subject, reason },
3370            completed,
3371            outputs,
3372            cursor,
3373            case_id,
3374            quota,
3375        )
3376        .await
3377    }
3378
3379    /// Refuse a run whose tenant is at a ceiling.
3380    ///
3381    /// Fails **closed**: an unreachable quota store refuses rather than admits,
3382    /// because a ceiling that yields when its accounting is down is a ceiling an
3383    /// attacker removes by taking the accounting down.
3384    ///
3385    /// Live admission only, and replay and resume are skipped for **different**
3386    /// reasons. Re-checking during replay would let a run that happened produce
3387    /// a different history when it is re-read: a ceiling crossed since admission
3388    /// would rewrite the past into a refusal. Resume is not that — it decides
3389    /// only what happens next — and it is ungated because refusing it strands a
3390    /// run mid-saga with its reversals unrun. What that costs the ceiling is
3391    /// stated where the ceiling is defined ([`crate::quota`]); it is a real cost
3392    /// rather than an absent one.
3393    async fn check_quota(
3394        &self,
3395        run: RunId,
3396        governed_by: Option<&crate::journal::AgentIdentity>,
3397        chain: Option<&crate::core::Delegation>,
3398        at: crate::core::Timestamp,
3399        (budget, width): (&Budget, usize),
3400    ) -> Result<QuotaPass, RuntimeError> {
3401        let Some(quotas) = self.quotas.as_ref() else {
3402            return Ok(QuotaPass::disabled());
3403        };
3404        let pass = QuotaPass::at(&self.quota, at, true).sized_for(width);
3405
3406        // The halt is checked **before** the unlimited shortcut, because an
3407        // emergency stop is not a ceiling and a tenant with no ceilings is
3408        // exactly the one an operator is most likely to need to stop. Reading it
3409        // fails closed for the same reason the ceilings do: a switch that yields
3410        // when its store is unreachable is a switch an attacker throws by taking
3411        // the store down.
3412        //
3413        // Every standing halt in one read, and the **narrowest match wins the
3414        // message**: an operator who stopped one agent and whoever is refused
3415        // are looking for different sentences, and "the tenant is halted" told
3416        // to the caller of agent 12 sends them to the wrong incident. Ordering
3417        // the scopes puts `Subject` before `Revision` before `Agent` before
3418        // `Tenant`, so the reason reported is the most specific one that
3419        // actually covers this run — and a withdrawn authority outranks every
3420        // workload scope, because it is the answer that tells the caller their
3421        // credential is the problem rather than the agent.
3422        match quotas.halts().await {
3423            Ok(halts) => {
3424                if let Some(halt) = halts
3425                    .into_iter()
3426                    .filter(|halt| halt.scope.covers(governed_by, chain))
3427                    .max_by(|a, b| a.scope.cmp(&b.scope))
3428                {
3429                    return Err(RuntimeError::QuotaExceeded(
3430                        crate::quota::QuotaError::Halted {
3431                            tenant: self.tenant.as_str().to_owned(),
3432                            scope: halt.scope,
3433                            reason: halt.reason,
3434                        },
3435                    ));
3436                }
3437            }
3438            Err(e) => {
3439                return Err(RuntimeError::QuotaExceeded(
3440                    crate::quota::QuotaError::Unavailable(e.to_string()),
3441                ));
3442            }
3443        }
3444
3445        // What the run can cost at most, held against the period in the same
3446        // store transaction that checks the ceiling and takes the slot. A run
3447        // that leaves a bounded unit unbounded is refused here: admitted, it
3448        // would hold nothing against a period it can spend without limit.
3449        let hold = match crate::quota::reservation(&self.quota, budget, width as u64) {
3450            Ok(amount) => {
3451                amount
3452                    .zip(pass.period())
3453                    .map(|(amount, period)| crate::quota::SpendHold {
3454                        period: period.to_owned(),
3455                        amount,
3456                    })
3457            }
3458            Err(field) => {
3459                return Err(RuntimeError::QuotaExceeded(
3460                    crate::quota::QuotaError::Unbounded {
3461                        tenant: self.tenant.as_str().to_owned(),
3462                        unit: if field.contains("minor") {
3463                            "money"
3464                        } else {
3465                            "tokens"
3466                        },
3467                        field,
3468                    },
3469                ));
3470            }
3471        };
3472
3473        quotas
3474            .reserve(run, &self.quota, hold.as_ref(), at)
3475            .await
3476            .map_err(RuntimeError::QuotaExceeded)?;
3477        Ok(pass)
3478    }
3479
3480    /// Atomically settle this pass's spend and admission slot.
3481    ///
3482    /// Required before physical sealing and lease release. A failure keeps the
3483    /// lease owned so the abandonment sweep retries from journal evidence; a
3484    /// best-effort write here would turn a transient store outage into permanent
3485    /// under-accounting. The period is the one captured when this pass started,
3486    /// never recomputed here.
3487    async fn settle_quota(
3488        &self,
3489        run: RunId,
3490        epoch: crate::core::Epoch,
3491        spend: Spend,
3492        pass: &QuotaPass,
3493        concludes: bool,
3494    ) -> Result<(), RuntimeError> {
3495        let Some(quotas) = self.quotas.as_ref() else {
3496            return Ok(());
3497        };
3498        let pending = |error: crate::core::StoreError| RuntimeError::QuotaSettlementPending {
3499            run: run.to_string(),
3500            epoch,
3501            detail: error.to_string(),
3502        };
3503        if !pass.enabled {
3504            // A conclusion no pass opened — an abandonment decided while the
3505            // run was stopped — still ends the run, and a run that has ended
3506            // holds nothing against the period.
3507            if concludes {
3508                quotas.release(run).await.map_err(pending)?;
3509            }
3510            return Ok(());
3511        }
3512        let settlement = pass
3513            .settlement(run, epoch, spend, concludes)
3514            .expect("an enabled quota pass has a settlement");
3515        quotas.settle(&settlement).await.map_err(pending)
3516    }
3517
3518    /// Release a reservation for an admission whose journal never existed.
3519    pub(crate) async fn release_empty_quota_reservation(
3520        &self,
3521        run: RunId,
3522    ) -> Result<(), crate::core::StoreError> {
3523        match self.quotas.as_ref() {
3524            Some(quotas) => quotas.release(run).await,
3525            None => Ok(()),
3526        }
3527    }
3528
3529    /// Settle and release the admission slots sealed runs still hold.
3530    ///
3531    /// A plane with no quota store may seal a run whose admission holds a
3532    /// slot in this ledger, and no resume or recovery selects a sealed run
3533    /// again. Settlement is idempotent, so a slot another path already settled
3534    /// is not billed twice. Returns how many slots were given back.
3535    pub(crate) async fn release_sealed_slots(&self, limit: usize) -> Result<usize, RuntimeError> {
3536        let Some(quotas) = self.quotas.as_ref() else {
3537            return Ok(0);
3538        };
3539        let held = quotas
3540            .running_runs(limit)
3541            .await
3542            .map_err(RuntimeError::Store)?;
3543        let positions = self
3544            .store
3545            .log_positions(&held)
3546            .await
3547            .map_err(RuntimeError::from_store)?;
3548        let mut released = 0;
3549        for (run, sealed) in held.into_iter().zip(positions) {
3550            if sealed.is_none() {
3551                continue;
3552            }
3553            let records = self
3554                .store
3555                .read(run, 1)
3556                .await
3557                .map_err(RuntimeError::from_store)?;
3558            match self.settle_recorded_quota_passes(run, &records).await {
3559                Ok(()) => released += 1,
3560                Err(error) => tracing::warn!(
3561                    %run,
3562                    %error,
3563                    "a sealed run's admission slot could not be released; the next sweep retries"
3564                ),
3565            }
3566        }
3567        Ok(released)
3568    }
3569
3570    /// Seal the lease-free runs whose sealing conclusion is written and whose
3571    /// seal is not.
3572    ///
3573    /// Each [`LEASE_FREE_OUTCOMES`] writer appends its conclusion and then
3574    /// seals in a second call, and holds no lease, so recovery never selects
3575    /// a run whose writer died between the two. The conclusion alone already
3576    /// lists the run under its outcome, so a history reads it as done while
3577    /// the audit finds it in no log. The conclusion is what was decided; the
3578    /// seal adds nothing to it but the log position, so finishing it here
3579    /// writes no decision the writer did not make.
3580    ///
3581    /// Looks at the newest `limit` runs of each outcome. Returns how many it
3582    /// sealed.
3583    pub(crate) async fn seal_unsealed_conclusions(
3584        &self,
3585        limit: usize,
3586    ) -> Result<usize, RuntimeError> {
3587        let mut sealed = 0;
3588        for &outcome in LEASE_FREE_OUTCOMES {
3589            let runs = self
3590                .store
3591                .runs_by_outcome(outcome, limit)
3592                .await
3593                .map_err(RuntimeError::from_store)?;
3594            let positions = self
3595                .store
3596                .log_positions(&runs)
3597                .await
3598                .map_err(RuntimeError::from_store)?;
3599            for (run, position) in runs.into_iter().zip(positions) {
3600                if position.is_some() {
3601                    continue;
3602                }
3603                match self.store.seal(run, LEASE_FREE_EPOCH, outcome).await {
3604                    Ok(_) => sealed += 1,
3605                    Err(error) => tracing::warn!(
3606                        %run,
3607                        %error,
3608                        "a concluded run's seal could not be finished; the next sweep retries"
3609                    ),
3610                }
3611            }
3612        }
3613        Ok(sealed)
3614    }
3615
3616    /// Settle every quota pass present in history, idempotently.
3617    ///
3618    /// Reached before a resume can dispatch. A normally settled pass is a cheap
3619    /// receipt lookup; a pass whose process died is accrued here. Conclusions
3620    /// carry the exact in-memory live total, including a known outcome whose
3621    /// terminal effect record failed before a later append recovered. An
3622    /// unconcluded crash has only durable terminal effect records to sum.
3623    async fn settle_recorded_quota_passes(
3624        &self,
3625        run: RunId,
3626        records: &[Record],
3627    ) -> Result<(), RuntimeError> {
3628        // A plane with no ledger resumes a run only when no pass of it accrued
3629        // spend to a period, which would otherwise go unbilled. It cannot give
3630        // back the concurrency slot of an admission whose process died
3631        // mid-pass; the ledger's own plane does, on its next sweep once the
3632        // run is sealed.
3633        let Some(quotas) = self.quotas.as_ref() else {
3634            if records.iter().any(|record| {
3635                matches!(
3636                    record.kind(),
3637                    RecordKind::QuotaPassStarted {
3638                        period: Some(_),
3639                        ..
3640                    }
3641                )
3642            }) {
3643                return Err(RuntimeError::PlanContract(
3644                    "this run has passes that accrued spend to a quota period, and the resuming \
3645                     plane has no quota store to settle them"
3646                        .to_owned(),
3647                ));
3648            }
3649            return Ok(());
3650        };
3651
3652        for record in records {
3653            let RecordKind::QuotaPassStarted {
3654                period,
3655                release_slot,
3656            } = record.kind()
3657            else {
3658                continue;
3659            };
3660            let epoch = record.body.epoch;
3661            let spend = records
3662                .iter()
3663                .rev()
3664                .find_map(|candidate| (candidate.body.epoch == epoch).then_some(candidate.kind()))
3665                .and_then(|kind| match kind {
3666                    RecordKind::RunConcluded { live_spend, .. } => Some(*live_spend),
3667                    _ => None,
3668                })
3669                .unwrap_or_else(|| spend_recorded_in_epoch(records, epoch));
3670            quotas
3671                .settle(&crate::quota::QuotaSettlement {
3672                    run,
3673                    epoch,
3674                    period: period.clone(),
3675                    spend,
3676                    release_slot: *release_slot,
3677                    concludes: concluded_in(records, epoch),
3678                })
3679                .await
3680                .map_err(|error| RuntimeError::QuotaSettlementPending {
3681                    run: run.to_string(),
3682                    epoch,
3683                    detail: error.to_string(),
3684                })?;
3685        }
3686        // A run whose history has ended holds nothing against the period,
3687        // whichever path ended it — including one no pass marker opened.
3688        if records.iter().any(is_sealing_conclusion) {
3689            quotas
3690                .release(run)
3691                .await
3692                .map_err(|error| RuntimeError::QuotaSettlementPending {
3693                    run: run.to_string(),
3694                    epoch: records.last().map_or(0, |r| r.body.epoch),
3695                    detail: error.to_string(),
3696                })?;
3697        }
3698        Ok(())
3699    }
3700
3701    /// Repair quota receipts, and finish a conclusion an owner died after.
3702    ///
3703    /// Under recovery, a run whose last record is a conclusion lost its owner
3704    /// between concluding and handing the lease back. What remains is the seal and the release, never the business
3705    /// work: recovery is not permission to retry a failure, with or without a
3706    /// quota store. A run that crashed before concluding continues through
3707    /// replay.
3708    async fn recover_quota_before_resume(
3709        &self,
3710        run: RunId,
3711        records: &[Record],
3712        lease: &crate::journal::Lease,
3713        recovering: bool,
3714    ) -> Result<Option<RunOutcome>, RuntimeError> {
3715        self.settle_recorded_quota_passes(run, records).await?;
3716        if !recovering {
3717            return Ok(None);
3718        }
3719        let Some(last) = records.last() else {
3720            return Ok(None);
3721        };
3722        let RecordKind::RunConcluded {
3723            outcome,
3724            reason,
3725            exhaustion,
3726            ..
3727        } = last.kind()
3728        else {
3729            return Ok(None);
3730        };
3731
3732        let status = recorded_status(outcome, reason.as_deref(), exhaustion.as_ref(), records);
3733        let chain_head = if SEALED_OUTCOMES.contains(&outcome.as_str()) {
3734            let head = self
3735                .store
3736                .seal(run, lease.epoch, outcome)
3737                .await
3738                .map_err(RuntimeError::from_store)?;
3739            // A repaired seal is a conclusion like any other.
3740            self.retire_waits(run).await;
3741            head
3742        } else {
3743            self.store
3744                .head(run)
3745                .await
3746                .map_err(RuntimeError::from_store)?
3747                .hash
3748        };
3749        Ok(Some(RunOutcome {
3750            failed_effect: failure_of(&status, records),
3751            run_id: run,
3752            status,
3753            chain_head,
3754            consumed: Consumed::default(),
3755            output: None,
3756        }))
3757    }
3758
3759    /// The effect this run ended on, read from its records, when it failed.
3760    ///
3761    /// Read only for a failure: every other ending answers `None` without a
3762    /// read.
3763    async fn failed_effect(
3764        &self,
3765        run: RunId,
3766        status: &RunStatus,
3767    ) -> Result<Option<crate::journal::FailedEffect>, RuntimeError> {
3768        if !matches!(status, RunStatus::Failed(_)) {
3769            return Ok(None);
3770        }
3771        let records = self
3772            .store
3773            .read(run, 1)
3774            .await
3775            .map_err(RuntimeError::from_store)?;
3776        Ok(failure_of(status, &records))
3777    }
3778
3779    /// Return a closed run without continuing it, repairing a missing physical
3780    /// seal when a crash landed after the conclusion append.
3781    async fn closed_resume_outcome(
3782        &self,
3783        run: RunId,
3784        records: &[Record],
3785        lease: &crate::journal::Lease,
3786    ) -> Result<Option<RunOutcome>, RuntimeError> {
3787        let Some(status) = resume_is_closed(records) else {
3788            return Ok(None);
3789        };
3790        let outcome = recorded_conclusion(records).unwrap_or_else(|| status.as_str().to_owned());
3791        let chain_head = if SEALED_OUTCOMES.contains(&outcome.as_str()) {
3792            let head = self
3793                .store
3794                .seal(run, lease.epoch, &outcome)
3795                .await
3796                .map_err(RuntimeError::from_store)?;
3797            // The crash this repairs may have landed before the conclusion
3798            // retired the run's waits; retiring twice is harmless.
3799            self.retire_waits(run).await;
3800            head
3801        } else {
3802            self.store
3803                .head(run)
3804                .await
3805                .map_err(RuntimeError::from_store)?
3806                .hash
3807        };
3808        Ok(Some(RunOutcome {
3809            failed_effect: failure_of(&status, records),
3810            run_id: run,
3811            status,
3812            chain_head,
3813            output: None,
3814            consumed: Consumed::default(),
3815        }))
3816    }
3817
3818    /// Finish an abandonment a decision asked for and a crash interrupted.
3819    ///
3820    /// The instruction is journaled before it is acted on, so this is what
3821    /// makes the ending arrive whatever happened in between — and it is the
3822    /// same path the verb itself takes, rather than a repair beside it.
3823    ///
3824    /// Nothing is unwound and no step runs: the run seals holding whatever it
3825    /// left in the world, which is the whole content of the decision.
3826    async fn abandon_if_decided(
3827        &self,
3828        run: RunId,
3829        records: &[Record],
3830        lease: &crate::journal::Lease,
3831    ) -> Result<Option<RunOutcome>, RuntimeError> {
3832        let Some(status) = pending_abandonment(records) else {
3833            return Ok(None);
3834        };
3835        self.conclude(
3836            run,
3837            lease.epoch,
3838            status,
3839            None,
3840            true,
3841            recorded_case_id(records),
3842            Consumed::default(),
3843            Spend::ZERO,
3844            &QuotaPass::disabled(),
3845        )
3846        .await
3847        .map(Some)
3848    }
3849
3850    /// This plane, writing through a journal that puts `quota`'s marker on
3851    /// the pass's first append — so a pass that writes nothing leaves no
3852    /// marker. `None` for a pass with no marker.
3853    fn marking_pass(&self, run: RunId, quota: &QuotaPass) -> Option<Self> {
3854        let marker = quota.started()?;
3855        Some(Self {
3856            store: Arc::new(super::pass_journal::PassJournal::new(
3857                Arc::clone(&self.store),
3858                run,
3859                marker,
3860            )),
3861            ..self.clone()
3862        })
3863    }
3864
3865    /// Open the quota identity of the live tail a resume is about to execute.
3866    ///
3867    /// Its marker is not written here: it rides the pass's first append (see
3868    /// [`super::pass_journal`]), so a resume that writes nothing leaves none.
3869    async fn start_replay_quota_pass(
3870        &self,
3871        run: RunId,
3872        mode: Pass,
3873        width: usize,
3874    ) -> Result<QuotaPass, RuntimeError> {
3875        let (Pass::Resume, Some(quotas)) = (mode, self.quotas.as_ref()) else {
3876            return Ok(QuotaPass::disabled());
3877        };
3878        let pass = QuotaPass::at(&self.quota, now_for_admission(), false).sized_for(width);
3879        // The run spends in the period it resumes in, so what it still holds
3880        // moves there — never refused, because a resume is not.
3881        if let Some(period) = pass.period() {
3882            quotas
3883                .carry(run, period)
3884                .await
3885                .map_err(RuntimeError::Store)?;
3886        }
3887        Ok(pass)
3888    }
3889
3890    /// The ceilings a run gets: its agent's, or the plane's if it has no agent.
3891    ///
3892    /// Per agent, because that is who declared them. A plane-wide budget would
3893    /// let one agent's generosity bound another's runs, which is the whole
3894    /// reason a declaration belongs to an identity rather than to a process.
3895    fn budget_for(&self, target: &str) -> Budget {
3896        #[cfg(feature = "manifest")]
3897        if let Ok(skill) = self.resolve(target)
3898            && let Some(m) = self.governing(skill.as_ref())
3899        {
3900            return m.budget();
3901        }
3902        let _ = target;
3903        self.budget
3904    }
3905
3906    /// Which declaration governs runs of this capability, if a declared agent
3907    /// does.
3908    ///
3909    /// Resolved the same way the budget is, and for the same reason: a run is
3910    /// governed by the agent that answers its entry capability. A commissioned
3911    /// sub-run opens its own run and records its own governor, so every run in a
3912    /// room has exactly one — there is no case where this has to pick.
3913    ///
3914    /// A manifest that cannot produce a digest is one that could not have been
3915    /// published; recording `None` would claim the run was ungoverned, so the
3916    /// failure is surfaced as an absent identity rather than a false one.
3917    #[cfg(feature = "manifest")]
3918    fn identity_for(&self, target: &str) -> Option<crate::journal::AgentIdentity> {
3919        let skill = self.resolve(target).ok()?;
3920        self.identity_of(self.governing(skill.as_ref())?.as_ref())
3921    }
3922
3923    /// What the agent answering `capability` may do, as its declaration says.
3924    ///
3925    /// `None` when nothing on this plane answers it; a reach with no
3926    /// declaration when something does and no declaration governs it.
3927    #[cfg(feature = "manifest")]
3928    pub(crate) fn reach_of(&self, capability: &str) -> Option<crate::core::Reach> {
3929        let skill = self.resolve(capability).ok()?;
3930        let declaration = self.governing(skill.as_ref()).and_then(|manifest| {
3931            let digest = manifest.digest().ok()?;
3932            Some(super::reach::declared(&manifest, digest, |server| {
3933                self.peers.as_ref().is_some_and(|wiring| {
3934                    wiring
3935                        .registry
3936                        .grant(&crate::peers::PeerId::new(server))
3937                        .is_some()
3938                })
3939            }))
3940        });
3941        Some(crate::core::Reach {
3942            capability: capability.to_owned(),
3943            declaration,
3944        })
3945    }
3946
3947    /// The declaration's identity, as every gate reports it.
3948    ///
3949    /// One construction, because admission and each effect gate must name the
3950    /// same revision the same way: two spellings of *which agent is acting*
3951    /// would let a rule permitted at the door mean something else at the sink.
3952    #[cfg(feature = "manifest")]
3953    fn identity_of(&self, m: &crate::manifest::Manifest) -> Option<crate::journal::AgentIdentity> {
3954        Some(crate::journal::AgentIdentity {
3955            name: m.metadata.name.clone(),
3956            version: m.metadata.version.clone(),
3957            digest: m.digest().ok()?,
3958            publisher: self.published_by.get(&m.metadata.name).cloned(),
3959        })
3960    }
3961
3962    /// Whether anything on this plane answers `target`.
3963    ///
3964    /// For a catalogue built before the first call: a surface that advertises a
3965    /// capability nothing provides offers a caller — often a model — a verb
3966    /// that fails at admission every time, and the failure names the plane
3967    /// rather than the catalogue that lied about it.
3968    ///
3969    /// Gated on the feature that builds a catalogue, so a build without one
3970    /// answers *this question is not asked here* rather than carrying a method
3971    /// nothing reaches.
3972    #[cfg(feature = "mcp-server")]
3973    pub(crate) fn provides(&self, target: &str) -> bool {
3974        self.resolve(target).is_ok()
3975    }
3976
3977    /// Whether a grant's server component names a registered A2A peer — a
3978    /// call the run may wait on.
3979    #[cfg(feature = "mcp-server")]
3980    pub(crate) fn wires_peer(&self, server: &str) -> bool {
3981        self.peers.as_ref().is_some_and(|wiring| {
3982            wiring
3983                .registry
3984                .grant(&crate::peers::PeerId::new(server))
3985                .is_some()
3986        })
3987    }
3988
3989    fn resolve(&self, target: &str) -> Result<Arc<dyn Skill>, RuntimeError> {
3990        if let Some(s) = self.skills.get(target) {
3991            return Ok(Arc::clone(s));
3992        }
3993        let cap = Capability::new(target);
3994        if let Some(name) = self.by_capability.get(&cap)
3995            && let Some(s) = self.skills.get(name)
3996        {
3997            return Ok(Arc::clone(s));
3998        }
3999        // Capabilities rather than skill names: `run` is documented to take the
4000        // capability, and listing the names would answer a question nobody
4001        // asked with vocabulary that does not work in the call that failed.
4002        let mut available: Vec<String> = self
4003            .by_capability
4004            .keys()
4005            .map(std::string::ToString::to_string)
4006            .collect();
4007        available.sort();
4008        Err(RuntimeError::NoProvider {
4009            target: target.to_owned(),
4010            available,
4011        })
4012    }
4013
4014    /// Start a run: a capability a skill answers, or a plan.
4015    ///
4016    /// The one door. What the run is bound to — a case, a caller's chain, a
4017    /// pinned declaration — goes in [`RunRequest::terms`]; whether it runs at
4018    /// most once goes in [`RunRequest::once`], which changes what the answer
4019    /// is; and the terminal says whether to wait for it.
4020    ///
4021    /// ```no_run
4022    /// # use std::sync::Arc;
4023    /// # use agentplane::core::Tainted;
4024    /// # use agentplane::runtime::{Runtime, RunTerms};
4025    /// # async fn f(rt: Arc<Runtime>, input: Tainted<serde_json::Value>) -> Result<(), agentplane::RuntimeError> {
4026    /// let outcome = rt.start("order.settle", input.clone()).run().await?;
4027    /// let admission = rt
4028    ///     .start("order.settle", input)
4029    ///     .terms(RunTerms::default().correlated("order", &[]))
4030    ///     .once("billing:msg-7")
4031    ///     .run()
4032    ///     .await?;
4033    /// # Ok(()) }
4034    /// ```
4035    ///
4036    /// The input arrives **labelled**, and that is the whole of the decision.
4037    /// [`Tainted::trusted`](crate::core::Tainted::trusted) is what an operator
4038    /// writes for a literal, a constant or a configuration value they vouch for;
4039    /// anything that came from outside — an inbound event, a queue message, a
4040    /// counterparty's payload — keeps the label it arrived with. A bare value
4041    /// admitted as trusted would satisfy `require_trusted` fields with
4042    /// attacker-chosen data and give the egress ceiling nothing to join with.
4043    pub fn start(
4044        &self,
4045        target: impl Into<Target>,
4046        input: Tainted<Value>,
4047    ) -> RunRequest<'_, Unkeyed> {
4048        RunRequest {
4049            runtime: self,
4050            target: target.into(),
4051            input,
4052            terms: RunTerms::default(),
4053            key: Unkeyed,
4054        }
4055    }
4056
4057    /// This plane as the handle a background run keeps.
4058    ///
4059    /// Every `Runtime` is built inside an `Arc` that holds `self_ref`, and a
4060    /// borrow of it is a borrow of that `Arc`, so the upgrade cannot fail
4061    /// while `&self` exists.
4062    fn owned(&self) -> Arc<Self> {
4063        self.self_ref
4064            .upgrade()
4065            .expect("a runtime is only reachable through the Arc its builder made")
4066    }
4067
4068    /// Admit and execute, blocking until the run rests.
4069    async fn admit_target(
4070        &self,
4071        target: Target,
4072        input: Tainted<Value>,
4073        terms: RunTerms,
4074    ) -> Result<RunOutcome, RuntimeError> {
4075        match target {
4076            Target::Capability(capability) => self.admit(&capability, input, terms).await,
4077            Target::Plan(plan) => self.admit_plan(plan, input, terms).await,
4078        }
4079    }
4080
4081    /// Admit, and let the run proceed in the background.
4082    async fn spawn_target(
4083        self: &Arc<Self>,
4084        target: Target,
4085        input: Tainted<Value>,
4086        terms: RunTerms,
4087    ) -> Result<RunId, RuntimeError> {
4088        // Resolved before the id is minted: an unknown capability is the
4089        // caller's mistake and must be an error, not a run that exists and
4090        // immediately fails. A plan is validated inside admission, which also
4091        // precedes the id becoming visible.
4092        let plan = match target {
4093            Target::Capability(capability) => {
4094                let skill = self.resolve(&capability)?;
4095                PlanIR::single(first_capability(&skill.descriptor()))
4096            }
4097            Target::Plan(plan) => plan,
4098        };
4099
4100        let run = RunId::generate();
4101        // Registered before admission, not after. Admission is several store
4102        // writes, and a drain beginning inside them would take its snapshot
4103        // without this run, report a complete stop, and let the process exit
4104        // while the run was still being admitted. Registered first, both orders
4105        // work out: a drain that starts after this waits for the run, and one
4106        // that starts before it is seen by the gate inside `admit_only`, which
4107        // refuses and drops the ticket on the way out.
4108        let ticket = self.inflight.enter(run);
4109        let admitted = self
4110            .admit_only(run, plan, input, terms, Entry::Outside)
4111            .await?;
4112
4113        let plane = Arc::clone(self);
4114        // In the caller's span, as a blocking run is: a background run's
4115        // trace is still the trace of whatever started it.
4116        tokio::spawn(
4117            async move {
4118                let _ticket = ticket;
4119                plane.execute_admitted(admitted).await
4120            }
4121            .in_current_span(),
4122        );
4123        Ok(run)
4124    }
4125
4126    /// Admission under a key: the one copy, for blocking and background,
4127    /// capability and plan.
4128    ///
4129    /// Two gates, and only the second arbitrates. The read below is an
4130    /// optimisation — it saves a redelivery the cost of a policy evaluation, a
4131    /// quota reservation, a lease and a correlation lookup — and settles
4132    /// nothing, since two instances racing both see nothing here. The
4133    /// constraint inside `append` decides, on the same division
4134    /// `CaseStore::correlate_or_open` runs on.
4135    async fn claimed<T, F>(&self, key: &str, admit: F) -> Result<Claimed<T>, RuntimeError>
4136    where
4137        F: Future<Output = Result<T, RuntimeError>>,
4138    {
4139        validate_admission_key(key)?;
4140        if let Some(held) = self.holder_of(key).await? {
4141            return Ok(Claimed::Held(held));
4142        }
4143        match admit.await {
4144            Ok(fresh) => Ok(Claimed::Fresh(fresh)),
4145            Err(RuntimeError::Store(crate::core::StoreError::DuplicateAdmission {
4146                run, ..
4147            })) => Ok(Claimed::Held(parse_holder(&run)?)),
4148            Err(e) => Err(e),
4149        }
4150    }
4151
4152    /// Wait until `run` rests and answer with its outcome — the value
4153    /// re-derived by strict replay when it concluded.
4154    ///
4155    /// A run whose lease is free is resumed here: its holder has gone. One
4156    /// whose lease is held is followed on the journal until it rests or the
4157    /// lease lapses, so a holder that dies mid-wait is taken over rather than
4158    /// waited for forever.
4159    async fn settled(&self, run: RunId) -> Result<RunOutcome, RuntimeError> {
4160        loop {
4161            if let Some(rested) = self.recorded_outcome(run).await? {
4162                return self.with_output(rested).await;
4163            }
4164            // Subscribed before the attempt, so an append between the two
4165            // still wakes the wait below.
4166            let mut follower = self.store.watch(run).await;
4167            match self.replay(run, Mode::Resume).await {
4168                Err(RuntimeError::LeaseHeld { remaining_secs, .. }) => {
4169                    let lapse = Duration::from_secs(remaining_secs.max(1));
4170                    match follower.as_mut() {
4171                        Some(follower) => {
4172                            let _ = tokio::time::timeout(lapse, follower.changed()).await;
4173                        }
4174                        None => tokio::time::sleep(lapse).await,
4175                    }
4176                }
4177                resumed => return resumed,
4178            }
4179        }
4180    }
4181
4182    /// A rested run's outcome with its value: strict replay re-derives what a
4183    /// concluded run answered, performing nothing. A suspension has no answer
4184    /// yet, and is returned as recorded.
4185    async fn with_output(&self, rested: RunOutcome) -> Result<RunOutcome, RuntimeError> {
4186        if rested.status == RunStatus::Succeeded {
4187            self.replay(rested.run_id, Mode::Strict).await
4188        } else {
4189            Ok(rested)
4190        }
4191    }
4192
4193    /// Which run holds this admission key, if any.
4194    async fn holder_of(&self, key: &str) -> Result<Option<RunId>, RuntimeError> {
4195        self.store
4196            .admitted_as(key)
4197            .await
4198            .map_err(RuntimeError::from_store)
4199    }
4200
4201    /// Answer a duplicate with what the original run has on the record.
4202    async fn answer_with(&self, held: RunId) -> Result<Admission, RuntimeError> {
4203        Ok(match self.recorded_outcome(held).await? {
4204            Some(outcome) => Admission::Replayed(outcome),
4205            None => Admission::InFlight(held),
4206        })
4207    }
4208
4209    /// What a run has recorded, **without resuming it**.
4210    ///
4211    /// `None` while the run is still working — it has records but has reached
4212    /// neither a conclusion nor a suspension.
4213    ///
4214    /// No lease is taken and nothing is appended. The original is frequently
4215    /// live when its redelivery lands, and a read that acquired a lease would
4216    /// fence a healthy run mid-step in order to report on it.
4217    ///
4218    /// The **last** record decides, never any record: a run that suspended, was
4219    /// resumed and carried on has a suspension in its history and is not
4220    /// suspended now. Same rule as the A2A status projection, so the two
4221    /// surfaces cannot disagree about one run.
4222    ///
4223    /// # Errors
4224    ///
4225    /// If the journal cannot be read.
4226    pub async fn recorded_outcome(&self, run: RunId) -> Result<Option<RunOutcome>, RuntimeError> {
4227        let records = self
4228            .store
4229            .read(run, 1)
4230            .await
4231            .map_err(RuntimeError::from_store)?;
4232        let Some(last) = records.last() else {
4233            return Ok(None);
4234        };
4235        let Some(status) = observed_status(&records) else {
4236            return Ok(None);
4237        };
4238        Ok(Some(RunOutcome {
4239            failed_effect: failure_of(&status, &records),
4240            run_id: run,
4241            status,
4242            chain_head: last.hash,
4243            // Reading a record performs nothing, so it consumes nothing — the
4244            // same answer the closed-run path gives, and for the same reason:
4245            // the spend belongs to the pass that did the work.
4246            consumed: Consumed::default(),
4247            // A step's result is reconstructed by replay rather than stored, so
4248            // there is none to hand back here. See `Admission::Replayed`.
4249            output: None,
4250        }))
4251    }
4252
4253    /// Rebuild a run's case binding **from its history**, never by re-deriving.
4254    ///
4255    /// Two facts are read back rather than recomputed, and each has its own
4256    /// failure if it is not.
4257    ///
4258    /// The **case id**, because re-correlating could land on a different case if
4259    /// the keys were since released — silently rewriting which business fact the
4260    /// run belongs to.
4261    ///
4262    /// The **business keys**, because a case accumulates them over months. A run
4263    /// that resolved `memory.formation.subject: $correlation/meter` would
4264    /// otherwise resolve it against a set the live run never saw, write a second
4265    /// memory under a second subject, and produce a history that disagrees with
4266    /// itself with nothing on the record explaining why.
4267    fn recorded_case(&self, records: &[Record]) -> Option<CaseContext> {
4268        let (case_id, correlation) = records.iter().find_map(|r| match r.kind() {
4269            RecordKind::CaseBound { correlation, .. } => {
4270                r.body.case.map(|case_id| (case_id, correlation.clone()))
4271            }
4272            _ => None,
4273        })?;
4274        self.cases.as_ref().map(|cases| CaseContext {
4275            cases: Arc::clone(cases),
4276            tasks: self.tasks.clone(),
4277            events: self.events.clone(),
4278            calendar: Arc::clone(&self.calendar),
4279            case_id,
4280            correlation,
4281        })
4282    }
4283
4284    /// The contract this runtime enforces on every plan.
4285    pub(crate) fn contract(&self) -> crate::plan::Contract {
4286        let contract = crate::plan::Contract::new(self.by_capability.keys().cloned());
4287        if self.require_verifier {
4288            contract.require_verifier()
4289        } else {
4290            contract
4291        }
4292    }
4293
4294    async fn admit(
4295        &self,
4296        target: &str,
4297        input: Tainted<Value>,
4298        terms: RunTerms,
4299    ) -> Result<RunOutcome, RuntimeError> {
4300        // A bare target is the degenerate plan: one node, terminal.
4301        let skill = self.resolve(target)?;
4302        let capability = first_capability(&skill.descriptor());
4303        self.admit_plan(PlanIR::single(capability), input, terms)
4304            .await
4305    }
4306
4307    async fn admit_plan(
4308        &self,
4309        plan: PlanIR,
4310        input: Tainted<Value>,
4311        terms: RunTerms,
4312    ) -> Result<RunOutcome, RuntimeError> {
4313        self.admit_plan_as(RunId::generate(), plan, input, terms, Entry::Outside)
4314            .await
4315    }
4316
4317    /// Admit and set working the sub-run a started commission orders, under
4318    /// `key`, answering with its id.
4319    ///
4320    /// A key already held answers with the run that holds it: the start was
4321    /// re-performed after a crash and its sub-run exists. A commission, not an
4322    /// outside admission, so the drain gate does not see it — but the sub-run
4323    /// is tracked in flight, so a drain waits for it.
4324    pub(crate) async fn start_commission_run(
4325        &self,
4326        parent: RunId,
4327        target: &str,
4328        input: Tainted<Value>,
4329        terms: RunTerms,
4330        key: &str,
4331    ) -> Result<RunId, RuntimeError> {
4332        let skill = self.resolve(target)?;
4333        let plan = PlanIR::single(first_capability(&skill.descriptor()));
4334        let run = RunId::generate();
4335        let ticket = self.inflight.enter(run);
4336        let admit = Box::pin(self.admit_only(run, plan, input, terms.keyed(key), Entry::Nested));
4337        match self.claimed(key, admit).await? {
4338            Claimed::Fresh(admitted) => {
4339                let plane = self.owned();
4340                let working = tokio::spawn(
4341                    async move {
4342                        let _ticket = ticket;
4343                        plane.execute_admitted(admitted).await
4344                    }
4345                    .in_current_span(),
4346                );
4347                self.commissions
4348                    .lock()
4349                    .unwrap_or_else(std::sync::PoisonError::into_inner)
4350                    .insert(run, (parent, working));
4351                Ok(run)
4352            }
4353            Claimed::Held(held) => Ok(held),
4354        }
4355    }
4356
4357    /// A commissioned sub-run once it rests, with its answer and everything
4358    /// it spent.
4359    ///
4360    /// From the task that executed it, when this process set it working: that
4361    /// pass did all its work, so its outcome is the whole. Otherwise — the
4362    /// commissioning run resumed after a crash, or another instance started
4363    /// it — the run is settled and read back by strict replay, which reads the
4364    /// answer and every pass's spend from the journal; a resumed pass reports
4365    /// only its own.
4366    pub(crate) async fn rested_commission(&self, run: RunId) -> Result<RunOutcome, RuntimeError> {
4367        let working = self
4368            .commissions
4369            .lock()
4370            .unwrap_or_else(std::sync::PoisonError::into_inner)
4371            .remove(&run);
4372        if let Some((_, working)) = working {
4373            return working.await.map_err(|e| {
4374                RuntimeError::Store(crate::core::StoreError::Backend(format!(
4375                    "the commissioned run's task ended without an outcome: {e}"
4376                )))
4377            })?;
4378        }
4379        self.settled(run).await?;
4380        self.replay(run, Mode::Strict).await
4381    }
4382
4383    /// Let go of the tasks running `parent`'s unawaited commissions, once its
4384    /// pass ends: the sub-runs carry on, and a later await reads them back
4385    /// from the journal.
4386    fn forget_commissions_of(&self, parent: RunId) {
4387        self.commissions
4388            .lock()
4389            .unwrap_or_else(std::sync::PoisonError::into_inner)
4390            .retain(|_, (started_by, _)| *started_by != parent);
4391    }
4392
4393    /// Record the chain this run acts under, beside the plan it authorizes.
4394    ///
4395    /// The two are read back together: a chain without its plan says nothing
4396    /// about what it was allowed to do, and a plan without its chain says
4397    /// nothing about who was allowed to run it.
4398    fn bind_identity(
4399        run: RunId,
4400        chain: Option<&crate::core::Delegation>,
4401        records: &mut Vec<Append>,
4402    ) {
4403        if let Some(chain) = chain {
4404            records.push(Append::new(
4405                run,
4406                RecordKind::IdentityBound {
4407                    chain: chain.links().cloned().collect(),
4408                },
4409            ));
4410        }
4411    }
4412
4413    /// Check the plan against the delegation chain's authority.
4414    ///
4415    /// The plan is the authorization graph, so this is where authority belongs:
4416    /// a plan that names a capability outside the chain's scope must never
4417    /// start, rather than failing at whichever step happens to reach it first.
4418    /// Checking here also makes the refusal deterministic — it depends only on
4419    /// the frozen plan and the chain, both of which are recorded.
4420    ///
4421    /// The chain's other two bounds are checked beside it, and only here: a
4422    /// chain issued for another plane or past its validity is refused before
4423    /// the run exists, and a run admitted under a live chain is never re-judged
4424    /// — replay reads the recorded chain back.
4425    ///
4426    /// `acts_under` is whether the run acts under `ceiling` or is only bounded
4427    /// by it. A refusal names the chain's subject only in the first case: a
4428    /// caller bounded by the plane's chain is not its subject, and a denial
4429    /// naming the operator would say the operator was refused.
4430    fn authorize_scope(
4431        &self,
4432        plan: &PlanIR,
4433        ceiling: Option<&crate::core::Delegation>,
4434        acts_under: bool,
4435    ) -> Result<(), RuntimeError> {
4436        let Some(chain) = ceiling else {
4437            return Ok(());
4438        };
4439        chain.admissible(self.tenant.as_str(), now_for_admission())?;
4440        let scope = chain.effective_scope();
4441        for node in &plan.nodes {
4442            if !scope.permits(&node.capability) {
4443                return Err(RuntimeError::PolicyDenied(
4444                    crate::core::PolicyError::Denied {
4445                        principal: if acts_under {
4446                            chain.subject().id.clone()
4447                        } else {
4448                            node.capability.to_string()
4449                        },
4450                        action: crate::core::ACTION_ADMIT.to_owned(),
4451                        resource: node.capability.to_string(),
4452                    },
4453                ));
4454            }
4455        }
4456        Ok(())
4457    }
4458
4459    /// Authorize starting a run, before the run exists.
4460    ///
4461    /// A denial here leaves no journal at all, which is correct: nothing
4462    /// happened, and a run record for something that was never allowed to start
4463    /// would be a run nobody can explain.
4464    fn authorize_admission(
4465        &self,
4466        capability: &str,
4467        governed_by: Option<&crate::journal::AgentIdentity>,
4468        chain: Option<&crate::core::Delegation>,
4469        input: &Value,
4470    ) -> Result<(), RuntimeError> {
4471        let Some(engine) = self.policy.as_ref() else {
4472            return Ok(());
4473        };
4474        // Who is acting, and what is being asked for. Passing one string as both
4475        // made every admission rule a tautology — `principal == resource` cannot
4476        // express "this agent may not run that capability", which is the whole
4477        // question at admission on a plane hosting several agents.
4478        //
4479        // The principal is an **authenticated** identity or it is nothing. The
4480        // scope check a few lines above already denies under the delegation
4481        // subject, so anything else here would give one refused run two answers
4482        // to "who was refused" in the same error type.
4483        //
4484        // The agent's `metadata.name` is deliberately *not* used, tempting as it
4485        // is. A name is self-asserted — a manifest is a file, and its name is
4486        // whatever the author typed — so a rule granting authority to a name
4487        // grants it to any file claiming that name. Worse, a fallback would be
4488        // *silent*: `principal == X` would mean an authenticated identity in a
4489        // deployment with a delegation chain and a self-asserted label in one
4490        // without, so the same rule would change meaning with the wiring.
4491        //
4492        // Rules that need to bind to the agent bind to `context.agent.digest`,
4493        // which is content-addressed and pins what the declaration actually
4494        // said — so "this exact declaration may admit, and an edited one may
4495        // not" is expressible. The capability is the fallback principal because
4496        // it claims nothing: it is what was asked for, not who asked.
4497        //
4498        // Built by the function `policy check` also calls, from the values
4499        // `RunAdmitted` and `IdentityBound` record.
4500        let request = crate::policy::requests::admission(
4501            &crate::policy::requests::Acting {
4502                tenant: self.tenant.as_str(),
4503                capability,
4504                agent: governed_by,
4505                chain,
4506            },
4507            input,
4508        );
4509        // Every refusal, not only a rule's. `let … else` over `Deny` alone
4510        // admits the run for every other decision, so a policy set that cannot
4511        // be evaluated would open the door it exists to hold — the fail-open a
4512        // widened vocabulary creates for free unless the permit is the arm that
4513        // has to be named.
4514        let decision = engine.authorize(&request.as_request());
4515        let malformed = decision.is_malformed();
4516        let Some(reason) = decision.reason().map(ToOwned::to_owned) else {
4517            return Ok(());
4518        };
4519
4520        tracing::error!(
4521            target: telemetry::POLICY_DENIED,
4522            action = crate::core::ACTION_ADMIT,
4523            resource = %capability,
4524            policy_error = malformed,
4525            %reason,
4526        );
4527        self.meter
4528            .count(metrics::POLICY_DENIALS, crate::core::ACTION_ADMIT);
4529        Err(RuntimeError::PolicyDenied(
4530            crate::core::PolicyError::Denied {
4531                principal: request.principal,
4532                action: crate::core::ACTION_ADMIT.to_owned(),
4533                resource: capability.to_owned(),
4534            },
4535        ))
4536    }
4537
4538    /// The record that opens a run.
4539    ///
4540    /// Its own function because the label matters: journaled rather than
4541    /// recomputed, so a replay reaches the same verdict at every taint gate as
4542    /// the run it reproduces.
4543    fn admission(
4544        &self,
4545        capability: &str,
4546        governed_by: Option<crate::journal::AgentIdentity>,
4547        input: &Tainted<Value>,
4548        idempotency_key: Option<String>,
4549        admitted_by: Option<String>,
4550        acting_as: &Acting,
4551    ) -> RecordKind {
4552        RecordKind::RunAdmitted {
4553            capability: capability.to_owned(),
4554            governed_by: governed_by.map(Box::new),
4555            input: input.peek().clone(),
4556            input_label: input.label().clone(),
4557            input_field_labels: input
4558                .field_labels()
4559                .map(|(path, label)| (path.to_owned(), label.clone()))
4560                .collect(),
4561            policy_bundle: self.policy.as_ref().map(|p| Box::new(p.bundle())),
4562            canon: crate::core::canon::VERSION,
4563            idempotency_key,
4564            admitted_by,
4565            served_unchained: matches!(acting_as, Acting::Nobody),
4566            plane_chain: match acting_as {
4567                Acting::Plane => self.identity.is_some(),
4568                Acting::PlaneDelegate(_) => true,
4569                Acting::Caller(_) | Acting::Nobody => false,
4570            },
4571        }
4572    }
4573
4574    /// Everything up to and including the admission records, and no work.
4575    ///
4576    /// The seam a non-blocking submit needs. Admission is what makes a run
4577    /// *exist*: the policy gate, the lease, and the records that say what is
4578    /// about to happen. Splitting there means a caller can be told the run was
4579    /// accepted — or refused — before any of it runs, and a refusal stays an
4580    /// immediate answer rather than becoming a task that silently never appears.
4581    #[allow(clippy::too_many_lines)]
4582    async fn admit_only(
4583        &self,
4584        run: RunId,
4585        plan: PlanIR,
4586        input: Tainted<Value>,
4587        terms: RunTerms,
4588        entry: Entry,
4589    ) -> Result<Admitted, RuntimeError> {
4590        // First, and before the plan is even read: a draining instance is going
4591        // away, and everything below this line takes on work it may not be able
4592        // to finish. Refusing here leaves no lease, no quota slot and no
4593        // journal — the same nothing a policy denial leaves.
4594        if entry == Entry::Outside && self.inflight.is_draining() {
4595            return Err(RuntimeError::Draining);
4596        }
4597        crate::plan::validate(&plan, &self.contract())
4598            .map_err(|e| RuntimeError::PlanContract(e.to_string()))?;
4599
4600        let agent = plan
4601            .nodes
4602            .first()
4603            .map_or_else(|| "plan".to_owned(), |n| n.capability.to_string());
4604
4605        // Resolved once: the gate and the record must agree about which
4606        // declaration governs this run, and computing it twice is how they
4607        // start disagreeing.
4608        #[cfg(feature = "manifest")]
4609        let governed_by = self.identity_for(&agent);
4610        #[cfg(not(feature = "manifest"))]
4611        let governed_by: Option<crate::journal::AgentIdentity> = None;
4612
4613        if let Some(expected) = terms.expected_declaration {
4614            let found = governed_by.as_ref().map(|id| id.digest);
4615            if found != Some(expected) {
4616                return Err(RuntimeError::DeclarationPinMismatch {
4617                    agent,
4618                    expected,
4619                    found,
4620                });
4621            }
4622        }
4623
4624        // The caller's chain where one was presented, the plane's otherwise —
4625        // one choice, made once, so the gate, the policy context and the
4626        // record cannot disagree about whom this run acts for.
4627        let chain = terms.acting_as.resolve(self.identity.as_ref());
4628        self.authorize_scope(
4629            &plan,
4630            terms.acting_as.ceiling(self.identity.as_ref()),
4631            chain.is_some(),
4632        )?;
4633        self.authorize_admission(&agent, governed_by.as_ref(), chain, input.peek())?;
4634        let input = self.admit_content(&agent, input)?;
4635
4636        self.admit_reserved(run, plan, input, terms, agent, governed_by)
4637            .await
4638    }
4639
4640    /// Admission past the quota reservation: the lease, the records, the case
4641    /// binding, the outbox.
4642    ///
4643    /// Every error path hands the lease back. A lease left standing over a
4644    /// failed admission is not idle bookkeeping: it expires unreleased, which
4645    /// is the exact signature the recovery sweep reads as "an instance died
4646    /// holding this run" — so a half-admitted run (records appended, outbox
4647    /// registration failed) would be *executed* by the sweeper a TTL later,
4648    /// having never been admitted. The one error that concludes rather than
4649    /// merely unwinding is the post-append one, where records already exist:
4650    /// the run is concluded `failed` in its own chain first, so nothing later
4651    /// reads an open journal as work in progress.
4652    async fn admit_reserved(
4653        &self,
4654        run: RunId,
4655        plan: PlanIR,
4656        input: Tainted<Value>,
4657        terms: RunTerms,
4658        agent: String,
4659        governed_by: Option<crate::journal::AgentIdentity>,
4660    ) -> Result<Admitted, RuntimeError> {
4661        // Ownership precedes reservation. A process can die between any two
4662        // store calls; with this order, a slot reserved before the admission
4663        // batch always has an owned lease beside it, and the abandonment sweep
4664        // can release both. Reserving first could strand a slot over no run and
4665        // no lease — invisible to every recovery query.
4666        let lease = self
4667            .store
4668            .acquire(run, &self.owner, self.lease_ttl)
4669            .await
4670            .map_err(RuntimeError::from_store)?;
4671
4672        let budget = self.budget_for(&agent);
4673        // Boxed: the quota decision holds its store round trips across awaits,
4674        // and inlined it pushes every admission future past the size clippy
4675        // allows on the served surfaces that await one.
4676        let quota = match Box::pin(self.check_quota(
4677            run,
4678            governed_by.as_ref(),
4679            // **The chain this run acts under, not the plane's.** A served
4680            // surface admits each run under its caller's chain, so reading
4681            // the plane's here would check a withdrawal against the
4682            // operator's own subject and never match the caller's — a
4683            // refusal that does not happen, which leaves no trace anywhere.
4684            terms.acting_as.resolve(self.identity.as_ref()),
4685            now_for_admission(),
4686            (&budget, reserved_width(&budget, &plan)),
4687        ))
4688        .await
4689        {
4690            Ok(pass) => pass,
4691            Err(error) => {
4692                let _ = self.store.release_lease(run, lease.epoch).await;
4693                return Err(error);
4694            }
4695        };
4696
4697        let out = self
4698            .admit_under_lease(run, &lease, plan, input, terms, &agent, governed_by, quota)
4699            .await;
4700        if out.is_err() {
4701            if let Some(quotas) = self.quotas.as_ref()
4702                && let Err(error) = quotas.release(run).await
4703            {
4704                tracing::debug!(%run, %error, "could not release the quota slot of a failed admission");
4705            }
4706            if let Err(error) = self.store.release_lease(run, lease.epoch).await {
4707                tracing::debug!(
4708                    %run,
4709                    %error,
4710                    "could not release the lease of a failed admission; it will expire on its own"
4711                );
4712            }
4713        }
4714        out
4715    }
4716
4717    #[allow(clippy::too_many_arguments, clippy::too_many_lines)]
4718    async fn admit_under_lease(
4719        &self,
4720        run: RunId,
4721        lease: &crate::journal::Lease,
4722        plan: PlanIR,
4723        input: Tainted<Value>,
4724        terms: RunTerms,
4725        agent: &str,
4726        governed_by: Option<crate::journal::AgentIdentity>,
4727        quota: QuotaPass,
4728    ) -> Result<Admitted, RuntimeError> {
4729        let RunTerms {
4730            case,
4731            idempotency_key,
4732            acting_as,
4733            admitted_by,
4734            expected_declaration: _,
4735            subjects,
4736        } = terms;
4737        let chain = acting_as.resolve(self.identity.as_ref());
4738        let mut records = vec![
4739            Append::new(
4740                run,
4741                self.admission(
4742                    agent,
4743                    governed_by,
4744                    &input,
4745                    idempotency_key,
4746                    admitted_by,
4747                    &acting_as,
4748                ),
4749            ),
4750            // From here the plan is an authorization graph: compiled from
4751            // trusted input, frozen before anything untrusted was read, and
4752            // recorded so the journal that follows can be checked against it.
4753            Append::new(
4754                run,
4755                RecordKind::PlanFrozen {
4756                    steps: plan
4757                        .nodes
4758                        .iter()
4759                        .map(|n| n.capability.to_string())
4760                        .collect(),
4761                    plan: serde_json::to_value(&plan)?,
4762                },
4763            ),
4764        ];
4765
4766        Self::bind_identity(run, chain, &mut records);
4767        if let Some(started) = quota.started() {
4768            records.push(Append::new(run, started));
4769        }
4770
4771        #[cfg(feature = "manifest")]
4772        let declared: Vec<crate::journal::SubjectBinding> = self
4773            .resolve(agent)
4774            .ok()
4775            .and_then(|skill| self.governing(skill.as_ref()))
4776            .map(|m| {
4777                m.spec
4778                    .data_subjects
4779                    .iter()
4780                    .map(crate::manifest::DataSubject::binding)
4781                    .collect()
4782            })
4783            .unwrap_or_default();
4784        #[cfg(not(feature = "manifest"))]
4785        let declared: Vec<crate::journal::SubjectBinding> = Vec::new();
4786        // Before correlation, because `correlate_or_open` commits a case: a
4787        // binding that refuses the run must refuse it before anything outside
4788        // the journal names it. `$case` alone needs the case, and the case is
4789        // bound whenever the run asked for one, so it cannot refuse later.
4790        let bound = bind_subjects(&declared, subjects, &input, case.is_some())?;
4791
4792        // Correlation is deterministic and runs before planning: which case a
4793        // message belongs to is a matter of fact, settled by a lookup.
4794        let case_ctx = match (case, self.cases.as_ref()) {
4795            (Some(CaseBinding::Correlate { kind, keys }), Some(cases)) => {
4796                let correlation = cases
4797                    .correlate_or_open(&kind, &keys, now_for_admission())
4798                    .await
4799                    .map_err(RuntimeError::from_store)?;
4800                let case_id = correlation.case_id();
4801                // The case's own keys, not the ones this message carried. A run
4802                // that *joined* an existing case may have been admitted with one
4803                // key while the case holds three, and a binding naming any of
4804                // them must resolve — which is the difference between "the keys
4805                // in this request" and "the business facts this matter is
4806                // identified by".
4807                let bound_keys = case_correlation(cases.as_ref(), case_id, &keys).await?;
4808                // Stamp the case on the records already queued as well: every
4809                // record of a case-bound run carries its case, which is what
4810                // makes "show me everything about this matter" one range scan.
4811                for r in &mut records {
4812                    r.case = Some(case_id);
4813                }
4814                records.push(
4815                    Append::new(
4816                        run,
4817                        RecordKind::CaseBound {
4818                            case_kind: kind,
4819                            opened: correlation.is_new(),
4820                            correlation: bound_keys.clone(),
4821                        },
4822                    )
4823                    .case(case_id),
4824                );
4825                Some(CaseContext {
4826                    cases: Arc::clone(cases),
4827                    tasks: self.tasks.clone(),
4828                    events: self.events.clone(),
4829                    calendar: Arc::clone(&self.calendar),
4830                    case_id,
4831                    correlation: bound_keys,
4832                })
4833            }
4834            (Some(CaseBinding::Existing(case_id)), Some(cases)) => {
4835                let existing = cases
4836                    .case(case_id)
4837                    .await
4838                    .map_err(RuntimeError::from_store)?
4839                    .ok_or_else(|| {
4840                        RuntimeError::PlanContract(format!("no such case: {case_id}"))
4841                    })?;
4842                if existing.status.is_closed() {
4843                    return Err(RuntimeError::PlanContract(format!(
4844                        "case '{case_id}' is closed and cannot accept another run"
4845                    )));
4846                }
4847                for record in &mut records {
4848                    record.case = Some(case_id);
4849                }
4850                records.push(
4851                    Append::new(
4852                        run,
4853                        RecordKind::CaseBound {
4854                            case_kind: existing.kind,
4855                            opened: false,
4856                            correlation: existing.correlation.clone(),
4857                        },
4858                    )
4859                    .case(case_id),
4860                );
4861                Some(CaseContext {
4862                    cases: Arc::clone(cases),
4863                    tasks: self.tasks.clone(),
4864                    events: self.events.clone(),
4865                    calendar: Arc::clone(&self.calendar),
4866                    case_id,
4867                    correlation: existing.correlation,
4868                })
4869            }
4870            (Some(_), None) => {
4871                return Err(RuntimeError::PlanContract(
4872                    "this run was admitted with correlation keys but the runtime has no case \
4873                     store — build it with `.cases(store)`"
4874                        .into(),
4875                ));
4876            }
4877            (None, _) => None,
4878        };
4879
4880        let bound = fill_case_subjects(bound, case_ctx.as_ref());
4881        let subject_refs = subject_refs(run, &bound);
4882        if !bound.is_empty() {
4883            let mut append = Append::new(run, RecordKind::DataSubjectBound { bindings: bound });
4884            append.case = case_ctx.as_ref().map(CaseContext::id);
4885            records.push(append);
4886        }
4887
4888        // The append claims the admission key, so a key already held is refused
4889        // here, before anything outside the journal names the run.
4890        self.store
4891            .append(lease.epoch, records)
4892            .await
4893            .map_err(RuntimeError::from_store)?;
4894
4895        // Registered before the run is handed back, and its failure fails
4896        // admission: a case that does not list the run, or destinations that
4897        // never see its history.
4898        //
4899        // After the append rather than before: a registration written for a
4900        // run whose admission then failed — or whose process died — would name
4901        // a run no journal holds, and nothing could tell which case to take it
4902        // off. The price of that order is paid here instead: the admission
4903        // records above are already durable, so a registration failure must
4904        // **conclude** the run rather than merely return — an open journal
4905        // under a lease that then lapses is exactly what the recovery sweep
4906        // reads as a crashed run, and it would execute a run whose admission
4907        // failed. Concluded `failed`, in the chain, with the reason beside it;
4908        // the caller's cleanup then releases the lease, and a resume registers
4909        // again before it executes.
4910        if let Err(open_failed) = self
4911            .register(run, case_ctx.as_ref().map(CaseContext::id))
4912            .await
4913        {
4914            let head = self
4915                .store
4916                .head(run)
4917                .await
4918                .map_err(RuntimeError::from_store)?;
4919            self.store
4920                .append(
4921                    lease.epoch,
4922                    vec![Append::new(
4923                        run,
4924                        RecordKind::RunConcluded {
4925                            outcome: RunStatus::Failed(String::new()).as_str().to_owned(),
4926                            reason: Some(format!(
4927                                "the run's registration was refused ({open_failed}), so this \
4928                                 run was concluded without ever executing"
4929                            )),
4930                            exhaustion: None,
4931                            live_spend: Spend::default(),
4932                            chain_head: head.hash,
4933                        },
4934                    )],
4935                )
4936                .await
4937                .map_err(RuntimeError::from_store)?;
4938            return Err(RuntimeError::from_store(open_failed));
4939        }
4940
4941        Ok(Admitted {
4942            run,
4943            epoch: lease.epoch,
4944            quota,
4945            budget: self.budget_for(agent),
4946            agent: agent.to_owned(),
4947            plan,
4948            input,
4949            case: case_ctx,
4950            identity: chain.cloned(),
4951            subjects: subject_refs,
4952        })
4953    }
4954
4955    /// Execute a run that has already been admitted.
4956    async fn execute_admitted(&self, a: Admitted) -> Result<RunOutcome, RuntimeError> {
4957        let mut cursor = ReplayCursor::default();
4958        // Named, not `_`: `let _ = ` drops immediately, which would renew
4959        // nothing at all while looking exactly like this.
4960        let _heartbeat = self.heartbeat(a.run, a.epoch);
4961        self.execute(
4962            Execution {
4963                run: a.run,
4964                epoch: a.epoch,
4965                quota: a.quota,
4966                plan: &a.plan,
4967                input: a.input,
4968                mode: Pass::Live,
4969                case: a.case,
4970                budget: a.budget,
4971                agent: a.agent,
4972                identity: a.identity,
4973                subjects: a.subjects,
4974                refusal: None,
4975                withheld_subject: None,
4976                successors: Vec::new(),
4977                started: BTreeSet::new(),
4978                finished: BTreeSet::new(),
4979                succeeded: BTreeSet::new(),
4980                recorded_groups: BTreeMap::new(),
4981            },
4982            &mut cursor,
4983        )
4984        .await
4985    }
4986
4987    /// Admit and execute, which is what every blocking entry point does.
4988    ///
4989    /// The run id is the caller's rather than minted here, for batches: an
4990    /// item's run id is written to the batch store *before* the run starts, so
4991    /// that a crash leaves a reservation pointing at a journal that can be
4992    /// replayed rather than an item that must be guessed about.
4993    pub(crate) async fn admit_plan_as(
4994        &self,
4995        run: RunId,
4996        plan: PlanIR,
4997        input: Tainted<Value>,
4998        terms: RunTerms,
4999        entry: Entry,
5000    ) -> Result<RunOutcome, RuntimeError> {
5001        let admitted = self.admit_only(run, plan, input, terms, entry).await?;
5002        self.execute_admitted(admitted).await
5003    }
5004
5005    /// Why an open run may not cross its history frontier on this plane, if it
5006    /// may not.
5007    ///
5008    /// Two facts recorded at admission are compared with what this plane holds:
5009    /// the policy bundle (who may authorize an effect) and, for a declarative
5010    /// agent, the declaration (what the agent *is*). A resume under either one
5011    /// changed would continue one run's journal under another's rules. The
5012    /// answer is a refusal to journal rather than an error to raise; the caller
5013    /// decides where it goes.
5014    fn resume_refusal(&self, records: &[Record]) -> Result<Option<RuntimeError>, RuntimeError> {
5015        if let Some(refusal) = self.policy_bundle_refusal(records)? {
5016            return Ok(Some(refusal));
5017        }
5018        #[cfg(feature = "manifest")]
5019        if let Some(refusal) = self.declaration_refusal(records)? {
5020            return Ok(Some(refusal));
5021        }
5022        Ok(None)
5023    }
5024
5025    /// [`RuntimeError::NoProvider`] when a capability any of the run's plans
5026    /// names resolves to nothing on this plane.
5027    ///
5028    /// Every plan the run froze is checked, not only the one in force: an
5029    /// unwind resolves the steps that ran, and those may belong to a plan a
5030    /// replan replaced.
5031    fn refuse_undrivable<'p>(
5032        &self,
5033        plans: impl IntoIterator<Item = &'p PlanIR>,
5034    ) -> Result<(), RuntimeError> {
5035        for node in plans.into_iter().flat_map(|p| &p.nodes) {
5036            self.resolve(&node.capability.0)?;
5037        }
5038        Ok(())
5039    }
5040
5041    /// Name the run where it is watched from outside its journal: on its case,
5042    /// and to every outbox destination.
5043    ///
5044    /// Only after the admission's append, and idempotent — an existing case
5045    /// row and an existing delivery cursor are kept — so every writing resume
5046    /// repeats it.
5047    async fn register(
5048        &self,
5049        run: RunId,
5050        case: Option<crate::core::CaseId>,
5051    ) -> Result<(), crate::core::StoreError> {
5052        if let (Some(case), Some(cases)) = (case, self.cases.as_ref()) {
5053            cases.attach_run(case, run).await?;
5054        }
5055        #[cfg(feature = "push")]
5056        if let Some(outbox) = &self.outbox {
5057            outbox.open(run).await?;
5058        }
5059        Ok(())
5060    }
5061
5062    /// [`RuntimeError::NoCaseStore`] when the run is bound to a case and this
5063    /// plane has no case store.
5064    ///
5065    /// Everything a resume writes belongs to the case, and without the store
5066    /// it would land under the run alone, where erasing the case misses it.
5067    /// About this plane rather than the run, so another plane may resume it.
5068    fn refuse_caseless(&self, run: RunId, records: &[Record]) -> Result<(), RuntimeError> {
5069        match recorded_case_id(records) {
5070            Some(case) if self.cases.is_none() => Err(RuntimeError::NoCaseStore {
5071                run: run.to_string(),
5072                case: case.to_string(),
5073            }),
5074            _ => Ok(()),
5075        }
5076    }
5077
5078    /// Journal a resume refusal as the run's quarantine, naming what moved.
5079    async fn quarantine_refused(
5080        &self,
5081        run: RunId,
5082        records: &[Record],
5083        lease: &crate::journal::Lease,
5084        refusal: &RuntimeError,
5085    ) -> Result<RunOutcome, RuntimeError> {
5086        self.conclude(
5087            run,
5088            lease.epoch,
5089            RunStatus::Quarantined(refusal.to_string()),
5090            None,
5091            true,
5092            recorded_case_id(records),
5093            Consumed::default(),
5094            Spend::ZERO,
5095            &QuotaPass::disabled(),
5096        )
5097        .await
5098    }
5099
5100    /// The bundle half of [`resume_refusal`](Self::resume_refusal).
5101    ///
5102    /// Resume can dispatch new effects after it reaches the end of history.
5103    /// They must be judged by the same complete bundle recorded at admission,
5104    /// or one run would claim one policy while later effects were authorized
5105    /// by another.
5106    fn policy_bundle_refusal(
5107        &self,
5108        records: &[Record],
5109    ) -> Result<Option<RuntimeError>, RuntimeError> {
5110        let recorded = records
5111            .iter()
5112            .find_map(|record| match record.kind() {
5113                RecordKind::RunAdmitted { policy_bundle, .. } => {
5114                    Some(policy_bundle.as_deref().cloned())
5115                }
5116                _ => None,
5117            })
5118            .ok_or_else(|| {
5119                RuntimeError::PlanContract("journal has no RunAdmitted record".into())
5120            })?;
5121        let configured = self.policy.as_ref().map(|policy| policy.bundle());
5122        if recorded != configured {
5123            return Ok(Some(RuntimeError::PolicyBundleChanged {
5124                recorded: recorded.as_ref().map(PolicyBundleIdentity::digest),
5125                configured: configured.as_ref().map(PolicyBundleIdentity::digest),
5126            }));
5127        }
5128        Ok(None)
5129    }
5130
5131    /// The declaration half of [`resume_refusal`](Self::resume_refusal).
5132    ///
5133    /// For a declarative agent the manifest is the whole behaviour, and a resume
5134    /// under an edited one runs a different program over the first one's
5135    /// journal — which replay would eventually catch as a key mismatch, several
5136    /// effects in and without being able to say what changed. The digest is
5137    /// already on the record at admission, so refusing here costs a comparison
5138    /// and turns a late diagnosis into a named remedy.
5139    ///
5140    /// **Silent in two cases, and deliberately.** A run admitted with no
5141    /// declaration is a coded skill, whose behaviour is the embedder's binary:
5142    /// this crate cannot identify that and will not pretend to. And a plane that
5143    /// holds no manifest under the recorded name cannot run the agent
5144    /// declaratively at all, so the refusal it needs is the one plan resolution
5145    /// already gives.
5146    #[cfg(feature = "manifest")]
5147    fn declaration_refusal(
5148        &self,
5149        records: &[Record],
5150    ) -> Result<Option<RuntimeError>, RuntimeError> {
5151        let Some(recorded) = records.iter().find_map(|record| match record.kind() {
5152            RecordKind::RunAdmitted { governed_by, .. } => governed_by.as_deref().cloned(),
5153            _ => None,
5154        }) else {
5155            return Ok(None);
5156        };
5157        let Some(current) = self
5158            .governed_by
5159            .values()
5160            .find(|m| m.metadata.name == recorded.name)
5161        else {
5162            return Ok(None);
5163        };
5164        let configured = current
5165            .digest()
5166            .map_err(|e| RuntimeError::PlanContract(e.to_string()))?;
5167        if configured != recorded.digest {
5168            return Ok(Some(RuntimeError::DeclarationChanged {
5169                agent: recorded.name,
5170                recorded: recorded.digest,
5171                configured,
5172            }));
5173        }
5174        Ok(None)
5175    }
5176
5177    /// A plan left sealed is erased only where a key ring could have opened
5178    /// it.
5179    ///
5180    /// A wired ring opens every payload whose key still exists and fails the
5181    /// read when it cannot be reached, so a payload still sealed behind one was
5182    /// erased. Behind no ring it says nothing about the key at all: the data
5183    /// is intact, and telling a reader it is gone sends them to the wrong
5184    /// remedy.
5185    fn sealed_or_erased(&self, error: RuntimeError) -> RuntimeError {
5186        match error {
5187            RuntimeError::PayloadsErased { run } if !self.holds_key_ring() => {
5188                RuntimeError::PayloadsSealed { run }
5189            }
5190            other => other,
5191        }
5192    }
5193
5194    /// Whether this plane was handed a key ring to open sealed payloads with.
5195    #[cfg_attr(not(feature = "keyring"), allow(clippy::unused_self))]
5196    const fn holds_key_ring(&self) -> bool {
5197        #[cfg(feature = "keyring")]
5198        {
5199            self.keyring.is_some()
5200        }
5201        #[cfg(not(feature = "keyring"))]
5202        {
5203            false
5204        }
5205    }
5206
5207    /// Re-execute a recorded run from its journal.
5208    ///
5209    /// * [`Mode::Strict`] verifies determinism: every effect must match, and the
5210    ///   run must not want any effect the journal lacks.
5211    /// * [`Mode::Resume`] recovers a crashed run: history is replayed, then
5212    ///   execution continues live from wherever the record ends.
5213    ///
5214    /// Either way no external effect is performed for anything already in the
5215    /// journal. That is the whole point — a resumed run does not re-issue the
5216    /// invoice it already issued.
5217    pub async fn replay(&self, run: RunId, mode: Mode) -> Result<RunOutcome, RuntimeError> {
5218        let mode = Pass::from(mode);
5219        // ── Ownership before observation ───────────────────────────────────
5220        //
5221        // Resume writes, so it takes the lease *before* it reads. The old
5222        // order — snapshot the records, then acquire — left a window in which
5223        // a live owner could append past the snapshot; the resume then
5224        // replayed a prefix of the true history and continued live from a
5225        // frontier that was not the frontier, re-deriving keys the owner had
5226        // already written. Claiming first means the snapshot below is taken
5227        // under this instance's own fence. Strict verifies and never writes,
5228        // so it needs no lease — and must not take one, or every regression
5229        // check would fence a run somebody may be legitimately running.
5230        let lease = if mode == Pass::Resume {
5231            Some(
5232                self.store
5233                    .acquire(run, &self.owner, self.lease_ttl)
5234                    .await
5235                    .map_err(RuntimeError::from_store)?,
5236            )
5237        } else {
5238            None
5239        };
5240
5241        self.replay_releasing(run, mode, lease, Claim::Standing)
5242            .await
5243    }
5244
5245    /// Replay a recorded run strictly and say what it found.
5246    ///
5247    /// [`replay`](Self::replay) with [`Mode::Strict`], answered as a
5248    /// [`Verdict`](super::Verdict) rather than a status: *verified* when every
5249    /// recorded effect was asked for again and the run reached the ending it
5250    /// recorded, whatever that ending was; *diverged* at the first effect this
5251    /// build asked for differently; *cannot replay* when the run cannot be
5252    /// re-derived here, for a reason that is not the declaration. Both
5253    /// revisions are named on every answer — the one the run was admitted
5254    /// under and the one this plane holds by that name.
5255    ///
5256    /// Writes nothing, takes no lease and dispatches nothing, as a strict
5257    /// replay never does. It is the question an author asks of an edited
5258    /// manifest: replayed under the edit, this run's history either reaches
5259    /// nothing it changed or names the first effect it did.
5260    ///
5261    /// # Errors
5262    ///
5263    /// If the run does not exist or its journal does not verify, or on a store
5264    /// failure — never for a divergence or a reason the verdict names.
5265    pub async fn verify(&self, run: RunId) -> Result<super::Verdict, RuntimeError> {
5266        use super::{CannotReplay, Finding};
5267
5268        let records = self
5269            .store
5270            .read(run, 1)
5271            .await
5272            .map_err(RuntimeError::from_store)?;
5273        if records.is_empty() {
5274            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
5275                run.to_string(),
5276            )));
5277        }
5278        let recorded = records.iter().find_map(|r| match r.kind() {
5279            RecordKind::RunAdmitted { governed_by, .. } => governed_by.as_deref().cloned(),
5280            _ => None,
5281        });
5282        let candidate = self.candidate_revision(recorded.as_ref());
5283        let verdict = |finding| super::Verdict {
5284            run,
5285            recorded: recorded.clone(),
5286            candidate: candidate.clone(),
5287            finding,
5288        };
5289
5290        // A property of the history rather than of how it ended, so it is
5291        // named whatever the run's state: every key moves under another rule.
5292        if let Err(RuntimeError::CanonicalizationChanged {
5293            recorded,
5294            implemented,
5295        }) = ensure_replayable_canon(&records)
5296        {
5297            return Ok(verdict(Finding::CannotReplay(
5298                CannotReplay::CanonicalizationChanged {
5299                    recorded,
5300                    implemented,
5301                },
5302            )));
5303        }
5304        let Some(ending) = recorded_ending(&records) else {
5305            return Ok(verdict(Finding::CannotReplay(CannotReplay::NotConcluded)));
5306        };
5307        if ended_by_an_act(&ending) {
5308            return Ok(verdict(Finding::CannotReplay(CannotReplay::EndedByAnAct {
5309                outcome: ending,
5310            })));
5311        }
5312
5313        let mut dispatched = false;
5314        let mut divergence = None;
5315        let replayed = self
5316            .replay_under(
5317                run,
5318                Pass::Strict,
5319                None,
5320                Claim::Standing,
5321                &mut dispatched,
5322                &mut divergence,
5323            )
5324            .await;
5325        let finding = match (replayed, divergence) {
5326            (Ok(_), Some(d)) => Finding::Diverged(d),
5327            (Ok(outcome), None) if outcome.status.as_str() == ending => {
5328                Finding::Verified { outcome: ending }
5329            }
5330            (Ok(outcome), None) => Finding::OutcomeDiffers {
5331                recorded: ending,
5332                replayed: outcome.status.as_str().to_owned(),
5333            },
5334            (Err(RuntimeError::PayloadsErased { .. }), _) => {
5335                Finding::CannotReplay(CannotReplay::Erased)
5336            }
5337            (Err(RuntimeError::PayloadsSealed { .. }), _) => {
5338                Finding::CannotReplay(CannotReplay::KeyAbsent)
5339            }
5340            (Err(RuntimeError::NoProvider { target, .. }), _) => {
5341                Finding::CannotReplay(CannotReplay::EntryPointRemoved { capability: target })
5342            }
5343            (Err(other), _) => return Err(other),
5344        };
5345        Ok(verdict(finding))
5346    }
5347
5348    /// The declaration this plane holds under the recorded agent's name.
5349    #[cfg(feature = "manifest")]
5350    fn candidate_revision(
5351        &self,
5352        recorded: Option<&crate::journal::AgentIdentity>,
5353    ) -> Option<crate::journal::AgentIdentity> {
5354        let name = &recorded?.name;
5355        let held = self
5356            .governed_by
5357            .values()
5358            .find(|m| &m.metadata.name == name)?;
5359        self.identity_of(held)
5360    }
5361
5362    /// Without the manifest there are no declarations to name.
5363    #[cfg(not(feature = "manifest"))]
5364    #[allow(clippy::unused_self)]
5365    const fn candidate_revision(
5366        &self,
5367        _recorded: Option<&crate::journal::AgentIdentity>,
5368    ) -> Option<crate::journal::AgentIdentity> {
5369        None
5370    }
5371
5372    /// Resume a run under a lease the caller already holds.
5373    ///
5374    /// The wake path's handover: a timer or event delivery records the wake
5375    /// under its own lease and then resumes the run. Releasing that lease and
5376    /// letting the resume acquire a fresh one opened a window — a crash
5377    /// between the two left a *released* lease over a run whose timer was
5378    /// disarmed and whose subscription was retired, which no queue in the
5379    /// system names: the abandonment sweep looks only for leases that expired
5380    /// while still holding an owner. Handing the lease over closes the window;
5381    /// the run is owned continuously from wake to conclusion, and a crash
5382    /// anywhere in between leaves an expired *owned* lease the sweep drains.
5383    ///
5384    /// A resume that fails does the same thing a crash does, and for the same
5385    /// reason: the recorded wake took the run off the waiting listing, so the
5386    /// lease is the only thing left that names it.
5387    pub(crate) async fn resume_holding(
5388        &self,
5389        run: RunId,
5390        lease: crate::journal::Lease,
5391    ) -> Result<RunOutcome, RuntimeError> {
5392        self.replay_releasing(run, Pass::Resume, Some(lease), Claim::Unlisted)
5393            .await
5394    }
5395
5396    /// Recover an owner-abandoned run without turning settlement repair into
5397    /// an operator-requested retry of a recorded conclusion.
5398    ///
5399    /// A recovery that fails for a reason no retry changes — the journal, not
5400    /// the moment, is what refuses — quarantines the run under the lease the
5401    /// failed pass kept, naming the reason. Left alone it would lapse owned
5402    /// and be recovered, and refused, every lease period for ever; a
5403    /// quarantine is listed, counted, and answered by a person.
5404    pub(crate) async fn recover_abandoned_run(
5405        &self,
5406        run: RunId,
5407    ) -> Result<RunOutcome, RuntimeError> {
5408        let lease = self
5409            .store
5410            .acquire(run, &self.owner, self.lease_ttl)
5411            .await
5412            .map_err(RuntimeError::from_store)?;
5413        match self
5414            .replay_releasing(run, Pass::Resume, Some(lease.clone()), Claim::Recovering)
5415            .await
5416        {
5417            Err(error) if recurs_on_every_resume(&error) => {
5418                let records = self
5419                    .store
5420                    .read(run, 1)
5421                    .await
5422                    .map_err(RuntimeError::from_store)?;
5423                let refusal = RuntimeError::PlanContract(format!(
5424                    "recovery refused, and would be refused the same way on every attempt: \
5425                     {error}"
5426                ));
5427                self.quarantine_refused(run, &records, &lease, &refusal)
5428                    .await
5429            }
5430            other => other,
5431        }
5432    }
5433
5434    async fn replay_releasing(
5435        &self,
5436        run: RunId,
5437        mode: Pass,
5438        lease: Option<crate::journal::Lease>,
5439        claim: Claim,
5440    ) -> Result<RunOutcome, RuntimeError> {
5441        let mut dispatched = false;
5442        let outcome = self
5443            .replay_under(run, mode, lease.as_ref(), claim, &mut dispatched, &mut None)
5444            .await;
5445
5446        // Idempotent: a resume that reached a conclusion already handed its
5447        // lease back in `conclude`, and re-releasing the same epoch changes
5448        // nothing. Released on success, and on a failure only where the run
5449        // is still listed somewhere else: the caller claimed a run whose
5450        // journal already says where it stands, and the refusal came before
5451        // anything past the frontier was written. Every other failure keeps
5452        // the lease so it lapses owned, which is what puts the run on the
5453        // abandonment listing the recovery sweep drains. A wake or a delivery
5454        // has already recorded the awaited result, so the run is off the
5455        // waiting listing; a recovery holds a run nothing else lists; and a
5456        // pass that failed mid-flight wrote records past a conclusion that no
5457        // longer describes it. Releasing any of those strands the run with no
5458        // driver and on no listing.
5459        let release = match &outcome {
5460            Ok(_) => true,
5461            // Settlement is retried by recovery, which finds the run through
5462            // this lease.
5463            Err(RuntimeError::QuotaSettlementPending { .. }) => false,
5464            Err(_) => claim == Claim::Standing && !dispatched,
5465        };
5466        if release
5467            && let Some(lease) = &lease
5468            && let Err(e) = self.store.release_lease(run, lease.epoch).await
5469        {
5470            tracing::debug!(
5471                %run,
5472                error = %e,
5473                "could not hand back the resume lease; it will expire on its own"
5474            );
5475        }
5476        outcome
5477    }
5478
5479    #[allow(clippy::too_many_lines)]
5480    async fn replay_under(
5481        &self,
5482        run: RunId,
5483        mode: Pass,
5484        lease: Option<&crate::journal::Lease>,
5485        claim: Claim,
5486        dispatched: &mut bool,
5487        divergence: &mut Option<crate::journal::Divergence>,
5488    ) -> Result<RunOutcome, RuntimeError> {
5489        let records = self
5490            .store
5491            .read(run, 1)
5492            .await
5493            .map_err(RuntimeError::from_store)?;
5494        if records.is_empty() {
5495            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
5496                run.to_string(),
5497            )));
5498        }
5499
5500        // Never trust a journal that does not verify. A tampered or truncated
5501        // history would let replay "confirm" something that never happened.
5502        //
5503        // Per-record hashes are already checked on read, so a single altered
5504        // record fails before we get here; this catches the structural attacks
5505        // that survive individually-valid records — deletion, reordering, and
5506        // splicing history from another run.
5507        Record::verify_chain(&records, Digest::ZERO).map_err(RuntimeError::from_store)?;
5508
5509        ensure_replayable_canon(&records)?;
5510
5511        if mode == Pass::Resume
5512            && let Some(outcome) = self
5513                .recover_quota_before_resume(
5514                    run,
5515                    &records,
5516                    lease.expect("resume holds a lease"),
5517                    claim == Claim::Recovering,
5518                )
5519                .await?
5520        {
5521            return Ok(outcome);
5522        }
5523
5524        if mode == Pass::Resume
5525            && let Some(outcome) = self
5526                .closed_resume_outcome(run, &records, lease.expect("resume holds a lease"))
5527                .await?
5528        {
5529            return Ok(outcome);
5530        }
5531
5532        // Placed *before* the compensation guard below on purpose: a run that
5533        // quarantined partway through an unwind is exactly the one an operator
5534        // has no other way to close, and it must not be the one this refuses to
5535        // close.
5536        if mode == Pass::Resume
5537            && let Some(outcome) = self
5538                .abandon_if_decided(run, &records, lease.expect("resume holds a lease"))
5539                .await?
5540        {
5541            return Ok(outcome);
5542        }
5543
5544        if mode == Pass::Resume {
5545            refuse_resume_over_undone_work(run, &records)?;
5546        }
5547
5548        // **Read after the conclusions above, and that ordering is the rule.**
5549        // A plan is what a run needs to *execute*; concluding one needs no
5550        // plan, and a cancellation and an abandonment are recorded in the
5551        // clear. Reading here rather than at the top of this function is what
5552        // leaves an operator a verb when the plan can no longer be read at all
5553        // — which is an ordinary state, because erasing a case's key erases
5554        // the plan of every run in it.
5555        let input = records
5556            .iter()
5557            .find_map(Record::admitted_input)
5558            .ok_or_else(|| {
5559                RuntimeError::PlanContract("journal has no RunAdmitted record".into())
5560            })?;
5561        let (plan, successors) =
5562            frozen_plans(run, &records).map_err(|e| self.sealed_or_erased(e))?;
5563
5564        // Only a plane that can drive the run judges it. A plane holding no
5565        // provider for the run's steps — an operator's terminal, a fleet
5566        // member serving other agents — answers `NoProvider` before comparing
5567        // its policy or declarations with the run's: a mismatch there says
5568        // nothing about the run, and quarantining it would stop a run this
5569        // plane could never have continued.
5570        if mode == Pass::Resume {
5571            self.refuse_undrivable(std::iter::once(&plan).chain(&successors))?;
5572            self.refuse_caseless(run, &records)?;
5573        }
5574
5575        // Resume can dispatch new effects past the end of history, so it is
5576        // held to the bundle and the declaration recorded at admission. Strict
5577        // replay performs no effects and remains usable as an offline verifier
5578        // without loading the historical engine.
5579        //
5580        // A refusal here is journaled as a quarantine rather than returned. The
5581        // callers that reach this unattended — a wake, a delivery, the recovery
5582        // sweep — have already recorded the result that took the run off its
5583        // waiting listing, and an error there reaches nobody: the run would
5584        // sit on no listing, or be retried every tick into the same refusal.
5585        // A quarantine is the state that already means *the runtime cannot go
5586        // on and a person must decide*, it is listed and counted, and both of
5587        // its answers work here — reopen on a plane holding the recorded
5588        // revision, or abandon.
5589        if mode == Pass::Resume
5590            && let Some(refusal) = self.resume_refusal(&records)?
5591        {
5592            let lease = lease.expect("resume holds a lease");
5593            return self
5594                .quarantine_refused(run, &records, lease, &refusal)
5595                .await;
5596        }
5597
5598        // A resume executes the run, so it registers it first: what it adds is
5599        // whatever an admission that failed or died partway never wrote.
5600        if mode == Pass::Resume {
5601            self.register(run, recorded_case_id(&records))
5602                .await
5603                .map_err(RuntimeError::from_store)?;
5604        }
5605
5606        *dispatched = true;
5607        // The agent recorded at admission, so a replay is bounded by the
5608        // ceilings the run actually had — and dispatched no wider than the
5609        // width its reservation was sized for.
5610        let budget = self.budget_for(&recorded_agent(&records));
5611        let quota = self
5612            .start_replay_quota_pass(run, mode, reserved_width(&budget, &plan))
5613            .await?;
5614        let through = self.marking_pass(run, &quota);
5615        let this = through.as_ref().unwrap_or(self);
5616
5617        let case_ctx = self.recorded_case(&records);
5618
5619        let mut cursor = ReplayCursor::from_records(&records);
5620
5621        // Strict verification must not write, and it does not: it holds no
5622        // lease, and appends happen only past the end of history, which
5623        // Strict mode refuses to reach. Resume writes under the lease it
5624        // claimed before reading.
5625        let epoch = lease.map_or_else(|| records.last().map_or(1, |r| r.body.epoch), |l| l.epoch);
5626
5627        // Strict verification never writes, so it holds no lease to renew.
5628        let _heartbeat = lease.map(|l| self.heartbeat(run, l.epoch));
5629        this.execute(
5630            Execution {
5631                run,
5632                epoch,
5633                quota,
5634                plan: &plan,
5635                input,
5636                mode,
5637                case: case_ctx,
5638                budget,
5639                agent: recorded_agent(&records),
5640                identity: recorded_chain(&records)?,
5641                subjects: recorded_subjects(run, &records),
5642                // A step-level refusal has no effect key, so it cannot ride the
5643                // replay cursor like an effect's does. It is lifted here from
5644                // the records `replay` has already read.
5645                refusal: recorded_step_refusal(&records),
5646                withheld_subject: standing_withholding(&records),
5647                successors,
5648                started: recorded_started_steps(&records),
5649                finished: recorded_finished_steps(&records),
5650                succeeded: recorded_succeeded_steps(&records),
5651                recorded_groups: recorded_groups(&records),
5652            },
5653            &mut cursor,
5654        )
5655        .await
5656        .inspect(|_| *divergence = cursor.divergence().cloned())
5657    }
5658
5659    /// The trace root. Every span below is a child, so "what did this run do"
5660    /// is one query rather than a correlation exercise across logs.
5661    ///
5662    /// Instrumented rather than entered: an `Entered` guard held across an
5663    /// `.await` belongs to the thread, and with concurrent dispatch that
5664    /// reparents whatever runs next onto this span.
5665    async fn execute(
5666        &self,
5667        plan: Execution<'_>,
5668        cursor: &mut ReplayCursor,
5669    ) -> Result<RunOutcome, RuntimeError> {
5670        let span = tracing::info_span!(
5671            telemetry::RUN_SPAN,
5672            { telemetry::GEN_AI_OPERATION } = telemetry::GEN_AI_INVOKE_AGENT,
5673            { telemetry::RUN_ID } = tracing::field::display(plan.run),
5674            // Which declaration this run is executing. A plane serving several
5675            // agents cannot otherwise separate them, and separating them is
5676            // the first thing an operator does.
5677            { telemetry::GEN_AI_AGENT_NAME } = plan.agent.as_str(),
5678            { telemetry::MODE } = telemetry::mode_str(plan.mode),
5679            { telemetry::GEN_AI_CONVERSATION_ID } = plan
5680                .case
5681                .as_ref()
5682                .map(|c| super::ctx::CaseContext::id(c).to_string()),
5683            // Empty under the default policy, which is what `Omitted` means.
5684            // Recorded rather than declared so the attribute is absent rather
5685            // than blank — a collector cannot read *not disclosed* as *no
5686            // tenant* if there is no field at all.
5687            { telemetry::TENANT } = tracing::field::Empty,
5688            { telemetry::OUTCOME } = tracing::field::Empty,
5689            { telemetry::SEMCONV } = telemetry::SEMCONV_VERSION,
5690            { telemetry::OTEL_NAME } = format!("{} {}", telemetry::GEN_AI_INVOKE_AGENT, plan.agent),
5691            { telemetry::OTEL_KIND } = "internal",
5692        );
5693        if !self.meter.tenant().is_empty() {
5694            span.record(telemetry::TENANT, self.meter.tenant());
5695        }
5696        self.execute_inner(plan, cursor).instrument(span).await
5697    }
5698
5699    /// The driver loop: ready set, admit, dispatch, apply, repeat.
5700    ///
5701    /// Long by line count and deliberately not split further. Every *step* is
5702    /// already its own method — `admit_ready`, `dispatch`, `collect`, `apply`,
5703    /// `adopt_successor`, `stop`. What is left is the order they happen in, and
5704    /// that order is the algorithm. Breaking it up again would scatter one
5705    /// readable sequence across functions that exist only to satisfy a line
5706    /// count, which is the opposite of the thing the lint is protecting.
5707    #[allow(clippy::too_many_lines)]
5708    async fn execute_inner(
5709        &self,
5710        plan: Execution<'_>,
5711        cursor: &mut ReplayCursor,
5712    ) -> Result<RunOutcome, RuntimeError> {
5713        let Execution {
5714            run,
5715            epoch,
5716            quota,
5717            plan: ir,
5718            input,
5719            mode,
5720            case,
5721            budget,
5722            agent,
5723            identity,
5724            subjects,
5725            refusal: recorded_refusal,
5726            withheld_subject,
5727            successors,
5728            started,
5729            finished,
5730            succeeded,
5731            recorded_groups,
5732        } = plan;
5733        // Every value derived from what the run took in carries its subjects.
5734        let input = input.attributed(&subjects);
5735        let writing = !matches!(mode, Pass::Strict);
5736        let case_id = case.as_ref().map(super::ctx::CaseContext::id);
5737        let stamp = |a: Append| match case_id {
5738            Some(c) => a.case(c),
5739            None => a,
5740        };
5741
5742        // The plan in force. Owned rather than borrowed, because a replan
5743        // replaces it and a reference into the version list could not
5744        // survive that. `recorded_successors` is empty on a live run and seeded
5745        // from the journal on a replay, so a successor is read back rather than
5746        // re-synthesised.
5747        let mut current: PlanIR = ir.clone();
5748        let mut replans: u32 = 0;
5749        // A withholding the recorded history left standing, if it did. Seeded
5750        // from the records for the same reason the step refusal is: it is a fact
5751        // about *this* run that a resume has to answer, and asking the journal
5752        // again at every boundary would be a read per ready set for a question
5753        // whose answer only this loop can change.
5754        let mut withheld: Option<String> = withheld_subject;
5755        let recorded_successors = successors;
5756
5757        // One ledger for the run. A step never gets its own allowance to blow.
5758        let ledger = Arc::new(std::sync::Mutex::new(Ledger::new(budget)));
5759
5760        // Steps already completed, and what they produced. Rebuilt from the
5761        // journal on replay so a resumed run knows where it got to.
5762        let mut done: BTreeSet<StepId> = BTreeSet::new();
5763
5764        // The same steps as `done`, in the order they finished, **with the
5765        // capability that actually ran**. Unwinding needs both: a set has no
5766        // order to reverse, and after a replan the current plan may have
5767        // different work — or nothing at all — at a completed step's id.
5768        // Resolving the compensation from the live plan then undoes something
5769        // that never ran, which is a refund for a charge nobody made.
5770        let mut completed: Vec<(StepId, Capability)> = Vec::new();
5771        // The node each completed step ran as. A successor may carry a
5772        // completed step over only as this exact node: `done` is by id, so a
5773        // redeclared node would count as done on the strength of other work.
5774        let mut ran: BTreeMap<StepId, PlanNode> = BTreeMap::new();
5775        let mut outputs: BTreeMap<StepId, Tainted<Value>> = BTreeMap::new();
5776
5777        // ── Ready-set scheduling ───────────────────────────────────────────
5778        //
5779        // Dispatch order is a deterministic total order (ascending step id), so
5780        // replay reproduces it exactly. A plan with parallelism that
5781        // dispatched in completion order would replay differently every time.
5782        // The new work of a ready set whose recorded successes replay first.
5783        let mut deferred: Option<Vec<StepId>> = None;
5784        loop {
5785            let mut ready = current.ready(&done);
5786            if let Some(rest) = deferred.take() {
5787                let held: Vec<StepId> =
5788                    ready.iter().copied().filter(|s| rest.contains(s)).collect();
5789                if !held.is_empty() {
5790                    ready = held;
5791                }
5792            }
5793            // A resume's ready set can hold a step the record shows succeeding
5794            // beside one it does not — parallel work cut off mid-flight. The
5795            // recorded steps replay as a set of their own first, so the checks
5796            // below meet the frontier with their outputs already restored; the
5797            // rest is the next set, with nothing their replay made ready
5798            // joining it.
5799            if writing
5800                && ready.iter().any(|s| succeeded.contains(s))
5801                && ready.iter().any(|s| !succeeded.contains(s))
5802            {
5803                let (recorded, rest) = ready.into_iter().partition(|s| succeeded.contains(s));
5804                ready = recorded;
5805                deferred = Some(rest);
5806            }
5807            // ── The stop check, at a step boundary and nowhere else ────────
5808            //
5809            // Between boundaries an effect may be announced and not yet
5810            // recorded, and interrupting there manufactures the in-doubt case
5811            // the whole protocol exists to avoid. Checking here costs one
5812            // store read per ready set and buys a cancellation that can never
5813            // strand an effect.
5814            //
5815            // Skipped while replaying: a recorded run's history already
5816            // contains whatever stop it received, and re-reading the live
5817            // request would let a cancellation arriving *today* rewrite what a
5818            // run did last year.
5819            //
5820            // A resume asks only at its frontier — once the ready set holds a
5821            // step the record does not show succeeding. Before that, the
5822            // replayed steps have not yet refilled `completed` and `outputs`,
5823            // and an unwind from there would hand each compensation nothing
5824            // and walk the steps in id order rather than in reverse of the
5825            // order they finished.
5826            let at_frontier = ready.is_empty() || ready.iter().any(|s| !succeeded.contains(s));
5827            if writing
5828                && at_frontier
5829                && let Some(c) = self
5830                    .store
5831                    .cancellation(run)
5832                    .await
5833                    .map_err(RuntimeError::from_store)?
5834            {
5835                let status = RunStatus::Cancelled {
5836                    actor: c.actor.clone(),
5837                    reason: c.reason.clone(),
5838                };
5839                // Journaled *before* unwinding, so the reason the run stopped is
5840                // in the chain even if compensation then fails and quarantines
5841                // it. An operator reading a half-unwound run must be able to see
5842                // that somebody asked for this.
5843                self.store
5844                    .append(
5845                        epoch,
5846                        vec![stamp(Append::new(
5847                            run,
5848                            RecordKind::RunCancelled {
5849                                actor: c.actor,
5850                                reason: c.reason,
5851                            },
5852                        ))],
5853                    )
5854                    .await
5855                    .map_err(RuntimeError::from_store)?;
5856                // A step that paused with its group open holds work only that
5857                // step can take back: its reversals are built from the members'
5858                // outputs as the step runs. Re-run it once to abandon the group
5859                // — members read back, the pause aborts instead of waiting —
5860                // and only then unwind the steps before it.
5861                let abandon: Vec<StepId> = ready
5862                    .iter()
5863                    .copied()
5864                    .filter(|&s| {
5865                        recorded_groups.iter().any(|((step, phase, _), g)| {
5866                            *step == s && *phase == Phase::Forward && g.opened > g.settled
5867                        })
5868                    })
5869                    .collect();
5870                if !abandon.is_empty() {
5871                    let dispatched = self
5872                        .dispatch(
5873                            &abandon,
5874                            cursor,
5875                            Batch {
5876                                agent: &agent,
5877                                identity: identity.as_ref(),
5878                                subjects: &subjects,
5879                                run,
5880                                epoch,
5881                                ir: &current,
5882                                mode,
5883                                case: &case,
5884                                ledger: &ledger,
5885                                writing,
5886                                stamp: &stamp,
5887                                input: &input,
5888                                outputs: &outputs,
5889                                started: &started,
5890                                finished: &finished,
5891                                recorded_groups: &recorded_groups,
5892                                parallelism: 1,
5893                                abandoning: true,
5894                            },
5895                        )
5896                        .await;
5897                    collect(dispatched, &abandon, cursor)?;
5898                }
5899                return self
5900                    .stop(
5901                        Unwind {
5902                            agent: &agent,
5903                            identity: identity.as_ref(),
5904                            subjects: &subjects,
5905                            run,
5906                            epoch,
5907                            ir: &current,
5908                            mode,
5909                            case: case.clone(),
5910                            ledger: &ledger,
5911                            writing,
5912                            stamp: &stamp,
5913                        },
5914                        status,
5915                        &completed,
5916                        &outputs,
5917                        cursor,
5918                        case_id,
5919                        &quota,
5920                    )
5921                    .await;
5922            }
5923
5924            // ── The withdrawal check, at the same boundary ─────────────────
5925            //
5926            // Beside the stop check and for its reasons. Unlike a cancellation
5927            // this is a **pause**: `stop` is reached with a status
5928            // `maybe_unwind` passes straight through, as exhaustion is.
5929            if writing && at_frontier {
5930                let standing = self
5931                    .withdrawal_against(identity.as_ref())
5932                    .await
5933                    .map_err(RuntimeError::Store)?;
5934                match (standing, withheld.take()) {
5935                    // Lifted since the run was withheld: that decision is a
5936                    // fact about the run and goes on the record, beside the
5937                    // withholding it supersedes rather than instead of it.
5938                    // Without it a strict replay would stop at the old pause and
5939                    // report `withheld` about a run whose own later records show
5940                    // it finishing.
5941                    (None, Some(prior)) => {
5942                        self.store
5943                            .append(
5944                                epoch,
5945                                vec![stamp(Append::new(
5946                                    run,
5947                                    RecordKind::AuthorityRestored { subject: prior },
5948                                ))],
5949                            )
5950                            .await
5951                            .map_err(RuntimeError::from_store)?;
5952                    }
5953                    // Still withdrawn. The standing record already says so — no
5954                    // second one, for the reason a re-exhausted run writes no
5955                    // second refusal.
5956                    // The prior is dropped rather than restored: this pass
5957                    // returns, and the *records* are what a later resume reads
5958                    // its standing state from.
5959                    (Some(withdrawal), Some(_prior)) => {
5960                        return self
5961                            .withhold(
5962                                Unwind {
5963                                    agent: &agent,
5964                                    identity: identity.as_ref(),
5965                                    subjects: &subjects,
5966                                    run,
5967                                    epoch,
5968                                    ir: &current,
5969                                    mode,
5970                                    case: case.clone(),
5971                                    ledger: &ledger,
5972                                    writing,
5973                                    stamp: &stamp,
5974                                },
5975                                withdrawal,
5976                                false,
5977                                &completed,
5978                                &outputs,
5979                                cursor,
5980                                case_id,
5981                                &quota,
5982                            )
5983                            .await;
5984                    }
5985                    (Some(withdrawal), None) => {
5986                        return self
5987                            .withhold(
5988                                Unwind {
5989                                    agent: &agent,
5990                                    identity: identity.as_ref(),
5991                                    subjects: &subjects,
5992                                    run,
5993                                    epoch,
5994                                    ir: &current,
5995                                    mode,
5996                                    case: case.clone(),
5997                                    ledger: &ledger,
5998                                    writing,
5999                                    stamp: &stamp,
6000                                },
6001                                withdrawal,
6002                                true,
6003                                &completed,
6004                                &outputs,
6005                                cursor,
6006                                case_id,
6007                                &quota,
6008                            )
6009                            .await;
6010                    }
6011                    (None, None) => {}
6012                }
6013            }
6014            if ready.is_empty() {
6015                break;
6016            }
6017
6018            // Admission first, and deliberately not concurrent: which step a
6019            // ceiling refuses must be a property of the plan, not of which
6020            // future happened to poll first.
6021            let (admitted, refused) = self
6022                .admit_ready(
6023                    &ready,
6024                    &ledger,
6025                    mode,
6026                    recorded_refusal.as_ref(),
6027                    cursor,
6028                    Journalling {
6029                        run,
6030                        epoch,
6031                        writing,
6032                        stamp: &stamp,
6033                    },
6034                )
6035                .await?;
6036
6037            let dispatched = self
6038                .dispatch(
6039                    &admitted,
6040                    cursor,
6041                    Batch {
6042                        agent: &agent,
6043                        identity: identity.as_ref(),
6044                        subjects: &subjects,
6045                        run,
6046                        epoch,
6047                        ir: &current,
6048                        mode,
6049                        case: &case,
6050                        ledger: &ledger,
6051                        writing,
6052                        stamp: &stamp,
6053                        input: &input,
6054                        outputs: &outputs,
6055                        started: &started,
6056                        finished: &finished,
6057                        recorded_groups: &recorded_groups,
6058                        parallelism: quota.parallelism(&budget),
6059                        abandoning: false,
6060                    },
6061                )
6062                .await;
6063            let outcomes = collect(dispatched, &ready, cursor)?;
6064
6065            // ── Strict verification consumes everything, per step ──────────
6066            //
6067            // Divergence has two directions, and the ordered key comparison
6068            // only sees one of them. A build that asks for a *different*
6069            // effect fails at the mismatch; a build that asks for **fewer**
6070            // effects than the record sails past the remainder and would
6071            // report the step verified — a false confirmation about exactly
6072            // the history a verification pass exists to check. So a step that
6073            // finished with journaled effects still unread is the same
6074            // quarantine-grade finding as a key mismatch, named by the first
6075            // effect nothing requested.
6076            if mode == Pass::Strict {
6077                for &(step, ref status, _) in &outcomes {
6078                    // A step that stopped — suspended, failed, refused — ended
6079                    // where its history says it ended, and the stop itself is
6080                    // the verified finding. Only a step that claims to have
6081                    // *finished* owes a fully consumed slice.
6082                    if !matches!(status, RunStatus::Succeeded) {
6083                        continue;
6084                    }
6085                    if let Some(missing) = cursor.unconsumed_in(step, Phase::Forward) {
6086                        cursor.note_divergence(crate::journal::Divergence::unconsumed(
6087                            step,
6088                            Phase::Forward,
6089                            &missing,
6090                        ));
6091                        self.meter.count(metrics::DIVERGENCES, "");
6092                        tracing::error!(
6093                            target: telemetry::NONDETERMINISM,
6094                            %step, key = %missing.key, unconsumed = true,
6095                        );
6096                        return self
6097                            .conclude(
6098                                run,
6099                                epoch,
6100                                RunStatus::Quarantined(format!(
6101                                    "strict replay verified less than the run recorded: \
6102                                     step {step} finished with journaled effect {missing} \
6103                                     never requested — this build performs fewer effects than the \
6104                                     recorded one"
6105                                )),
6106                                None,
6107                                writing,
6108                                case_id,
6109                                Consumed::default(),
6110                                Spend::default(),
6111                                &quota,
6112                            )
6113                            .await;
6114                    }
6115                }
6116            }
6117
6118            // A step that stops for any reason stops the run: whatever remains
6119            // either depended on it, or will be dispatched when it resumes.
6120            // Anything already done may need undoing first.
6121            let mut stopped = apply(
6122                &current,
6123                outcomes,
6124                &mut done,
6125                &mut completed,
6126                &mut ran,
6127                &mut outputs,
6128            )
6129            .or_else(|| refused.map(RunStatus::Exhausted));
6130
6131            if let Some(RunStatus::Replanning(reason)) = &stopped {
6132                let recorded = recorded_successors.get(replans as usize);
6133                let journal = Journalling {
6134                    run,
6135                    epoch,
6136                    writing: writing && recorded.is_none(),
6137                    stamp: &stamp,
6138                };
6139                let cx = Replan {
6140                    run,
6141                    current: &current,
6142                    ran: &ran,
6143                    reason,
6144                    already_replanned: replans,
6145                    max_replans: budget.max_replans,
6146                    recorded,
6147                };
6148                match self
6149                    .adopt_successor(cx, journal, &outputs, &completed)
6150                    .await?
6151                {
6152                    Ok(next) => {
6153                        replans += 1;
6154                        current = next;
6155                        continue;
6156                    }
6157                    Err(refusal) => stopped = Some(refusal),
6158                }
6159            }
6160
6161            if let Some(status) = stopped {
6162                return self
6163                    .stop(
6164                        Unwind {
6165                            agent: &agent,
6166                            identity: identity.as_ref(),
6167                            subjects: &subjects,
6168                            run,
6169                            epoch,
6170                            ir: &current,
6171                            mode,
6172                            case: case.clone(),
6173                            ledger: &ledger,
6174                            writing,
6175                            stamp: &stamp,
6176                        },
6177                        status,
6178                        &completed,
6179                        &outputs,
6180                        cursor,
6181                        case_id,
6182                        &quota,
6183                    )
6184                    .await;
6185            }
6186        }
6187
6188        // ── Strict verification consumes everything, at the end ────────────
6189        //
6190        // The per-batch check above covers steps this build dispatched; this
6191        // covers the ones it never asked for at all — a step a changed plan no
6192        // longer reaches, a compensation slice nothing requested. Either way the
6193        // record holds work this build cannot account for, and "verified" must
6194        // not be the answer.
6195        if mode == Pass::Strict
6196            && let Some((step, phase, missing)) = cursor.first_unconsumed()
6197        {
6198            cursor.note_divergence(crate::journal::Divergence::unconsumed(
6199                step, phase, &missing,
6200            ));
6201            self.meter.count(metrics::DIVERGENCES, "");
6202            tracing::error!(
6203                target: telemetry::NONDETERMINISM,
6204                %step, key = %missing.key, unconsumed = true,
6205            );
6206            return self
6207                .conclude(
6208                    run,
6209                    epoch,
6210                    RunStatus::Quarantined(format!(
6211                        "strict replay verified less than the run recorded: journaled \
6212                         effect {missing} at step {step}, {phase:?} phase, was never \
6213                         requested — this build performs fewer effects than the recorded one"
6214                    )),
6215                    None,
6216                    writing,
6217                    case_id,
6218                    Consumed::default(),
6219                    Spend::default(),
6220                    &quota,
6221                )
6222                .await;
6223        }
6224
6225        // Read into a value first. A `MutexGuard` built inline as an argument
6226        // lives until the end of the full expression — which here is *after*
6227        // the await — so the lock would be held across a suspension. That is
6228        // the same shape as the `Span::enter()` bug: a guard whose
6229        // scope is wider than it looks, and invisible until something else
6230        // needs the lock.
6231        let (consumed, live_spend) = {
6232            let l = ledger.lock().expect("budget mutex");
6233            (l.consumed(), l.live_spend())
6234        };
6235        self.conclude(
6236            run,
6237            epoch,
6238            completion(&current, &done),
6239            run_output(&current, &outputs),
6240            writing,
6241            case_id,
6242            consumed,
6243            live_spend,
6244            &quota,
6245        )
6246        .await
6247    }
6248
6249    /// Read the clock into the ledger, when a wall-clock ceiling is declared.
6250    ///
6251    /// Separate so the guard's scope is two statements that cannot span the
6252    /// await between them, and so the condition is asked once rather than
6253    /// spelled at the caller.
6254    async fn observe_clock(
6255        cx: &mut StepCtx<'_>,
6256        ledger: &Arc<std::sync::Mutex<Ledger>>,
6257    ) -> Result<(), crate::core::StepError> {
6258        if !ledger.lock().expect("budget mutex").tracks_wallclock() {
6259            return Ok(());
6260        }
6261        let at = cx.now().await?;
6262        ledger.lock().expect("budget mutex").observe_clock(at);
6263        Ok(())
6264    }
6265
6266    /// Run a ready set concurrently, at most `max_parallel_steps` at a time.
6267    ///
6268    /// The ready set is every node whose predecessors are done, so nothing in
6269    /// it depends on anything else in it — running them one at a time is a
6270    /// choice, and the wrong one when steps are waiting on models and networks.
6271    ///
6272    /// Each step takes its own slice of history, which is what makes this sound:
6273    /// a step touches only its own effects, so no shared mutable state is left
6274    /// between them, and the per-step replay cursor verifies each one's order
6275    /// independently of how the journal happened to interleave them.
6276    ///
6277    /// The width is bounded because it is not otherwise bounded by anything:
6278    /// a fan-out is as wide as its author wrote it, and every step in flight
6279    /// holds a connection, a provider's rate limit and one operation's worth
6280    /// of spend the metered ceilings have checked and not yet billed. Ordered
6281    /// rather than completion-ordered — `buffered`, not `buffer_unordered` —
6282    /// so the outcomes still apply in ready order and a plan's dispatch is a
6283    /// property of the plan rather than of which future finished first.
6284    async fn dispatch(
6285        &self,
6286        admitted: &[StepId],
6287        cursor: &mut ReplayCursor,
6288        batch: Batch<'_>,
6289    ) -> Vec<Dispatched> {
6290        use futures_util::StreamExt;
6291
6292        let slices: Vec<(StepId, StepCursor)> = admitted
6293            .iter()
6294            .map(|&s| (s, cursor.take(s, Phase::Forward)))
6295            .collect();
6296        // Floored at one because a width of zero would poll nothing forever.
6297        // A zero ceiling is refused at parse and at build, so this is the
6298        // second half of a rule rather than the only half — and the failure it
6299        // guards against is a hang, which is the one shape a later reader
6300        // cannot diagnose from the outside.
6301        let width = batch.parallelism.max(1);
6302
6303        futures_util::stream::iter(slices.into_iter().map(|(step, slice)| async move {
6304            let node = batch
6305                .ir
6306                .node(step)
6307                .ok_or_else(|| RuntimeError::PlanContract(format!("no node {step}")))?;
6308            let (status, out, slice) = self
6309                .run_step(
6310                    StepRun {
6311                        agent: batch.agent,
6312                        identity: batch.identity,
6313                        subjects: batch.subjects,
6314                        run: batch.run,
6315                        epoch: batch.epoch,
6316                        node,
6317                        phase: Phase::Forward,
6318                        mode: batch.mode,
6319                        case: batch.case.clone(),
6320                        ledger: batch.ledger,
6321                        writing: batch.writing,
6322                        stamp: batch.stamp,
6323                        already_started: batch.started.contains(&step),
6324                        already_finished: batch.finished.contains(&step),
6325                        recorded_groups: batch
6326                            .recorded_groups
6327                            .iter()
6328                            .filter(|((s, p, _), _)| *s == step && *p == Phase::Forward)
6329                            .map(|((_, _, name), n)| (name.clone(), *n))
6330                            .collect(),
6331                        abandoning: batch.abandoning,
6332                    },
6333                    batch.input,
6334                    batch.outputs,
6335                    slice,
6336                )
6337                .await?;
6338            Ok((step, status, out, slice))
6339        }))
6340        .buffered(width)
6341        .collect()
6342        .await
6343    }
6344
6345    /// Take the successor a step asked for: check it, record it, announce it.
6346    ///
6347    /// The nested result separates two failures. The outer `RuntimeError` is the
6348    /// runtime itself failing — the journal would not accept the plan — and is
6349    /// never recoverable. The inner `RunStatus` is the *request* being refused,
6350    /// which is an ordinary outcome the run reports.
6351    async fn adopt_successor(
6352        &self,
6353        cx: Replan<'_>,
6354        journal: Journalling<'_>,
6355        outputs: &BTreeMap<StepId, Tainted<Value>>,
6356        completed: &[(StepId, Capability)],
6357    ) -> Result<Result<PlanIR, RunStatus>, RuntimeError> {
6358        // What each step that started and journaled an effect ran as, read
6359        // from the journal only when a successor is about to be checked — a
6360        // replay reads the successor back instead.
6361        let started = if cx.recorded.is_none() {
6362            let records = self
6363                .store
6364                .read(cx.run, 1)
6365                .await
6366                .map_err(RuntimeError::from_store)?;
6367            let effectful: BTreeSet<StepId> = records
6368                .iter()
6369                .filter(|r| r.body.phase.is_forward())
6370                .filter_map(|r| match r.kind() {
6371                    RecordKind::EffectStarted { .. } => r.body.step,
6372                    _ => None,
6373                })
6374                .collect();
6375            let mut started = started_skills(&records);
6376            started.retain(|step, _| effectful.contains(step));
6377            started
6378        } else {
6379            BTreeMap::new()
6380        };
6381        let next = match self.successor(cx, outputs, completed, &started).await {
6382            Ok(next) => next,
6383            Err(refusal) => return Ok(Err(refusal)),
6384        };
6385        if journal.writing {
6386            self.freeze(journal.run, journal.epoch, &next, journal.stamp)
6387                .await?;
6388        }
6389        announce_replan(
6390            &self.meter,
6391            journal.run,
6392            &next,
6393            next.reason.as_deref().unwrap_or(""),
6394        );
6395        Ok(Ok(next))
6396    }
6397
6398    /// Record a plan version in the journal.
6399    ///
6400    /// A successor is frozen exactly as a first plan is, so replay reads it back
6401    /// rather than asking a planner that may since have changed its mind.
6402    async fn freeze(
6403        &self,
6404        run: RunId,
6405        epoch: u64,
6406        plan: &PlanIR,
6407        stamp: &(dyn Fn(Append) -> Append + Send + Sync),
6408    ) -> Result<(), RuntimeError> {
6409        self.store
6410            .append(
6411                epoch,
6412                vec![stamp(Append::new(
6413                    run,
6414                    RecordKind::PlanFrozen {
6415                        steps: plan.nodes.iter().map(|n| n.capability.0.clone()).collect(),
6416                        plan: serde_json::to_value(plan)?,
6417                    },
6418                ))],
6419            )
6420            .await
6421            .map_err(RuntimeError::from_store)?;
6422        Ok(())
6423    }
6424
6425    /// Produce the successor plan a step asked for, or say why not.
6426    ///
6427    /// Three gates, and the first is not negotiable.
6428    ///
6429    /// **Provenance.** The frozen plan is an authorization graph compiled from
6430    /// trusted input only. A replan *changes that graph*, so once any
6431    /// untrusted value has reached working memory, anything shaping the new plan
6432    /// may be attacker-chosen — and choosing the authorization graph is the
6433    /// whole game. `plan-then-execute` is enforced here, structurally. A run
6434    /// that wants a different plan after reading untrusted input is describing
6435    /// exactly the attack.
6436    ///
6437    /// **Budget.** A run that replans without bound has stopped making progress
6438    /// and started thrashing.
6439    ///
6440    /// **On replay, the successor is read back, never re-synthesised.** A
6441    /// planner asked twice can answer differently — a changed router, a
6442    /// different model — and replay would then verify the run against a plan
6443    /// that never governed it. Same rule as the first plan, for the same
6444    /// reason.
6445    async fn successor(
6446        &self,
6447        cx: Replan<'_>,
6448        outputs: &BTreeMap<StepId, Tainted<Value>>,
6449        completed: &[(StepId, Capability)],
6450        started: &BTreeMap<StepId, String>,
6451    ) -> Result<PlanIR, RunStatus> {
6452        if let Some(source) = untrusted_in(outputs) {
6453            return Err(RunStatus::Failed(format!(
6454                "replanning refused: untrusted data from {source} is already in working memory, and the plan is an authorization graph — letting it change now would let that data choose what runs next ({})",
6455                cx.reason
6456            )));
6457        }
6458
6459        let spent = cx.already_replanned;
6460        if let Some(max) = cx.max_replans
6461            && spent >= max
6462        {
6463            return Err(RunStatus::Exhausted(crate::core::BudgetExceeded::Replans {
6464                allowed: max,
6465            }));
6466        }
6467
6468        // Replay: the successor is in the journal, at the position this replan
6469        // reached. Reading it back is what keeps a re-planned run replayable.
6470        if let Some(recorded) = cx.recorded {
6471            return Ok(recorded.clone());
6472        }
6473
6474        let replanner = self.replanner.as_ref().ok_or_else(|| {
6475            RunStatus::Failed(format!(
6476                "a step asked to replan and this runtime has no planner — build it with `.replanner(..)` ({})",
6477                cx.reason
6478            ))
6479        })?;
6480
6481        let next = replanner
6482            .replan(cx.current, cx.reason, completed)
6483            .await
6484            .map_err(|e| RunStatus::Failed(format!("replanning failed: {e}")))?;
6485
6486        // A successor faces the same contract a first plan does. One that fails
6487        // validation stops the run rather than half-applying.
6488        crate::plan::validate(&next, &self.contract())
6489            .map_err(|e| RunStatus::Failed(format!("the successor plan is invalid: {e}")))?;
6490
6491        // A completed step carries over only as the node it ran as, or not at
6492        // all. Other work at its id breaks replay — effect keys derive from the
6493        // step id — and has the saga undo something that never ran. The same
6494        // capability redeclared is refused too: `done` is by id, so a node with
6495        // new arguments, dependencies or flags would count as done on the
6496        // strength of work done before any of them held — a terminal verifier
6497        // satisfied without ever seeing what it verifies.
6498        for (step, node_ran) in cx.ran {
6499            let Some(node) = next.node(*step) else {
6500                continue;
6501            };
6502            if node.capability != node_ran.capability {
6503                return Err(RunStatus::Failed(format!(
6504                    "the successor plan reuses step {step} — which already ran \
6505                     as '{}' — for '{}'. Keep a completed step's node or leave \
6506                     the step out; effect keys are derived from the step id, so \
6507                     new work at a used id cannot be replayed",
6508                    node_ran.capability.0, node.capability.0
6509                )));
6510            }
6511            if node != node_ran {
6512                return Err(RunStatus::Failed(format!(
6513                    "the successor plan redeclares completed step {step} ('{}') \
6514                     with different arguments, dependencies or flags. A \
6515                     completed step counts as done by its id, so the new node \
6516                     would be satisfied by work done under the old one — carry \
6517                     it over unchanged, or give the new work a fresh id",
6518                    node.capability.0
6519                )));
6520            }
6521        }
6522
6523        // A step that started, journaled an effect and did not complete holds
6524        // keys derived from its id just the same — it may have charged before
6525        // asking to replan — so its id may not be reused for work another
6526        // skill would do. One that journaled nothing, the step that asked for
6527        // a fallback before touching anything, leaves its id free.
6528        for (step, skill) in started {
6529            if cx.ran.contains_key(step) {
6530                continue;
6531            }
6532            let Some(node) = next.node(*step) else {
6533                continue;
6534            };
6535            let resolved = self
6536                .resolve(&node.capability.0)
6537                .map(|s| s.descriptor().name)
6538                .ok();
6539            if resolved.as_deref() != Some(skill.as_str()) {
6540                return Err(RunStatus::Failed(format!(
6541                    "the successor plan reuses step {step} — which already \
6542                     started as '{skill}' and journaled effects — for '{}'. Keep the step's \
6543                     capability or leave the step out; effect keys are derived \
6544                     from the step id, and the unwind undoes a started step as \
6545                     the skill that ran",
6546                    node.capability.0
6547                )));
6548            }
6549        }
6550
6551        if next.derived_from != Some(cx.current.digest()) {
6552            return Err(RunStatus::Failed(
6553                "the successor plan does not name its predecessor — use `PlanIR::succeed_with`, or the audit trail has a hole where the lineage should be"
6554                    .into(),
6555            ));
6556        }
6557
6558        // The third leg of the lineage `succeed_with` sets. A successor with
6559        // no reason would freeze into the journal as a plan that replaced
6560        // another with nothing on the record saying why — the half of an
6561        // incident (*what changed the run's mind*) that versioned replanning
6562        // exists to keep.
6563        if next.reason.as_deref().is_none_or(str::is_empty) {
6564            return Err(RunStatus::Failed(
6565                "the successor plan carries no reason — use `PlanIR::succeed_with`, or the \
6566                 journal freezes a plan that replaced another with nothing saying why"
6567                    .into(),
6568            ));
6569        }
6570
6571        Ok(next)
6572    }
6573
6574    /// Decide which of a ready set may start, in ready order.
6575    ///
6576    /// Three regimes, one per mode, and the middle one is the subtle case:
6577    ///
6578    /// * **Strict** consumes recorded verdicts and recomputes none. A run
6579    ///   replayed under a larger budget stopped where it stopped; recomputing
6580    ///   made a step-limited run replay as *succeeded* — a false audit result.
6581    /// * **Resume** takes each verdict from wherever the truth lives. A step
6582    ///   with unconsumed history was admitted by the run that recorded it,
6583    ///   and the history *is* the verdict. A step at the frontier is about to
6584    ///   run **live**, and every live step is metered — keying this on "is
6585    ///   the mode a replay" instead of "will this step dispatch live" let a
6586    ///   resumed run cross `max_steps` unmetered for as long as it kept
6587    ///   running. A *recorded refusal* is re-evaluated against the ledger now
6588    ///   in force: exhaustion is a pause, and raising the ceiling and
6589    ///   resuming is how it un-pauses.
6590    /// * **Live** asks the ledger, which is the ordinary case.
6591    async fn admit_ready(
6592        &self,
6593        ready: &[StepId],
6594        ledger: &Arc<std::sync::Mutex<Ledger>>,
6595        mode: Pass,
6596        recorded: Option<&(StepId, String, String)>,
6597        cursor: &ReplayCursor,
6598        journal: Journalling<'_>,
6599    ) -> Result<(Vec<StepId>, Option<crate::core::BudgetExceeded>), RuntimeError> {
6600        let mut admitted = Vec::new();
6601
6602        for &step in ready {
6603            // ── A recorded step refusal at this step ───────────────────────
6604            if let Some((at, limit, used)) = recorded
6605                && *at == step
6606            {
6607                if mode == Pass::Strict {
6608                    // Verbatim: the recorded run stopped here, and a
6609                    // verification pass reaches the same conclusion without
6610                    // consulting today's ledger.
6611                    return Ok((
6612                        admitted,
6613                        Some(crate::core::BudgetExceeded::Recorded {
6614                            limit: limit.clone(),
6615                            used: used.clone(),
6616                        }),
6617                    ));
6618                }
6619                // Resume: the replayed prefix has already billed the journaled
6620                // historical usage into this ledger, so the question is "does
6621                // the budget now in force admit one more step over what the
6622                // run already spent". Still no → the run concludes exhausted
6623                // again, and the standing refusal already says so — no second
6624                // record. Yes → the re-admission is its own record, beside
6625                // the refusal it supersedes, so a strict replay of this
6626                // history reads refusal-then-continuation as a decision
6627                // somebody made rather than as divergence.
6628                if let Err(exceeded) = ledger
6629                    .lock()
6630                    .expect("budget mutex")
6631                    .admit_step(admitted.len())
6632                {
6633                    return Ok((admitted, Some(exceeded)));
6634                }
6635                if journal.writing {
6636                    self.store
6637                        .append(
6638                            journal.epoch,
6639                            vec![(journal.stamp)(
6640                                Append::new(
6641                                    journal.run,
6642                                    RecordKind::BudgetReadmitted {
6643                                        limit: limit.clone(),
6644                                    },
6645                                )
6646                                .step(step),
6647                            )],
6648                        )
6649                        .await
6650                        .map_err(RuntimeError::from_store)?;
6651                }
6652                admitted.push(step);
6653                continue;
6654            }
6655
6656            let verdict = match mode {
6657                Pass::Strict => Ok(()),
6658                Pass::Resume if !cursor.exhausted(step, Phase::Forward) => Ok(()),
6659                Pass::Live | Pass::Resume => ledger
6660                    .lock()
6661                    .expect("budget mutex")
6662                    .admit_step(admitted.len()),
6663            };
6664
6665            let Err(exceeded) = verdict else {
6666                admitted.push(step);
6667                continue;
6668            };
6669
6670            if journal.writing {
6671                // The refusal goes in the journal under the step it refused, so
6672                // replay reads the verdict rather than recomputing it.
6673                let used = format!("{:?}", ledger.lock().expect("budget mutex").consumed());
6674                self.store
6675                    .append(
6676                        journal.epoch,
6677                        vec![(journal.stamp)(
6678                            Append::new(
6679                                journal.run,
6680                                RecordKind::BudgetRefused {
6681                                    limit: exceeded.to_string(),
6682                                    used,
6683                                },
6684                            )
6685                            .step(step),
6686                        )],
6687                    )
6688                    .await
6689                    .map_err(RuntimeError::from_store)?;
6690            }
6691            return Ok((admitted, Some(exceeded)));
6692        }
6693
6694        Ok((admitted, None))
6695    }
6696
6697    /// End a run that stopped early: undo what warrants undoing, then seal.
6698    ///
6699    /// Both places a run can stop short go through here, so the unwind can never
6700    /// be attached to one of them and forgotten on the other.
6701    #[allow(clippy::too_many_arguments)]
6702    async fn stop(
6703        &self,
6704        cx: Unwind<'_>,
6705        status: RunStatus,
6706        completed: &[(StepId, Capability)],
6707        outputs: &BTreeMap<StepId, Tainted<Value>>,
6708        cursor: &mut ReplayCursor,
6709        case_id: Option<crate::core::CaseId>,
6710        quota: &QuotaPass,
6711    ) -> Result<RunOutcome, RuntimeError> {
6712        let (run, epoch, writing, ir) = (cx.run, cx.epoch, cx.writing, cx.ir);
6713        let output = run_output(ir, outputs);
6714        // Cloned before `maybe_unwind` consumes `cx`; read *after* it, because an
6715        // item's cost is what the whole attempt consumed, compensation included.
6716        let ledger = cx.ledger.clone();
6717        let unwound = self
6718            .maybe_unwind(cx, status, completed, outputs, cursor)
6719            .await?;
6720        // A strict pass that stops early is still a verification, and its
6721        // verdict must not outrun what it consumed. The ready loop's
6722        // unconsumed-effects check runs only when the loop drains normally,
6723        // and every early conclusion — a failure, an unwind, a recorded
6724        // suspension — comes through here instead; without this check a build
6725        // that performs fewer effects than the record (a compensation that
6726        // shrank, a step that now suspends earlier) would return the recorded
6727        // status as "verified" over effects it never requested. Checked after
6728        // `maybe_unwind` because a strict pass consumes recorded compensation
6729        // slices there, and what remains after that is genuinely unclaimed.
6730        //
6731        // A run that already quarantined keeps its own reason. Divergence
6732        // leaves history unconsumed as a matter of course — the step stopped
6733        // where it disagreed — so reporting the leftovers here would replace
6734        // the precise finding ("expected this effect, recomputed that one")
6735        // with a vaguer restatement of its consequence, and an operator would
6736        // be told the build performs fewer effects when what it actually did
6737        // was perform a different one.
6738        if !writing
6739            && !matches!(unwound, RunStatus::Quarantined(_))
6740            && let Some((step, phase, missing)) = cursor.first_unconsumed()
6741        {
6742            cursor.note_divergence(crate::journal::Divergence::unconsumed(
6743                step, phase, &missing,
6744            ));
6745            self.meter.count(metrics::DIVERGENCES, "");
6746            tracing::error!(
6747                target: telemetry::NONDETERMINISM,
6748                %step, key = %missing.key, unconsumed = true,
6749            );
6750            return self
6751                .conclude(
6752                    run,
6753                    epoch,
6754                    RunStatus::Quarantined(format!(
6755                        "strict replay verified less than the run recorded: journaled \
6756                         effect {missing} at step {step}, {phase:?} phase, was never \
6757                         requested — this build performs fewer effects than the recorded one"
6758                    )),
6759                    None,
6760                    writing,
6761                    case_id,
6762                    Consumed::default(),
6763                    Spend::default(),
6764                    quota,
6765                )
6766                .await;
6767        }
6768        // Scoped before the await — see `execute_inner` on why an inline guard
6769        // outlives the call it is passed to.
6770        let (consumed, live_spend) = {
6771            let l = ledger.lock().expect("budget mutex");
6772            (l.consumed(), l.live_spend())
6773        };
6774        self.conclude(
6775            run, epoch, unwound, output, writing, case_id, consumed, live_spend, quota,
6776        )
6777        .await
6778    }
6779
6780    /// Undo the completed steps, if the way the run stopped calls for it.
6781    ///
6782    /// # When a run does *not* unwind
6783    ///
6784    /// * **Quarantined.** The run holds an effect whose outcome is unknown, and
6785    ///   you cannot safely undo around one: compensating a payment that may
6786    ///   never have gone out creates a refund for money nobody took. Everything
6787    ///   stays exactly where it is until a person supplies the fact the journal
6788    ///   lacks — [`Runtime::reconcile_effect`](super::Runtime::reconcile_effect)
6789    ///   and [`decide_quarantine`](super::Runtime::decide_quarantine) — and
6790    ///   giving up
6791    ///   ([`Abandon`](crate::core::QuarantineDecision::Abandon)) still unwinds
6792    ///   nothing, because impatience is not evidence. This is the rule that
6793    ///   separates a saga that is honest about distributed systems from one that
6794    ///   tidies up and hopes.
6795    /// * **Suspended.** The run is healthy and waiting. Nothing has failed.
6796    ///
6797    /// A failure *after the pivot* also stops the unwind at the pivot: once the
6798    /// business has committed, reversing the decisions leading up to it would
6799    /// contradict something the outside world has already acted on.
6800    ///
6801    /// # Two rules every unwind takes
6802    ///
6803    /// **It must not unwind around an unknown outcome.** The same rule
6804    /// quarantine already enforces, applied at the unwind's own door. For a
6805    /// cancelled run the check is unconditional — cancellation gets no second
6806    /// pass, so an unresolved outcome stays unresolved forever. For a failed
6807    /// run it is conditional on the unwind list: an orphan under a failure is
6808    /// resolvable, because the announcement is journaled, the effect declared
6809    /// a `Recovery`, and resuming resolves it — but only while nothing was
6810    /// compensated, since compensation closes the run to resume. A failure
6811    /// that would compensate around doubt therefore quarantines instead; a
6812    /// failure with nothing to unwind stays a failure and keeps its resume.
6813    ///
6814    /// **It must undo the steps it interrupted.** Compensation walks
6815    /// *completed* steps, which covers the step that failed only if it never
6816    /// mutated. A stop arrives from outside while a step is typically
6817    /// suspended — holding effects it performed and never completing — and a
6818    /// failure in one concurrent sibling interrupts the others the same way:
6819    /// severity ordering can conclude the run while a suspended sibling holds
6820    /// a landed mutation. Unwinding only completed steps would leave exactly
6821    /// that work standing — a run that posted to a ledger and then suspended
6822    /// for approval would conclude with the posting still in the world and,
6823    /// once anything else was compensated, no resume left to finish it.
6824    async fn maybe_unwind(
6825        &self,
6826        cx: Unwind<'_>,
6827        status: RunStatus,
6828        completed: &[(StepId, Capability)],
6829        outputs: &BTreeMap<StepId, Tainted<Value>>,
6830        cursor: &mut ReplayCursor,
6831    ) -> Result<RunStatus, RuntimeError> {
6832        match status {
6833            // A cancelled run unwinds exactly as a failed one does. Stopping a
6834            // run that has already moved money and leaving the movement in
6835            // place is not stopping it — and the operator who asked is entitled
6836            // to assume "stop" means the world is put back, not that the
6837            // process merely exited.
6838            //
6839            // **Exhausted is deliberately not here.** Exhaustion is a pause,
6840            // not a fault: the run did what it was told, and what it was told
6841            // included a ceiling. Its mutations stand, because the operator's
6842            // two honest options both need them standing — raise the ceiling
6843            // and resume (which continues *over* the completed work), or
6844            // cancel (which unwinds through this same function under a status
6845            // that does). Unwinding on exhaustion would make the three ends of
6846            // an exhausted run contradict each other: the work reversed, the
6847            // run still resumable, and the resume reporting success over a
6848            // world where the work no longer stands.
6849            RunStatus::Failed(_) | RunStatus::Cancelled { .. } => {}
6850            other => return Ok(other),
6851        }
6852
6853        let (list, evidence) = match self.gated_unwind_list(&cx, &status, completed).await? {
6854            Ok(scope) => scope,
6855            Err(quarantine) => return Ok(quarantine),
6856        };
6857        let UnwindEvidence {
6858            mutated,
6859            started: _,
6860            undone: already_undone,
6861            recorded_groups,
6862            failed: _,
6863        } = evidence;
6864        let completed = &list[..];
6865
6866        for (step, capability) in completed.iter().rev().cloned() {
6867            // Resolved from what ran, not from the plan in force. After a replan
6868            // the two can differ, and undoing whatever now occupies that slot is
6869            // how a saga compensates work that never happened.
6870            let skill = self.resolve(&capability.0)?;
6871            let declared = skill.compensation();
6872
6873            match declared {
6874                // The point of no return. Everything from here back stays.
6875                crate::core::Compensation::Pivot => break,
6876                crate::core::Compensation::Unnecessary => continue,
6877                crate::core::Compensation::Undeclared => {
6878                    if !mutated.contains(&step) {
6879                        // Nothing to undo, and the journal proves it.
6880                        continue;
6881                    }
6882                    return Ok(RunStatus::Quarantined(format!(
6883                        "step {step} ('{}') changed external state and declares no \
6884                         compensation, so the run cannot be safely unwound — \
6885                         declare Compensation on it, or resolve this by hand",
6886                        capability.0
6887                    )));
6888                }
6889                crate::core::Compensation::Compensatable => {}
6890            }
6891
6892            let result = self
6893                .run_compensation(
6894                    &cx,
6895                    step,
6896                    skill.as_ref(),
6897                    outputs,
6898                    cursor,
6899                    recorded_groups
6900                        .iter()
6901                        .filter(|((s, p, _), _)| *s == step && *p == Phase::Compensating)
6902                        .map(|((_, _, name), n)| (name.clone(), *n))
6903                        .collect(),
6904                )
6905                .await;
6906
6907            // A compensation may legitimately need to wait — a refund that needs
6908            // four eyes is still a refund. Suspension is not failure: the run is
6909            // healthy, its frame is durable, and it will finish unwinding when
6910            // the answer arrives. Reporting it as a failure would quarantine a
6911            // run doing exactly the right thing and tell the operator the
6912            // compensation had broken.
6913            if let Err(crate::core::SkillError::Step(crate::core::StepError::Suspended(reason))) =
6914                &result
6915            {
6916                return Ok(RunStatus::Suspended(reason.clone()));
6917            }
6918
6919            let outcome = match &result {
6920                Ok(()) => "compensated".to_owned(),
6921                Err(e) => e.to_string(),
6922            };
6923
6924            // Re-run but do not re-record. On resume the compensation executes
6925            // again with every effect served from the journal, which is what
6926            // keeps strict verification meaningful — but a second
6927            // `StepCompensated` would report one compensation as two.
6928            if result.is_ok() {
6929                tracing::info!(target: telemetry::COMPENSATED, run = %cx.run, %step);
6930                self.meter.count(metrics::COMPENSATIONS, "done");
6931            }
6932            if cx.writing && !already_undone.contains(&step) {
6933                self.store
6934                    .append(
6935                        cx.epoch,
6936                        vec![(cx.stamp)(
6937                            Append::new(
6938                                cx.run,
6939                                RecordKind::StepCompensated {
6940                                    compensation: declared,
6941                                    outcome: outcome.clone(),
6942                                },
6943                            )
6944                            .step(step)
6945                            .phase(Phase::Compensating),
6946                        )],
6947                    )
6948                    .await
6949                    .map_err(RuntimeError::from_store)?;
6950            }
6951
6952            if let Err(fault) = &result {
6953                tracing::error!(
6954                    target: telemetry::COMPENSATION_FAILED,
6955                    run = %cx.run,
6956                    %step,
6957                    error_type = fault.class(),
6958                    reason_digest = %telemetry::reason_digest(&outcome),
6959                );
6960                self.meter.count(metrics::COMPENSATIONS, "failed");
6961                // Not a problem more compensation solves. Unwinding further
6962                // would undo steps *before* one that is now in an unknown
6963                // state, which is strictly worse than stopping and saying so.
6964                return Ok(RunStatus::Quarantined(format!(
6965                    "compensation failed for step {step} ('{}'): {outcome} — the run is \
6966                     partially unwound and needs an operator",
6967                    capability.0
6968                )));
6969            }
6970        }
6971
6972        Ok(status)
6973    }
6974
6975    /// Whether `run`'s journal holds a landed mutation no compensation undid.
6976    ///
6977    /// The attention roll-up's question about a failed run: `failed` alone may
6978    /// be cleared by a resume, but one standing on landed work leaves that
6979    /// work in the world until somebody finishes or unwinds it.
6980    pub(super) async fn holds_landed_work(&self, run: RunId) -> Result<bool, RuntimeError> {
6981        let evidence = self.unwind_evidence(run).await?;
6982        Ok(!evidence.mutated.is_subset(&evidence.undone))
6983    }
6984
6985    /// What the journal knows about an unwind before it starts:
6986    /// `(steps that changed something, steps already compensated)`.
6987    ///
6988    /// Evidence, not bookkeeping. The journal already knows both, it knows the
6989    /// same thing on replay, and nothing has to be threaded through the executor
6990    /// to keep a parallel copy honest.
6991    ///
6992    /// **A mutating effect that provably did not happen does not mark its
6993    /// step.** The announcement alone is not the evidence: an effect whose
6994    /// terminal record classifies it `DidNotHappen` — a peer that refused
6995    /// before acting, a call that never left — changed nothing, and
6996    /// compensating it is a refund for money nobody took. Everything else
6997    /// counts, and the defaults lean that way: an announcement with no
6998    /// terminal record is an orphan and therefore in doubt, and each new
6999    /// attempt starts from doubt again, so a retry that lands after a refusal
7000    /// is still evidence the step touched the world.
7001    ///
7002    /// **Nor does a member of a group that settled `Aborted`.** That settlement
7003    /// is written only once every landed member was reversed, so the group's
7004    /// effects — members and reversals alike — left nothing standing, and
7005    /// counting them marked a cleanly aborted step as mutated: a quarantine for
7006    /// a step declaring no compensation, a second undo for one that did.
7007    ///
7008    /// What this does **not** establish is that a `DidNotHappen` classification
7009    /// is true — that is the driver's claim, and a driver that mislabels a
7010    /// landed call leaves a mutation nothing here will undo.
7011    async fn unwind_evidence(&self, run: RunId) -> Result<UnwindEvidence, RuntimeError> {
7012        let records = self
7013            .store
7014            .read(run, 1)
7015            .await
7016            .map_err(RuntimeError::from_store)?;
7017
7018        // Per mutating forward effect: the step it belongs to, and whether the
7019        // record leaves it capable of having touched the world.
7020        let mut touching: BTreeMap<crate::core::EffectKey, (StepId, bool)> = BTreeMap::new();
7021        // The mutating effects of the group each step has open. A step holds
7022        // at most one open group, so the step is the key.
7023        let mut grouped: BTreeMap<StepId, Vec<crate::core::EffectKey>> = BTreeMap::new();
7024        let mut undone = BTreeSet::new();
7025        let mut failed = BTreeSet::new();
7026        for r in &records {
7027            let Some(step) = r.body.step else { continue };
7028            let key = r.effect_key();
7029            match r.kind() {
7030                RecordKind::StepFinished { outcome } if r.body.phase.is_forward() => {
7031                    if outcome == RunStatus::Succeeded.as_str() {
7032                        failed.remove(&step);
7033                    } else {
7034                        failed.insert(step);
7035                    }
7036                }
7037                RecordKind::EffectStarted { mutates: true, .. } if r.body.phase.is_forward() => {
7038                    if let Some(key) = key {
7039                        // A fresh attempt is undecided until its own terminal
7040                        // record says otherwise, so it supersedes whatever the
7041                        // previous attempt concluded.
7042                        touching.insert(key, (step, true));
7043                        if let Some(members) = grouped.get_mut(&step) {
7044                            members.push(key);
7045                        }
7046                    }
7047                }
7048                RecordKind::GroupOpened { .. } if r.body.phase.is_forward() => {
7049                    grouped.insert(step, Vec::new());
7050                }
7051                RecordKind::GroupSettled { outcome, .. } if r.body.phase.is_forward() => {
7052                    let members = grouped.remove(&step).unwrap_or_default();
7053                    if *outcome == crate::core::GroupOutcome::Aborted {
7054                        for member in members {
7055                            touching.remove(&member);
7056                        }
7057                    }
7058                }
7059                RecordKind::EffectFailed { disposition, .. }
7060                | RecordKind::EffectReconciled { disposition, .. } => {
7061                    if let Some(entry) = key.and_then(|k| touching.get_mut(&k)) {
7062                        entry.1 = *disposition != crate::core::Disposition::DidNotHappen;
7063                    }
7064                }
7065                RecordKind::StepCompensated { .. } => {
7066                    undone.insert(step);
7067                }
7068                _ => {}
7069            }
7070        }
7071        let mutated = touching
7072            .into_values()
7073            .filter_map(|(step, touched)| touched.then_some(step))
7074            .collect();
7075        Ok(UnwindEvidence {
7076            mutated,
7077            started: started_skills(&records),
7078            undone,
7079            recorded_groups: recorded_groups(&records),
7080            failed,
7081        })
7082    }
7083
7084    /// The half of a step's frame that belongs to the **plane** rather than to
7085    /// the step.
7086    ///
7087    /// One place, for both a forward step and a compensation: a plane
7088    /// capability wired into one of the two and not the other is a control
7089    /// that applies going forward and not while unwinding.
7090    fn plane_frame(&self, step: StepFrame) -> super::ctx::Frame {
7091        let StepFrame {
7092            run,
7093            epoch,
7094            step,
7095            phase,
7096            mode,
7097            case,
7098            ledger,
7099            identity,
7100            subjects,
7101            agent,
7102            #[cfg(feature = "manifest")]
7103            manifest,
7104            recorded_groups,
7105        } = step;
7106        super::ctx::Frame {
7107            run,
7108            epoch,
7109            step,
7110            phase,
7111            mode,
7112            case,
7113            timers: self.timers.clone(),
7114            blobs: self.blobs.clone(),
7115            memories: self.memories.clone(),
7116            semantic: self.semantic.clone(),
7117            authorities: self.authorities.clone(),
7118            peers: self.peers.clone(),
7119            #[cfg(feature = "manifest")]
7120            tools: self.tools.clone(),
7121            journal_ceiling: self.journal_ceiling,
7122            #[cfg(feature = "manifest")]
7123            egress: self.egress.clone(),
7124            #[cfg(feature = "manifest")]
7125            checkers: self.checkers.clone(),
7126            streams: self.streams.clone(),
7127            #[cfg(feature = "manifest")]
7128            rates: self.rates.clone(),
7129            meter: self.meter.clone(),
7130            #[cfg(feature = "keyring")]
7131            keyring: self.keyring.clone(),
7132            tenant: self.tenant.clone(),
7133            ledger,
7134            policy: self.policy.clone(),
7135            identity,
7136            subjects,
7137            agent,
7138            plane: self.self_ref.clone(),
7139            #[cfg(feature = "manifest")]
7140            declaration: manifest.as_deref().and_then(|m| self.identity_of(m)),
7141            #[cfg(feature = "manifest")]
7142            manifest,
7143            signer: self.signer.clone(),
7144            recorded_groups,
7145        }
7146    }
7147
7148    /// Run one step's `compensate`, in its own phase and cursor slice.
7149    ///
7150    /// Split out so the unwind reads as the policy it is. The step gets a full
7151    /// `StepCtx` on purpose: compensating effects are journaled, retried,
7152    /// reconciled and replayed exactly like forward ones, and may suspend for a
7153    /// human — a refund that needs four eyes is still a refund.
7154    async fn run_compensation(
7155        &self,
7156        cx: &Unwind<'_>,
7157        step: StepId,
7158        skill: &dyn Skill,
7159        outputs: &BTreeMap<StepId, Tainted<Value>>,
7160        cursor: &mut ReplayCursor,
7161        recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
7162    ) -> Result<(), crate::core::SkillError> {
7163        // A step with no recorded output still gets compensated: the absence of
7164        // a result says nothing about whether it changed anything, and the
7165        // compensation is what knows.
7166        let output = outputs
7167            .get(&step)
7168            .cloned()
7169            .unwrap_or_else(|| Tainted::trusted(Value::Null));
7170
7171        let mut ctx = StepCtx::new(
7172            &self.store,
7173            cursor.take(step, Phase::Compensating),
7174            self.plane_frame(StepFrame {
7175                run: cx.run,
7176                epoch: cx.epoch,
7177                step,
7178                phase: Phase::Compensating,
7179                mode: cx.mode,
7180                case: cx.case.clone(),
7181                ledger: Arc::clone(cx.ledger),
7182                identity: cx.identity.cloned(),
7183                subjects: cx.subjects.clone(),
7184                agent: cx.agent.to_owned(),
7185                #[cfg(feature = "manifest")]
7186                manifest: self.governing(skill),
7187                recorded_groups,
7188            }),
7189        );
7190
7191        let result = skill.compensate(&mut ctx, &output).await;
7192        let result = moved_blob_wins(&ctx, result);
7193        let mut slice = ctx.into_cursor();
7194        if let Err(crate::core::SkillError::Step(error)) = &result
7195            && let Some(divergence) =
7196                crate::journal::Divergence::of(step, Phase::Compensating, error, &slice)
7197        {
7198            slice.note_divergence(divergence);
7199        }
7200        cursor.restore(step, Phase::Compensating, slice);
7201        result
7202    }
7203
7204    /// What the unwind walks, gated — or the quarantine that replaces it.
7205    ///
7206    /// Reads the journal's evidence and applies the stop gates. The gates run
7207    /// exactly when this conclusion closes the run to resume: always for a
7208    /// cancellation (it gets no second pass), and for a failure precisely
7209    /// when the unwind will compensate something — compensation is what
7210    /// closes resume, and a failure that compensates nothing stays open, so
7211    /// its orphans resolve and its suspended siblings re-register on the
7212    /// resume instead of being unwound around.
7213    async fn gated_unwind_list(
7214        &self,
7215        cx: &Unwind<'_>,
7216        status: &RunStatus,
7217        completed: &[(StepId, Capability)],
7218    ) -> Result<Result<(Vec<(StepId, Capability)>, UnwindEvidence), RunStatus>, RuntimeError> {
7219        // Which steps actually changed something outside. Read from the
7220        // journal rather than tracked in memory: it is the same evidence live
7221        // and on replay, and it is what lets an *undeclared* step be judged
7222        // on what it did instead of on what nobody said about it.
7223        let evidence = self.unwind_evidence(cx.run).await?;
7224        // The question is asked of the interrupted siblings too: a sibling
7225        // holding landed work the failure stranded is work the unwind must
7226        // reach, and asked of completed steps alone, a failure with nothing
7227        // completed to compensate left it standing. The failing step itself
7228        // is not asked — its own landed work leaves the run resumable.
7229        let candidates = Self::with_interrupted_siblings(completed, &evidence);
7230        let list = if status.is_cancelled() || self.will_compensate(&candidates, &evidence.mutated)
7231        {
7232            match self
7233                .stop_list(cx.run, completed, &evidence.started, &evidence.mutated)
7234                .await?
7235            {
7236                Ok(list) => list,
7237                Err(quarantine) => return Ok(Err(quarantine)),
7238            }
7239        } else {
7240            completed.to_vec()
7241        };
7242        Ok(Ok((list, evidence)))
7243    }
7244
7245    /// The unwind list for a stop, or the quarantine that replaces it.
7246    ///
7247    /// Both rules in `maybe_unwind`'s doc comment, applied in order: refuse to
7248    /// unwind while an outcome is unknown, and extend the list with the steps
7249    /// the stop interrupted. Called only for a conclusion that closes the run
7250    /// to resume — a cancellation, or a failure whose unwind will compensate
7251    /// — so doubt quarantines unconditionally here: there is no later pass on
7252    /// which an unresolved outcome could still resolve, and compensating
7253    /// around the one call nobody can account for would destroy the evidence
7254    /// an operator needs to resolve it by hand.
7255    async fn stop_list(
7256        &self,
7257        run: RunId,
7258        completed: &[(StepId, Capability)],
7259        started: &BTreeMap<StepId, String>,
7260        mutated: &BTreeSet<StepId>,
7261    ) -> Result<Result<Vec<(StepId, Capability)>, RunStatus>, RuntimeError> {
7262        if let Some(step) = self.undecided_effect(run).await? {
7263            return Ok(Err(RunStatus::Quarantined(format!(
7264                "step {step} holds a mutating effect whose outcome is unknown — it \
7265                 was announced and never concluded, or concluded in doubt with no \
7266                 reconciliation — so the run cannot be unwound: compensating around \
7267                 it would undo everything except the one thing nobody can account for"
7268            ))));
7269        }
7270        Ok(Ok(Self::with_interrupted_steps(
7271            completed, started, mutated,
7272        )))
7273    }
7274
7275    /// Whether an unwind over these completed steps would compensate anything.
7276    ///
7277    /// The question that decides whether a failure closes its run: a declared
7278    /// `Compensatable` step will be compensated, and an `Undeclared` step
7279    /// that provably mutated quarantines inside the loop — either way the
7280    /// conclusion is not one a resume can reopen. `Pivot` and `Unnecessary`
7281    /// steps compensate nothing. Deliberately conservative on a capability
7282    /// that no longer resolves: an unresolvable skill is treated as
7283    /// compensating, so the unwind path — which will surface the resolution
7284    /// error loudly — is taken rather than silently skipped.
7285    fn will_compensate(
7286        &self,
7287        completed: &[(StepId, Capability)],
7288        mutated: &BTreeSet<StepId>,
7289    ) -> bool {
7290        completed.iter().any(|(step, capability)| {
7291            self.resolve(&capability.0)
7292                .map_or(true, |skill| match skill.compensation() {
7293                    crate::core::Compensation::Compensatable => true,
7294                    crate::core::Compensation::Undeclared => mutated.contains(step),
7295                    crate::core::Compensation::Pivot | crate::core::Compensation::Unnecessary => {
7296                        false
7297                    }
7298                })
7299        })
7300    }
7301
7302    /// The completed steps, plus every sibling a failure interrupted holding
7303    /// landed work: a step that mutated, never completed, and did not itself
7304    /// fail. Undone as the skill its `StepStarted` names, as
7305    /// [`with_interrupted_steps`](Self::with_interrupted_steps) undoes it.
7306    fn with_interrupted_siblings(
7307        completed: &[(StepId, Capability)],
7308        evidence: &UnwindEvidence,
7309    ) -> Vec<(StepId, Capability)> {
7310        let mut out = completed.to_vec();
7311        let done: BTreeSet<StepId> = out.iter().map(|(s, _)| *s).collect();
7312        for step in evidence
7313            .mutated
7314            .iter()
7315            .filter(|s| !done.contains(s) && !evidence.failed.contains(s))
7316        {
7317            if let Some(skill) = evidence.started.get(step) {
7318                out.push((*step, Capability::new(skill.as_str())));
7319            }
7320        }
7321        out
7322    }
7323
7324    /// The unwind list for a stop: completed steps, plus any step that mutated
7325    /// without completing — the one a cancellation interrupted, or the sibling
7326    /// a failure stranded mid-suspension. The interrupted steps go last, so
7327    /// the caller's reverse walk undoes them first.
7328    ///
7329    /// An interrupted step is undone as the skill its `StepStarted` names —
7330    /// the capability that ran — never as whatever the plan in force has at
7331    /// its id. A step that charged and then asked to replan is interrupted
7332    /// under a successor that may drop it or put other work at its id, and
7333    /// either reading would leave the charge standing or refund it through a
7334    /// skill that never ran.
7335    ///
7336    /// A step whose compensation is already on the record stays in the list,
7337    /// exactly as a completed step does. The walk re-runs its compensation
7338    /// with every effect served from the journal and the `StepCompensated`
7339    /// dedup keeps the record single — while filtering it out here made a
7340    /// *replayed* unwind walk fewer steps than the recorded one, so a strict
7341    /// pass over an honestly-unwound run left the interrupted step's
7342    /// compensation slice unconsumed and quarantined a history that was
7343    /// telling the truth.
7344    fn with_interrupted_steps(
7345        completed: &[(StepId, Capability)],
7346        started: &BTreeMap<StepId, String>,
7347        mutated: &BTreeSet<StepId>,
7348    ) -> Vec<(StepId, Capability)> {
7349        let mut out = completed.to_vec();
7350        let done: BTreeSet<StepId> = out.iter().map(|(s, _)| *s).collect();
7351        for step in mutated.iter().filter(|s| !done.contains(s)) {
7352            if let Some(skill) = started.get(step) {
7353                out.push((*step, Capability::new(skill.as_str())));
7354            }
7355        }
7356        out
7357    }
7358
7359    /// The lowest step holding an effect this journal cannot decide, if any.
7360    ///
7361    /// Ordinarily the run is already `Quarantined` when this is true, and a
7362    /// quarantined run never unwinds. Cancellation opens a second door into the
7363    /// unwind, and it has to be shut the same way — otherwise an operator's
7364    /// stop compensates every step *around* the one nobody can account for,
7365    /// which is precisely the refund for money nobody took, arriving through a
7366    /// control meant to make things safer.
7367    ///
7368    /// The rule itself is [`undecided_effects`](crate::journal::undecided_effects),
7369    /// shared with the operator API and the offline audit. This adds only the
7370    /// question the unwind asks of it: *how far back does the doubt reach*.
7371    async fn undecided_effect(&self, run: RunId) -> Result<Option<StepId>, RuntimeError> {
7372        let records = self
7373            .store
7374            .read(run, 1)
7375            .await
7376            .map_err(RuntimeError::from_store)?;
7377        Ok(crate::journal::undecided_effects(&records)
7378            .into_iter()
7379            .map(|u| u.step)
7380            .min())
7381    }
7382
7383    /// Execute one plan node.
7384    async fn run_step(
7385        &self,
7386        ctx: StepRun<'_>,
7387        run_input: &Tainted<Value>,
7388        outputs: &BTreeMap<StepId, Tainted<Value>>,
7389        cursor: crate::journal::StepCursor,
7390    ) -> Result<
7391        (
7392            RunStatus,
7393            Option<Tainted<Value>>,
7394            crate::journal::StepCursor,
7395        ),
7396        RuntimeError,
7397    > {
7398        let span = tracing::info_span!(
7399            telemetry::STEP_SPAN,
7400            { telemetry::STEP } = tracing::field::display(ctx.node.id),
7401            { telemetry::CAPABILITY } = tracing::field::display(&ctx.node.capability.0),
7402            { telemetry::PHASE } = ctx.phase.as_str(),
7403            { telemetry::MODE } = telemetry::mode_str(ctx.mode),
7404            { telemetry::OUTCOME } = tracing::field::Empty,
7405        );
7406        self.run_step_inner(ctx, run_input, outputs, cursor)
7407            .instrument(span)
7408            .await
7409    }
7410
7411    #[allow(clippy::too_many_lines)]
7412    async fn run_step_inner(
7413        &self,
7414        ctx: StepRun<'_>,
7415        run_input: &Tainted<Value>,
7416        outputs: &BTreeMap<StepId, Tainted<Value>>,
7417        cursor: crate::journal::StepCursor,
7418    ) -> Result<
7419        (
7420            RunStatus,
7421            Option<Tainted<Value>>,
7422            crate::journal::StepCursor,
7423        ),
7424        RuntimeError,
7425    > {
7426        let StepRun {
7427            identity,
7428            subjects,
7429            run,
7430            epoch,
7431            node,
7432            phase,
7433            mode,
7434            case,
7435            ledger,
7436            writing,
7437            stamp,
7438            agent,
7439            already_started,
7440            already_finished,
7441            recorded_groups,
7442            abandoning,
7443        } = ctx;
7444        let step = node.id;
7445        let skill = self.resolve(&node.capability.0)?;
7446
7447        // Assemble this step's input from its declared sources. Labels join, so
7448        // provenance flows through the graph without anyone threading it by hand.
7449        let step_input = assemble(node, run_input, outputs)?;
7450
7451        // Announce a step doing new work, once: always in `Live`, and on a
7452        // resume only for a step the journal never announced — a frontier
7453        // step running for the first time. Keying this on the mode alone got
7454        // both directions wrong: resumed frontier steps went unannounced, and
7455        // the `!Strict` finish gate below duplicated `StepFinished` for every
7456        // fully replayed step on every resume.
7457        let announce = writes_step_record(mode, !already_started);
7458        if announce {
7459            self.store
7460                .append(
7461                    epoch,
7462                    vec![stamp(
7463                        Append::new(
7464                            run,
7465                            RecordKind::StepStarted {
7466                                skill: skill.descriptor().name,
7467                            },
7468                        )
7469                        .step(step),
7470                    )],
7471                )
7472                .await
7473                .map_err(RuntimeError::from_store)?;
7474        }
7475
7476        let mut cx = StepCtx::new(
7477            &self.store,
7478            cursor,
7479            self.plane_frame(StepFrame {
7480                run,
7481                epoch,
7482                step,
7483                phase,
7484                mode,
7485                case,
7486                ledger: Arc::clone(ledger),
7487                identity: identity.cloned(),
7488                subjects: subjects.clone(),
7489                agent: agent.to_owned(),
7490                #[cfg(feature = "manifest")]
7491                manifest: self.governing(skill.as_ref()),
7492                recorded_groups,
7493            }),
7494        );
7495        cx.abandoning = abandoning;
7496        // The one place elapsed time is measured, and it is measured with a
7497        // journaled clock read like every other observation — a wall-clock
7498        // ceiling decided from an ambient clock would give a replayed run a
7499        // different verdict every time it was looked at.
7500        //
7501        // Paid only by a run that asked for the ceiling, which is why the
7502        // step's first effect is this one when a wall-clock budget is declared
7503        // and the skill's own when it is not. The consequence is worth stating:
7504        // that history can be replayed under a *raised* wall-clock ceiling, and
7505        // not under a build that removed it, because the recorded run performs
7506        // a read this one would not request.
7507        let result = match Self::observe_clock(&mut cx, ledger).await {
7508            Ok(()) => skill.invoke(&mut cx, step_input).await,
7509            Err(e) => Err(crate::core::SkillError::Step(e)),
7510        };
7511        let result = settle_abandoned_group(&mut cx, result).await;
7512        let result = moved_blob_wins(&cx, result);
7513        let wrote = cx.wrote_records();
7514        let mut cursor = cx.into_cursor();
7515        if let Err(crate::core::SkillError::Step(error)) = &result
7516            && let Some(divergence) = crate::journal::Divergence::of(step, phase, error, &cursor)
7517        {
7518            cursor.note_divergence(divergence);
7519        }
7520        ledger.lock().expect("budget mutex").record_step();
7521
7522        let fault = result.as_ref().err().map(crate::core::SkillError::class);
7523        let (status, output) = classify(&self.meter, result);
7524        tracing::Span::current().record(telemetry::OUTCOME, status.as_str());
7525        if let RunStatus::Quarantined(why) = &status {
7526            tracing::error!(
7527                target: telemetry::QUARANTINED,
7528                %run,
7529                %step,
7530                error_type = fault.unwrap_or("quarantined"),
7531                reason_digest = %telemetry::reason_digest(why),
7532            );
7533        }
7534
7535        // Record the ending only where the journal does not already hold it:
7536        // every live step; a resumed step that did new work; and a resumed
7537        // step reaching an ending the record never captured — how a suspended
7538        // step that finishes purely from replayed history (its awaited event
7539        // recorded by the delivery) still gets its one `StepFinished`. What
7540        // this rules out is a duplicate ending for every fully replayed
7541        // completed step, appended on every resume of a run doing nothing new.
7542        let record_ending = writes_step_record(mode, announce || wrote || !already_finished);
7543        if writing && record_ending {
7544            // A suspended step has not finished, so it records why it stopped
7545            // rather than claiming an outcome.
7546            let record = match &status {
7547                RunStatus::Suspended(reason) => RecordKind::RunSuspended {
7548                    reason: reason.clone(),
7549                },
7550                other => RecordKind::StepFinished {
7551                    outcome: other.as_str().to_owned(),
7552                },
7553            };
7554            self.store
7555                .append(epoch, vec![stamp(Append::new(run, record).step(step))])
7556                .await
7557                .map_err(RuntimeError::from_store)?;
7558        }
7559
7560        Ok((status, output, cursor))
7561    }
7562
7563    /// Whether the chain's last record is already this conclusion. `head` is
7564    /// the chain's last sequence.
7565    async fn last_is_conclusion(
7566        &self,
7567        run: RunId,
7568        head: crate::core::Seq,
7569        status: &RunStatus,
7570    ) -> Result<bool, RuntimeError> {
7571        if head < 1 {
7572            return Ok(false);
7573        }
7574        let page = self
7575            .store
7576            .read_page(run, head, 1)
7577            .await
7578            .map_err(RuntimeError::from_store)?;
7579        Ok(matches!(
7580            page.first().map(Record::kind),
7581            Some(RecordKind::RunConcluded { outcome, .. }) if outcome == status.as_str()
7582        ))
7583    }
7584
7585    /// Seal the run and report.
7586    /// Eight arguments, and each is a distinct fact about how the run ended
7587    /// that the caller already holds. Bundling them into a struct would move the
7588    /// same fields one indirection away without removing a single one.
7589    #[allow(clippy::too_many_arguments)]
7590    async fn conclude(
7591        &self,
7592        run: RunId,
7593        epoch: u64,
7594        status: RunStatus,
7595        output: Option<Tainted<Value>>,
7596        writing: bool,
7597        case: Option<crate::core::CaseId>,
7598        consumed: Consumed,
7599        live_spend: Spend,
7600        quota: &QuotaPass,
7601    ) -> Result<RunOutcome, RuntimeError> {
7602        // Loud toward the operator, ordinary toward the caller. A failed run is
7603        // a conclusion a resume can honestly answer, so it is not an incident —
7604        // but without this event nothing says why one failed except the
7605        // journal and an index that needs the HTTP surface mounted.
7606        if let RunStatus::Failed(reason) = &status {
7607            tracing::warn!(
7608                target: telemetry::RUN_FAILED,
7609                %run,
7610                error_type = "failed",
7611                reason_digest = %telemetry::reason_digest(reason),
7612            );
7613        }
7614
7615        // A suspended run is not sealed: its chain is going to be extended the
7616        // moment whatever it waits for arrives.
7617        let chain_head = if writing && !status.is_suspended() {
7618            // The conclusion goes *in* the chain — before the chain is closed
7619            // over it, where the conclusion is one that closes it. Two things
7620            // follow, and both were missing while the outcome lived only in a
7621            // side table: tamper detection covers how the run ended, and a
7622            // resumed run can read that fact from the same history it verifies
7623            // rather than inferring it from the last step that happened to
7624            // finish. The stores also derive the outcome index from this
7625            // record, last conclusion wins — which is what keeps a
7626            // failed-then-resumed-then-succeeded run from being listed as
7627            // failed forever.
7628            let before = self
7629                .store
7630                .head(run)
7631                .await
7632                .map_err(RuntimeError::from_store)?;
7633            // A resume that did nothing new reaches the same open conclusion
7634            // the record already carries, and appending it again would grow
7635            // the journal by one identical conclusion per resume. The chain's
7636            // own last record is the authority: only when it already *is*
7637            // this conclusion is the append skipped. A resume that worked and then concluded the same way
7638            // has its work between the two conclusions, so both are kept.
7639            // Scoped to open conclusions, because a sealing one is
7640            // unreachable twice: the seal refuses further appends and a
7641            // sealed run's replay is a no-op long before here.
7642            let repeated =
7643                !status.seals() && self.last_is_conclusion(run, before.seq, &status).await?;
7644            if repeated {
7645                before.hash
7646            } else {
7647                let mut sealed = Append::new(
7648                    run,
7649                    RecordKind::RunConcluded {
7650                        outcome: status.as_str().to_owned(),
7651                        reason: status.own_account().map(std::borrow::Cow::into_owned),
7652                        exhaustion: match &status {
7653                            RunStatus::Exhausted(limit) => Some(limit.clone()),
7654                            _ => None,
7655                        },
7656                        live_spend,
7657                        chain_head: before.hash,
7658                    },
7659                );
7660                if let Some(c) = case {
7661                    sealed = sealed.case(c);
7662                }
7663                let concluded = self
7664                    .store
7665                    .append(epoch, vec![sealed])
7666                    .await
7667                    .map_err(RuntimeError::from_store)?;
7668
7669                concluded.last().map_or(before.hash, |r| r.hash)
7670            }
7671        } else {
7672            self.store
7673                .head(run)
7674                .await
7675                .map_err(RuntimeError::from_store)?
7676                .hash
7677        };
7678
7679        // Hand the lease back rather than letting it time out.
7680        //
7681        // Whatever the outcome — sealed, suspended, exhausted — this instance is
7682        // finished with the run. Holding the lease until expiry would make every
7683        // failover wait out the TTL for nothing.
7684        //
7685        // Best-effort on purpose. A release that fails costs a TTL of patience;
7686        // turning it into a run failure would convert a tidiness problem into a
7687        // correctness one, after the work is already done and journaled.
7688        //
7689        // Only when this pass held a lease at all. A strict pass acquires
7690        // nothing — it is a read — and both stores release on epoch match
7691        // alone, so a strict verification of a run whose owner is live under
7692        // that same epoch would mark the *owner's* lease released: its
7693        // heartbeat stops renewing and any delivery or sweep may fence a
7694        // healthy run mid-step. The same guard keeps a read-only pass out of
7695        // the quota ledger and the metrics: a verification is not work the
7696        // tenant did today, and accruing a historical run's spend into the
7697        // current period on every audit would bill the past once per look.
7698        self.forget_commissions_of(run);
7699        if writing {
7700            // Settlement precedes both physical sealing and lease release. If
7701            // it fails, the conclusion remains open under an owned lease; once
7702            // that lease expires, recovery derives this pass from the journal
7703            // and retries the idempotent receipt.
7704            //
7705            // Whatever the outcome this instance is finished with the run, so
7706            // a **suspended** run gives its slot back too — it costs a row,
7707            // not a thread, and holding the slot would mean a tenant waiting
7708            // on a hundred approvals could start nothing. The figure accrued
7709            // is what *this pass* dispatched, not the run's cumulative spend,
7710            // which would bill the replayed prefix once per resume.
7711            self.settle_quota(run, epoch, live_spend, quota, status.seals())
7712                .await?;
7713
7714            let chain_head = if status.seals() {
7715                let head = self
7716                    .store
7717                    .seal(run, epoch, status.as_str())
7718                    .await
7719                    .map_err(RuntimeError::from_store)?;
7720                self.retire_waits(run).await;
7721                head
7722            } else {
7723                chain_head
7724            };
7725
7726            if let Err(e) = self.store.release_lease(run, epoch).await {
7727                tracing::debug!(
7728                    %run,
7729                    error = %e,
7730                    "could not hand back the lease; it will expire on its own"
7731                );
7732            }
7733
7734            announce(&self.meter, run, &status);
7735
7736            return Ok(RunOutcome {
7737                failed_effect: self.failed_effect(run, &status).await?,
7738                run_id: run,
7739                status,
7740                chain_head,
7741                consumed,
7742                output,
7743            });
7744        }
7745
7746        Ok(RunOutcome {
7747            failed_effect: self.failed_effect(run, &status).await?,
7748            run_id: run,
7749            status,
7750            chain_head,
7751            consumed,
7752            // The caller is outside the lattice, so the label is dropped at the
7753            // boundary rather than inside the graph.
7754            output,
7755        })
7756    }
7757}
7758
7759/// The effect a run ended on, when it failed on one.
7760fn failure_of(status: &RunStatus, records: &[Record]) -> Option<crate::journal::FailedEffect> {
7761    matches!(status, RunStatus::Failed(_))
7762        .then(|| crate::journal::failed_effect(records))
7763        .flatten()
7764}
7765
7766/// The run's result: the terminal step's output.
7767///
7768/// Not "whichever step finished last". That coincides with the terminal step
7769/// only while dispatch is sequential, and stops being well-defined the moment
7770/// two steps run at once. Lowest id wins when a plan has several terminals, so
7771/// the answer is a property of the plan rather than of the schedule.
7772fn run_output(ir: &PlanIR, outputs: &BTreeMap<StepId, Tainted<Value>>) -> Option<Tainted<Value>> {
7773    ir.nodes
7774        .iter()
7775        .filter(|n| n.terminal)
7776        .map(|n| n.id)
7777        .min()
7778        .and_then(|id| outputs.get(&id).cloned())
7779        // A run that stopped before any terminal step still has something to
7780        // report: the furthest output it did produce.
7781        .or_else(|| outputs.iter().next_back().map(|(_, v)| v.clone()))
7782}
7783
7784/// Effects a run announced and never recorded an outcome for — the waits a
7785/// closed run leaves behind.
7786fn unanswered_waits(records: &[Record]) -> Vec<crate::core::EffectKey> {
7787    let settled: BTreeSet<_> = records
7788        .iter()
7789        .filter(|r| {
7790            matches!(
7791                r.kind(),
7792                RecordKind::EffectDone { .. }
7793                    | RecordKind::EffectFailed { .. }
7794                    | RecordKind::EffectReconciled { .. }
7795            )
7796        })
7797        .filter_map(Record::effect_key)
7798        .collect();
7799    records
7800        .iter()
7801        .filter(|r| matches!(r.kind(), RecordKind::EffectStarted { .. }))
7802        .filter_map(Record::effect_key)
7803        .filter(|k| !settled.contains(k))
7804        .collect()
7805}
7806
7807/// Whether `epoch` holds the conclusion that sealed the run — the pass whose
7808/// settlement releases what the run still held.
7809fn concluded_in(records: &[Record], epoch: crate::core::Epoch) -> bool {
7810    records
7811        .iter()
7812        .any(|r| r.body.epoch == epoch && is_sealing_conclusion(r))
7813}
7814
7815fn is_sealing_conclusion(record: &Record) -> bool {
7816    matches!(
7817        record.kind(),
7818        RecordKind::RunConcluded { outcome, .. } if SEALED_OUTCOMES.contains(&outcome.as_str())
7819    )
7820}
7821
7822/// Spend durably recorded by effects written under one live pass epoch.
7823fn spend_recorded_in_epoch(records: &[Record], epoch: crate::core::Epoch) -> Spend {
7824    records
7825        .iter()
7826        .filter(|record| record.body.epoch == epoch)
7827        .fold(Spend::default(), |total, record| {
7828            let spend = match record.kind() {
7829                RecordKind::EffectDone { spend, .. }
7830                | RecordKind::EffectFailed { spend, .. }
7831                | RecordKind::EffectReconciled { spend, .. } => *spend,
7832                _ => Spend::default(),
7833            };
7834            total.plus(spend)
7835        })
7836}
7837
7838/// Whether the plan actually finished.
7839///
7840/// Structural, never self-reported: a workload asserting it is done is not
7841/// evidence, so the runtime checks that every terminal node ran.
7842fn completion(ir: &PlanIR, done: &BTreeSet<StepId>) -> RunStatus {
7843    if ir.is_complete(done) {
7844        return RunStatus::Succeeded;
7845    }
7846    let missing: Vec<String> = ir
7847        .nodes
7848        .iter()
7849        .filter(|n| n.terminal && !done.contains(&n.id))
7850        .map(|n| n.id.to_string())
7851        .collect();
7852    RunStatus::Failed(format!(
7853        "plan did not complete: terminal step(s) {} never ran",
7854        missing.join(", ")
7855    ))
7856}
7857
7858/// Say that a run changed its plan, and what it changed from.
7859fn announce_replan(meter: &super::metrics::Meter, run: RunId, next: &PlanIR, reason: &str) {
7860    tracing::info!(
7861        target: telemetry::REPLANNED,
7862        %run,
7863        from = next.derived_from.map(Digest::to_hex),
7864        version = next.version,
7865        reason_digest = %telemetry::reason_digest(reason),
7866    );
7867    meter.count(metrics::REPLANS, "");
7868}
7869
7870/// Say how a run ended, on the run span and — for the loud ones — as an event.
7871fn announce(meter: &super::metrics::Meter, run: RunId, status: &RunStatus) {
7872    // Counted here and nowhere else. A step that quarantines also fails its run,
7873    // so counting at both levels would report one incident as two — and the
7874    // terminal status is the fact an operator is counting.
7875    meter.count(metrics::RUNS, status.as_str());
7876    match status {
7877        RunStatus::Quarantined(why) => {
7878            tracing::error!(
7879                target: telemetry::QUARANTINED,
7880                %run,
7881                error_type = "quarantined",
7882                reason_digest = %telemetry::reason_digest(why),
7883            );
7884            meter.count(metrics::QUARANTINES, "");
7885        }
7886        // Loud, though a person asked for it. The operator who decided already
7887        // knows; this reaches whoever answers for what the run left standing.
7888        RunStatus::Abandoned { actor, reason } => {
7889            tracing::error!(
7890                target: telemetry::ABANDONED,
7891                %run,
7892                %actor,
7893                reason_digest = %telemetry::reason_digest(reason),
7894            );
7895        }
7896        RunStatus::Exhausted(limit) => {
7897            tracing::warn!(target: telemetry::BUDGET_REFUSED, %run, %limit);
7898        }
7899        _ => {}
7900    }
7901    tracing::Span::current().record(telemetry::OUTCOME, status.as_str());
7902}
7903
7904/// Record a batch's successes, then report the first step that stopped.
7905///
7906/// **Every** success is recorded, including those of siblings dispatched
7907/// alongside the one that stopped. Returning early on the first failure loses
7908/// them — and a sibling that already performed a mutating effect would then
7909/// never be compensated, because `completed` is what the unwind reverses. The
7910/// work happened; the saga has to know about it.
7911///
7912/// When siblings stop for different reasons, **severity wins over ready order**.
7913/// A suspension is the run working; a failure is the run over. Letting one
7914/// sibling's wait mask another's failure would defer the unwind until an event
7915/// that may never arrive — leaving the failed sibling's mutations in place
7916/// indefinitely. Within one severity, ready order decides, so the choice stays a
7917/// property of the plan rather than of the schedule.
7918fn apply(
7919    plan: &PlanIR,
7920    outcomes: Vec<StepOutcome>,
7921    done: &mut BTreeSet<StepId>,
7922    completed: &mut Vec<(StepId, Capability)>,
7923    ran: &mut BTreeMap<StepId, PlanNode>,
7924    outputs: &mut BTreeMap<StepId, Tainted<Value>>,
7925) -> Option<RunStatus> {
7926    let mut stopped: Option<RunStatus> = None;
7927    for (step, status, output) in outcomes {
7928        let RunStatus::Succeeded = status else {
7929            if stopped
7930                .as_ref()
7931                .is_none_or(|held| severity(&status) > severity(held))
7932            {
7933                stopped = Some(status);
7934            }
7935            continue;
7936        };
7937        if let Some(v) = output {
7938            outputs.insert(step, v);
7939        }
7940        done.insert(step);
7941        if let Some(node) = plan.node(step) {
7942            completed.push((step, node.capability.clone()));
7943            ran.insert(step, node.clone());
7944        }
7945    }
7946    stopped
7947}
7948
7949/// How much a stop reason dominates a competing one.
7950///
7951/// `Quarantined` is highest because it is the only one that must *not* unwind:
7952/// something is undecidable, and compensating around it can make the damage
7953/// worse. `Suspended` is lowest because it is not a stop at all — the run is
7954/// healthy and waiting.
7955fn severity(status: &RunStatus) -> u8 {
7956    match status {
7957        RunStatus::Quarantined(_) => 3,
7958        // A stop ranks with a failure, not above it: both end the run, both
7959        // unwind, and when they arrive together the run is over either way.
7960        // Abandonment is not produced by a step and never competes here; it is
7961        // named so a new stop reason cannot be given a rank by omission.
7962        RunStatus::Cancelled { .. } | RunStatus::Abandoned { .. } | RunStatus::Failed(_) => 2,
7963        // A withdrawal ranks with exhaustion for the same reason: both pause a
7964        // healthy run, neither unwinds, and a sibling that actually failed has
7965        // decided more about the run than either.
7966        RunStatus::Exhausted(_) | RunStatus::Withheld { .. } => 1,
7967        // A replan request is the weakest signal in a batch: a sibling that
7968        // failed outright has already decided the run, and re-planning around a
7969        // failure is not what the requesting step was asking for.
7970        // Neither is reachable here at all — a sweep, a crossing and a lift are
7971        // sealed at birth with no steps to compete — and both are named for the reason
7972        // abandonment is: a rank given by omission is a decision nobody made.
7973        RunStatus::Replanning(_)
7974        | RunStatus::Suspended(_)
7975        | RunStatus::Succeeded
7976        | RunStatus::Swept
7977        // Unreachable for the same reason, and named rather than left to a
7978        // wildcard: an observed session has no steps of this plane's to
7979        // compete, because this plane ran none of it.
7980        | RunStatus::Observed
7981        | RunStatus::BrokeGlass { .. }
7982        | RunStatus::HaltLifted { .. }
7983        | RunStatus::HoldReleased { .. } => 0,
7984    }
7985}
7986
7987/// Gather a dispatched batch back into ready order.
7988///
7989/// Not completion order. `completed` is what the unwind reverses, and a saga
7990/// whose compensation order depended on which future finished first would undo
7991/// a plan differently on every run.
7992type Dispatched = Result<(StepId, RunStatus, Option<Tainted<Value>>, StepCursor), RuntimeError>;
7993
7994/// One step's result, once its history has been handed back.
7995type StepOutcome = (StepId, RunStatus, Option<Tainted<Value>>);
7996
7997fn collect(
7998    dispatched: Vec<Dispatched>,
7999    ready: &[StepId],
8000    cursor: &mut ReplayCursor,
8001) -> Result<Vec<StepOutcome>, RuntimeError> {
8002    let mut outcomes = Vec::with_capacity(dispatched.len());
8003    for result in dispatched {
8004        let (step, status, out, slice) = result?;
8005        cursor.restore(step, Phase::Forward, slice);
8006        outcomes.push((step, status, out));
8007    }
8008    // The key is an `Option`, and `None` would sort *first* — so the invariant
8009    // that makes this total is worth naming rather than trusting: every
8010    // dispatched step came from `admit_ready(&ready, ..)`, which returns a
8011    // subset of `ready`. A future that dispatched a step outside the ready set
8012    // would silently re-order this batch rather than fail, and the order is what
8013    // an unwind reverses.
8014    outcomes.sort_by_key(|(step, _, _)| ready.iter().position(|r| r == step));
8015    Ok(outcomes)
8016}
8017
8018/// What one ready set's dispatch needs.
8019struct Batch<'a> {
8020    agent: &'a str,
8021    identity: Option<&'a crate::core::Delegation>,
8022    subjects: &'a BTreeSet<crate::core::SubjectRef>,
8023    run: RunId,
8024    epoch: u64,
8025    ir: &'a PlanIR,
8026    mode: Pass,
8027    case: &'a Option<CaseContext>,
8028    ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8029    writing: bool,
8030    stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8031    input: &'a Tainted<Value>,
8032    outputs: &'a BTreeMap<StepId, Tainted<Value>>,
8033    /// Steps whose `StepStarted` the journal already holds. Empty on a live
8034    /// run.
8035    started: &'a BTreeSet<StepId>,
8036    /// Steps whose `StepFinished` the journal already holds. Empty on a live
8037    /// run.
8038    finished: &'a BTreeSet<StepId>,
8039    /// Group records the journal already holds, by writing step and phase.
8040    /// Empty on a live run.
8041    recorded_groups: &'a BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
8042    /// How many of this ready set may be in flight at once.
8043    parallelism: usize,
8044    /// Re-running steps only to abandon groups a pause left open.
8045    abandoning: bool,
8046}
8047
8048/// A standing withdrawal, as the step boundary meets it.
8049///
8050/// Carries the operator from the halt rather than only its reason: the
8051/// withholding record is the one durable trace of this act on the run it
8052/// stopped, and a pause nobody is named on cannot be asked about afterwards.
8053pub(crate) struct Withdrawal {
8054    pub(crate) subject: String,
8055    pub(crate) reason: String,
8056    pub(crate) by: crate::core::Operator,
8057}
8058
8059/// What the journal proves about a run an unwind is deciding over.
8060struct UnwindEvidence {
8061    /// Steps that announced a mutating forward effect.
8062    mutated: BTreeSet<StepId>,
8063    /// The skill each step that started ran as, from its `StepStarted`.
8064    started: BTreeMap<StepId, String>,
8065    /// Steps whose compensation is already on the record.
8066    undone: BTreeSet<StepId>,
8067    /// Group records already on the journal, by writing step and phase.
8068    recorded_groups: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
8069    /// Steps whose last forward ending is not a success: the failing step,
8070    /// which stays resumable, as opposed to a sibling the failure interrupted.
8071    failed: BTreeSet<StepId>,
8072}
8073
8074/// What producing a successor plan needs.
8075struct Replan<'a> {
8076    run: RunId,
8077    current: &'a PlanIR,
8078    /// The node each completed step ran as.
8079    ran: &'a BTreeMap<StepId, PlanNode>,
8080    reason: &'a str,
8081    already_replanned: u32,
8082    max_replans: Option<u32>,
8083    /// The successor this run produced when it first ran, if this is a replay.
8084    recorded: Option<&'a PlanIR>,
8085}
8086
8087/// The first untrusted value in working memory, if any.
8088///
8089/// Returns the source so the refusal can name it: "replanning refused" without
8090/// saying *what* made it unsafe sends an operator looking through the whole run.
8091fn untrusted_in(outputs: &BTreeMap<StepId, Tainted<Value>>) -> Option<String> {
8092    outputs.values().find_map(|v| {
8093        let label = v.label();
8094        label.is_untrusted().then(|| {
8095            label
8096                .provenance
8097                .first()
8098                .map_or_else(|| "an untrusted source".to_owned(), ToString::to_string)
8099        })
8100    })
8101}
8102
8103/// Where a refusal is recorded, when one is.
8104struct Journalling<'a> {
8105    run: RunId,
8106    epoch: u64,
8107    writing: bool,
8108    stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8109}
8110
8111/// What unwinding a run needs.
8112struct Unwind<'a> {
8113    identity: Option<&'a crate::core::Delegation>,
8114    subjects: &'a BTreeSet<crate::core::SubjectRef>,
8115    run: RunId,
8116    epoch: u64,
8117    ir: &'a PlanIR,
8118    mode: Pass,
8119    case: Option<CaseContext>,
8120    ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8121    writing: bool,
8122    stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8123    agent: &'a str,
8124}
8125
8126/// What one step's execution needs.
8127// Each flag is an independent fact about this dispatch; no combination is
8128// invalid, so an enum over them would satisfy the lint at the reader's expense.
8129#[allow(clippy::struct_excessive_bools)]
8130struct StepRun<'a> {
8131    identity: Option<&'a crate::core::Delegation>,
8132    subjects: &'a BTreeSet<crate::core::SubjectRef>,
8133    run: RunId,
8134    epoch: u64,
8135    node: &'a PlanNode,
8136    phase: Phase,
8137    mode: Pass,
8138    case: Option<CaseContext>,
8139    ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8140    writing: bool,
8141    stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8142    agent: &'a str,
8143    /// Whether the journal already announced this step. Always `false` live.
8144    already_started: bool,
8145    /// Whether the journal already records this step finishing. Always
8146    /// `false` live.
8147    already_finished: bool,
8148    /// Group records this step and phase already wrote. Always empty live.
8149    recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
8150    /// Whether this step is re-run only to abandon a group a pause left
8151    /// open, because the run was cancelled while paused.
8152    abandoning: bool,
8153}
8154
8155/// The half of a `StepCtx`'s frame that belongs to the **step**.
8156///
8157/// A struct rather than a dozen positional arguments, for the reason
8158/// [`StepRun`] is one: the other half comes from the plane, and a caller who
8159/// transposed `phase` and `mode` positionally would compile.
8160struct StepFrame {
8161    run: RunId,
8162    epoch: crate::core::Epoch,
8163    step: StepId,
8164    phase: Phase,
8165    mode: Pass,
8166    case: Option<CaseContext>,
8167    ledger: Arc<std::sync::Mutex<Ledger>>,
8168    identity: Option<crate::core::Delegation>,
8169    subjects: BTreeSet<crate::core::SubjectRef>,
8170    agent: String,
8171    #[cfg(feature = "manifest")]
8172    manifest: Option<Arc<crate::manifest::Manifest>>,
8173    recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
8174}
8175
8176/// Where the recorded run was refused by a *step* limit, if the refusal still
8177/// stands.
8178///
8179/// A step-level refusal has no effect key, so it cannot ride the replay cursor
8180/// the way an effect's does; it is lifted from the records instead.
8181///
8182/// **The last word wins.** An exhausted run may be resumed under a raised
8183/// ceiling; when the resume re-admits the refused step it journals a
8184/// `BudgetReadmitted` beside the old refusal, and from then on the refusal is
8185/// history that was *superseded*, not a verdict to re-serve. Without this, a
8186/// strict replay of the resumed history would stop at the old refusal and
8187/// report `Exhausted` about a run whose own later records show it finishing —
8188/// the refusal followed by the continuation would read as divergence.
8189fn recorded_step_refusal(records: &[Record]) -> Option<(StepId, String, String)> {
8190    let mut standing: Option<(StepId, String, String)> = None;
8191    for r in records {
8192        match r.kind() {
8193            RecordKind::BudgetRefused { limit, used } if r.effect_key().is_none() => {
8194                if let Some(step) = r.body.step {
8195                    standing = Some((step, limit.clone(), used.clone()));
8196                }
8197            }
8198            RecordKind::BudgetReadmitted { .. }
8199                if standing
8200                    .as_ref()
8201                    .is_some_and(|(s, _, _)| Some(*s) == r.body.step) =>
8202            {
8203                standing = None;
8204            }
8205            _ => {}
8206        }
8207    }
8208    standing
8209}
8210
8211/// The subject of a withholding the records left standing, if one stands.
8212///
8213/// **The last word wins**, exactly as it does for a step refusal: a withheld run
8214/// resumes once somebody lifts the halt, and the resume journals an
8215/// `AuthorityRestored` beside the withholding. From then on the withholding is
8216/// history that was *superseded* rather than a verdict to re-serve — without
8217/// this, a resumed run would be treated as still withheld and would conclude
8218/// that way forever, with each resume adding nothing.
8219fn standing_withholding(records: &[Record]) -> Option<String> {
8220    let mut standing: Option<String> = None;
8221    for r in records {
8222        match r.kind() {
8223            RecordKind::AuthorityWithheld { subject, .. } => standing = Some(subject.clone()),
8224            RecordKind::AuthorityRestored { .. } => standing = None,
8225            _ => {}
8226        }
8227    }
8228    standing
8229}
8230
8231/// The run's most recent recorded conclusion, if it has one.
8232///
8233/// `None` for a run that has only ever suspended — a suspension is not a
8234/// conclusion, and `conclude` writes no `RunSealed` for it.
8235fn recorded_conclusion(records: &[Record]) -> Option<String> {
8236    records.iter().rev().find_map(|r| match r.kind() {
8237        RecordKind::RunConcluded { outcome, .. } => Some(outcome.clone()),
8238        _ => None,
8239    })
8240}
8241
8242/// Whether this pass writes a step-level record: always live, never under
8243/// strict verification, and on a resume only when `new_fact` — the record is
8244/// not already in the journal, or the step did new work.
8245const fn writes_step_record(mode: Pass, new_fact: bool) -> bool {
8246    match mode {
8247        Pass::Live => true,
8248        Pass::Resume => new_fact,
8249        Pass::Strict => false,
8250    }
8251}
8252
8253/// The steps whose start is already on the record.
8254///
8255/// Read back so a resume neither re-announces a step history already announced
8256/// nor forgets to announce one it is starting for the first time.
8257fn recorded_started_steps(records: &[Record]) -> BTreeSet<StepId> {
8258    records
8259        .iter()
8260        .filter_map(|r| match r.kind() {
8261            RecordKind::StepStarted { .. } => r.body.step,
8262            _ => None,
8263        })
8264        .collect()
8265}
8266
8267/// The skill each step started as, from its forward `StepStarted`.
8268///
8269/// The capability that actually ran, which after a replan the plan in force
8270/// may no longer name at that id — or may name as other work.
8271fn started_skills(records: &[Record]) -> BTreeMap<StepId, String> {
8272    records
8273        .iter()
8274        .filter(|r| r.body.phase.is_forward())
8275        .filter_map(|r| match (r.kind(), r.body.step) {
8276            (RecordKind::StepStarted { skill }, Some(step)) => Some((step, skill.clone())),
8277            _ => None,
8278        })
8279        .collect()
8280}
8281
8282/// The steps whose ending is already on the record.
8283///
8284/// Read back so a resume records a `StepFinished` exactly for the steps that
8285/// reach an ending the journal does not yet hold: a fully replayed completed
8286/// step keeps its one record, and a suspended step that finally finishes —
8287/// even purely from replayed history, its awaited event having been recorded
8288/// by the delivery — gets the one it never had.
8289fn recorded_finished_steps(records: &[Record]) -> BTreeSet<StepId> {
8290    records
8291        .iter()
8292        .filter_map(|r| match r.kind() {
8293            RecordKind::StepFinished { .. } => r.body.step,
8294            _ => None,
8295        })
8296        .collect()
8297}
8298
8299/// The case the run is bound to, read from its `CaseBound` record.
8300fn recorded_case_id(records: &[Record]) -> Option<crate::core::CaseId> {
8301    records.iter().find_map(|r| match r.kind() {
8302        RecordKind::CaseBound { .. } => r.body.case,
8303        _ => None,
8304    })
8305}
8306
8307/// The steps whose latest forward ending on the record is a success.
8308fn recorded_succeeded_steps(records: &[Record]) -> BTreeSet<StepId> {
8309    let mut succeeded = BTreeSet::new();
8310    for r in records.iter().filter(|r| r.body.phase.is_forward()) {
8311        if let (RecordKind::StepFinished { outcome }, Some(step)) = (r.kind(), r.body.step) {
8312            if outcome == RunStatus::Succeeded.as_str() {
8313                succeeded.insert(step);
8314            } else {
8315                succeeded.remove(&step);
8316            }
8317        }
8318    }
8319    succeeded
8320}
8321
8322/// The group records already on the journal, by writing step and phase.
8323///
8324/// Read back for the same reason the step sets are: group records are not
8325/// effects, so the cursor cannot dedup them, and a resumed step that re-opens
8326/// its group or reaches its end at the frontier — cursor exhausted, writes
8327/// enabled — would otherwise write the same record a second time.
8328fn recorded_groups(
8329    records: &[Record],
8330) -> BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup> {
8331    let mut recorded: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup> =
8332        BTreeMap::new();
8333    for r in records {
8334        let (group, opened) = match r.kind() {
8335            RecordKind::GroupOpened { group, .. } => (group, true),
8336            RecordKind::GroupSettled { group, .. } => (group, false),
8337            _ => continue,
8338        };
8339        let Some(step) = r.body.step else { continue };
8340        let entry = recorded
8341            .entry((step, r.body.phase, group.clone()))
8342            .or_default();
8343        if opened {
8344            entry.opened += 1;
8345        } else {
8346            entry.settled += 1;
8347        }
8348    }
8349    recorded
8350}
8351
8352/// Every capability an agent advertises is provided by one of **its own**
8353/// skills.
8354///
8355/// An agent advertising a capability none of its skills provide is a card that
8356/// lies, and the caller who believed it finds out at dispatch — in production —
8357/// rather than here at startup.
8358///
8359/// This checked a plane-wide map, and the difference is not pedantry. A skill
8360/// registered on the *builder* rather than on the agent — `.agent(Agent::new(&m))`
8361/// followed by `.skill(s)` — satisfied a plane-wide check while being
8362/// **ungoverned**: `governed_by` is keyed from the agent's own skills, so that
8363/// skill gets no manifest. It runs under the plane's default budget instead of
8364/// the declared one, and `StepCtx::gate` never refuses a model or tool the file
8365/// did not list, because there is no file.
8366///
8367/// The plane built cleanly and the assertion's own message said "its skills",
8368/// so the only signal was a `None` from `cx.manifest()` that a skill has no
8369/// reason to check. That is a declaration reading as a control while governing
8370/// nothing, which is the one shape this codebase refuses everywhere.
8371#[cfg(feature = "manifest")]
8372fn check_declaration_matches_skills(
8373    m: &crate::manifest::Manifest,
8374    mine: &HashSet<Capability>,
8375) -> Result<(), BuildError> {
8376    let missing: Vec<String> = m
8377        .spec
8378        .capabilities
8379        .provides
8380        .iter()
8381        .filter(|c| !mine.contains(&Capability::new(c.as_str())))
8382        .cloned()
8383        .collect();
8384    if !missing.is_empty() {
8385        return Err(BuildError::AdvertisesWhatItCannotProvide {
8386            agent: m.metadata.name.clone(),
8387            missing,
8388        });
8389    }
8390
8391    // And the other direction, which is the one that leaves no trace. A skill
8392    // registered under a manifest is *governed* by it — the manifest's budget,
8393    // model grants, egress ceiling and policy identity all apply — so a
8394    // capability the skill answers and the declaration never names is a
8395    // reviewed surface with an unreviewed door in it. The declaration is the
8396    // artifact that gets read, digested and pinned, and the A2A card is built
8397    // from it, so the extra capability is served and advertised nowhere.
8398    //
8399    // It is the same argument the manifest already makes about prompts: a
8400    // system prompt composed in the deployer's code has no version, changes in
8401    // a deploy, and nothing connects the change to the runs it affected. A
8402    // capability added in code does the same to the agent's surface.
8403    let undeclared: Vec<String> = {
8404        let declared: HashSet<Capability> = m
8405            .spec
8406            .capabilities
8407            .provides
8408            .iter()
8409            .map(|c| Capability::new(c.as_str()))
8410            .collect();
8411        let mut extra: Vec<String> = mine
8412            .iter()
8413            .filter(|c| !declared.contains(c))
8414            .map(|c| c.0.clone())
8415            .collect();
8416        // Deterministic, so the message does not depend on hash order.
8417        extra.sort();
8418        extra
8419    };
8420    if !undeclared.is_empty() {
8421        return Err(BuildError::ProvidesWhatItDoesNotAdvertise {
8422            agent: m.metadata.name.clone(),
8423            undeclared,
8424        });
8425    }
8426    Ok(())
8427}
8428
8429/// Add one skill to the plane's two lookup tables, refusing a collision.
8430///
8431/// Both maps are plane-wide, and a bare `insert` would take a second
8432/// registration silently. That was tolerable when a plane was one agent and is
8433/// not now: dispatch resolves a capability to a skill *and to the manifest
8434/// governing it*, so a silent overwrite does not merely shadow the loser — it
8435/// moves work the loser still advertises out from under the loser's budget,
8436/// model grants and egress ceiling. Nothing in the journal would show it,
8437/// because the winner looks like the only claimant that ever existed.
8438///
8439/// Returns the skill's name, which is the key governance is recorded under.
8440///
8441/// # Errors
8442///
8443/// If another skill already holds this name, or another skill already claims one
8444/// of its capabilities.
8445fn register_skill(
8446    skill: Arc<dyn Skill>,
8447    caps: &mut HashMap<Capability, String>,
8448    skills: &mut HashMap<String, Arc<dyn Skill>>,
8449) -> Result<String, BuildError> {
8450    let d = skill.descriptor();
8451    if let Some(existing) = skills.get(&d.name)
8452        // Registering the *same* `Arc` twice is idempotent rather than a
8453        // mistake; two distinct skills under one name is the collision.
8454        && !Arc::ptr_eq(existing, &skill)
8455    {
8456        return Err(BuildError::DuplicateSkillName { name: d.name });
8457    }
8458    // `capabilities()`, not the raw field: a descriptor that declared nothing
8459    // answers its own name, and registering that here is what makes
8460    // `run("greet")` and an `agent/greet` grant resolve without a hello-world
8461    // program inventing a second name for its one skill.
8462    for cap in d.capabilities() {
8463        if let Some(first) = caps.get(&cap)
8464            && first != &d.name
8465        {
8466            return Err(BuildError::CapabilityClaimedTwice {
8467                capability: cap.0,
8468                first: first.clone(),
8469                second: d.name,
8470            });
8471        }
8472        caps.insert(cap, d.name.clone());
8473    }
8474    skills.insert(d.name.clone(), skill);
8475    Ok(d.name)
8476}
8477
8478/// The capability a bare `run(target, ..)` admits: the first declared, or the
8479/// skill's own name — [`SkillDescriptor::capabilities`] is never empty.
8480fn first_capability(descriptor: &SkillDescriptor) -> Capability {
8481    descriptor
8482        .capabilities()
8483        .into_iter()
8484        .next()
8485        .unwrap_or_else(|| Capability::new(descriptor.name.clone()))
8486}
8487
8488/// The business keys a case is identified by, for the binding record.
8489///
8490/// Read from the case rather than taken from the request, because a run joining
8491/// an existing case is admitted with whichever key its message carried while the
8492/// case may be identified by several — and a manifest binding naming any of them
8493/// must resolve.
8494///
8495/// Falls back to the admitted keys if the case cannot be read back. That is not
8496/// defensive padding: `correlate_or_open` has already committed the binding, so
8497/// a failure here is a read that lost a race with nothing, and the keys the run
8498/// was admitted with are a true subset of the case's. Recording them is strictly
8499/// better than recording none, and *nothing* is what would silently make a
8500/// binding unresolvable for a run whose case is perfectly well identified.
8501async fn case_correlation(
8502    cases: &dyn CaseStore,
8503    case_id: crate::core::CaseId,
8504    admitted: &[CorrelationKey],
8505) -> Result<Vec<CorrelationKey>, RuntimeError> {
8506    let stored = cases
8507        .case(case_id)
8508        .await
8509        .map_err(RuntimeError::from_store)?
8510        .map(|case| case.correlation)
8511        .unwrap_or_default();
8512    let mut keys = if stored.is_empty() {
8513        admitted.to_vec()
8514    } else {
8515        stored
8516    };
8517    // Canonical order, so the record is byte-stable for a given set and two
8518    // stores that answer in different orders produce the same journal.
8519    keys.sort();
8520    keys.dedup();
8521    Ok(keys)
8522}
8523
8524/// A lease owner that no other process will accidentally share.
8525///
8526/// The previous default was the constant `"agentplane"`, which every replica and
8527/// every restart used. Two consequences, both silent:
8528///
8529/// * Two replicas each saw the other's lease as their own and renewed it
8530///   without bumping the epoch — two writers on one run, which is the exact
8531///   situation fencing exists to make impossible.
8532/// * A process restarting after a crash "renewed" the dead process's lease
8533///   instead of waiting for expiry and fencing it, so a zombie still holding a
8534///   socket could keep writing under the same epoch as its replacement.
8535///
8536/// A per-process random identity turns both into the correct behaviour: a
8537/// different owner cannot renew, so it waits for expiry and takes over with
8538/// `epoch + 1`.
8539///
8540/// Not derived from a hostname or PID: containers reuse both. Randomness is the
8541/// property that matters; readability is what the `owner` override is for, and a
8542/// deployment with a real instance identity — a pod name — should pass it.
8543fn default_owner() -> String {
8544    use std::collections::hash_map::RandomState;
8545    use std::hash::{BuildHasher, Hasher};
8546    use std::sync::atomic::{AtomicU64, Ordering};
8547
8548    // OS entropy, and deliberately *not* the clock: this crate forbids reading
8549    // the wall clock outside a journaled effect, and rightly — a lease owner is
8550    // a poor reason to make an exception to a rule that keeps replay honest.
8551    // `RandomState` is seeded by the operating system, so two processes differ
8552    // even where a container has reused a PID.
8553    static SEED: std::sync::OnceLock<u64> = std::sync::OnceLock::new();
8554    // A counter beside it, so two runtimes built in one process — which tests do
8555    // constantly — never alias each other either.
8556    static SEQ: AtomicU64 = AtomicU64::new(0);
8557
8558    let seed = *SEED.get_or_init(|| RandomState::new().build_hasher().finish());
8559    let n = SEQ.fetch_add(1, Ordering::Relaxed);
8560    format!("agentplane-{seed:016x}-{n}")
8561}
8562
8563/// Resolve a run's data-subject bindings against its input, before the run
8564/// is bound to a case.
8565///
8566/// Declared bindings first, in declaration order, then the subjects the
8567/// embedder named; each entry's position is the index every reference to it
8568/// names. A binding that resolves to nothing refuses the run rather than
8569/// attributing its data to nobody. A `$case` binding is refused when the run
8570/// asked for no case, and otherwise left for [`fill_case_subjects`].
8571fn bind_subjects(
8572    declared: &[crate::journal::SubjectBinding],
8573    named: Vec<String>,
8574    input: &Tainted<Value>,
8575    in_case: bool,
8576) -> Result<Vec<crate::journal::BoundSubject>, RuntimeError> {
8577    use crate::journal::SubjectBinding;
8578    let refuse = |binding: &SubjectBinding, reason: &str| RuntimeError::SubjectUnbound {
8579        binding: binding.to_string(),
8580        reason: reason.to_owned(),
8581    };
8582    let mut resolved = Vec::new();
8583    let named = named
8584        .into_iter()
8585        .map(|subject| (SubjectBinding::Named, Some(subject)));
8586    for (position, (binding, given)) in declared
8587        .iter()
8588        .cloned()
8589        .map(|binding| (binding, None))
8590        .chain(named)
8591        .enumerate()
8592    {
8593        let (subject, trusted) = match (&binding, given) {
8594            (_, Some(subject)) => (subject, true),
8595            (SubjectBinding::Input { pointer }, None) => {
8596                let Some(selected) = input.project_pointer(pointer) else {
8597                    return Err(refuse(&binding, "it selects nothing in the run's input"));
8598                };
8599                let subject = match selected.peek() {
8600                    Value::String(text) => text.clone(),
8601                    Value::Number(number) => number.to_string(),
8602                    _ => {
8603                        return Err(refuse(&binding, "it selects neither a string nor a number"));
8604                    }
8605                };
8606                (
8607                    subject,
8608                    selected.label().trust == crate::core::Trust::Trusted,
8609                )
8610            }
8611            (SubjectBinding::Case, None) => {
8612                if !in_case {
8613                    return Err(refuse(&binding, "the run belongs to no case"));
8614                }
8615                (CASE_PENDING.to_owned(), true)
8616            }
8617            (SubjectBinding::Named, None) => {
8618                return Err(refuse(&binding, "no subject was named"));
8619            }
8620        };
8621        if subject.trim().is_empty() {
8622            return Err(refuse(&binding, "it resolves to an empty subject"));
8623        }
8624        let index = u16::try_from(position)
8625            .map_err(|_| refuse(&binding, "a run binds at most 65536 subjects"))?;
8626        resolved.push(crate::journal::BoundSubject {
8627            index,
8628            binding,
8629            trusted,
8630            subject,
8631        });
8632    }
8633    Ok(resolved)
8634}
8635
8636/// What a `$case` binding holds between [`bind_subjects`] and
8637/// [`fill_case_subjects`]: never journaled.
8638const CASE_PENDING: &str = "$case";
8639
8640/// Give each `$case` binding the id of the case the run was bound to.
8641fn fill_case_subjects(
8642    mut bound: Vec<crate::journal::BoundSubject>,
8643    case: Option<&CaseContext>,
8644) -> Vec<crate::journal::BoundSubject> {
8645    if let Some(case) = case {
8646        for entry in &mut bound {
8647            if entry.binding == crate::journal::SubjectBinding::Case {
8648                entry.subject = case.case_id.to_string();
8649            }
8650        }
8651    }
8652    bound
8653}
8654
8655/// The references a run's `DataSubjectBound` record makes available to every
8656/// value it reads.
8657fn subject_refs(
8658    run: RunId,
8659    bound: &[crate::journal::BoundSubject],
8660) -> BTreeSet<crate::core::SubjectRef> {
8661    bound
8662        .iter()
8663        .map(|b| crate::core::SubjectRef {
8664            run,
8665            index: b.index,
8666        })
8667        .collect()
8668}
8669
8670/// The references a recorded run's `DataSubjectBound` names, read back so a
8671/// resume attributes exactly as the live run did.
8672fn recorded_subjects(run: RunId, records: &[Record]) -> BTreeSet<crate::core::SubjectRef> {
8673    records
8674        .iter()
8675        .find_map(|r| match r.kind() {
8676            RecordKind::DataSubjectBound { bindings } => Some(subject_refs(run, bindings)),
8677            _ => None,
8678        })
8679        .unwrap_or_default()
8680}
8681
8682/// Refuse a history this build cannot re-derive.
8683///
8684/// Before recomputing anything, because every effect key a replay derives comes
8685/// out of the canonicalizer: a run written under another rule recomputes
8686/// different keys and would be quarantined as *non-determinism* — the most
8687/// serious conclusion this runtime reaches, reported for a healthy run because
8688/// the rule moved underneath it.
8689///
8690/// The chain itself is fine and always was: it hashes the bytes it stored rather
8691/// than re-canonicalizing them. What moved is everything *derived*, which is
8692/// exactly the class replay compares.
8693fn ensure_replayable_canon(records: &[Record]) -> Result<(), RuntimeError> {
8694    if let Some(recorded) = records.iter().find_map(recorded_canon)
8695        && recorded != crate::core::canon::VERSION
8696    {
8697        return Err(RuntimeError::CanonicalizationChanged {
8698            recorded,
8699            implemented: crate::core::canon::VERSION,
8700        });
8701    }
8702    Ok(())
8703}
8704
8705/// Which canonicalization rule wrote this run's derived digests.
8706fn recorded_canon(r: &Record) -> Option<u16> {
8707    match r.kind() {
8708        RecordKind::RunAdmitted { canon, .. } => Some(*canon),
8709        _ => None,
8710    }
8711}
8712
8713/// The principal a run was admitted as.
8714///
8715/// Read back rather than recomputed, for the same reason the plan is: the
8716/// principal a run was authorized as is a fact *about that run*, and deriving it
8717/// again from a plan that may since have been edited would silently re-attribute
8718/// history.
8719/// The chain a recorded run acted under, re-checked structurally on the way
8720/// back — never re-verified as a credential, for the reason `core::identity`
8721/// gives. `None` for a run admitted without one.
8722fn recorded_chain(records: &[Record]) -> Result<Option<crate::core::Delegation>, RuntimeError> {
8723    records
8724        .iter()
8725        .find_map(|r| match r.kind() {
8726            RecordKind::IdentityBound { chain } => Some(chain.clone()),
8727            _ => None,
8728        })
8729        .map(crate::core::Delegation::rehydrate)
8730        .transpose()
8731        .map_err(RuntimeError::Delegation)
8732}
8733
8734/// Whether a resume refused with `error` would be refused the same way on
8735/// every attempt: the journal it read is what refuses, not the store, a lease,
8736/// a quota or this plane's configuration.
8737///
8738/// Named rather than negated: a failure not listed here is retried, so a
8739/// transient one added later costs a retry loop rather than a quarantine.
8740/// `PlanContract`, `NoProvider` and `NoCaseStore` are not listed because they
8741/// are about this plane — a quota store, an agent or a case store it lacks —
8742/// and another plane may settle what this one cannot.
8743const fn recurs_on_every_resume(error: &RuntimeError) -> bool {
8744    matches!(
8745        error,
8746        RuntimeError::Encoding(_)
8747            | RuntimeError::ChainBroken { .. }
8748            | RuntimeError::CanonicalizationChanged { .. }
8749            | RuntimeError::Delegation(_)
8750    )
8751}
8752
8753fn recorded_agent(records: &[Record]) -> String {
8754    records
8755        .iter()
8756        .find_map(|r| match r.kind() {
8757            RecordKind::RunAdmitted { capability, .. } => Some(capability.clone()),
8758            _ => None,
8759        })
8760        .unwrap_or_default()
8761}
8762
8763/// Build a step's input from its declared argument sources.
8764///
8765/// Labels join across sources, so a step reading anything untrusted produces an
8766/// untrusted input without the plan author having to say so.
8767fn assemble(
8768    node: &PlanNode,
8769    run_input: &Tainted<Value>,
8770    outputs: &BTreeMap<StepId, Tainted<Value>>,
8771) -> Result<Tainted<Value>, RuntimeError> {
8772    // The common case — a single argument — passes the value through rather than
8773    // wrapping it in a one-key object, so simple plans stay legible.
8774    if node.args.len() == 1
8775        && let Some((_, only)) = node.args.iter().next()
8776    {
8777        return resolve_arg(node, only, run_input, outputs);
8778    }
8779
8780    let mut fields = Vec::with_capacity(node.args.len());
8781    for (name, source) in &node.args {
8782        let v = resolve_arg(node, source, run_input, outputs)?;
8783        fields.push((name.clone(), v));
8784    }
8785    Ok(Tainted::object(fields))
8786}
8787
8788fn resolve_arg(
8789    node: &PlanNode,
8790    source: &ArgSource,
8791    run_input: &Tainted<Value>,
8792    outputs: &BTreeMap<StepId, Tainted<Value>>,
8793) -> Result<Tainted<Value>, RuntimeError> {
8794    Ok(match source {
8795        // Projected as a step's output is: a field keeps the label it was
8796        // admitted with, and one admitted without its own inherits the whole
8797        // value's — the parts of an untrusted document are untrusted.
8798        ArgSource::RunInput { field } => project(run_input, field.as_deref()),
8799        ArgSource::Const { value } => Tainted::trusted(value.clone()),
8800        ArgSource::Node { step, field } => {
8801            // The contract already proved this is upstream, so a miss here means
8802            // the scheduler dispatched out of order — a bug worth naming rather
8803            // than papering over with a null.
8804            let upstream = outputs.get(step).ok_or_else(|| {
8805                RuntimeError::PlanContract(format!(
8806                    "step {} read step {step}, which has not produced a value",
8807                    node.id
8808                ))
8809            })?;
8810            project(upstream, field.as_deref())
8811        }
8812    })
8813}
8814
8815/// A value, or one of its top-level fields with its descendants' labels; an
8816/// absent field is a null under the whole value's label.
8817fn project(value: &Tainted<Value>, field: Option<&str>) -> Tainted<Value> {
8818    match field {
8819        Some(field) => value
8820            .project_field(field)
8821            .unwrap_or_else(|| Tainted::with_label(Value::Null, value.label().clone())),
8822        None => value.clone(),
8823    }
8824}
8825
8826/// A step that read a blob rewritten under its address ends unreproducible,
8827/// whatever the skill returned: its history cannot be reproduced, whether the
8828/// skill propagated the error, retried it or swallowed it.
8829fn moved_blob_wins<T>(
8830    cx: &StepCtx<'_>,
8831    result: Result<T, crate::core::SkillError>,
8832) -> Result<T, crate::core::SkillError> {
8833    match cx.moved_blob() {
8834        Some(what) => Err(crate::core::SkillError::Step(
8835            crate::core::StepError::Unreproducible {
8836                what: what.to_owned(),
8837                detail: "a pinned read returned other bytes under its address".to_owned(),
8838            },
8839        )),
8840        None => result,
8841    }
8842}
8843
8844/// Settle a group the skill left open, because `Drop` cannot.
8845///
8846/// A skill that fails with `?` never reaches `commit` or `abort`, so the handle
8847/// is dropped with members standing. Reversing them is async and `Drop` is not,
8848/// which is why the group lives on the context and the executor finishes what
8849/// the handle abandoned — the same relationship the executor already has with a
8850/// step's compensation.
8851///
8852/// Three situations, and they are not the same:
8853///
8854/// * **suspended** — the step has not ended. Its frame is persisted and it will
8855///   re-run from the top, rebuilding the group from the journal as it replays
8856///   the members. Reversing here would undo a run that is merely waiting.
8857/// * **failed** — abort, unless the failure leaves the world in doubt. Doubt is
8858///   the one condition under which nothing may be reversed.
8859/// * **succeeded with a group still open** — an author bug, and the safe
8860///   reading is that the group was never meant to take. It is reversed and the
8861///   step fails loudly, because a group that commits by being forgotten is
8862///   worse than one that does not commit at all.
8863async fn settle_abandoned_group(
8864    cx: &mut StepCtx<'_>,
8865    result: Result<Outcome, crate::core::SkillError>,
8866) -> Result<Outcome, crate::core::SkillError> {
8867    use crate::core::{SkillError, StepError};
8868
8869    let Some(name) = cx.open_group().map(|g| g.name.clone()) else {
8870        return result;
8871    };
8872    // A pause leaves the group open for the resumed step to settle — unless
8873    // the run was cancelled while paused and this pass exists to abandon it.
8874    if !cx.abandoning
8875        && matches!(&result, Err(SkillError::Step(e)) if crate::runtime::group::is_pause(e))
8876    {
8877        return result;
8878    }
8879
8880    // A member whose failure may have reached the world travels no further.
8881    // Reversing around a call that may have — or did — happen leaves the world
8882    // holding a write no `Aborted` settlement can honestly claim to have undone.
8883    let doubt = match &result {
8884        Err(SkillError::Step(e)) => crate::runtime::group::may_have_externalised(e),
8885        _ => false,
8886    };
8887    if doubt {
8888        let detail = match &result {
8889            Err(e) => e.to_string(),
8890            Ok(_) => String::new(),
8891        };
8892        let settled = cx
8893            .settle_open_group(crate::core::GroupOutcome::Quarantined, Some(&detail))
8894            .await;
8895        return match settled {
8896            Ok(()) => result,
8897            Err(e) => Err(SkillError::Step(e)),
8898        };
8899    }
8900
8901    match cx
8902        .abort_open_group("the step ended without settling the group")
8903        .await
8904    {
8905        // The abort itself could not be completed. That outranks whatever the
8906        // step was reporting: a partly unwound group is the more dangerous fact.
8907        Err(e) => Err(SkillError::Step(e)),
8908        Ok(()) => match result {
8909            // A reported failure keeps its own reason. `Outcome::Fail` is an
8910            // `Ok` at the type level and a failure in fact: leaving the group
8911            // to the runtime is the ordinary path there, not an author bug, and
8912            // overwriting the reason would tell an operator the step "returned
8913            // successfully" while hiding why it actually stopped.
8914            Err(e) => Err(e),
8915            failed @ Ok(Outcome::Fail { .. }) => failed,
8916            // Anything else claimed to make progress while leaving a group
8917            // unsettled.
8918            Ok(_) => Err(SkillError::Step(StepError::GroupAborted {
8919                what: format!(
8920                    "step made progress with group '{name}' still open — it was \
8921                     reversed, because a group that commits by being forgotten is worse \
8922                     than one that does not commit at all"
8923                ),
8924            })),
8925        },
8926    }
8927}
8928
8929/// Turn a step's result into a run status.
8930///
8931/// The distinction that matters is between an ordinary failure and a run whose
8932/// *history can no longer be trusted*. Divergence and orphaned effects are the
8933/// latter: they mean the journal no longer describes what this code does, so a
8934/// human has to look before anything else happens. Folding them into `Failed`
8935/// would put them in the same bucket as "the invoice was rejected", and they
8936/// would be retried like one.
8937fn classify(
8938    meter: &super::metrics::Meter,
8939    result: Result<Outcome, crate::core::SkillError>,
8940) -> (RunStatus, Option<Tainted<Value>>) {
8941    use crate::core::{SkillError, StepError};
8942
8943    match result {
8944        // The label travels with the value. Stripping it here would silently
8945        // launder provenance at every step boundary: a downstream step reading
8946        // an untrusted upstream output would receive it marked trusted, and the
8947        // taint gates further on would have nothing to act on.
8948        Ok(Outcome::Done(v)) => (RunStatus::Succeeded, Some(v)),
8949        Ok(Outcome::Fail { reason }) => (RunStatus::Failed(reason), None),
8950        // Not a failure: the executor decides whether a new plan is allowed,
8951        // because the answer depends on the run's provenance and budget, which
8952        // a step cannot see.
8953        Ok(Outcome::Replan { reason }) => (RunStatus::Replanning(reason), None),
8954        // Suspension is not a failure: the run is healthy and waiting. It
8955        // reaches here as an error only because that is how control leaves a
8956        // skill.
8957        Err(SkillError::Step(StepError::Suspended(reason))) => (RunStatus::Suspended(reason), None),
8958        // The ceiling did its job. Reporting this as a failure would have
8959        // operators debugging a system that behaved exactly as instructed.
8960        Err(SkillError::Step(StepError::Budget(exceeded))) => {
8961            (RunStatus::Exhausted(exceeded), None)
8962        }
8963        // A withdrawal found at a hop is the boundary's pause, reached
8964        // mid-step: the hop recorded the withholding before refusing.
8965        Err(SkillError::Step(StepError::Withheld { subject, reason })) => {
8966            (RunStatus::Withheld { subject, reason }, None)
8967        }
8968        Err(e) => {
8969            let msg = e.to_string();
8970            // Matched structurally, never on the message: a match on the word
8971            // "quarantined" would let a reworded error silently downgrade a run
8972            // to `Failed` — history kept, the flag that said not to trust it
8973            // lost.
8974            // Each of these is a failure P7 exists to make loud, and each gets
8975            // its own event so "did this happen" is a query rather than a grep.
8976            match &e {
8977                SkillError::Step(StepError::NonDeterminism {
8978                    seq,
8979                    expected,
8980                    actual,
8981                    detail,
8982                }) => {
8983                    tracing::error!(
8984                        target: telemetry::NONDETERMINISM,
8985                        %seq, %expected, %actual,
8986                        error_type = "nondeterminism",
8987                        reason_digest = %telemetry::reason_digest(detail),
8988                    );
8989                    meter.count(metrics::DIVERGENCES, "");
8990                }
8991                SkillError::Step(StepError::ReplayOverrun { actual, kind }) => {
8992                    tracing::error!(
8993                        target: telemetry::NONDETERMINISM,
8994                        %actual, %kind, overrun = true,
8995                    );
8996                    meter.count(metrics::DIVERGENCES, "");
8997                }
8998                SkillError::Step(StepError::Undecidable { key, detail, .. }) => {
8999                    tracing::error!(
9000                        target: telemetry::UNDECIDABLE,
9001                        %key,
9002                        error_type = "undecidable",
9003                        reason_digest = %telemetry::reason_digest(detail),
9004                    );
9005                    meter.count(metrics::UNDECIDABLE, "");
9006                }
9007                SkillError::Step(StepError::Unreproducible { what, detail }) => {
9008                    tracing::error!(
9009                        target: telemetry::UNREPRODUCIBLE,
9010                        error_type = "unreproducible",
9011                        reason_digest = %telemetry::reason_digest(&format!("{what}: {detail}")),
9012                    );
9013                    meter.count(metrics::UNREPRODUCIBLE, "");
9014                }
9015                _ => {}
9016            }
9017
9018            let untrustworthy = matches!(
9019                e,
9020                SkillError::Step(
9021                    StepError::NonDeterminism { .. }
9022                        | StepError::ReplayOverrun { .. }
9023                        | StepError::Undecidable { .. }
9024                        | StepError::GroupUnsettled { .. }
9025                        | StepError::Unreproducible { .. }
9026                )
9027            );
9028            if untrustworthy {
9029                (RunStatus::Quarantined(msg), None)
9030            } else {
9031                (RunStatus::Failed(msg), None)
9032            }
9033        }
9034    }
9035}
9036
9037/// Everything one execution needs, gathered so the executor is not called with
9038/// eight positional arguments — two of which are `u64`-shaped and would swap
9039/// silently.
9040struct Execution<'a> {
9041    /// Where the recorded run was refused by a step limit, if it was. `None`
9042    /// for a live run, which has no history to consult.
9043    refusal: Option<(StepId, String, String)>,
9044    /// The subject of a withholding the recorded history left standing. `None`
9045    /// for a live run, and for a resumed one whose withdrawal was already
9046    /// lifted and recorded.
9047    withheld_subject: Option<String>,
9048    /// Successor plans the recorded run produced, oldest first. Empty on a live
9049    /// run. Read back rather than re-synthesised, because a planner asked twice
9050    /// can answer differently.
9051    successors: Vec<PlanIR>,
9052    /// Steps whose `StepStarted` is already on the record. Empty on a live
9053    /// run. Read back so a resume neither re-announces a step nor forgets to
9054    /// announce one it starts for the first time.
9055    started: BTreeSet<StepId>,
9056    /// Steps whose `StepFinished` is already on the record. Empty on a live
9057    /// run.
9058    finished: BTreeSet<StepId>,
9059    /// Steps whose latest forward ending on the record is a success. Empty on
9060    /// a live run. A resume replays these before it asks whether to stop, so
9061    /// an unwind sees their outputs and the order they finished in.
9062    succeeded: BTreeSet<StepId>,
9063    /// Group records already on the journal, keyed by the step and phase
9064    /// that wrote them. Empty on a live run. Read back for the same reason
9065    /// `finished` is: group records are not effects, so the cursor cannot
9066    /// dedup them, and a resumed step that re-opens or re-settles its group
9067    /// at the frontier — where its cursor is exhausted and writes are
9068    /// enabled — would report one group as two.
9069    recorded_groups: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
9070    run: RunId,
9071    epoch: u64,
9072    quota: QuotaPass,
9073    plan: &'a PlanIR,
9074    input: Tainted<Value>,
9075    mode: Pass,
9076    case: Option<CaseContext>,
9077    budget: Budget,
9078    /// The run's data-subject references: from the `DataSubjectBound` just
9079    /// written on a live run, read back from it on a replay.
9080    subjects: BTreeSet<crate::core::SubjectRef>,
9081    /// Who is acting, for the policy principal. Read back from `RunAdmitted` on
9082    /// a replay rather than recomputed, for the same reason the plan is: the
9083    /// principal a run was authorized as is a fact about that run.
9084    agent: String,
9085    /// On whose behalf, for the policy context and for what this run
9086    /// commissions. Read back from `IdentityBound` on a replay for the same
9087    /// reason: a resumed run acting under the plane's chain instead of the one
9088    /// recorded for it would reach different verdicts than the live run did.
9089    identity: Option<crate::core::Delegation>,
9090}
9091
9092/// What a conclusion says when nothing on the chain says who ended the run.
9093///
9094/// Quarantine rather than a placeholder name, for the reason the exhausted arm
9095/// beside it quarantines when the typed ceiling verdict is missing: the record
9096/// is not trustworthy, and *not trustworthy* is a different bucket from *try
9097/// again*. A name invented here would be served through the operator API as the
9098/// person who did it.
9099const UNATTRIBUTED: &str = "the chain records this ending and not who asked for it — an \
9100                            operator act with nobody on it cannot be answered by reading further";
9101
9102/// The recorded status of a run that must not be resumed.
9103///
9104/// The outcomes that close a run to recovery:
9105///
9106/// * **Succeeded** — there is nothing outstanding. Re-executing would repeat
9107///   work that is not an effect (a case-state write, say), which is the same
9108///   class of bug the effect protocol prevents, arriving through a side door.
9109/// * **Cancelled** — a stopped run stays stopped, or the next inbound event
9110///   resumes it and it carries on doing the thing somebody intervened to
9111///   prevent.
9112/// * **Abandoned** — a person already decided there will never be an answer.
9113/// * **Quarantined, until a person answers it.** Resuming an unanswered
9114///   quarantine would re-hit whatever could not be decided, and burying that in
9115///   a retry loop is exactly how an undecidable situation becomes an unnoticed
9116///   one. A recorded [`QuarantineDecided`](RecordKind::QuarantineDecided)
9117///   lifts it — the *precondition* is that a named person looked, and the
9118///   verdict is still the runtime's: a reopened run re-decides from a history
9119///   that now holds whatever they established, and reaches the same quarantine
9120///   again if they were wrong or incomplete.
9121///
9122/// A **failed** run is deliberately *not* terminal here: a process that died
9123/// mid-flight records a failure, and recovering it is the entire point.
9124///
9125/// # Why the seal, and not the last step
9126///
9127/// Scanning backwards for a `StepFinished` and reading its outcome is wrong
9128/// twice, and the second is severe:
9129///
9130/// * A step's outcome is not the run's. They coincide only in a one-step plan.
9131/// * `find_map` **skips** a record it does not recognise and keeps looking. A
9132///   run whose last step failed after earlier steps succeeded would match an
9133///   *earlier* `StepFinished { outcome: "succeeded" }` and be reported
9134///   closed-and-succeeded: every multi-step run suspended after a failure —
9135///   every saga waiting on an approval to finish unwinding — could never be
9136///   resumed, and would report success while stuck.
9137///
9138/// `RunSealed` is written by `conclude` for exactly the runs that reached a
9139/// conclusion, and never for a suspended one. That is the fact this needs, so
9140/// it is the fact it reads.
9141pub(crate) fn resume_is_closed(records: &[Record]) -> Option<RunStatus> {
9142    let outcome = records.iter().rev().find_map(|r| match r.kind() {
9143        RecordKind::RunConcluded { outcome, .. } => Some(outcome.as_str()),
9144        _ => None,
9145    })?;
9146
9147    match outcome {
9148        "succeeded" => Some(RunStatus::Succeeded),
9149        "quarantined" => quarantine_decision(records).is_none().then(|| {
9150            RunStatus::Quarantined(
9151                "recorded as quarantined; a named person has to answer it before it can run \
9152                 again — reopen it once the doubt is resolved, or abandon it"
9153                    .into(),
9154            )
9155        }),
9156        // A conclusion whose attribution record is missing is **quarantined**,
9157        // not attributed to a person called "unknown". The arm for an exhausted
9158        // run with no typed ceiling verdict already reads this way; these three
9159        // fabricated a name instead, and a fabricated one is worse than a
9160        // missing one because it is indistinguishable from a real operator with
9161        // that name and it is served through the operator API as fact.
9162        //
9163        // The reason is the one the person gave, from the record that holds
9164        // it, rather than a sentence written here: an operator refused a
9165        // resume is asking *why this run is closed*, and a canned restatement
9166        // of the outcome answers a question they did not ask.
9167        "abandoned" => Some(recorded_abandonment(records).map_or_else(
9168            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9169            |(actor, reason)| RunStatus::Abandoned { actor, reason },
9170        )),
9171        // A stopped run stays stopped. Otherwise the next inbound event resumes
9172        // it and it carries on doing the thing somebody intervened to prevent —
9173        // and the intervention would look, from the journal, like it worked.
9174        "cancelled" => Some(recorded_cancellation(records).map_or_else(
9175            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9176            |(actor, reason)| RunStatus::Cancelled { actor, reason },
9177        )),
9178        // The two conclusions that deliberately do not close a run: a failed
9179        // run resumes with its completed effects read back from history, and an
9180        // exhausted one continues once somebody raises the ceiling.
9181        //
9182        // Their relationship to [`RunStatus::seals`] is the load-bearing part
9183        // and is pinned by `a_sealing_conclusion_is_never_resumable`. The
9184        // direction that matters is *no sealing status may be resumable*: a
9185        // status that seals froze the journal and published a Merkle leaf, so
9186        // resuming it would grow the history past the leaf every later
9187        // checkpoint attests. The reverse direction is deliberately **not** an
9188        // equality — `Suspended` and `Replanning` also do not seal, and never
9189        // reach here at all, because neither is ever a recorded conclusion.
9190        // Stating that as "the two that do not seal" was wrong; there are four,
9191        // and only two of them can be a `RunSealed` outcome.
9192        // A withdrawal joins them, and is the clearest case of the three: the
9193        // run did nothing wrong, its mutations stand, and lifting the halt is
9194        // what continues it. Closing it would turn *stop acting under this
9195        // credential* into *destroy this work*, which is not what an operator
9196        // withdrawing a credential asked for — the operator who wants that
9197        // cancels, and cancelling unwinds.
9198        "failed" | "exhausted" | WITHHELD_OUTCOME => None,
9199        // Fail closed. An outcome this build does not recognise — a sweep's
9200        // `swept`, a future variant, a corrupted string — is not permission to
9201        // resume; it is a run whose recorded ending this code cannot interpret,
9202        // and continuing it would graft new behaviour onto a history that says
9203        // it ended.
9204        other => Some(RunStatus::Quarantined(format!(
9205            "recorded as '{other}', which this build does not recognise as resumable"
9206        ))),
9207    }
9208}
9209
9210/// Refuse a resume that would report success over work its own history reversed.
9211///
9212/// A run whose journal holds compensation-phase records is not resumable once it
9213/// stands concluded. The unwind *reversed* work — a hold released, a refund
9214/// issued — and a resume that replays the forward history and continues would
9215/// conclude success over a world where the work was undone: the journal would
9216/// say "done" about effects its own later records say were taken back.
9217///
9218/// Scoped to runs with a standing conclusion, deliberately: a run that is
9219/// mid-unwind (a compensation suspended for four eyes, a crash between
9220/// reversals) has concluded nothing, and resuming it is how the unwind finishes.
9221fn refuse_resume_over_undone_work(run: RunId, records: &[Record]) -> Result<(), RuntimeError> {
9222    let Some(outcome) = recorded_conclusion(records) else {
9223        return Ok(());
9224    };
9225    if !records
9226        .iter()
9227        .any(|r| matches!(r.kind(), RecordKind::StepCompensated { .. }))
9228    {
9229        return Ok(());
9230    }
9231    Err(RuntimeError::PlanContract(format!(
9232        "run {run} concluded '{outcome}' after compensating completed steps — the work was \
9233         reversed, and resuming over undone work would report success about a world where it no \
9234         longer stands. Start a fresh run"
9235    )))
9236}
9237
9238/// The person's answer standing against this run's quarantine, if there is one.
9239///
9240/// Scoped to what follows the **last** conclusion, so an answer is spent by the
9241/// pass it authorized. A reopened run that quarantines again is quarantined
9242/// again: the earlier decision was made about a history that has since moved,
9243/// and reading it as standing would turn one person's judgement into a
9244/// permanent licence to retry.
9245fn quarantine_decision(records: &[Record]) -> Option<(crate::core::QuarantineDecision, &Record)> {
9246    records
9247        .iter()
9248        .rev()
9249        .take_while(|r| !matches!(r.kind(), RecordKind::RunConcluded { .. }))
9250        .find_map(|r| match r.kind() {
9251            RecordKind::QuarantineDecided { decision, .. } => Some((*decision, r)),
9252            _ => None,
9253        })
9254}
9255
9256/// The abandonment this run's history asks for but has not yet recorded.
9257///
9258/// A decision is written before it is acted on, so a crash in between leaves
9259/// the instruction standing rather than lost. This is what the next resume
9260/// reads to finish the job — and it is why abandonment is not simply performed
9261/// inline by the verb that requests it.
9262fn pending_abandonment(records: &[Record]) -> Option<RunStatus> {
9263    let (decision, record) = quarantine_decision(records)?;
9264    if decision != crate::core::QuarantineDecision::Abandon {
9265        return None;
9266    }
9267    match record.kind() {
9268        RecordKind::QuarantineDecided {
9269            decider, reason, ..
9270        } => Some(RunStatus::Abandoned {
9271            actor: decider.clone(),
9272            reason: reason.clone(),
9273        }),
9274        _ => None,
9275    }
9276}
9277
9278/// The longest admission key this runtime accepts.
9279///
9280/// Bounded because the key is a **counterparty-controlled** string that becomes
9281/// part of a storage key: unbounded, it is an index entry an emitter chooses the
9282/// size of. 512 bytes is far above what an identity needs — a `CloudEvents`
9283/// `source` and `id` together are tens of bytes — and far below what a btree
9284/// index refuses, so the refusal is this crate's with a reason rather than the
9285/// backend's with an error code.
9286pub const MAX_ADMISSION_KEY_BYTES: usize = 512;
9287
9288/// Refuse a key that cannot mean what the caller thinks it means.
9289///
9290/// An **empty** key is the important one, and it is refused rather than
9291/// accepted as a value. A missing header or an unset variable arrives here as
9292/// `""`, and `""` is a perfectly good key: the first message claims it and every
9293/// later message — a *different* message — is answered with the first one's run.
9294/// That is silent, total message loss produced by a configuration mistake, and
9295/// it is exactly what an at-most-once control must not do quietly.
9296fn validate_admission_key(key: &str) -> Result<(), RuntimeError> {
9297    if key.trim().is_empty() {
9298        return Err(RuntimeError::PlanContract(
9299            "an admission key is empty — an unset header or variable arrives here as \
9300             an empty string, and accepting it would answer every later message with \
9301             the first one's run. Use the message's own identity, such as \
9302             `InboundEvent::dedup_key()`"
9303                .into(),
9304        ));
9305    }
9306    if key.len() > MAX_ADMISSION_KEY_BYTES {
9307        return Err(RuntimeError::PlanContract(format!(
9308            "an admission key of {} bytes exceeds the {MAX_ADMISSION_KEY_BYTES}-byte \
9309             limit — the key is chosen by the emitter and becomes part of a storage \
9310             key, so its size is not theirs to decide",
9311            key.len()
9312        )));
9313    }
9314    Ok(())
9315}
9316
9317/// The run id a store named as an admission key's holder.
9318///
9319/// An unparseable holder is corruption in a derived index. Refused rather than
9320/// admitted around: "the index is unreadable" is not evidence the key is free.
9321fn parse_holder(run: &str) -> Result<RunId, RuntimeError> {
9322    RunId::parse(run).map_err(|e| {
9323        RuntimeError::PlanContract(format!(
9324            "an admission key is held by run '{run}', which does not parse ({e}) — refusing rather than admitting a second run under a key whose holder this build cannot read"
9325        ))
9326    })
9327}
9328
9329/// What a run's history says its state is, if it says anything.
9330///
9331/// **One reader, for every surface that answers the question.** Idempotent
9332/// admission, the operator API and the A2A projection are three views of one
9333/// fact, and three copies of this match agree right up until somebody adds a
9334/// record kind or a conclusion arrives that this build cannot read — after
9335/// which a caller polling and the same caller streaming are told different
9336/// things about one run, which is worse than either answer being wrong. The
9337/// laxest copy is the one that decides what an operator sees, so there is one.
9338///
9339/// `None` means the run is still going: the last record is neither a suspension
9340/// nor a conclusion. It is not this function's business whether that means
9341/// *running* or *never existed* — an empty history answers `None` too, and the
9342/// caller knows which it asked for.
9343///
9344/// **The last record, not any record.** A run that waited, was resumed and
9345/// carried on has a suspension in its history and is not suspended now.
9346pub(crate) fn observed_status(records: &[Record]) -> Option<RunStatus> {
9347    Some(match records.last()?.kind() {
9348        RecordKind::RunSuspended { reason } => RunStatus::Suspended(reason.clone()),
9349        RecordKind::RunConcluded {
9350            outcome,
9351            reason,
9352            exhaustion,
9353            ..
9354        } => recorded_status(outcome, reason.as_deref(), exhaustion.as_ref(), records),
9355        _ => return None,
9356    })
9357}
9358
9359/// A sealed run's status, rebuilt from its recorded conclusion.
9360///
9361/// Distinct from `resume_is_closed`, which asks the narrower *may this run be
9362/// resumed* and returns `None` for the conclusions that stay resumable. Every
9363/// conclusion has an answer here: a failed run reported as "still working"
9364/// would send a redelivery into a second admission.
9365fn recorded_status(
9366    outcome: &str,
9367    reason: Option<&str>,
9368    exhaustion: Option<&crate::core::BudgetExceeded>,
9369    records: &[Record],
9370) -> RunStatus {
9371    // A conclusion that carries no reason gets a statement of that fact rather
9372    // than an empty string: the empty summary is what `RunStatus::reason`
9373    // exists to prevent, and this is the surface a retried delivery reads.
9374    let said = || {
9375        reason.map_or_else(
9376            || "concluded without a recorded reason".to_owned(),
9377            ToOwned::to_owned,
9378        )
9379    };
9380    match outcome {
9381        "succeeded" => RunStatus::Succeeded,
9382        "failed" => RunStatus::Failed(said()),
9383        "replanning" => RunStatus::Replanning(said()),
9384        "exhausted" => exhaustion.cloned().map_or_else(
9385            || {
9386                RunStatus::Quarantined(
9387                    "recorded as exhausted without the typed ceiling verdict".to_owned(),
9388                )
9389            },
9390            RunStatus::Exhausted,
9391        ),
9392        "quarantined" => RunStatus::Quarantined(said()),
9393        "cancelled" => recorded_cancellation(records).map_or_else(
9394            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9395            |(actor, reason)| RunStatus::Cancelled { actor, reason },
9396        ),
9397        "abandoned" => recorded_abandonment(records).map_or_else(
9398            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9399            |(actor, reason)| RunStatus::Abandoned { actor, reason },
9400        ),
9401        // A withdrawal, read from the record that holds both halves. Without
9402        // this arm the catch-all answered *quarantined, and this build does
9403        // not recognise 'withheld'* — the plane calling its own record
9404        // foreign, on the surface an operator uses to decide whether to lift a
9405        // halt or to cancel.
9406        WITHHELD_OUTCOME => recorded_withholding(records).map_or_else(
9407            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9408            |(subject, reason)| RunStatus::Withheld { subject, reason },
9409        ),
9410        super::sweeper::SWEEP_OUTCOME => RunStatus::Swept,
9411        // Read back as itself, like every other ending this build writes. The
9412        // catch-all below would answer `Quarantined` — this plane calling its
9413        // own record foreign, on a run whose whole point is that somebody else
9414        // ran it.
9415        OBSERVED_OUTCOME => RunStatus::Observed,
9416        BREAK_GLASS_OUTCOME => recorded_crossing(records).map_or_else(
9417            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9418            |(actor, reason)| RunStatus::BrokeGlass { actor, reason },
9419        ),
9420        HALT_LIFTED_OUTCOME => recorded_halt_lift(records).map_or_else(
9421            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9422            |actor| RunStatus::HaltLifted { actor },
9423        ),
9424        HOLD_RELEASED_OUTCOME => recorded_hold_release(records).map_or_else(
9425            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9426            |actor| RunStatus::HoldReleased { actor },
9427        ),
9428        // Fail closed, as the resume path does: a conclusion this build cannot
9429        // interpret is not permission to treat the run as ordinary. Every
9430        // outcome this build *writes* has an arm above, and a guard holds the
9431        // two lists together — an ending answered here as unrecognised would be
9432        // this plane calling its own record foreign, and the surface it reaches
9433        // is the one an operator uses to read a break-glass crossing.
9434        other => RunStatus::Quarantined(format!(
9435            "recorded as '{other}', which this build does not recognise"
9436        )),
9437    }
9438}
9439
9440/// Whether this value is a sealed payload rather than a readable one.
9441///
9442/// Always false without `keyring`: nothing seals, so nothing can be sealed, and
9443/// a build with no key ring must not carry a dependency on the module that
9444/// defines the wrapper.
9445fn sealed_payload(value: &serde_json::Value) -> bool {
9446    #[cfg(feature = "keyring")]
9447    {
9448        crate::journal::payload::is_sealed(value)
9449    }
9450    #[cfg(not(feature = "keyring"))]
9451    {
9452        let _ = value;
9453        false
9454    }
9455}
9456
9457/// How the run's history says it last ended, or that it is waiting.
9458///
9459/// The last conclusion or suspension on the record, as the outcome word a
9460/// replay's own status reports, so the two compare directly. `None` for a run
9461/// that has neither: it is still in flight, and a strict replay of it would
9462/// stop at its frontier and call that divergence.
9463fn recorded_ending(records: &[Record]) -> Option<String> {
9464    records.iter().rev().find_map(|r| match r.kind() {
9465        RecordKind::RunConcluded { outcome, .. } => Some(outcome.clone()),
9466        RecordKind::RunSuspended { .. } => Some("suspended".to_owned()),
9467        _ => None,
9468    })
9469}
9470
9471/// Whether an ending was an act rather than something the run's steps reached.
9472///
9473/// A cancellation, an abandonment, a sweep, a break-glass crossing, a lift, a
9474/// withdrawal and an observation are each recorded by whoever took them; a
9475/// strict replay re-derives steps, and no step decided these.
9476fn ended_by_an_act(outcome: &str) -> bool {
9477    matches!(outcome, "cancelled" | "abandoned")
9478        || [
9479            super::sweeper::SWEEP_OUTCOME,
9480            BREAK_GLASS_OUTCOME,
9481            HALT_LIFTED_OUTCOME,
9482            HOLD_RELEASED_OUTCOME,
9483            WITHHELD_OUTCOME,
9484            OBSERVED_OUTCOME,
9485        ]
9486        .contains(&outcome)
9487}
9488
9489/// The plan the run was admitted under, and each successor a replan produced,
9490/// oldest first. Read back rather than recompiled, which could produce a graph
9491/// that never governed the run.
9492///
9493/// All or nothing. Successors are walked by position, so skipping one this
9494/// build cannot read would put every later replan on the wrong plan.
9495///
9496/// # Errors
9497///
9498/// [`RuntimeError::PayloadsErased`] when a plan is still sealed, named before
9499/// it is parsed: a sealed payload deserialises into a *missing field*, and that
9500/// sentence sends an operator looking for a corrupt journal rather than telling
9501/// them the data was erased on request.
9502fn frozen_plans(run: RunId, records: &[Record]) -> Result<(PlanIR, Vec<PlanIR>), RuntimeError> {
9503    let mut plans = records
9504        .iter()
9505        .filter_map(|r| match r.kind() {
9506            RecordKind::PlanFrozen { plan, .. } => Some(plan),
9507            _ => None,
9508        })
9509        .map(|plan| {
9510            if sealed_payload(plan) {
9511                return Err(RuntimeError::PayloadsErased {
9512                    run: run.to_string(),
9513                });
9514            }
9515            serde_json::from_value(plan.clone()).map_err(RuntimeError::Encoding)
9516        })
9517        .collect::<Result<Vec<PlanIR>, _>>()?
9518        .into_iter();
9519    let admitted = plans
9520        .next()
9521        .ok_or_else(|| RuntimeError::PlanContract("journal has no PlanFrozen record".into()))?;
9522    Ok((admitted, plans.collect()))
9523}
9524
9525/// Whether this run's history is sealed to a key that no longer exists.
9526///
9527/// Read from the frozen plan, which every admitted run has and which is sealed
9528/// under the run's own erasure scope — so it is gone exactly when the run's
9529/// data is gone. Checking a payload rather than asking the key ring keeps this
9530/// a property of the records in hand: the same answer offline, on a restored
9531/// copy, and in a build with no ring wired.
9532fn payloads_erased(records: &[Record]) -> bool {
9533    records.iter().any(|r| match r.kind() {
9534        RecordKind::PlanFrozen { plan, .. } => sealed_payload(plan),
9535        _ => false,
9536    })
9537}
9538
9539/// Who asked for the stop, read back from the chain.
9540///
9541/// Read rather than remembered, for the same reason every other fact about a run
9542/// is: the journal gives the same answer on every subsequent read, and an
9543/// operator asking "who stopped this?" six weeks later is asking history.
9544///
9545/// **Both halves, from the one record that holds them.** The conclusion does
9546/// not repeat the reason — see [`RecordKind::RunConcluded`]'s `reason` for why
9547/// a second copy of it would be the worse copy.
9548fn recorded_cancellation(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9549    records.iter().rev().find_map(|r| match r.kind() {
9550        RecordKind::RunCancelled { actor, reason } => Some((actor.clone(), reason.clone())),
9551        _ => None,
9552    })
9553}
9554
9555/// Who answered the quarantine and what they said, read back from the chain.
9556fn recorded_abandonment(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9557    records.iter().rev().find_map(|r| match r.kind() {
9558        RecordKind::QuarantineDecided {
9559            decider, reason, ..
9560        } => Some((decider.clone(), reason.clone())),
9561        _ => None,
9562    })
9563}
9564
9565/// Who crossed into this tenant and why, read back from the chain.
9566fn recorded_crossing(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9567    records.iter().rev().find_map(|r| match r.kind() {
9568        RecordKind::BreakGlass { actor, reason, .. } => Some((actor.clone(), reason.clone())),
9569        _ => None,
9570    })
9571}
9572
9573/// Who lifted the halt this run records, read back from its lift record.
9574fn recorded_halt_lift(records: &[Record]) -> Option<crate::core::Operator> {
9575    records.iter().rev().find_map(|r| match r.kind() {
9576        RecordKind::HaltLifted { by, .. } => Some(by.clone()),
9577        _ => None,
9578    })
9579}
9580
9581/// Who released the hold this run records, read back from its release record.
9582fn recorded_hold_release(records: &[Record]) -> Option<crate::core::Operator> {
9583    records.iter().rev().find_map(|r| match r.kind() {
9584        RecordKind::HoldReleased { by, .. } => Some(by.clone()),
9585        _ => None,
9586    })
9587}
9588
9589/// Whose authority was withdrawn, and what the operator said when withdrawing
9590/// it — read back from the withholding record.
9591///
9592/// The last one standing: a withdrawal may be lifted and thrown again within a
9593/// run's history, and the conclusion is about the one that stopped it.
9594fn recorded_withholding(records: &[Record]) -> Option<(String, String)> {
9595    records.iter().rev().find_map(|r| match r.kind() {
9596        RecordKind::AuthorityWithheld {
9597            subject, reason, ..
9598        } => Some((subject.clone(), reason.clone())),
9599        _ => None,
9600    })
9601}
9602
9603/// Wall-clock read for the case's `opened_at` stamp, and for a rate
9604/// reservation's instant.
9605///
9606/// Admission happens before any step exists, so there is no `StepCtx` to
9607/// journal through. Both values are store metadata and never participate in
9608/// replay: a rate refusal is journaled on the run and read back, never
9609/// re-counted — run-visible time still goes through `StepCtx::now`, which
9610/// journals it.
9611#[allow(clippy::disallowed_methods)]
9612pub(super) fn now_for_admission() -> crate::core::Timestamp {
9613    crate::core::Timestamp::now_utc()
9614}
9615
9616/// A lift was recorded in `run` and the register's row was not removed.
9617///
9618/// The control still stands; the operator is told the lift failed and which
9619/// run recorded the attempt.
9620fn removal_failed(run: RunId, e: &crate::core::StoreError) -> RuntimeError {
9621    RuntimeError::ControlStands {
9622        run: run.to_string(),
9623        detail: e.to_string(),
9624    }
9625}
9626
9627/// An operator's lift or release that found a control standing and recorded
9628/// ending it.
9629#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9630pub struct ControlLifted {
9631    /// The run holding the record.
9632    pub record: RunId,
9633    /// Whether this call removed the register row. `false` when another lift
9634    /// or release removed it between this call's read and its removal: the
9635    /// control is gone either way, and the record still stands.
9636    pub removed: bool,
9637}
9638
9639/// The epoch an operator act's own run is written under.
9640///
9641/// Such a run has no competing writer to fence against — it is created,
9642/// written and sealed in one call — so a constant is honest here for the same
9643/// reason it is in the sweeper.
9644const BREAK_GLASS_EPOCH: crate::core::Epoch = LEASE_FREE_EPOCH;
9645
9646/// How a break-glass run ends. Not a run status: it neither succeeded nor
9647/// failed at a goal, which is why a sweep seals as `swept` rather than
9648/// borrowing one.
9649const BREAK_GLASS_OUTCOME: &str = "broke-glass";
9650
9651/// How a run recording an operator lifting a halt ends.
9652pub const HALT_LIFTED_OUTCOME: &str = "halt-lifted";
9653
9654/// How a run recording an operator releasing a legal hold ends.
9655pub const HOLD_RELEASED_OUTCOME: &str = "hold-released";
9656
9657/// How a run ends when the authority it acts under is withdrawn under it.
9658///
9659/// Resumable, so it is deliberately absent from both
9660/// [`SEALED_OUTCOMES`] and [`OUTCOMES_OF_RECORD`]: the run has not ended, it is
9661/// waiting for somebody to lift a halt or to cancel it properly.
9662pub const WITHHELD_OUTCOME: &str = "withheld";
9663
9664/// The outcome a session this plane only **observed** is sealed under.
9665///
9666/// Its own word, and not `succeeded`: a session this runtime did not execute
9667/// has no success to report, and answering with one would put somebody else's
9668/// agent's work in the column an operator reads as *this plane did it*. The
9669/// distinction is the whole reason the status exists.
9670pub const OBSERVED_OUTCOME: &str = "observed";
9671
9672/// The epoch every run written without a lease is written under: a sweep's
9673/// record, an operator act's, an observed session's. Nothing takes such a run
9674/// over, so there is no ownership for the epoch to arbitrate, and one constant
9675/// lets the pass that finishes their seals use the epoch they were written at.
9676pub(crate) const LEASE_FREE_EPOCH: crate::core::Epoch = 1;
9677
9678/// The sealing outcomes whose runs are written without a lease.
9679///
9680/// Recovery selects a run by its lapsed lease, so it never finds one of these
9681/// whose writer died between conclusion and seal;
9682/// [`Runtime::seal_unsealed_conclusions`] is what finishes them.
9683pub(crate) const LEASE_FREE_OUTCOMES: &[&str] = &[
9684    super::sweeper::SWEEP_OUTCOME,
9685    BREAK_GLASS_OUTCOME,
9686    HALT_LIFTED_OUTCOME,
9687    HOLD_RELEASED_OUTCOME,
9688    OBSERVED_OUTCOME,
9689];
9690
9691/// One governed identity: a declaration and the skills that serve it.
9692///
9693/// A runtime **runs** agents; it is not one. It owns the journal, the stores,
9694/// the model drivers and the policy engine — infrastructure, shared. An agent
9695/// owns a manifest and its skills — governance, per-identity. Several agents on
9696/// one plane share a journal and are still separately declared, separately
9697/// bounded, and separately answerable.
9698///
9699/// Conflating the two forced a runtime per agent, which meant a lease owner per
9700/// agent for what is one process, a model driver registered once per agent, and
9701/// nowhere in the journal to record *which* agent governed a run.
9702#[cfg(feature = "manifest")]
9703#[derive(Debug, Default)]
9704pub struct Agent {
9705    manifest: Option<Arc<crate::manifest::Manifest>>,
9706    /// Who vouched for the declaration, when it came from a verified resolution.
9707    publisher: Option<crate::core::KeyId>,
9708    skills: Vec<Arc<dyn Skill>>,
9709}
9710
9711#[cfg(feature = "manifest")]
9712impl Agent {
9713    /// An agent governed by this declaration.
9714    ///
9715    /// Nobody has vouched for it. Prefer [`Agent::published_by`] where the
9716    /// manifest came from a verified registry resolution.
9717    #[must_use]
9718    pub fn new(manifest: &crate::manifest::Manifest) -> Self {
9719        Self {
9720            manifest: Some(Arc::new(manifest.clone())),
9721            publisher: None,
9722            skills: Vec::new(),
9723        }
9724    }
9725
9726    /// Record who vouched for this declaration.
9727    ///
9728    /// Takes the [`KeyId`](crate::core::KeyId) that
9729    /// [`Registry::resolve_verified`](crate::manifest::Registry::resolve_verified)
9730    /// returned beside the manifest — which is otherwise dropped on the floor,
9731    /// so a verified resolution and a parsed file become indistinguishable the
9732    /// moment they reach the runtime.
9733    ///
9734    /// # Why this is the grouping a policy wants
9735    ///
9736    /// A rule has to name *a set of agents*, and the obvious candidates do not
9737    /// survive contact with a deployment:
9738    ///
9739    /// * the **workload identity** is per-instance, so a rule naming one is a
9740    ///   rule rewritten on every deploy;
9741    /// * the agent **name**, its **role**, or any group label in the manifest is
9742    ///   self-asserted — a file claims it, so a rule granting authority to one
9743    ///   grants it to any file that types the same string;
9744    /// * the **digest** is unforgeable but names exactly one revision, so every
9745    ///   edit is a policy change.
9746    ///
9747    /// A publisher key is the only one that is both a group — many agents, many
9748    /// versions — and impossible to claim without holding the key. Bind the rule
9749    /// to the publisher, keep the digest for "this exact revision", and leave
9750    /// the name for humans reading logs.
9751    #[must_use]
9752    pub fn published_by(mut self, key_id: impl Into<crate::core::KeyId>) -> Self {
9753        self.publisher = Some(key_id.into());
9754        self
9755    }
9756
9757    /// Give it a skill.
9758    #[must_use]
9759    pub fn skill(mut self, skill: impl Skill + 'static) -> Self {
9760        self.skills.push(Arc::new(skill));
9761        self
9762    }
9763}
9764
9765/// Receives every model call's live output, with the run it belongs to.
9766///
9767/// What [`ModelStreamObserver`](crate::model::ModelStreamObserver) receives for
9768/// one call, for every call a plane makes — so a surface can show a run's model
9769/// writing while the journal holds only the finished answer.
9770pub trait RunStreamObserver: Send + Sync + std::fmt::Debug {
9771    /// Delivery is advisory and must not block the call.
9772    fn event(&self, run: RunId, event: Tainted<crate::model::ModelStreamEvent>);
9773}
9774
9775/// Assembles a [`Runtime`].
9776#[derive(Debug)]
9777pub struct RuntimeBuilder {
9778    store: Arc<dyn JournalStore>,
9779    /// The witnesses this plane submits checkpoints to, and the bar a round
9780    /// is held to. Both or neither: a witness list with no declared quorum is
9781    /// witnessing whose evidence is whatever happened to arrive.
9782    witnesses: Vec<Arc<dyn crate::journal::Witness>>,
9783    quorum: Option<crate::journal::WitnessQuorum>,
9784    witness_interval: Option<std::time::Duration>,
9785    signer: Option<Arc<dyn crate::core::Signer>>,
9786    skills: Vec<Arc<dyn Skill>>,
9787    #[cfg(feature = "manifest")]
9788    tools: Option<(
9789        Arc<crate::tools::ToolCatalog>,
9790        Arc<dyn crate::tools::ToolClient>,
9791    )>,
9792    #[cfg(feature = "manifest")]
9793    egress: Option<Arc<crate::core::Egress>>,
9794    #[cfg(feature = "manifest")]
9795    checkers: crate::content::Checkers,
9796    streams: Option<Arc<dyn RunStreamObserver>>,
9797    journal_ceiling: Option<crate::core::Sensitivity>,
9798    tenant: crate::core::TenantId,
9799    owner: Option<String>,
9800    lease_ttl: Duration,
9801    memories: Option<Arc<dyn crate::memory::MemoryStore>>,
9802    semantic: Option<Arc<super::SemanticMemory>>,
9803    authorities: Option<Arc<dyn crate::authority::AuthorityStore>>,
9804    peers: Option<Arc<PeerWiring>>,
9805    tenant_label: super::telemetry::TenantLabel,
9806    quotas: Option<Arc<dyn crate::quota::QuotaStore>>,
9807    quota: crate::quota::TenantQuota,
9808    budget: Budget,
9809    require_verifier: bool,
9810    /// Typed tools whose coherence with every agent is checked at `build`.
9811    #[cfg(feature = "manifest")]
9812    toolbox: Option<crate::tools::ToolBox>,
9813    /// Tool servers reached by some transport other than the box, by name.
9814    #[cfg(feature = "manifest")]
9815    tool_servers: Vec<(String, Arc<dyn crate::tools::ToolClient>)>,
9816    /// Agents registered on this plane, each with its own declaration.
9817    #[cfg(feature = "manifest")]
9818    agents: Vec<Agent>,
9819    /// Drivers by the name a manifest calls them.
9820    #[cfg(feature = "manifest")]
9821    providers: HashMap<String, Arc<dyn crate::model::ModelProvider>>,
9822    cases: Option<Arc<dyn CaseStore>>,
9823    events: Option<Arc<dyn EventStore>>,
9824    tasks: Option<Arc<dyn TaskStore>>,
9825    timers: Option<Arc<dyn TimerStore>>,
9826    blobs: Option<Arc<dyn crate::blob::BlobStore>>,
9827    disclosures: Option<Arc<dyn crate::disclosure::DisclosureRegister>>,
9828    #[cfg(feature = "push")]
9829    push: Option<Arc<dyn crate::push::PushStore>>,
9830    #[cfg(feature = "keyring")]
9831    keyring: Option<Arc<dyn crate::keyring::KeyRing>>,
9832    batches: Option<Arc<dyn crate::batch::BatchStore>>,
9833    policy: Option<Arc<dyn crate::core::PolicyEngine>>,
9834    identity: Option<crate::core::Delegation>,
9835    replanner: Option<Arc<dyn crate::plan::Replanner>>,
9836    calendar: Option<Arc<dyn Calendar>>,
9837    /// Destinations this deployment sends its own run events to.
9838    #[cfg(feature = "push")]
9839    outbox: Option<Arc<crate::push::Outbox>>,
9840}
9841
9842impl RuntimeBuilder {
9843    #[must_use]
9844    pub fn skill(mut self, s: impl Skill) -> Self {
9845        self.skills.push(Arc::new(s));
9846        self
9847    }
9848
9849    /// The workload identity this plane signs its outward claims with.
9850    ///
9851    /// What it buys is that a tool or peer can *check* who called it. Without a
9852    /// signer the provenance block still travels — a server can correlate on it
9853    /// — but it is an assertion any intermediary could have written, and a
9854    /// callee must not authorize on it.
9855    ///
9856    /// Give the store the same signer ([`signing_as`] there) so records and
9857    /// outward claims carry one identity. They are separate settings because a
9858    /// plane can legitimately have one without the other.
9859    ///
9860    /// [`signing_as`]: crate::store::RedbStore::signing_as
9861    #[must_use]
9862    pub fn signing_as(mut self, signer: Arc<dyn crate::core::Signer>) -> Self {
9863        self.signer = Some(signer);
9864        self
9865    }
9866
9867    /// How long this plane's run leases last.
9868    ///
9869    /// The trade is recovery speed against tolerance for a slow instance: a
9870    /// crashed owner's runs stay unclaimable for this long, and a live owner
9871    /// must renew within it. The runtime heartbeats while a run executes, so
9872    /// this bounds *crash* detection rather than how long a run may take.
9873    ///
9874    /// A TTL below [`MIN_LEASE_TTL`] is refused at
9875    /// [`build`](Self::build)/[`try_build`](Self::try_build) as
9876    /// [`BuildError::LeaseUnrenewable`](crate::runtime::BuildError::LeaseUnrenewable).
9877    /// Both stores keep lease expiry in **whole seconds** and treat
9878    /// `expires_at <= now` as lapsed, so a one-second lease expires the moment
9879    /// the clock ticks past the second it was written in — no matter how often
9880    /// it is renewed. Such a lease cannot be held by a live run, and a run that
9881    /// cannot hold its lease is one any instance may take away mid-flight.
9882    /// Refused where a manifest-serving plane can report it as a diagnostic
9883    /// rather than left as a panic in a setter, which `try_build`'s callers
9884    /// could never see as a value.
9885    #[must_use]
9886    pub fn lease_ttl(mut self, ttl: Duration) -> Self {
9887        self.lease_ttl = ttl;
9888        self
9889    }
9890
9891    /// Put this plane's tenant on what it reports about itself.
9892    ///
9893    /// Off by default, and one policy for both signals: the `tenant` field of
9894    /// every metric event, and `agentplane.tenant` on the run span. Read
9895    /// [`telemetry::TenantLabel`](super::telemetry::TenantLabel) before turning
9896    /// it on — a tenant name is often a customer name, and an observability
9897    /// backend is usually the least protected system in a deployment.
9898    ///
9899    /// Cardinality is bounded by construction — the label is *this plane's*
9900    /// tenant, so the number of streams is the number of planes configured, and
9901    /// no request can grow it.
9902    #[must_use]
9903    pub const fn tenant_label(mut self, label: super::telemetry::TenantLabel) -> Self {
9904        self.tenant_label = label;
9905        self
9906    }
9907
9908    /// Give this plane's agents a memory.
9909    ///
9910    /// Optional, and absent by default: an agent with no memory is a normal
9911    /// agent, and one that quietly gained persistent state because a store was
9912    /// wired for something else would be a surprise.
9913    ///
9914    /// Read [`crate::memory`] before wiring one. Writable memory is delayed
9915    /// code: what is written today is read into a context window tomorrow, where
9916    /// a model treats it as established fact.
9917    #[must_use]
9918    pub fn memory(mut self, memories: Arc<dyn crate::memory::MemoryStore>) -> Self {
9919        self.memories = Some(memories);
9920        self
9921    }
9922
9923    /// Give this plane a semantic index, and the embedder that speaks its
9924    /// language.
9925    ///
9926    /// Both together, never one at a time: `build` holds
9927    /// [`Embedder::revision`] to the [`IndexIdentity::query_revision`] the
9928    /// index declares it accepts, and refuses the pair otherwise
9929    /// ([`BuildError::EmbeddingSpaceMismatch`] says what a mismatch costs).
9930    ///
9931    /// Also needs [`memory`](Self::memory): the index holds only commitments,
9932    /// and every hit is materialised from the authoritative store before its
9933    /// content is exposed.
9934    ///
9935    /// [`Embedder::revision`]: crate::memory::Embedder::revision
9936    /// [`IndexIdentity::query_revision`]: crate::memory::IndexIdentity::query_revision
9937    /// [`BuildError::EmbeddingSpaceMismatch`]: crate::runtime::BuildError::EmbeddingSpaceMismatch
9938    #[must_use]
9939    pub fn semantic_memory(
9940        mut self,
9941        embedder: Arc<dyn crate::memory::Embedder>,
9942        retriever: Arc<dyn crate::memory::SemanticRetriever>,
9943    ) -> Self {
9944        self.semantic = Some(Arc::new(super::SemanticMemory {
9945            embedder,
9946            retriever,
9947        }));
9948        self
9949    }
9950
9951    /// Whether the plane's budget, or any registered agent's, caps money.
9952    fn states_a_money_ceiling(&self) -> bool {
9953        #[cfg(feature = "manifest")]
9954        if self
9955            .agents
9956            .iter()
9957            .filter_map(|agent| agent.manifest.as_ref())
9958            .any(|manifest| manifest.budget().max_minor_units.is_some())
9959        {
9960            return true;
9961        }
9962        self.budget.max_minor_units.is_some()
9963    }
9964
9965    /// Attach durable standing-authority accounting.
9966    ///
9967    /// The ceiling neither of the other two can express. A budget bounds one
9968    /// run; a quota bounds a tenant over a billing period. A standing authority
9969    /// bounds *an authorization* — what one customer approved, spanning as many
9970    /// runs as it takes, revocable when they change their mind.
9971    ///
9972    /// Without one, [`StepCtx::draw`](crate::runtime::StepCtx::draw) refuses
9973    /// rather than falling back to an in-process counter. That fallback would
9974    /// fail **open** the moment a second instance started, which is exactly when
9975    /// a shared ceiling was needed.
9976    #[must_use]
9977    pub fn authorities(mut self, authorities: Arc<dyn crate::authority::AuthorityStore>) -> Self {
9978        self.authorities = Some(authorities);
9979        self
9980    }
9981
9982    /// Bound what this tenant may consume, durably.
9983    ///
9984    /// Budgets bound one run; this bounds the tenant. Both are needed: a caller
9985    /// that can start runs can start a thousand, each within its own ceiling.
9986    ///
9987    /// The accounting lives in the store, so the ceiling survives a second
9988    /// instance — an in-process counter would silently double the moment
9989    /// somebody scales out, which is exactly when it was needed.
9990    ///
9991    /// Read [`crate::quota`] for what each ceiling does and does not bound; a
9992    /// limit believed to bound something it does not is worse than none.
9993    #[must_use]
9994    pub fn quota(
9995        mut self,
9996        quotas: Arc<dyn crate::quota::QuotaStore>,
9997        quota: crate::quota::TenantQuota,
9998    ) -> Self {
9999        self.quotas = Some(quotas);
10000        self.quota = quota;
10001        self
10002    }
10003
10004    /// This **process instance's** identity, as it appears in run leases.
10005    ///
10006    /// Not the agent's name. Fencing rests on the epoch — a claim never renews,
10007    /// and a renewal names the exact epoch it holds — so a shared string costs
10008    /// no safety; it costs the answer to *which process holds this run*, which
10009    /// is what an operator asks of a stuck one and what `LeaseHeld` reports.
10010    ///
10011    /// So keep it unique per running process, which is what the default is —
10012    /// override it only with a better instance identity than a random one,
10013    /// such as a pod name. An agent's *name* is
10014    /// [`Manifest::metadata`](crate::manifest::Metadata::name).
10015    ///
10016    /// The owner lives in the lease table and never in the chain, so it has no
10017    /// bearing on replay.
10018    #[must_use]
10019    pub fn owner(mut self, o: impl Into<String>) -> Self {
10020        self.owner = Some(o.into());
10021        self
10022    }
10023
10024    /// Cap what a run may consume.
10025    ///
10026    /// Defaults to [`Budget::unlimited`], which is right for a runtime whose
10027    /// effects are all free and local, and wrong the moment one of them calls a
10028    /// metered API.
10029    #[must_use]
10030    pub fn budget(mut self, budget: Budget) -> Self {
10031        self.budget = budget;
10032        self
10033    }
10034
10035    /// Refuse a plan in which nothing checks the work.
10036    ///
10037    /// A node that declares [`verifies`](crate::core::PlanNode::verifies) must
10038    /// already depend on what it checks; this makes carrying one a condition of
10039    /// admission. It binds every plan this runtime validates, and the one that
10040    /// matters is the one the embedder did not write: a
10041    /// [`Replanner`](crate::plan::Replanner)'s successor is a graph proposed
10042    /// mid-run, and a control that held for the first plan and not for its
10043    /// replacement is a control a replan removes.
10044    #[must_use]
10045    pub fn require_verifier(mut self) -> Self {
10046        self.require_verifier = true;
10047        self
10048    }
10049
10050    /// Attach long-lived case storage, enabling correlation and deadlines.
10051    #[must_use]
10052    pub fn cases(mut self, cases: Arc<dyn CaseStore>) -> Self {
10053        self.cases = Some(cases);
10054        self
10055    }
10056
10057    /// Send this plane's own events to destinations the deployment configured.
10058    ///
10059    /// Every run registers each destination **at admission**, so there is no
10060    /// window in which a run exists and nothing is watching it. Delivery is a
10061    /// separate, operator-scheduled sweep — see
10062    /// [`DeliveryWorker`](crate::push::DeliveryWorker) — reading each run's
10063    /// journal past a cursor that advances only on 2xx.
10064    ///
10065    /// This is the mirror image of A2A push, and the mirror is the point: there
10066    /// the *caller* names a URL and three controls exist because of it. Here the
10067    /// deployment names it, and the run's own history is the outbox rather than
10068    /// a queue that can fall out of sync with it.
10069    #[cfg(feature = "push")]
10070    #[must_use]
10071    pub fn outbox(mut self, outbox: Arc<crate::push::Outbox>) -> Self {
10072        self.outbox = Some(outbox);
10073        self
10074    }
10075
10076    /// Attach the webhook registration store.
10077    ///
10078    /// The same handle the A2A server and the [`Outbox`](crate::push::Outbox)
10079    /// are given: this is the *durable* half of push, and it belongs to the
10080    /// plane because its failures do. A receiver that answers permanently, or
10081    /// stays silent past the retry ceiling, has its registration
10082    /// [parked](crate::push::PushStore::park) rather than deleted — the cursor
10083    /// is the only record of how far that receiver got — and a parked row is a
10084    /// backlog an operator reads and re-arms on the operator API. Without this
10085    /// the plane serves the delivery and cannot answer for it.
10086    #[cfg(feature = "push")]
10087    #[must_use]
10088    pub fn push(mut self, push: Arc<dyn crate::push::PushStore>) -> Self {
10089        self.push = Some(push);
10090        self
10091    }
10092
10093    /// Attach inbound-event storage, enabling durable waits.
10094    #[must_use]
10095    pub fn events(mut self, events: Arc<dyn EventStore>) -> Self {
10096        self.events = Some(events);
10097        self
10098    }
10099
10100    /// Attach a worklist, enabling human tasks.
10101    #[must_use]
10102    pub fn tasks(mut self, tasks: Arc<dyn TaskStore>) -> Self {
10103        self.tasks = Some(tasks);
10104        self
10105    }
10106
10107    /// Supply the planner that produces successor plans.
10108    ///
10109    /// Without one, a step asking to replan fails with that as the reason —
10110    /// which is the honest outcome, not a silent no-op.
10111    #[must_use]
10112    pub fn replanner(mut self, r: Arc<dyn crate::plan::Replanner>) -> Self {
10113        self.replanner = Some(r);
10114        self
10115    }
10116
10117    /// Supply the durable-timer store.
10118    ///
10119    /// Needed by `StepCtx::sleep` and `sleep_until`, and by the sweep that wakes
10120    /// them. A runtime without one refuses to sleep rather than falling back to
10121    /// an in-process wait that a restart would forget.
10122    #[must_use]
10123    pub fn timers(mut self, timers: Arc<dyn TimerStore>) -> Self {
10124        self.timers = Some(timers);
10125        self
10126    }
10127
10128    /// Register an agent on this plane.
10129    ///
10130    /// A runtime runs agents; it is not one. This is where a declaration and
10131    /// its skills arrive together, so several agents can share one journal, one
10132    /// set of drivers and one process identity while each stays separately
10133    /// governed.
10134    ///
10135    /// The declaration **binds** for that agent's steps: an effect naming a
10136    /// model or tool its manifest never listed is refused before dispatch and
10137    /// journaled, and the egress and delegation ceilings combine with the sink's
10138    /// own — the stricter wins. Its budget bounds its runs. Architectural
10139    /// injection patterns are deliberately absent from the schema, because this
10140    /// runtime cannot prove that arbitrary skill code follows one.
10141    ///
10142    /// An agent declaring `spec.execution` needs no skill: the runtime supplies
10143    /// the behaviour. See [`provider`](Self::provider) for the driver mapping it
10144    /// needs.
10145    ///
10146    /// It does **not** set the lease owner. That identifies a *process*, and one
10147    /// plane running four agents is still one process — see
10148    /// [`owner`](Self::owner).
10149    ///
10150    /// # This call never fails
10151    ///
10152    /// It records the agent and returns. Every refusal an agent can cause — a
10153    /// capability none of its skills provide, a provider no driver is registered
10154    /// for, oversight with nowhere to put a decision — is raised at
10155    /// [`build`](Self::build), which panics, or returned by
10156    /// [`try_build`](Self::try_build), which does not.
10157    ///
10158    /// That split matters for a daemon assembling a plane from files it did not
10159    /// write: a bad manifest is an *input* there, and a panic takes down every
10160    /// other tenant in the process to report it. A `# Panics` section here,
10161    /// describing `build`'s refusals, would send readers looking for a
10162    /// fallible variant of the wrong call — the refusals are documented where
10163    /// they fire.
10164    #[cfg(feature = "manifest")]
10165    #[must_use]
10166    pub fn agent(mut self, agent: Agent) -> Self {
10167        self.agents.push(agent);
10168        self
10169    }
10170
10171    /// Register a model driver under the name a manifest uses for it.
10172    ///
10173    /// The seam a declarative agent needs. A manifest says `provider: anthropic`
10174    /// — a string a reviewer can read — and something has to map that to a
10175    /// driver holding a credential. That mapping is deployment wiring, not a
10176    /// property of the agent, which is exactly why it lives here and not in the
10177    /// file: an agent's declaration should not change when its API key does.
10178    ///
10179    /// Required only for [`ExecutionKind::Completion`] and the other declarative
10180    /// kinds. A hand-written skill constructs its own `ModelCall` and never
10181    /// consults this.
10182    ///
10183    /// [`ExecutionKind::Completion`]: crate::manifest::ExecutionKind::Completion
10184    #[cfg(feature = "manifest")]
10185    #[must_use]
10186    pub fn provider(
10187        mut self,
10188        name: impl Into<String>,
10189        provider: Arc<dyn crate::model::ModelProvider>,
10190    ) -> Self {
10191        self.providers.insert(name.into(), provider);
10192        self
10193    }
10194
10195    /// Supply content-addressed blob storage.
10196    ///
10197    /// Needed by `StepCtx::store_blob`, which is how bytes too large for a
10198    /// journal record get somewhere durable while the chain keeps only their
10199    /// digest. A runtime without one refuses rather than silently inlining
10200    /// megabytes into an append-only chain that can never take them back.
10201    #[must_use]
10202    pub fn blobs(mut self, blobs: Arc<dyn crate::blob::BlobStore>) -> Self {
10203        self.blobs = Some(blobs);
10204        self
10205    }
10206
10207    /// Where disclosures of a matter are recorded, so a retention pass names
10208    /// the copies of what it erased.
10209    #[must_use]
10210    pub fn disclosures(mut self, register: Arc<dyn crate::disclosure::DisclosureRegister>) -> Self {
10211        self.disclosures = Some(register);
10212        self
10213    }
10214
10215    /// The operator's tool catalogue, and the client that reaches those tools.
10216    ///
10217    /// Required by a `tool-calling` agent and by nothing else: a skill that
10218    /// calls tools builds its own [`ToolCall`](crate::tools::ToolCall), because
10219    /// it knows which client it means. A declarative agent has no code to make
10220    /// that choice, so the plane makes it once.
10221    ///
10222    /// The catalogue is the authority. A manifest grants a subset of it, the
10223    /// model is offered exactly that subset, and a name the model returns is
10224    /// matched against it byte for byte.
10225    #[cfg(feature = "manifest")]
10226    #[must_use]
10227    pub fn tools(
10228        mut self,
10229        catalog: Arc<crate::tools::ToolCatalog>,
10230        client: Arc<dyn crate::tools::ToolClient>,
10231    ) -> Self {
10232        self.tools = Some((catalog, client));
10233        self
10234    }
10235
10236    /// Typed tools, with their catalogue derived and their coherence enforced.
10237    ///
10238    /// The one-call form, and the reason it exists is not brevity. Deriving the
10239    /// catalogue and checking it against every agent's manifest were both
10240    /// possible before and both **optional**, and a control a caller may forget
10241    /// is not a control — it is advice that reads like one.
10242    ///
10243    /// So this does three things that were three things:
10244    ///
10245    /// * derives the catalogue from each agent's declaration, so a grant, its
10246    ///   ceiling and its protected fields are stated once;
10247    /// * refuses to build if the tools this binary implements and the manifests
10248    ///   a reviewer approved have drifted apart;
10249    /// * wires the box as the client.
10250    ///
10251    /// The work happens in [`build`](Self::build) rather than here, and that is
10252    /// the whole reason it is trustworthy: checking on this call would check
10253    /// against the agents registered *so far*, so `.toolbox(..).agent(..)` would
10254    /// pass by having nothing to disagree with. An enforcement that depends on
10255    /// the order a builder was written is not one.
10256    #[cfg(feature = "manifest")]
10257    #[must_use]
10258    pub fn toolbox(mut self, tools: crate::tools::ToolBox) -> Self {
10259        self.toolbox = Some(tools);
10260        self
10261    }
10262
10263    /// A tool server this plane reaches over some transport of its own.
10264    ///
10265    /// An MCP connection is the usual one. Registering it does three things:
10266    ///
10267    /// * a plane may reach **several** servers, because the router resolves the
10268    ///   `tool://server/name` a grant carries rather than handing every id to one
10269    ///   client;
10270    /// * typed in-process tools and remote servers can be used by the *same*
10271    ///   agent, which is the ordinary shape;
10272    /// * a grant naming a server nobody wired is refused at build, in the same
10273    ///   breath as a grant nothing implements — both mean the model would be
10274    ///   offered a tool that fails when chosen.
10275    ///
10276    /// Composes with [`toolbox`](Self::toolbox); the box answers for the servers
10277    /// its own tools name and these answer for theirs. A server claimed twice is
10278    /// a panic, because registration order deciding which transport carries a
10279    /// call is the defect [`ToolRouter`](crate::tools::ToolRouter) exists to
10280    /// remove.
10281    #[cfg(feature = "manifest")]
10282    #[must_use]
10283    pub fn tool_server(
10284        mut self,
10285        name: impl Into<String>,
10286        client: Arc<dyn crate::tools::ToolClient>,
10287    ) -> Self {
10288        self.tool_servers.push((name.into(), client));
10289        self
10290    }
10291
10292    /// Where this plane's **tool transports** may connect.
10293    ///
10294    /// A `ToolClient` is the embedder's code, and the split is the one every
10295    /// other outbound path here makes: **the client owns the connection, the
10296    /// plane owns the destination.** A transport says where it goes with
10297    /// [`ToolClient::destination`](crate::tools::ToolClient::destination), and
10298    /// this decides whether that is somewhere it may go — before the effect
10299    /// exists, so nothing leaves, nothing is journaled and nothing is metered.
10300    ///
10301    /// A [`Destination::Local`](crate::tools::Destination::Local) transport is
10302    /// not judged: tools compiled into the binary and a stdio child process
10303    /// contact no host of this plane's choosing, so there is nothing for an
10304    /// allowlist to decide. What that does not claim is that the far side
10305    /// reaches nothing — the same residual a compromised allowlisted endpoint
10306    /// carries.
10307    ///
10308    /// Unset means no egress control on the tool path, spelled the way an
10309    /// absent policy engine is: by configuring nothing, rather than by
10310    /// configuring something that looks like a control and is not.
10311    ///
10312    /// ```ignore
10313    /// let plane = Runtime::builder(store)
10314    ///     .tool_server("tickets", mcp_over_https)
10315    ///     .egress(Egress::new().allow("mcp.tickets.example"))
10316    ///     .build();
10317    /// ```
10318    #[cfg(feature = "manifest")]
10319    #[must_use]
10320    pub fn egress(mut self, egress: crate::core::Egress) -> Self {
10321        self.egress = Some(Arc::new(egress));
10322        self
10323    }
10324
10325    /// Forward every model call's live output, with the run it belongs to.
10326    ///
10327    /// Advisory and live-only: the journal's completion stays the canonical
10328    /// answer, a replay performs no call and so streams nothing, and an
10329    /// observer that drops events changes nothing a run does.
10330    #[must_use]
10331    pub fn observe_model_streams(mut self, observer: Arc<dyn RunStreamObserver>) -> Self {
10332        self.streams = Some(observer);
10333        self
10334    }
10335
10336    /// Register a content checker, by its [`name`](crate::content::ContentChecker::name).
10337    ///
10338    /// A manifest check naming a checker no builder registered, or mapping a
10339    /// category the checker does not declare, refuses the build: a declared
10340    /// check nothing could run is a control a reviewer approved that does not
10341    /// exist.
10342    #[cfg(feature = "manifest")]
10343    #[must_use]
10344    pub fn content_checker(mut self, checker: Arc<dyn crate::content::ContentChecker>) -> Self {
10345        Arc::make_mut(&mut self.checkers).insert(checker.name().to_owned(), checker);
10346        self
10347    }
10348
10349    /// The highest sensitivity this plane will write into an append-only record.
10350    ///
10351    /// # What this is, and what the key ring is
10352    ///
10353    /// Two different answers to *this data must be erasable*, and they compose.
10354    /// [`keyring`](Self::keyring) is the **seal it** half: payloads become
10355    /// ciphertext whose key an erasure destroys, which reaches every replica
10356    /// and every backup at once. This is the **refuse it** half: an argument
10357    /// more sensitive than the deployment is willing to make permanent is
10358    /// refused at dispatch, before it reaches the chain.
10359    ///
10360    /// The plane-level twin of `spec.security.max_sensitivity_journaled`, for
10361    /// a plane of hand-written skills that runs under no manifest. The default
10362    /// is the unsafe one — a journal keeps everything, indefinitely — so a
10363    /// plane must be able to state the ceiling whether or not it is
10364    /// declarative. It is an enforcement point at the same gate the manifest's
10365    /// field reaches, not a warning; where both are present **the stricter
10366    /// wins**, as a reviewed tool grant may only tighten what the operator's
10367    /// catalogue allows.
10368    ///
10369    /// Absent means no ceiling, spelled the way an absent policy engine is.
10370    ///
10371    /// ```ignore
10372    /// let plane = Runtime::builder(store)
10373    ///     // Personal data goes in a blob and the chain gets the digest.
10374    ///     .max_sensitivity_journaled(Sensitivity::Internal)
10375    ///     .skill(Triage)
10376    ///     .build();
10377    /// ```
10378    #[must_use]
10379    pub const fn max_sensitivity_journaled(mut self, ceiling: crate::core::Sensitivity) -> Self {
10380        self.journal_ceiling = Some(ceiling);
10381        self
10382    }
10383
10384    /// Which tenant this plane runs as.
10385    ///
10386    /// **One plane, one tenant — but one process, many planes.** A plane is the
10387    /// unit that is bound to a tenant; serving several is
10388    /// [`Planes`](crate::api::Planes)' job, and it resolves the plane from the
10389    /// authenticated caller's tenant rather than from the request, so a handler
10390    /// cannot reach a store it did not resolve. An unregistered tenant is
10391    /// refused rather than defaulted.
10392    ///
10393    /// The name scopes **data keys**, so one tenant's cryptographic erasure
10394    /// cannot reach another's bytes, and it reaches the **policy request**, so a
10395    /// rule can be written per tenant.
10396    ///
10397    /// It does **not** scope the store — that is a separate handle, scoped by
10398    /// `RedbStore::for_tenant` or `PostgresStore::for_tenant`. Two tenants may
10399    /// share one store, because the tenant is a key component of every row on
10400    /// both backends rather than a filter. Setting one and not the other is
10401    /// refused at [`build`](Self::build) rather than discovered later: a plane
10402    /// whose store is scoped elsewhere works perfectly and writes its runs into
10403    /// somebody else's keyspace.
10404    ///
10405    /// Defaults to `default`, which is a real tenant rather than an absence: the
10406    /// single-tenant path is then the same code as the multi-tenant one, and a
10407    /// special "no tenant" case is a second path that would not get tested.
10408    #[must_use]
10409    pub fn tenant(mut self, tenant: crate::core::TenantId) -> Self {
10410        self.tenant = tenant;
10411        self
10412    }
10413
10414    /// Seal payload bytes, and make erasure reach copies deletion cannot.
10415    ///
10416    /// With a key ring configured, everything written through
10417    /// [`StepCtx::blobs`](crate::runtime::StepCtx::blobs) — including
10418    /// [`store_blob`](crate::runtime::StepCtx::store_blob) and governed media —
10419    /// is encrypted under a data key belonging to the run's **case**. Erasing
10420    /// that case destroys the key, so every copy of those bytes becomes
10421    /// unreadable at once: the live store, the replicas, and every backup ever
10422    /// taken. Expiring blobs only reaches the first of those.
10423    ///
10424    /// The case is the erasure unit because it is already the retention unit —
10425    /// bytes are linked to their case at write time, and a second, differently
10426    /// shaped unit for keys would let the two disagree about what an erasure
10427    /// covered.
10428    ///
10429    /// It also seals the stores the plane holds — the journal's payloads, case
10430    /// state, task proposals and buffered event payloads — at
10431    /// [`build`](Self::build), so the order they were registered in cannot lose
10432    /// the guarantee — a store registered after this call is sealed just the
10433    /// same.
10434    ///
10435    /// **Governed memory is the one store this does not reach.**
10436    /// [`EncryptedMemoryStore`](crate::keyring::EncryptedMemoryStore)
10437    /// serialises subject erasure against writes and legal-hold changes with a
10438    /// process-local mutex, so it holds its contract on a single-writer
10439    /// deployment and nowhere else; wrapping it here would hand that adapter to
10440    /// an active-active `PostgreSQL` plane, where the mutex coordinates nothing
10441    /// and the hold race it exists to prevent is the result. Its erasure unit
10442    /// is one item version (`memory-item/<id>@<version>`) and outlives every
10443    /// case, so `erase_case` is not the act that reaches it either. Wrap it
10444    /// yourself, where the deployment's topology is visible:
10445    ///
10446    /// ```ignore
10447    /// let memories = EncryptedMemoryStore::new(inner, keys.clone(), tenant.clone());
10448    /// Runtime::builder(store).memory(Arc::new(memories)).keyring(keys).build()
10449    /// ```
10450    ///
10451    /// Without one, bytes are stored as given and erasure remains deletion.
10452    #[cfg(feature = "keyring")]
10453    #[must_use]
10454    pub fn keyring(mut self, keyring: Arc<dyn crate::keyring::KeyRing>) -> Self {
10455        self.keyring = Some(keyring);
10456        self
10457    }
10458
10459    /// Wrap every store a key ring can seal, at **build** time.
10460    ///
10461    /// One call, one guarantee. A deployer who configures a key ring reads it
10462    /// as *this plane is encrypted*, so the journal, the case store, the
10463    /// worklist, the event buffer and the blobs are sealed together. Five
10464    /// independent wrapping calls would be a control that can be forgotten
10465    /// four times, and forgetting looks exactly like remembering.
10466    ///
10467    /// Wrapped here rather than inside `keyring()` for the reason the tool
10468    /// catalogue is checked here: a wrap applied when the ring is supplied
10469    /// would cover only the stores registered so far, so `keyring(..)` before
10470    /// `cases(..)` would silently seal less than `keyring(..)` after it. An
10471    /// enforcement a reordering can lose is one a reformatter can delete.
10472    ///
10473    /// Every decorator is handed `self.tenant` rather than reading a name back
10474    /// out of the store it wraps, so all four derive the erasure scope from one
10475    /// value. `keyring()`'s own documentation names the one store deliberately
10476    /// left out, and why.
10477    #[cfg(feature = "keyring")]
10478    fn seal_stores(&mut self) {
10479        let Some(keys) = self.keyring.clone() else {
10480            return;
10481        };
10482        let tenant = self.tenant.clone();
10483        self.store = crate::keyring::SealedJournal::wrap(
10484            Arc::clone(&self.store),
10485            Arc::clone(&keys),
10486            tenant.clone(),
10487        );
10488        if let Some(cases) = self.cases.take() {
10489            self.cases = Some(crate::keyring::SealedCases::wrap(
10490                cases,
10491                Arc::clone(&keys),
10492                tenant.clone(),
10493            ));
10494        }
10495        if let Some(events) = self.events.take() {
10496            self.events = Some(crate::keyring::SealedEvents::wrap(
10497                events,
10498                Arc::clone(&keys),
10499                tenant.clone(),
10500            ));
10501        }
10502        // The outbox's store keeps the operator destinations' bearer tokens —
10503        // credentials like any caller's, and the one store here reached through
10504        // a handle the embedder built rather than registered. Sealing it in the
10505        // same sweep is what keeps `keyring()` one decision rather than five
10506        // and a sixth.
10507        #[cfg(feature = "push")]
10508        if let Some(outbox) = self.outbox.take() {
10509            self.outbox = Some(Arc::new(
10510                outbox
10511                    .as_ref()
10512                    .clone()
10513                    .sealed(Arc::clone(&keys), tenant.clone()),
10514            ));
10515        }
10516        if let Some(tasks) = self.tasks.take() {
10517            self.tasks = Some(crate::keyring::SealedTasks::wrap(tasks, keys, tenant));
10518        }
10519    }
10520
10521    /// Without the `keyring` feature there is nothing to seal.
10522    ///
10523    /// `&mut self` matches the sealing arm above, which does mutate. Taking
10524    /// `&self` here would make the signature depend on a feature flag, and
10525    /// every caller with it.
10526    #[cfg(not(feature = "keyring"))]
10527    #[allow(clippy::unused_self, clippy::needless_pass_by_ref_mut)]
10528    fn seal_stores(&mut self) {}
10529
10530    /// Supply the store that tracks batch items.
10531    ///
10532    /// Only needed for [`Runtime::run_batch`]; a plane that runs no batches does
10533    /// not need one, and asking for it unconditionally would make the common
10534    /// case carry the uncommon one's setup.
10535    #[must_use]
10536    pub fn batches(mut self, batches: Arc<dyn crate::batch::BatchStore>) -> Self {
10537        self.batches = Some(batches);
10538        self
10539    }
10540
10541    /// Supply the authorization engine.
10542    ///
10543    /// Without one there is no policy layer — the information-flow gates still
10544    /// apply, but nothing asks whether the principal was allowed. That is a
10545    /// deliberate absence rather than a permissive default: see `core::policy`
10546    /// on why there is no `AllowAll` to configure by mistake.
10547    ///
10548    /// The engine's complete immutable bundle identity is recorded at admission,
10549    /// so both whether policy was on and exactly which executable semantics
10550    /// governed the run are answerable from the journal. An open run may resume
10551    /// only under that same identity.
10552    #[must_use]
10553    pub fn policy(mut self, policy: Arc<dyn crate::core::PolicyEngine>) -> Self {
10554        self.policy = Some(policy);
10555        self
10556    }
10557
10558    /// The chain this plane's own runs act under.
10559    ///
10560    /// The identity of a run the embedder starts in-process — checked against
10561    /// the plan at admission (the plan is the authorization graph, so one that
10562    /// exceeds the chain's authority never starts), held to the chain's
10563    /// audience and validity, and journaled, so "on whose behalf" is
10564    /// answerable from history rather than reconstructed from timestamps.
10565    ///
10566    /// It is **not** the identity of a run a served surface admits for a
10567    /// caller: those act under the chain the caller presented, or under none
10568    /// when it presented none ([`RunTerms::served`]). A plane that bound its own
10569    /// chain to every peer's run would be an ambient credential.
10570    ///
10571    /// Verification of the *credential* belongs to whatever authenticates the
10572    /// caller; what arrives here is already a chain, and its attenuation is
10573    /// guaranteed by its own constructors however it was obtained.
10574    #[must_use]
10575    pub fn acting_as(mut self, chain: crate::core::Delegation) -> Self {
10576        self.identity = Some(chain);
10577        self
10578    }
10579
10580    /// The peers this plane may call, and the transport that reaches them.
10581    ///
10582    /// The registry is wiring — what each peer is granted, which credential
10583    /// reaches it, how a failed call recovers. What a run may *ask* a peer
10584    /// for is a manifest grant, `tool://<peer>/<capability>`, dispatched
10585    /// through [`StepCtx::call_peer`](crate::runtime::StepCtx::call_peer).
10586    /// Refused at build: a peer named `agent` or like a wired tool server,
10587    /// and a grant naming a capability outside the peer's registry scope.
10588    #[must_use]
10589    pub fn peers(
10590        mut self,
10591        registry: crate::peers::PeerRegistry,
10592        client: Arc<dyn crate::peers::PeerClient>,
10593    ) -> Self {
10594        self.peers = Some(Arc::new(PeerWiring { registry, client }));
10595        self
10596    }
10597
10598    /// Submit this plane's checkpoints to witnesses, and hold each round to a
10599    /// quorum.
10600    ///
10601    /// This is the only control here whose value comes from *not* being this
10602    /// plane. The hash chain proves no record was edited, the signatures say
10603    /// which workload wrote them, and the Merkle log proves no sealed run was
10604    /// removed — but every input to all three comes from the party an auditor
10605    /// is being asked to trust. None of them detects a whole run deleted
10606    /// before anyone took a checkpoint, or two internally perfect histories
10607    /// shown to two auditors. A witness keeps the last checkpoint it saw and
10608    /// cosigns only a checkpoint that provably extends it, so those two stop
10609    /// being invisible.
10610    ///
10611    /// Availability never waits on it: submission runs on
10612    /// [`sweep`](Runtime::sweep), after sealing and off the run path, and a
10613    /// round that falls short is a report rather than a refusal. A run whose
10614    /// witnesses are unreachable finished long ago, and making the plane's
10615    /// availability depend on a third party is the wrong trade for evidence
10616    /// that is read after the fact.
10617    ///
10618    /// Point it at witnesses somebody else runs. A witness hosted beside the
10619    /// history it vouches for is an anchor one compromise removes, and
10620    /// [`MemoryWitness`](crate::journal::MemoryWitness) says so at its own
10621    /// definition.
10622    ///
10623    /// # Errors
10624    ///
10625    /// [`build`](RuntimeBuilder::build) refuses a quorum larger than the
10626    /// number of witnesses configured, and a witness list with no quorum.
10627    #[must_use]
10628    pub fn witnesses(
10629        mut self,
10630        witnesses: Vec<Arc<dyn crate::journal::Witness>>,
10631        quorum: crate::journal::WitnessQuorum,
10632    ) -> Self {
10633        self.witnesses = witnesses;
10634        self.quorum = Some(quorum);
10635        self
10636    }
10637
10638    /// Declare the maximum time between checkpoint submissions to each
10639    /// witness: [`sweep`](Runtime::sweep) re-submits an unchanged checkpoint
10640    /// once it has passed since the last round that met quorum, so an idle
10641    /// plane still carries a fresh witness timestamp. Without it an unchanged
10642    /// log is never re-submitted.
10643    ///
10644    /// The interval is kept only as often as `sweep` runs; a deployment
10645    /// sweeping on its own scheduler owns that cadence.
10646    ///
10647    /// # Errors
10648    ///
10649    /// [`build`](RuntimeBuilder::build) refuses an interval with no witnesses.
10650    #[must_use]
10651    pub fn witness_interval(mut self, interval: std::time::Duration) -> Self {
10652        self.witness_interval = Some(interval);
10653        self
10654    }
10655
10656    /// Supply the calendar that resolves deadline descriptions to instants.
10657    ///
10658    /// Defaults to [`WallClock`], which understands plain offsets and refuses
10659    /// anything it does not know rather than approximating it. Domain calendars
10660    /// — working days, holidays, cut-off hours — are the adapter's job.
10661    #[must_use]
10662    pub fn calendar(mut self, calendar: Arc<dyn Calendar>) -> Self {
10663        self.calendar = Some(calendar);
10664        self
10665    }
10666
10667    /// Derive the tool catalogue from the agents and refuse a disagreement.
10668    ///
10669    /// Every agent, not the first: a plane may host several, and a tool granted
10670    /// to none of them is still a tool this binary can be asked for.
10671    ///
10672    /// # Errors
10673    ///
10674    /// If the box and any agent's manifest disagree. A build-time
10675    /// misconfiguration has a fix and no recovery, and it is refused the way the
10676    /// tenant mismatch beside it is: before anything runs.
10677    #[cfg(feature = "manifest")]
10678    fn settle_toolbox(&mut self) -> Result<(), BuildError> {
10679        let servers = std::mem::take(&mut self.tool_servers);
10680        let tools = self.toolbox.take();
10681        // A peer's name is a server name a grant can carry, so it must mean
10682        // exactly one thing on this plane: not `agent`, and not a name some
10683        // tool transport also answers for.
10684        let peer_names: std::collections::BTreeSet<String> = self
10685            .peers
10686            .as_ref()
10687            .map(|wiring| wiring.registry.peers().map(ToString::to_string).collect())
10688            .unwrap_or_default();
10689        if peer_names.contains(crate::tools::AGENT_SERVER) {
10690            return Err(BuildError::ReservedToolServer);
10691        }
10692        for name in &peer_names {
10693            if servers.iter().any(|(server, _)| server == name)
10694                || tools
10695                    .as_ref()
10696                    .is_some_and(|t| t.servers().any(|s| s == name))
10697            {
10698                return Err(BuildError::PeerIsAlsoAToolServer {
10699                    server: name.clone(),
10700                });
10701            }
10702        }
10703        if tools.is_none() && servers.is_empty() {
10704            return Ok(());
10705        }
10706        let tools = tools.unwrap_or_default();
10707        let remote_servers: std::collections::BTreeSet<String> =
10708            servers.iter().map(|(name, _)| name.clone()).collect();
10709        // What a grant may name without this box implementing it: a wired
10710        // transport, or a registered peer.
10711        let reachable_elsewhere: std::collections::BTreeSet<String> =
10712            remote_servers.union(&peer_names).cloned().collect();
10713        // `agent` names agents on this plane, and only them. A transport or a
10714        // typed tool under that name would let a deployment decide whether a
10715        // reviewed grant means "an agent here" or "somebody's server".
10716        if remote_servers.contains(crate::tools::AGENT_SERVER)
10717            || tools
10718                .servers()
10719                .any(|server| server == crate::tools::AGENT_SERVER)
10720        {
10721            return Err(BuildError::ReservedToolServer);
10722        }
10723        if remote_servers.len() != servers.len() {
10724            // The set lost an entry, so some name appears twice. Naming it beats
10725            // reporting a count a reader then has to go and diff by hand.
10726            let mut seen = std::collections::BTreeSet::new();
10727            let duplicate = servers
10728                .iter()
10729                .map(|(name, _)| name)
10730                .find(|name| !seen.insert((*name).clone()))
10731                .cloned()
10732                .unwrap_or_default();
10733            return Err(BuildError::DuplicateToolServer { server: duplicate });
10734        }
10735        // Both forms wired is not a merge and must not silently be one. The
10736        // hand-built catalogue is the operator saying something deliberate; the
10737        // derived one is the agent's declaration. Overwriting either with the
10738        // other would run a plane under grants nobody chose.
10739        if self.tools.is_some() {
10740            return Err(BuildError::ToolsWiredTwice);
10741        }
10742        let mut catalog = crate::tools::ToolCatalog::new();
10743        let mut declared = 0usize;
10744        // Which agent's declaration a tool's catalogue entry came from, so a
10745        // second agent declaring the same tool *differently* is a build-time
10746        // refusal rather than a silent overwrite.
10747        //
10748        // A plane has one catalogue and its agents have one manifest each, so
10749        // two agents granting `tool://ledger/read` with different protected
10750        // fields cannot both be satisfied. Merging by last-writer would resolve
10751        // it by **registration order** — the one thing this builder already
10752        // says an enforcement must never depend on — and it fails at a
10753        // distance: `declared` compares each agent's manifest against the
10754        // catalogue-derived descriptor exactly, so the agent that lost the race
10755        // is refused *every* call to that tool, in production, with a message
10756        // blaming a code-versus-manifest drift that neither file exhibits.
10757        let mut source: BTreeMap<crate::tools::ToolId, (String, crate::tools::ToolSafety)> =
10758            BTreeMap::new();
10759        for agent in &self.agents {
10760            let Some(manifest) = agent.manifest.as_ref() else {
10761                continue;
10762            };
10763            declared += 1;
10764            tools
10765                .check_against(manifest, &reachable_elsewhere)
10766                .map_err(|problems| BuildError::ToolDrift {
10767                    agent: manifest.metadata.name.clone(),
10768                    problems,
10769                })?;
10770            for (id, safety) in crate::tools::ToolCatalog::from_manifest(manifest).entries() {
10771                if let Some((first, existing)) = source.get(&id) {
10772                    if existing != &safety {
10773                        return Err(BuildError::ToolDeclaredTwoWays {
10774                            tool: id.reference(),
10775                            first: first.clone(),
10776                            second: manifest.metadata.name.clone(),
10777                        });
10778                    }
10779                    continue;
10780                }
10781                source.insert(id.clone(), (manifest.metadata.name.clone(), safety.clone()));
10782                catalog = catalog.allow(id, safety);
10783            }
10784        }
10785        // A box with nothing to be coherent *with* is the same defect one step
10786        // earlier: tools wired to a plane where no declaration admits them, so
10787        // nothing a reviewer reads describes what this binary can reach.
10788        if declared == 0 {
10789            return Err(BuildError::ToolsWithoutDeclaration);
10790        }
10791        // The typed argument type is the schema source. Overlay its
10792        // presentation only after every manifest has been checked, so the
10793        // model sees exactly what the body will deserialize rather than the
10794        // old permissive `{ type: object }` fallback.
10795        for id in tools.ids() {
10796            let (description, schema, _) = tools
10797                .declared(id)
10798                .expect("every registered typed tool has a declaration");
10799            let reviewed_description = catalog
10800                .declaration(id)
10801                .map_or_else(|| description.to_owned(), |(text, _)| text.to_owned());
10802            catalog = catalog.declare(id.clone(), reviewed_description, schema.clone());
10803        }
10804        // One client per server, resolved by the name a grant carries. A single
10805        // client handed every id could not tell `tool://ledger/read` from
10806        // `tool://tickets/read`, and a transport that never reads the server
10807        // component answers both — from whichever server it happens to hold.
10808        let router = servers.into_iter().fold(
10809            crate::tools::ToolRouter::new().toolbox(&Arc::new(tools)),
10810            |router, (name, client)| router.server(name, client),
10811        );
10812        self.tools = Some((
10813            Arc::new(catalog),
10814            Arc::new(router) as Arc<dyn crate::tools::ToolClient>,
10815        ));
10816        Ok(())
10817    }
10818
10819    /// Settle the tool catalogue: derive it if asked, then hold it to the
10820    /// declarations.
10821    ///
10822    /// One call because the two halves are one decision seen from either side.
10823    /// [`settle_toolbox`](Self::settle_toolbox) covers the derived catalogue,
10824    /// where code and manifest could disagree; the check after it covers the
10825    /// stated one, where operator and manifest could. Whichever way the
10826    /// catalogue arrived, it is checked before anything runs.
10827    #[cfg(feature = "manifest")]
10828    fn settle_tools(&mut self) -> Result<(), BuildError> {
10829        self.settle_toolbox()?;
10830        self.check_catalogue_not_laxer_than_grants()
10831    }
10832
10833    /// Refuse a stated catalogue that is **laxer** than a reviewed grant.
10834    ///
10835    /// `toolbox(..)` derives the catalogue from the manifests, so the two
10836    /// cannot drift. `tools(..)` states it by hand, and there the operator's
10837    /// entry and the agent's declaration are two copies of one decision — with
10838    /// nothing, until this, that noticed them disagreeing.
10839    ///
10840    /// Only one direction is a defect, the same one
10841    /// [`ToolBox::check_against`](crate::tools::ToolBox::check_against) refuses.
10842    /// An operator being **more** cautious than the declaration is fine and
10843    /// often right. An operator being **less** cautious is not, and it changes
10844    /// two things at once:
10845    ///
10846    /// * the whole-value taint gate stops firing, so model-chosen arguments
10847    ///   reach something that changes the world;
10848    /// * `ToolSafety::read_only` carries `Recovery::Retry`, so a timed-out call
10849    ///   to a money-moving tool is sent a second time.
10850    ///
10851    /// The dispatch gates already take the stricter `mutates` of the two, so
10852    /// the first is contained at runtime. The second is not — recovery is read
10853    /// from the catalogue alone — and neither should have to be, because this
10854    /// is a wiring mistake with a fix and no recovery. It is refused here,
10855    /// beside the rest of them.
10856    #[cfg(feature = "manifest")]
10857    fn check_catalogue_not_laxer_than_grants(&self) -> Result<(), BuildError> {
10858        let Some((catalog, _)) = self.tools.as_ref() else {
10859            return Ok(());
10860        };
10861        let mut problems = Vec::new();
10862        for agent in &self.agents {
10863            let Some(manifest) = agent.manifest.as_ref() else {
10864                continue;
10865            };
10866            for grant in &manifest.spec.tools {
10867                if !grant.mutates {
10868                    continue;
10869                }
10870                let Some(id) = crate::tools::ToolId::parse(&grant.reference) else {
10871                    continue;
10872                };
10873                if catalog.safety(&id).is_some_and(|s| !s.mutates) {
10874                    problems.push(format!(
10875                        "agent '{}' grants '{}' as mutating and the stated catalogue \
10876                         calls it read-only",
10877                        manifest.metadata.name, grant.reference
10878                    ));
10879                }
10880            }
10881        }
10882        if problems.is_empty() {
10883            Ok(())
10884        } else {
10885            Err(BuildError::CatalogueLaxerThanGrant { problems })
10886        }
10887    }
10888
10889    /// Assemble the runtime, or panic naming the wiring mistake.
10890    ///
10891    /// The ordinary entry point. Every refusal below is a bug in code the author
10892    /// is looking at, so propagating it through `?` to a `main` that prints it
10893    /// is ceremony around an abort — and each is caught here at startup rather
10894    /// than at dispatch, in production, where the cost is a run that has already
10895    /// begun.
10896    ///
10897    /// Use [`try_build`](Self::try_build) where a manifest arrives at *runtime*
10898    /// — read from disk, pinned by a registry, or supplied per tenant. There a
10899    /// bad declaration is an input rather than a bug, and a panic would take
10900    /// every other tenant in the process down to report it.
10901    ///
10902    /// # Panics
10903    ///
10904    /// On any [`BuildError`]:
10905    ///
10906    /// * A manifest declares a capability in `spec.capabilities.provides` that
10907    ///   no registered skill provides. **An agent has skills**, so a declaration
10908    ///   advertising one it cannot perform is a card that lies.
10909    /// * Two agents claim the same capability. Dispatch resolves a capability to
10910    ///   one skill *and to the manifest governing it*, so a second claim would
10911    ///   silently take the first's work out from under the first's budget,
10912    ///   model grants and egress ceiling.
10913    /// * Two skills share a name. A name is what a capability resolves to and
10914    ///   what governance is keyed on; two of them make both lookups arbitrary.
10915    /// * A stated catalogue calls a tool read-only that a reviewed manifest
10916    ///   grants as mutating. That exemption drops the whole-value taint gate
10917    ///   and makes a timed-out money-moving call retryable — the one direction
10918    ///   an operator cannot be right about.
10919    /// * A declarative agent names a provider no driver is registered for, or
10920    ///   declares `spec.execution` without a privileged model to call.
10921    /// * A plane and its store — or its blob store — are scoped to different
10922    ///   tenants. The two are set
10923    ///   separately — this builder's tenant scopes data keys and the policy
10924    ///   request, `for_tenant` scopes the store's keys — and the mismatch does
10925    ///   not show up at runtime. It *works*, and writes this tenant's runs into
10926    ///   another's keyspace while every erasure and every policy request names
10927    ///   the right one.
10928    ///
10929    /// # Long-running services want [`try_build`](Self::try_build)
10930    ///
10931    /// This panics, which is the honest answer for a binary wiring its own
10932    /// skills: every variant above is a bug in code the author is looking at,
10933    /// and aborting reports it at the moment it can be fixed. A daemon is a
10934    /// different case — it wants to exit with a diagnostic, and a plane
10935    /// assembled from a manifest that arrived at runtime is handling an *input*,
10936    /// where a panic reports one tenant's typo by killing every other tenant's
10937    /// in-flight run. `try_build` returns the same `BuildError` instead. One
10938    /// implementation underneath both, so they cannot disagree about what is
10939    /// refused.
10940    #[must_use]
10941    pub fn build(self) -> Arc<Runtime> {
10942        // Not `expect`. That formats the error with `Debug`, which would print
10943        // `AdvertisesWhatItCannotProvide { agent: "…", missing: [...] }` — the
10944        // variant's *shape* — while the sentence explaining what to do about it
10945        // lives in `Display`. Panicking with `{error}` keeps the two entry
10946        // points telling one story, which is the whole point of `build` being
10947        // `try_build` underneath.
10948        match self.try_build() {
10949            Ok(runtime) => runtime,
10950            Err(error) => panic!("{error}"),
10951        }
10952    }
10953
10954    /// The driver for a declarative agent's privileged model — named rather
10955    /// than defaulted, because falling back to some other registered driver
10956    /// would run the agent on a model its own declaration does not name.
10957    /// Every declared content check names a registered checker, and maps only
10958    /// categories that checker declares.
10959    #[cfg(feature = "manifest")]
10960    fn check_content_checkers(
10961        checkers: &crate::content::Checkers,
10962        m: &crate::manifest::Manifest,
10963    ) -> Result<(), BuildError> {
10964        let Some(content) = m.spec.security.content.as_ref() else {
10965            return Ok(());
10966        };
10967        for check in &content.checks {
10968            let refused = |detail: String| BuildError::ContentCheck {
10969                agent: m.metadata.name.clone(),
10970                check: check.id.clone(),
10971                detail,
10972            };
10973            let Some(checker) = checkers.get(&check.checker) else {
10974                return Err(refused(format!(
10975                    "no checker named '{}' is registered on this plane",
10976                    check.checker
10977                )));
10978            };
10979            if let Some(unknown) = check
10980                .on
10981                .keys()
10982                .find(|category| !checker.categories().contains(*category))
10983            {
10984                return Err(refused(format!(
10985                    "'{unknown}' is not a category checker '{}' declares",
10986                    check.checker
10987                )));
10988            }
10989        }
10990        Ok(())
10991    }
10992
10993    #[cfg(feature = "manifest")]
10994    fn declared_provider(
10995        providers: &HashMap<String, Arc<dyn crate::model::ModelProvider>>,
10996        m: &crate::manifest::Manifest,
10997    ) -> Result<Arc<dyn crate::model::ModelProvider>, BuildError> {
10998        let model = m
10999            .spec
11000            .models
11001            .as_ref()
11002            .and_then(|x| x.privileged.as_ref())
11003            .ok_or_else(|| BuildError::DeclarativeWithoutModel {
11004                agent: m.metadata.name.clone(),
11005            })?;
11006        providers
11007            .get(&model.provider)
11008            .map(Arc::clone)
11009            .ok_or_else(|| BuildError::UnknownProvider {
11010                agent: m.metadata.name.clone(),
11011                provider: model.provider.clone(),
11012            })
11013    }
11014
11015    /// Assemble the runtime, or say why it cannot be.
11016    ///
11017    /// The same checks as [`build`](Self::build), returned rather than raised.
11018    /// One implementation behind both, so they cannot come to disagree about
11019    /// what is refused.
11020    ///
11021    /// # Errors
11022    ///
11023    /// Any [`BuildError`] — see [`build`](Self::build) for what each means.
11024    // `mut` is for `settle_toolbox`, which only exists when manifests do.
11025    #[cfg_attr(not(feature = "manifest"), allow(unused_mut))]
11026    #[allow(clippy::too_many_lines)]
11027    pub fn try_build(mut self) -> Result<Arc<Runtime>, BuildError> {
11028        if self.lease_ttl < MIN_LEASE_TTL {
11029            return Err(BuildError::LeaseUnrenewable {
11030                ttl: self.lease_ttl,
11031                minimum: MIN_LEASE_TTL,
11032            });
11033        }
11034        // A ceiling of zero is a budget already spent, and it refuses the
11035        // first effect of every run this plane will ever make. The manifest
11036        // refuses one at parse; a plane wired in Rust reaches the same `Budget`
11037        // without passing a parser, so the rule is applied at both doors from
11038        // the one definition in `Budget::bricked_ceiling`.
11039        if let Some(field) = self.budget.bricked_ceiling() {
11040            return Err(BuildError::BudgetPermitsNothing { field });
11041        }
11042        if let Some(engine) = self.policy.as_ref() {
11043            preflight_policy(engine.as_ref(), self.identity.as_ref())?;
11044        }
11045        // Before `seal_stores`, necessarily: sealing wraps each handle in one
11046        // scoped to the *plane's* tenant, after which every store agrees with
11047        // the plane and the question cannot be asked.
11048        // Both halves of push, because either can be wired to another
11049        // tenant's rows and the consequence is the same: registrations,
11050        // cursors and credentials filed under a scope this plane's erasure
11051        // never names.
11052        #[cfg(feature = "push")]
11053        let push = self
11054            .push
11055            .as_ref()
11056            .map(|s| s.tenant())
11057            .or_else(|| self.outbox.as_ref().map(|o| o.store_tenant()));
11058        #[cfg(not(feature = "push"))]
11059        let push: Option<&str> = None;
11060        check_same_tenant(
11061            self.store.as_ref(),
11062            self.blobs.as_ref(),
11063            self.memories.as_ref(),
11064            &[
11065                ("case", self.cases.as_ref().map(|s| s.tenant())),
11066                ("event", self.events.as_ref().map(|s| s.tenant())),
11067                ("task", self.tasks.as_ref().map(|s| s.tenant())),
11068                ("memory", self.memories.as_ref().map(|s| s.tenant())),
11069                ("quota", self.quotas.as_ref().map(|s| s.tenant())),
11070                ("timer", self.timers.as_ref().map(|s| s.tenant())),
11071                ("batch", self.batches.as_ref().map(|s| s.tenant())),
11072                ("authority", self.authorities.as_ref().map(|s| s.tenant())),
11073                ("push", push),
11074            ],
11075            &self.tenant,
11076        )?;
11077        // The one wiring mistake here that would otherwise never fail: a
11078        // cross-space query ranks rather than refusing.
11079        if let Some(semantic) = &self.semantic {
11080            let embedder = semantic.embedder.revision();
11081            let index = semantic.retriever.index().query_revision;
11082            if embedder != index {
11083                return Err(BuildError::EmbeddingSpaceMismatch { embedder, index });
11084            }
11085            if semantic.embedder.pricing().is_none() && self.states_a_money_ceiling() {
11086                return Err(BuildError::UnpricedEmbedder { embedder });
11087            }
11088            let Some(memories) = self.memories.take() else {
11089                return Err(BuildError::SemanticMemoryWithoutStore);
11090            };
11091            // Every erasure through the plane's store — and through a sealed
11092            // store's own subject erasure — reaches the index built from it.
11093            let reached = memories.erasure_index();
11094            let indexed: Arc<dyn crate::memory::MemoryStore> = match reached {
11095                Some(index) if Arc::ptr_eq(&index, &semantic.retriever) => memories,
11096                Some(_) => return Err(BuildError::MemoryIndexedElsewhere),
11097                None if memories.seals() => {
11098                    return Err(BuildError::SealedMemoryMissesIndex);
11099                }
11100                None => Arc::new(crate::memory::IndexedMemoryStore::new(
11101                    Arc::clone(&memories),
11102                    Arc::clone(&semantic.retriever),
11103                )),
11104            };
11105            self.memories = Some(indexed);
11106        }
11107        self.seal_stores();
11108        #[cfg(feature = "manifest")]
11109        self.settle_tools()?;
11110
11111        let mut skills = HashMap::new();
11112        let mut by_capability = HashMap::new();
11113        #[cfg(feature = "manifest")]
11114        let mut governed_by: HashMap<String, Arc<crate::manifest::Manifest>> = HashMap::new();
11115        #[cfg(feature = "manifest")]
11116        let mut published_by: HashMap<String, crate::core::KeyId> = HashMap::new();
11117
11118        // Skills registered directly belong to the plane's anonymous agent: no
11119        // declaration, so nothing to enforce against them beyond the runtime's
11120        // own budget. That is a legitimate shape — not every agent needs a
11121        // manifest — and it is why `skill()` still exists beside `agent()`.
11122        for s in self.skills {
11123            register_skill(s, &mut by_capability, &mut skills)?;
11124        }
11125
11126        #[cfg(feature = "manifest")]
11127        for agent in self.agents {
11128            if let (Some(m), Some(key)) = (agent.manifest.as_ref(), agent.publisher.clone()) {
11129                published_by.insert(m.metadata.name.clone(), key);
11130            }
11131            let Some(m) = agent.manifest.clone() else {
11132                for s in agent.skills {
11133                    register_skill(s, &mut by_capability, &mut skills)?;
11134                }
11135                continue;
11136            };
11137
11138            // The capabilities this agent's *own* skills provide, which is what
11139            // its declaration is checked against below.
11140            let mut mine: HashSet<Capability> = HashSet::new();
11141            for s in agent.skills {
11142                mine.extend(s.descriptor().capabilities());
11143                let name = register_skill(s, &mut by_capability, &mut skills)?;
11144                governed_by.insert(name, Arc::clone(&m));
11145            }
11146
11147            // A declarative agent needs no skill: the runtime supplies the
11148            // behaviour its manifest asked for.
11149            if let Some(execution) = &m.spec.execution {
11150                // A `call` calls no model, so it needs no provider.
11151                let provider = if execution.kind == crate::manifest::ExecutionKind::Call {
11152                    None
11153                } else {
11154                    Some(Self::declared_provider(&self.providers, &m)?)
11155                };
11156                if m.spec.capabilities.provides.is_empty() {
11157                    return Err(BuildError::DeclarativeProvidesNothing {
11158                        agent: m.metadata.name.clone(),
11159                    });
11160                }
11161                // A plane whose only tools are agents needs no toolbox and no
11162                // transport: the catalogue is derived from the declaration and
11163                // dispatch is `commission`. The empty router is deliberate —
11164                // an agent-server call never reaches it, and anything else
11165                // arriving there is refused as unreachable rather than
11166                // silently absorbed.
11167                let tools = self.tools.clone().or_else(|| {
11168                    // Agents on this plane and registered peers: both
11169                    // dispatch through the runtime's own effects rather than
11170                    // a tool transport, so a declaration made only of them
11171                    // needs no box and no router.
11172                    let all_agent = !m.spec.tools.is_empty()
11173                        && m.spec.tools.iter().all(|g| {
11174                            crate::tools::ToolId::parse(&g.reference).is_some_and(|id| {
11175                                id.server == crate::tools::AGENT_SERVER
11176                                    || self.peers.as_ref().is_some_and(|wiring| {
11177                                        wiring
11178                                            .registry
11179                                            .grant(&crate::peers::PeerId::new(&id.server))
11180                                            .is_some()
11181                                    })
11182                            })
11183                        });
11184                    all_agent.then(|| {
11185                        (
11186                            Arc::new(crate::tools::ToolCatalog::from_manifest(&m)),
11187                            Arc::new(crate::tools::ToolRouter::new())
11188                                as Arc<dyn crate::tools::ToolClient>,
11189                        )
11190                    })
11191                });
11192                // A tool loop with nothing to reach fails on every run with the
11193                // same sentence, and the manifest that says so was read at
11194                // build. `planned` only needs a catalogue when it has grants;
11195                // `tool-calling` needs one to exist at all, since offering a
11196                // model no tools is not a tool loop.
11197                if tools.is_none() {
11198                    let needs = match execution.kind {
11199                        crate::manifest::ExecutionKind::ToolCalling => {
11200                            Some(if m.spec.tools.is_empty() {
11201                                "no tool grants".to_owned()
11202                            } else {
11203                                format!("{} tool grant(s)", m.spec.tools.len())
11204                            })
11205                        }
11206                        crate::manifest::ExecutionKind::Planned
11207                        | crate::manifest::ExecutionKind::Call
11208                            if !m.spec.tools.is_empty() =>
11209                        {
11210                            Some(format!("{} tool grant(s)", m.spec.tools.len()))
11211                        }
11212                        _ => None,
11213                    };
11214                    if let Some(grants) = needs {
11215                        return Err(BuildError::DeclarativeToolsUnreachable {
11216                            agent: m.metadata.name.clone(),
11217                            kind: execution.kind.as_str(),
11218                            grants,
11219                        });
11220                    }
11221                }
11222
11223                // Oversight needs somewhere to put the decision, and both
11224                // halves are known here. Left to run time it surfaces at the
11225                // first real approval — a person already waiting, on the code
11226                // path a test suite is least likely to reach.
11227                let declared = if m.spec.oversight.is_some() {
11228                    Some("`spec.oversight`".to_owned())
11229                } else if m.spec.tools.iter().any(|g| g.requires_approval) {
11230                    Some("a grant with `requires_approval: true`".to_owned())
11231                } else {
11232                    None
11233                };
11234                if let Some(declared) = declared {
11235                    // Ordered by what a reader fixes first: without a case
11236                    // there is nothing for a task to hang off.
11237                    //
11238                    // Timers are deliberately **not** required, and the test
11239                    // suite is why: four approval tests wire a case store and a
11240                    // worklist, no timers, and run an approval end to end. A
11241                    // task is opened, the run suspends, a person decides. What
11242                    // a timer store adds is the sweeper firing `on_expiry`, and
11243                    // refusing a configuration that works would be this check
11244                    // asserting more than it knows — the failure mode it exists
11245                    // to prevent, arriving from the other side.
11246                    let missing = if self.cases.is_none() {
11247                        Some(("case store", "cases"))
11248                    } else if self.tasks.is_none() {
11249                        Some(("worklist", "tasks"))
11250                    } else {
11251                        None
11252                    };
11253                    if let Some((missing, remedy)) = missing {
11254                        return Err(BuildError::OversightUnreachable {
11255                            agent: m.metadata.name.clone(),
11256                            declared,
11257                            missing,
11258                            remedy,
11259                        });
11260                    }
11261                }
11262
11263                // A recall reads before the first model call and formation
11264                // writes after the answer, so a missing store costs a run its
11265                // first effect or its whole budget depending on which half was
11266                // declared. Both facts are here: the file says memories are
11267                // reached, the plane says there is nowhere to reach.
11268                if let Some(memory) = &m.spec.memory {
11269                    let declared: [(&'static str, Option<&crate::manifest::MemorySubject>); 2] = [
11270                        (
11271                            "spec.memory.recall",
11272                            memory.recall.as_ref().map(|r| &r.subject),
11273                        ),
11274                        (
11275                            "spec.memory.formation",
11276                            memory.formation.as_ref().map(|f| &f.subject),
11277                        ),
11278                    ];
11279                    for (field, subject) in declared {
11280                        let Some(subject) = subject else { continue };
11281                        if self.memories.is_none() {
11282                            return Err(BuildError::MemoryWithoutStore {
11283                                agent: m.metadata.name.clone(),
11284                                declared: field,
11285                            });
11286                        }
11287                        if subject.needs_case() && self.cases.is_none() {
11288                            return Err(BuildError::MemorySubjectUnbindable {
11289                                agent: m.metadata.name.clone(),
11290                                subject: subject.as_written(),
11291                            });
11292                        }
11293                    }
11294                }
11295
11296                for cap in &m.spec.capabilities.provides {
11297                    let skill: Arc<dyn Skill> = Arc::new(super::declarative::Declarative::new(
11298                        execution.kind,
11299                        cap.clone(),
11300                        m.metadata.name.clone(),
11301                        provider.clone(),
11302                        tools.clone(),
11303                        execution.max_turns,
11304                    ));
11305                    mine.insert(Capability::new(cap.as_str()));
11306                    let name = register_skill(skill, &mut by_capability, &mut skills)?;
11307                    governed_by.insert(name, Arc::clone(&m));
11308                }
11309            }
11310
11311            check_declaration_matches_skills(&m, &mine)?;
11312        }
11313
11314        // Agent grants are validated against the finished plane, because the
11315        // capability they name may belong to an agent registered *later* — a
11316        // check inside the loop would pass or fail on registration order.
11317        #[cfg(feature = "manifest")]
11318        {
11319            let mut checked = std::collections::BTreeSet::new();
11320            for m in governed_by.values() {
11321                if !checked.insert(m.metadata.name.clone()) {
11322                    continue;
11323                }
11324                Self::check_content_checkers(&self.checkers, m)?;
11325                for grant in &m.spec.tools {
11326                    let Some(id) = crate::tools::ToolId::parse(&grant.reference) else {
11327                        continue;
11328                    };
11329                    // A grant naming a registered peer is a delegating hop,
11330                    // and both things that would refuse it on every run are
11331                    // knowable here: a capability the registry never gave the
11332                    // peer, and a granting agent whose own ceiling is zero.
11333                    if let Some(wiring) = self.peers.as_ref()
11334                        && let Some(peer_grant) = wiring
11335                            .registry
11336                            .grant(&crate::peers::PeerId::new(&id.server))
11337                    {
11338                        if !peer_grant.scope.permits(&Capability::new(id.tool.as_str())) {
11339                            return Err(BuildError::PeerGrantOutsideScope {
11340                                agent: m.metadata.name.clone(),
11341                                peer: id.server,
11342                                capability: id.tool,
11343                            });
11344                        }
11345                        let cannot_delegate = m
11346                            .spec
11347                            .topology
11348                            .as_ref()
11349                            .is_some_and(|t| t.role == crate::manifest::Role::Specialist)
11350                            || m.spec.security.max_delegation_depth == Some(0);
11351                        if cannot_delegate {
11352                            return Err(BuildError::PeerGrantOnASpecialist {
11353                                agent: m.metadata.name.clone(),
11354                                peer: id.server,
11355                            });
11356                        }
11357                        continue;
11358                    }
11359                    if id.server != crate::tools::AGENT_SERVER {
11360                        continue;
11361                    }
11362                    if m.spec.capabilities.provides.contains(&id.tool) {
11363                        return Err(BuildError::AgentToolSelfReference {
11364                            agent: m.metadata.name.clone(),
11365                            capability: id.tool,
11366                        });
11367                    }
11368                    if !by_capability.contains_key(&Capability::new(id.tool.as_str())) {
11369                        return Err(BuildError::AgentToolUnknownCapability {
11370                            agent: m.metadata.name.clone(),
11371                            capability: id.tool,
11372                        });
11373                    }
11374                }
11375            }
11376        }
11377
11378        // Every ceiling any declaration on this plane states, by tool: the
11379        // count is per tool, so the tightest binds every agent calling it.
11380        #[cfg(feature = "manifest")]
11381        let rates = {
11382            let mut declarations: Vec<&Arc<crate::manifest::Manifest>> =
11383                governed_by.values().collect();
11384            declarations.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
11385            let mut ceilings: BTreeMap<String, Vec<crate::quota::RateCeiling>> = BTreeMap::new();
11386            let mut first = None;
11387            for m in declarations {
11388                for grant in &m.spec.tools {
11389                    let Some(rate) = grant.rate_limit else {
11390                        continue;
11391                    };
11392                    first.get_or_insert_with(|| (m.metadata.name.clone(), grant.reference.clone()));
11393                    let stated = ceilings.entry(grant.reference.clone()).or_default();
11394                    let ceiling = crate::quota::RateCeiling::from(rate);
11395                    if !stated.contains(&ceiling) {
11396                        stated.push(ceiling);
11397                        stated.sort();
11398                    }
11399                }
11400            }
11401            if ceilings.is_empty() {
11402                None
11403            } else if let Some(store) = self.quotas.clone() {
11404                Some(Arc::new(super::ctx::Rates { store, ceilings }))
11405            } else {
11406                let (agent, grant) = first.unwrap_or_default();
11407                return Err(BuildError::RateLimitWithoutQuotaStore { agent, grant });
11408            }
11409        };
11410
11411        // Both or neither, checked here because a builder cannot see the other
11412        // half at the moment either is set.
11413        if self.witness_interval.is_some() && self.witnesses.is_empty() {
11414            return Err(BuildError::IntervalWithoutWitnesses);
11415        }
11416        let witnesses = match (self.witnesses.is_empty(), self.quorum) {
11417            (true, None) => None,
11418            (false, Some(quorum)) if quorum.required() <= self.witnesses.len() => {
11419                Some(Arc::new(Witnessing {
11420                    witnesses: self.witnesses,
11421                    quorum,
11422                    submitted: std::sync::atomic::AtomicU64::new(0),
11423                    interval: self.witness_interval,
11424                    last_met: std::sync::Mutex::new(None),
11425                }))
11426            }
11427            (empty, quorum) => {
11428                return Err(BuildError::WitnessQuorumUnreachable {
11429                    declared: quorum.map_or(0, |q| q.required()),
11430                    configured: if empty { 0 } else { self.witnesses.len() },
11431                });
11432            }
11433        };
11434
11435        Ok(Arc::new_cyclic(|self_ref| Runtime {
11436            self_ref: self_ref.clone(),
11437            witnesses,
11438            inflight: Arc::default(),
11439            commissions: Arc::default(),
11440            signer: self.signer,
11441            store: self.store,
11442            skills,
11443            by_capability,
11444            #[cfg(feature = "manifest")]
11445            published_by,
11446            #[cfg(feature = "manifest")]
11447            providers: self.providers,
11448            meter: super::metrics::Meter::new(self.tenant_label, &self.tenant),
11449            tenant: self.tenant,
11450            owner: self.owner.unwrap_or_else(default_owner),
11451            lease_ttl: self.lease_ttl,
11452            memories: self.memories,
11453            semantic: self.semantic,
11454            authorities: self.authorities,
11455            peers: self.peers,
11456            #[cfg(feature = "manifest")]
11457            tools: self.tools,
11458            journal_ceiling: self.journal_ceiling,
11459            #[cfg(feature = "manifest")]
11460            egress: self.egress,
11461            #[cfg(feature = "manifest")]
11462            checkers: self.checkers,
11463            streams: self.streams,
11464            quotas: self.quotas,
11465            #[cfg(feature = "manifest")]
11466            rates,
11467            quota: self.quota,
11468            budget: self.budget,
11469            require_verifier: self.require_verifier,
11470            cases: self.cases,
11471            events: self.events,
11472            tasks: self.tasks,
11473            timers: self.timers,
11474            blobs: self.blobs,
11475            disclosures: self.disclosures,
11476            #[cfg(feature = "keyring")]
11477            keyring: self.keyring,
11478            batches: self.batches,
11479            policy: self.policy,
11480            identity: self.identity,
11481            replanner: self.replanner,
11482            calendar: self.calendar.unwrap_or_else(|| Arc::new(WallClock)),
11483            #[cfg(feature = "push")]
11484            push: self.push,
11485            #[cfg(feature = "push")]
11486            outbox: self.outbox,
11487            #[cfg(feature = "manifest")]
11488            governed_by,
11489        }))
11490    }
11491}
11492
11493/// Ask the policy set the questions this plane will ask, before it asks them
11494/// for real.
11495///
11496/// Cedar evaluates **every** rule against **every** request, so a rule reading
11497/// an attribute a request does not carry does not quietly fail to match — it
11498/// errors, and an unevaluable rule may be the `forbid` that would have stopped
11499/// the call, so the gate refuses. One unguarded rule therefore denies every
11500/// effect of every run, from a policy set that compiled cleanly and validated
11501/// against its schema.
11502///
11503/// The probes cover every shape a runtime request takes: an effect that reads
11504/// and one that mutates, each with and without a label (only `sink` carries
11505/// one), a release, and an admission. `mutates` matters on its own: `&&`
11506/// short-circuits, so `context.mutates && context.label.trust == ..` evaluates
11507/// cleanly on a read and errors on every mutating call that carries no label.
11508///
11509/// The delegation attributes follow the plane. A plane with a chain carries
11510/// `owner`, `scope` and `delegation_depth` on every request, and a rule reading
11511/// them unguarded is correct there. A plane without one never carries them on
11512/// its own runs — and still admits runs under a caller's chain from a served
11513/// surface, so both shapes are asked.
11514///
11515/// What this does **not** establish: that the rules are *right*. A set that
11516/// permits everything passes here, as does one whose rules error only on a
11517/// shape carrying an attribute of an unexpected type. It answers one question
11518/// — can this plane's own requests be evaluated at all — which is the question
11519/// whose wrong answer looks like a working plane that refuses everything. The
11520/// served surfaces ask their own actions with their own contexts, and check
11521/// those where they are built.
11522#[cfg_attr(not(feature = "manifest"), allow(dead_code))]
11523fn preflight_policy(
11524    engine: &dyn crate::core::PolicyEngine,
11525    identity: Option<&crate::core::Delegation>,
11526) -> Result<(), BuildError> {
11527    let problems = runtime_policy_problems(engine, identity);
11528    if problems.is_empty() {
11529        return Ok(());
11530    }
11531    Err(BuildError::PolicyUnevaluable {
11532        problems: problems.join("; "),
11533    })
11534}
11535
11536/// The runtime tier's probes, answered by `engine`.
11537fn runtime_policy_problems(
11538    engine: &dyn crate::core::PolicyEngine,
11539    identity: Option<&crate::core::Delegation>,
11540) -> Vec<String> {
11541    use crate::policy::requests;
11542
11543    // Values are placeholders shaped like the real ones: a rule reading
11544    // `context.label.trust` on a release must find a label there, or the probe
11545    // reports a defect the real request would never hit. Nulls are absent for
11546    // the same reason — no gate sends one, and the adapter strips them.
11547    let run = RunId(ulid::Ulid::nil());
11548    let step = StepId(0);
11549    let label = crate::core::Label::untrusted(crate::core::SourceId::new("preflight"));
11550    let release = crate::core::Release::whole(
11551        crate::core::ReleaseScope::trust(),
11552        "preflight",
11553        "tool://preflight/probe",
11554        ["preflight"],
11555    );
11556    // A tool call's arguments are the one shape the runtime fixes for a kind,
11557    // so a rule scoped to `tool.call` may read `context.args.server` as-is.
11558    let tool_args = serde_json::json!({
11559        "server": "preflight",
11560        "tool": "preflight",
11561        "arguments": {},
11562        "protected_fields": [],
11563    });
11564
11565    // The chain shapes this plane's requests take. A chain of the plane's own
11566    // is on every request; without one, a run may still carry its caller's.
11567    let caller = crate::core::Delegation::root(crate::core::Principal::new(
11568        "preflight",
11569        crate::core::Scope::root(),
11570    ));
11571    let chains: Vec<Option<&crate::core::Delegation>> = match identity {
11572        Some(chain) => vec![Some(chain)],
11573        None => vec![None, Some(&caller)],
11574    };
11575
11576    // Built by the gates' own builders, so a probe cannot drift from the
11577    // shape it stands in for: an effect that reads and one that mutates, each
11578    // with and without a label (only `sink` carries one), a release, and an
11579    // admission.
11580    let mut built = Vec::new();
11581    for chain in chains {
11582        let acting = requests::Acting {
11583            tenant: "preflight",
11584            capability: "preflight.capability",
11585            agent: None,
11586            chain,
11587        };
11588        for (kind, args) in [
11589            ("preflight.effect", serde_json::json!({})),
11590            ("tool.call", tool_args.clone()),
11591        ] {
11592            for (mutates, labelled) in [(false, false), (true, false), (false, true), (true, true)]
11593            {
11594                built.push(requests::effect(
11595                    &acting,
11596                    run,
11597                    step,
11598                    kind,
11599                    &args,
11600                    mutates,
11601                    labelled.then_some(&label),
11602                ));
11603            }
11604        }
11605        built.push(requests::release(&acting, run, step, &release, &label));
11606        built.push(requests::admission(&acting, &serde_json::json!({})));
11607    }
11608    let requests: Vec<crate::core::PolicyRequest<'_>> = built
11609        .iter()
11610        .map(requests::GatedRequest::as_request)
11611        .collect();
11612
11613    // Asked through the engine's own preflight rather than by calling
11614    // `authorize` here. An engine written as Rust code has no missing-attribute
11615    // trap and nothing to report; the engine that has the trap implements the
11616    // method, and knows which resources its own rules name.
11617    let mut problems = engine.preflight(&requests);
11618    problems.sort();
11619    problems.dedup();
11620    problems
11621}
11622
11623/// Refuse a plane whose stores answer for somebody else.
11624///
11625/// The plane's tenant scopes its data keys and names its policy requests; each
11626/// store handle is scoped separately. Nothing at runtime looks wrong when the
11627/// two disagree — the work is written into another tenant's keyspace while
11628/// every erasure and every policy request names the right one — so the mismatch
11629/// is easy to make and invisible once made. Asking each store who it serves is
11630/// what turns it into a startup refusal.
11631///
11632/// **A store that does not override its accessor answers `default`.** That is
11633/// the honest answer for a single-tenant deployment and the reason the check
11634/// can be unconditional, but it is also why this catches a wiring mistake
11635/// rather than proving isolation: an implementation that is scoped to `acme`
11636/// and never overrode the accessor reports `default` and is refused, which is
11637/// the safe direction — the unsafe one would be an implementation that claims a
11638/// tenant it does not enforce, and no build-time question can settle that.
11639fn check_same_tenant(
11640    store: &dyn JournalStore,
11641    blobs: Option<&Arc<dyn crate::blob::BlobStore>>,
11642    memories: Option<&Arc<dyn crate::memory::MemoryStore>>,
11643    state: &[(&'static str, Option<&str>)],
11644    tenant: &crate::core::TenantId,
11645) -> Result<(), BuildError> {
11646    if let Some(blobs) = blobs
11647        && blobs.tenant() != tenant.as_str()
11648    {
11649        return Err(BuildError::BlobStoreTenant {
11650            plane: tenant.to_string(),
11651            store: blobs.tenant().to_owned(),
11652        });
11653    }
11654    if store.tenant() != tenant.as_str() {
11655        return Err(BuildError::JournalStoreTenant {
11656            plane: tenant.to_string(),
11657            store: store.tenant().to_owned(),
11658        });
11659    }
11660
11661    // The stores whose state a key ring seals, in the caller's fixed order so a
11662    // plane with two mismatches reports the same one every build — an error
11663    // that varies between runs of the same wiring sends an operator looking for
11664    // a race that is not there.
11665    for &(store, serves) in state {
11666        if let Some(serves) = serves
11667            && serves != tenant.as_str()
11668        {
11669            return Err(BuildError::StateStoreTenant {
11670                store,
11671                plane: tenant.to_string(),
11672                tenant: serves.to_owned(),
11673            });
11674        }
11675    }
11676
11677    // After the tenant checks, deliberately: a store scoped to the wrong tenant
11678    // is the more basic fault, and reporting the erasure lock first would send
11679    // an operator to fix the second-most-wrong thing.
11680    //
11681    // A process-local erasure lock beside a store two instances can write is a
11682    // control that reads as present and is not. The window it fails to close is
11683    // between an erasure's legal-hold check and its key destruction: the other
11684    // instance writes an item, that item is sealed under a scope about to stop
11685    // existing, and the erasure reports success. Both facts are here, so the
11686    // refusal is here.
11687    if store.is_shared() && memories.is_some_and(|m| m.erasure_is_distributed() == Some(false)) {
11688        return Err(BuildError::ErasureCoordinatorNotShared);
11689    }
11690    Ok(())
11691}
11692
11693/// Refuse an event from outside whose kind this plane mints for itself.
11694fn refuse_reserved_kind(event: &InboundEvent) -> Result<(), RuntimeError> {
11695    if crate::core::is_reserved_kind(&event.kind) {
11696        return Err(RuntimeError::ReservedEventKind {
11697            kind: event.kind.clone(),
11698        });
11699    }
11700    Ok(())
11701}
11702
11703/// Inbound event delivery.
11704impl Runtime {
11705    /// Deliver an inbound event, resuming whichever run was waiting for it.
11706    ///
11707    /// # Ordering
11708    ///
11709    /// The event is **stored before** anyone looks for a waiter. That ordering
11710    /// is the whole reason this works: a message can arrive before its run
11711    /// reaches the wait, and one that is matched-then-discarded leaves that run
11712    /// waiting forever for something that already happened.
11713    ///
11714    /// A [`Delivery::Buffered`] result is therefore normal and not an error —
11715    /// it means "held until someone asks". Only the sweep
11716    /// ([`EventStore::sweep_unclaimed`](crate::case::EventStore::sweep_unclaimed))
11717    /// decides an event is genuinely unroutable, because that is a claim about
11718    /// the future rather than about this instant.
11719    ///
11720    /// # Errors
11721    ///
11722    /// [`RuntimeError::ReservedEventKind`] for a kind in the `agentplane.`
11723    /// namespace, which only this plane mints; [`RuntimeError`] for a store
11724    /// failure or a refused resume.
11725    pub async fn deliver(&self, event: &InboundEvent) -> Result<Delivery, RuntimeError> {
11726        refuse_reserved_kind(event)?;
11727        self.deliver_buffered(event, true).await
11728    }
11729
11730    /// Deliver an event this plane minted itself — a worklist decision.
11731    ///
11732    /// The one path into the reserved namespace [`deliver`](Self::deliver)
11733    /// refuses from outside.
11734    pub(crate) async fn deliver_minted(
11735        &self,
11736        event: &InboundEvent,
11737    ) -> Result<Delivery, RuntimeError> {
11738        self.deliver_buffered(event, false).await
11739    }
11740
11741    /// Store `event`, then offer it to the oldest eligible waiter.
11742    ///
11743    /// `rematch` offers a duplicate again: the buffer and the match are two
11744    /// writes, and a counterparty's retry is the one chance to repair a crash
11745    /// between them. A minted decision is never re-offered, because its
11746    /// duplicate is the answer *already on record*, and offering that stored
11747    /// answer under a second decider's call would report it as theirs.
11748    async fn deliver_buffered(
11749        &self,
11750        event: &InboundEvent,
11751        rematch: bool,
11752    ) -> Result<Delivery, RuntimeError> {
11753        let events = self.events.as_ref().ok_or_else(|| {
11754            RuntimeError::PlanContract(
11755                "this runtime has no event store — build it with `.events(store)`".into(),
11756            )
11757        })?;
11758
11759        let now = now_for_admission();
11760
11761        // Durable first. Deduplication by event id makes a counterparty's retry
11762        // — and they all retry — harmless.
11763        let fresh = events
11764            .buffer(event, now)
11765            .await
11766            .map_err(RuntimeError::from_store)?;
11767
11768        // The match claims only an unclaimed, live row, so a retry of a
11769        // message already consumed matches nothing.
11770        if !fresh && !rematch {
11771            return Ok(Delivery::Duplicate);
11772        }
11773        let Some(sub) = events
11774            .match_waiter(event, now)
11775            .await
11776            .map_err(RuntimeError::from_store)?
11777        else {
11778            return Ok(if fresh {
11779                Delivery::Buffered
11780            } else {
11781                Delivery::Duplicate
11782            });
11783        };
11784
11785        self.resume_subscription(events, sub, event).await
11786    }
11787
11788    /// Deliver an inbound event to exactly `run`.
11789    ///
11790    /// This is the task-addressed counterpart to [`Runtime::deliver`]. It is
11791    /// used by protocols such as A2A where a follow-up carries a concrete task
11792    /// id. Correlation alone is insufficient there: two tasks may wait on the
11793    /// same business key, and resuming the oldest would violate the request.
11794    ///
11795    /// The event store atomically inserts and claims the event for this run. A
11796    /// run that is not waiting leaves no buffered event behind for another run.
11797    ///
11798    /// # Errors
11799    ///
11800    /// As [`deliver`](Self::deliver), and [`RuntimeError::PlanContract`] when
11801    /// `run` is not waiting for this input. Addressing a run by id is no way
11802    /// around the reserved namespace: a run waiting on a human task is waiting
11803    /// for exactly the kind this refuses.
11804    pub async fn deliver_to(
11805        &self,
11806        run: RunId,
11807        event: &InboundEvent,
11808    ) -> Result<Delivery, RuntimeError> {
11809        refuse_reserved_kind(event)?;
11810        let events = self.events.as_ref().ok_or_else(|| {
11811            RuntimeError::PlanContract(
11812                "this runtime has no event store — build it with `.events(store)`".into(),
11813            )
11814        })?;
11815        match events
11816            .deliver_to(run, event, now_for_admission())
11817            .await
11818            .map_err(RuntimeError::from_store)?
11819        {
11820            crate::case::TargetedDelivery::Duplicate => Ok(Delivery::Duplicate),
11821            crate::case::TargetedDelivery::NotWaiting => Err(RuntimeError::PlanContract(format!(
11822                "run {run} is not waiting for this input"
11823            ))),
11824            crate::case::TargetedDelivery::Matched(sub) => {
11825                self.resume_subscription(events, sub, event).await
11826            }
11827        }
11828    }
11829
11830    async fn resume_subscription(
11831        &self,
11832        events: &Arc<dyn crate::case::EventStore>,
11833        sub: crate::core::Subscription,
11834        event: &InboundEvent,
11835    ) -> Result<Delivery, RuntimeError> {
11836        // Record the event as the awaited effect's result, then let replay do
11837        // the rest: the resumed run reads it back like any other completed
11838        // effect, and none of the suspension machinery exists twice.
11839        //
11840        // The lease may be briefly held: an event can arrive in the window
11841        // between the run registering its subscription and its owner finishing
11842        // the suspension bookkeeping — the owner is *concluding*, and will
11843        // release within milliseconds. Failing that race would be fatal in the
11844        // quietest possible way: the delivery fails `LeaseHeld` **after** the
11845        // event is durably claimed for this run, the counterparty's retry
11846        // deduplicates against the claimed event, and nothing ever resumes the
11847        // run — a message that arrived in time, parked forever. So the claim
11848        // is retried under a bounded backoff; if the owner is genuinely
11849        // stalled past the bound, the event is *left claimed* for this run
11850        // and reported as buffered — the sweep's redelivery pass finds a
11851        // waiting subscription with a claimed event and finishes the job on a
11852        // later tick, so the bound bounds latency, never delivery.
11853        let mut backoff = Duration::from_millis(25);
11854        let lease = loop {
11855            match self
11856                .store
11857                .acquire(sub.run, &self.owner, self.lease_ttl)
11858                .await
11859            {
11860                Ok(lease) => break lease,
11861                Err(crate::core::StoreError::LeaseHeld { .. })
11862                    if backoff < Duration::from_secs(1) =>
11863                {
11864                    tokio::time::sleep(backoff).await;
11865                    backoff *= 2;
11866                }
11867                Err(crate::core::StoreError::LeaseHeld { .. }) => {
11868                    // The claim retired the subscription, so nothing lists
11869                    // this run as waiting any more — and the claimed event
11870                    // blocks every dedup'd retry of itself. Parking the pair
11871                    // is what keeps it findable: the sweep's redelivery pass
11872                    // walks parked waits, re-claims (a claim by the same run
11873                    // is returned, not filtered), and finishes the delivery
11874                    // once the owner concludes.
11875                    events
11876                        .park_wait(&sub, now_for_admission())
11877                        .await
11878                        .map_err(RuntimeError::from_store)?;
11879                    tracing::warn!(
11880                        run = %sub.run,
11881                        event = %event.id,
11882                        "an event was claimed for a run whose owner did not conclude \
11883                         within the retry window; the sweep will deliver it"
11884                    );
11885                    return Ok(Delivery::Buffered);
11886                }
11887                Err(e) => return Err(RuntimeError::from_store(e)),
11888            }
11889        };
11890
11891        let history = self
11892            .store
11893            .read(sub.run, 1)
11894            .await
11895            .map_err(RuntimeError::from_store)?;
11896        let already_recorded = history.iter().any(|record| {
11897            record.effect_key() == Some(sub.effect)
11898                && matches!(record.kind(), RecordKind::EffectDone { .. })
11899        });
11900        // A run whose conclusion is durable consumes nothing. An answer
11901        // appended after `RunConcluded` is one the run never acted on, and it
11902        // leaves a run its seal refuses forever. The resume below finishes it
11903        // instead — seals it and retires its waits, which hands this message,
11904        // still unconsumed, to the next waiter on its key.
11905        let closed = resume_is_closed(&history).is_some();
11906        if !already_recorded && !closed {
11907            // Judged here, where the output is recorded, under the run's own
11908            // declaration: the step that resumes reads the verdict back.
11909            let content = self.delivered_verdict(&history, &event.payload);
11910            self.store
11911                .append(
11912                    lease.epoch,
11913                    vec![{
11914                        let mut a = Append::new(
11915                            sub.run,
11916                            RecordKind::EffectDone {
11917                                output: event.payload.clone(),
11918                                // The sender, so a replayed run rebuilds the same
11919                                // provenance this delivery gave the value.
11920                                source: Some(event.source.clone()),
11921                                // And the operator who minted it, where one
11922                                // did: the same record on the same path, so a
11923                                // decision that resumes a suspended run is
11924                                // attributed exactly as one consumed inline.
11925                                by: event.by.clone(),
11926                                spend: crate::core::Spend::default(),
11927                                // An inbound payload is another party's data,
11928                                // which is exactly the point this lattice value
11929                                // names — the same one the run's own delivery
11930                                // path records.
11931                                declared: crate::core::DeclaredOutput::untrusted(),
11932                                content,
11933                                elapsed_ms: None,
11934                            },
11935                        )
11936                        .effect(sub.effect)
11937                        .step(sub.step)
11938                        .phase(sub.phase);
11939                        // Every record of a case-bound run carries its case, and a
11940                        // record written from outside the run is no exception.
11941                        if let Some(c) = sub.case {
11942                            a = a.case(c);
11943                        }
11944                        a
11945                    }],
11946                )
11947                .await
11948                .map_err(RuntimeError::from_store)?;
11949        }
11950
11951        if !closed {
11952            events
11953                .unsubscribe(sub.run, sub.effect)
11954                .await
11955                .map_err(RuntimeError::from_store)?;
11956        }
11957
11958        // The event is recorded as the awaited effect's result, and the resume
11959        // continues under the *same* lease. Releasing here and letting the
11960        // resume re-acquire opened the window this function's own doc names —
11961        // a crash between the release and the acquire left a released lease
11962        // over a run whose subscription was just retired: no driver, and
11963        // invisible to the abandonment queue, which lists only leases that
11964        // expired while still naming an owner. The handover keeps the run
11965        // owned continuously from delivery to conclusion.
11966        match self.resume_holding(sub.run, lease).await {
11967            // A `Fenced` means this lease lapsed mid-resume and somebody
11968            // claimed the run — a concurrent delivery, a recovery sweep. The
11969            // event is durably recorded, so whoever holds the lease replays
11970            // it; this delivery still delivered, and a sender told otherwise
11971            // retries into a duplicate.
11972            Ok(_) | Err(RuntimeError::Fenced { .. }) => {}
11973            // This plane holds no agent for the run — an operator's terminal
11974            // recording a decision. The answer is durable, and the lease is
11975            // kept so it lapses owned: that puts the run on the abandonment
11976            // listing, where the plane that runs the agent recovers it from
11977            // the recorded answer. Failing here would leave a recorded
11978            // decision reported as lost and the task it answers still claimed.
11979            // A plane holding no key ring for a sealed run, or no case store
11980            // for a case-bound one, is the same terminal: the plane that holds
11981            // it continues the run.
11982            Err(
11983                RuntimeError::NoProvider { .. }
11984                | RuntimeError::NoCaseStore { .. }
11985                | RuntimeError::PayloadsSealed { .. },
11986            ) => {
11987                return Ok(Delivery::Buffered);
11988            }
11989            Err(e) => return Err(e),
11990        }
11991        if closed {
11992            return Ok(Delivery::Buffered);
11993        }
11994        Ok(Delivery::Resumed { run: sub.run })
11995    }
11996
11997    /// Retire everything a run that concluded closed was still waiting on:
11998    /// its timers, its event subscriptions and its unanswered tasks.
11999    ///
12000    /// A sealed run can record nothing, so each of these left behind is work
12001    /// the plane keeps doing for nobody — a timer claimed and failed every
12002    /// lease period, the oldest waiter on a key taking the next event from a
12003    /// live run behind it, a task offered for a decision no answer reaches.
12004    ///
12005    /// Called after every seal of a concluded run — the conclusion's own and
12006    /// the one resume writes to repair a crash between conclusion and seal —
12007    /// and idempotent, so retiring twice is harmless.
12008    ///
12009    /// After the seal, and best-effort: the conclusion is already durable, and
12010    /// the stores pass over a sealed run's timers and subscriptions on their
12011    /// own, so a retirement that fails here costs clutter rather than a
12012    /// delivery. A task left pending by that failure is withdrawn by nothing
12013    /// else, which is why the failure is logged loudly.
12014    async fn retire_waits(&self, run: RunId) {
12015        if let Some(timers) = &self.timers
12016            && let Err(error) = timers.disarm_run(run).await
12017        {
12018            tracing::error!(%run, %error, "a closed run's timers could not be retired");
12019        }
12020        // The waits the run announced and never had answered: a message
12021        // claimed for one of them reached nobody and goes to the next waiter,
12022        // and a task opened for one is withdrawn. A task opened beside an
12023        // answer completes its own effect and is not among them: it is a
12024        // finding for a desk, and outlives the run by design.
12025        let unanswered = match self.store.read(run, 1).await {
12026            Ok(records) => unanswered_waits(&records),
12027            Err(error) => {
12028                tracing::error!(%run, %error, "a closed run's waits could not be read to retire them");
12029                return;
12030            }
12031        };
12032        if let Some(events) = &self.events {
12033            match events.unsubscribe_run(run, &unanswered).await {
12034                Ok(retired) => {
12035                    for event in retired.released {
12036                        self.offer_released(events, &event).await;
12037                    }
12038                }
12039                Err(error) => {
12040                    tracing::error!(%run, %error, "a closed run's subscriptions could not be retired");
12041                }
12042            }
12043        }
12044        if let Some(tasks) = &self.tasks {
12045            let awaited: Vec<_> = unanswered
12046                .iter()
12047                .map(|effect| crate::core::TaskId::derive(run, *effect))
12048                .collect();
12049            if let Err(error) = tasks.withdraw_run(run, &awaited).await {
12050                tracing::error!(%run, %error, "a closed run's open tasks could not be withdrawn");
12051            }
12052        }
12053    }
12054
12055    /// Offer a message a closed run held, and never consumed, to the next
12056    /// waiter on its key — the delivery it would have had if that run had not
12057    /// concluded first. Best effort, like the retirement that released it: a
12058    /// message no waiter takes now stays buffered for the next one.
12059    async fn offer_released(
12060        &self,
12061        events: &Arc<dyn crate::case::EventStore>,
12062        event: &InboundEvent,
12063    ) {
12064        let sub = match events.match_waiter(event, now_for_admission()).await {
12065            Ok(Some(sub)) => sub,
12066            Ok(None) => return,
12067            Err(error) => {
12068                tracing::error!(event = %event.id, %error, "a released message could not be offered");
12069                return;
12070            }
12071        };
12072        if let Err(error) = Box::pin(self.resume_subscription(events, sub, event)).await {
12073            tracing::error!(event = %event.id, %error, "a released message could not be delivered");
12074        }
12075    }
12076
12077    /// Finish deliveries that died between the claim and the resume.
12078    ///
12079    /// A delivery that raced a live owner leaves an event durably claimed for
12080    /// a run nothing resumed, and the counterparty's retries deduplicate
12081    /// against that claim — so no future delivery drives the run, and a
12082    /// message that arrived in time is stranded. The giving-up delivery parks
12083    /// the pair precisely so it stays findable; this pass walks the parked
12084    /// waits — never every registered wait, whose long legitimate members
12085    /// would fill the page ahead of a pair parked after them — re-claims
12086    /// (idempotently: a claim by the same run is returned, not filtered), and
12087    /// finishes the delivery. Runs whose owner is *still* live are skipped and
12088    /// found again next tick, so the delivery's retry bound bounds latency,
12089    /// never delivery.
12090    ///
12091    /// What this does **not** cover, honestly: a crash inside the store
12092    /// commit between an event's claim and the resume retires the
12093    /// subscription with nothing left to re-register it, and that window
12094    /// stays what it always was — recorded rather than closed, with later
12095    /// events for the same correlation dead-lettering as its symptom.
12096    pub(crate) async fn redeliver_claimed(
12097        &self,
12098        limit: usize,
12099    ) -> Result<crate::runtime::Redelivered, RuntimeError> {
12100        let Some(events) = self.events.as_ref() else {
12101            return Ok(crate::runtime::Redelivered::default());
12102        };
12103        let parked = events
12104            .parked_waits(limit)
12105            .await
12106            .map_err(RuntimeError::from_store)?;
12107        let examined = parked.len();
12108        let mut delivered = 0usize;
12109        for sub in parked {
12110            let Some(buffered) = events
12111                .claim_for(&sub, now_for_admission())
12112                .await
12113                .map_err(RuntimeError::from_store)?
12114            else {
12115                continue;
12116            };
12117            match self
12118                .resume_subscription(events, sub.clone(), &buffered.event)
12119                .await
12120            {
12121                Ok(Delivery::Resumed { .. }) => delivered += 1,
12122                // Still held live: the owner is working, and will either
12123                // consume the claim itself or conclude and be found next tick.
12124                Ok(_) => {}
12125                Err(error) => {
12126                    tracing::error!(
12127                        run = %sub.run,
12128                        %error,
12129                        "a claimed event's redelivery failed; retried next tick",
12130                    );
12131                }
12132            }
12133        }
12134        Ok(crate::runtime::Redelivered {
12135            finished: delivered,
12136            examined,
12137        })
12138    }
12139
12140    /// Retire events that nobody claimed within `grace`.
12141    ///
12142    /// A non-empty dead-letter list means a correlation key is wrong somewhere:
12143    /// the message arrived, was held, and no run ever asked for it. That is the
12144    /// failure which otherwise presents as a process silently never completing,
12145    /// so it is worth alerting on rather than logging.
12146    /// `grace` is a `std::time::Duration` for the reason
12147    /// [`StepCtx::deadline`](crate::runtime::StepCtx::deadline)'s `warn_before`
12148    /// is: a negative grace window is meaningless, and the signed type could
12149    /// express it — a cutoff moved *forward* of now, retiring events that had
12150    /// not yet had their chance. It is also the `Duration` the caller has.
12151    pub async fn sweep_events(&self, grace: std::time::Duration) -> Result<usize, RuntimeError> {
12152        let events = self
12153            .events
12154            .as_ref()
12155            .ok_or_else(|| RuntimeError::PlanContract("this runtime has no event store".into()))?;
12156        // Saturating rather than fallible, at both steps: a grace window
12157        // beyond what the calendar can hold means "retire nothing", and
12158        // refusing the call would be a worse answer to a caller asking for a
12159        // longer hold. The subtraction has to saturate too — `time` panics on
12160        // underflow, so converting the window to `Duration::MAX` and then
12161        // subtracting it aborts the tick that also breaches obligations and
12162        // recovers abandoned runs. The first instant the calendar can name is
12163        // a cutoff nothing is older than, which is exactly "retire nothing".
12164        let grace = time::Duration::try_from(grace).unwrap_or(time::Duration::MAX);
12165        let cutoff = now_for_admission()
12166            .checked_sub(grace)
12167            .unwrap_or_else(crate::core::first_instant);
12168        let retired = events
12169            .sweep_unclaimed(cutoff, "no run claimed this event within the grace window")
12170            .await
12171            .map_err(RuntimeError::from_store)?;
12172
12173        if retired > 0 {
12174            // A non-empty dead-letter list means a correlation key is wrong
12175            // somewhere: the message arrived, was held, and no run ever asked
12176            // for it. That is the failure which otherwise presents as a process
12177            // silently never completing.
12178            tracing::error!(target: telemetry::DEAD_LETTERED, count = retired, %cutoff);
12179            self.meter
12180                .count_by(metrics::DEAD_LETTERS, "", retired as u64);
12181        }
12182        Ok(retired)
12183    }
12184}
12185
12186/// Every `RunStatus`, so adding one forces a decision everywhere one is owed.
12187///
12188/// Written out rather than derived, because Rust cannot enumerate a
12189/// data-carrying enum. What keeps it honest is **not** the count assertions in
12190/// its consumers: those fire when somebody edits this list, which is the case
12191/// that needs no help. A variant added to the enum and forgotten here changes
12192/// nothing they can see, and every agreement test then walks a list that no
12193/// longer describes the type — passing over the one variant nobody decided
12194/// about.
12195///
12196/// `every_run_status_variant_is_listed` in `tests/guards` is what closes that:
12197/// it reads the enum's variants and this function's arms out of the source and
12198/// holds them to each other, which is the only check available for a list a
12199/// language cannot derive.
12200///
12201/// Crate-visible so those consumers share **one** list. Two copies would be two
12202/// places to remember, and the second would be the one that went stale — which
12203/// is the same failure the tests using it exist to catch.
12204#[cfg(test)]
12205pub(crate) fn every_status() -> Vec<RunStatus> {
12206    use crate::core::{BudgetExceeded, CorrelationKey, SuspendReason, Timestamp};
12207    vec![
12208        RunStatus::Succeeded,
12209        RunStatus::Failed("because".into()),
12210        RunStatus::Suspended(SuspendReason::AwaitingEvent {
12211            kind: "reply".into(),
12212            correlation: vec![CorrelationKey::new("claim", "CLM-1")],
12213            until: Timestamp::from_unix_timestamp(1_760_000_000).expect("time"),
12214        }),
12215        RunStatus::Exhausted(BudgetExceeded::Steps { allowed: 1 }),
12216        RunStatus::Quarantined("unknown outcome".into()),
12217        RunStatus::Replanning("try again".into()),
12218        RunStatus::Cancelled {
12219            actor: crate::core::Operator::asserted("ops").expect("a name"),
12220            reason: "stop".into(),
12221        },
12222        RunStatus::Abandoned {
12223            actor: crate::core::Operator::asserted("ops").expect("a name"),
12224            reason: "nobody could establish what happened".into(),
12225        },
12226        RunStatus::Swept,
12227        RunStatus::BrokeGlass {
12228            actor: crate::core::Operator::asserted("ops").expect("a name"),
12229            reason: "INC-42".into(),
12230        },
12231        RunStatus::HaltLifted {
12232            actor: crate::core::Operator::asserted("ops").expect("a name"),
12233        },
12234        RunStatus::HoldReleased {
12235            actor: crate::core::Operator::asserted("ops").expect("a name"),
12236        },
12237        RunStatus::Withheld {
12238            subject: "alice".into(),
12239            reason: "credential withdrawn".into(),
12240        },
12241        RunStatus::Observed,
12242    ]
12243}
12244
12245#[cfg(test)]
12246mod resume_agreement_tests {
12247    use super::{QuotaPass, RunStatus, resume_is_closed};
12248    use crate::journal::RecordKind;
12249
12250    // Through the crate re-export rather than `super::`, so the one path both
12251    // consumers use is exercised in every build. Reaching it directly here
12252    // would leave the re-export unused whenever `a2a-server` is off — which is
12253    // a lint failure in exactly the feature configuration nobody runs locally.
12254    use crate::runtime::every_status;
12255
12256    /// **A conclusion that sealed the journal may never be resumed.**
12257    ///
12258    /// `RunStatus::seals` decides whether a conclusion freezes the run and
12259    /// publishes a Merkle leaf; `resume_is_closed` decides whether a recorded
12260    /// conclusion may be continued. A status that did the first and permits the
12261    /// second would grow the history past the leaf every later checkpoint
12262    /// attests — the failure the "a conclusion is not a closure" work removed
12263    /// for `failed`, reachable again the moment the two disagree.
12264    ///
12265    /// The executor's own comment claimed "a test pins the agreement" while no
12266    /// such test existed, which is the unfalsifiable-guarantee shape this
12267    /// project treats as a defect in itself: the rule was real, checked by
12268    /// nobody, and deletable in silence.
12269    #[test]
12270    fn a_sealing_conclusion_is_never_resumable() {
12271        let statuses = every_status();
12272        assert_eq!(
12273            statuses.len(),
12274            14,
12275            "a RunStatus variant was added or removed — decide whether it seals \
12276             and whether a resume may continue from it, then update this list"
12277        );
12278
12279        for status in &statuses {
12280            let records = sealed_as(status.as_str());
12281            let verdict = resume_is_closed(&records);
12282            if status.seals() {
12283                assert!(
12284                    verdict.is_some(),
12285                    "'{}' seals the journal and enters the Merkle log, yet a resume \
12286                     is permitted from it — the resume would grow the history past \
12287                     the leaf every later checkpoint attests",
12288                    status.as_str()
12289                );
12290            }
12291        }
12292    }
12293
12294    /// The availability half: the conclusions that stay open really do.
12295    ///
12296    /// Asserted separately and by name rather than as the converse of the rule
12297    /// above, because the converse is false — `Suspended` and `Replanning` do
12298    /// not seal either, and neither is ever a recorded conclusion.
12299    ///
12300    /// `withheld` is here because it is a **pause**: the authority a run acts
12301    /// under was withdrawn, its mutations stand, and lifting the halt is what
12302    /// lets it continue. A withheld run that could not be resumed would make the
12303    /// withdrawal a destruction rather than a stop.
12304    #[test]
12305    fn a_failed_or_exhausted_run_may_still_be_resumed() {
12306        for outcome in ["failed", "exhausted", super::WITHHELD_OUTCOME] {
12307            assert!(
12308                resume_is_closed(&sealed_as(outcome)).is_none(),
12309                "'{outcome}' is a conclusion a resume must be able to continue \
12310                 from — its completed effects are read back from history, which \
12311                 is the point of having a journal"
12312            );
12313        }
12314    }
12315
12316    /// `SEALED_OUTCOMES` and `RunStatus::seals` are one rule in two spellings,
12317    /// and this holds them together variant by variant.
12318    ///
12319    /// The list exists so a caller wanting every sealed run — the export CLI —
12320    /// does not restate the rule as string literals in a binary, which is where
12321    /// a new sealing outcome would have been silently dropped from exactly the
12322    /// artifact an auditor asks for. Both directions matter: a sealing status
12323    /// missing from the list loses runs, and a non-sealing status present in it
12324    /// makes the export ask the outcome index about a backlog that drains.
12325    #[test]
12326    fn the_sealed_outcome_list_agrees_with_the_sealing_rule() {
12327        for status in super::every_status() {
12328            assert_eq!(
12329                super::SEALED_OUTCOMES.contains(&status.as_str()),
12330                status.seals(),
12331                "'{}' disagrees between RunStatus::seals and SEALED_OUTCOMES — \
12332                 one rule, two spellings, and the export reads the list",
12333                status.as_str()
12334            );
12335        }
12336    }
12337
12338    /// **Every outcome this build writes is one its own reader recognises.**
12339    ///
12340    /// The reader is a `match` over `&str` with a catch-all, so it goes on
12341    /// compiling forever while the writer's list grows — and the catch-all is
12342    /// not benign: it answers `Quarantined`, so a run this plane sealed itself
12343    /// reads back as *the runtime could not decide what happened*. Two endings
12344    /// sat there, `swept` and `broke-glass`, and a break-glass crossing — the
12345    /// one control whose entire content is who crossed and why — answered the
12346    /// operator surface with the words "this build does not recognise it" in
12347    /// place of the reason.
12348    ///
12349    /// Held from the writer's side, and the writer's side is
12350    /// [`every_status`](super::every_status) — every variant the type can
12351    /// take. `OUTCOMES_OF_RECORD` is the *export's* list and deliberately
12352    /// omits the endings that stay resumable, so a guard reading it asks about
12353    /// a subset of what the writer writes and passes while the rest fall
12354    /// through the catch-all. `withheld` sat there: the one ending whose whole
12355    /// content is *whose credential was withdrawn*, answering the operator API
12356    /// with "this build does not recognise it".
12357    #[test]
12358    fn every_outcome_this_build_writes_is_one_it_can_read_back() {
12359        for written in super::every_status() {
12360            // A suspension is not a conclusion; it has its own record and its
12361            // own arm in `observed_status`.
12362            if written.is_suspended() {
12363                continue;
12364            }
12365            let outcome = written.as_str();
12366            let status = super::recorded_status(
12367                outcome,
12368                written.own_account().as_deref(),
12369                match &written {
12370                    RunStatus::Exhausted(limit) => Some(limit),
12371                    _ => None,
12372                },
12373                &sealed_as(outcome),
12374            );
12375            assert_eq!(
12376                status.as_str(),
12377                outcome,
12378                "this build concludes runs as '{outcome}' and reads that back as \
12379                 '{}' — a catch-all answering `Quarantined` about a record this \
12380                 plane wrote itself",
12381                status.as_str()
12382            );
12383        }
12384    }
12385
12386    /// **An ending a person caused reports the words that person gave.**
12387    ///
12388    /// The conclusion does not carry them: one sentence, one record. So the
12389    /// reader has to reach the attribution record for the reason exactly as it
12390    /// already does for the actor, and a reader that only learned the second
12391    /// half would answer an incident review with a blank where the operator
12392    /// wrote why.
12393    #[test]
12394    fn an_ending_a_person_caused_reports_their_own_words() {
12395        for (outcome, expected) in [
12396            ("cancelled", "stop"),
12397            ("abandoned", "nobody could establish what happened"),
12398            (super::BREAK_GLASS_OUTCOME, "INC-42"),
12399            (super::WITHHELD_OUTCOME, "credential withdrawn"),
12400        ] {
12401            // `None`, as the conclusion this build writes now carries.
12402            let status = super::recorded_status(outcome, None, None, &sealed_as(outcome));
12403            assert_eq!(
12404                status.reason().as_deref(),
12405                Some(expected),
12406                "'{outcome}' lost the reason the person gave: {status:?}"
12407            );
12408        }
12409    }
12410
12411    /// **An ending nobody is named on is quarantined, not given a name.**
12412    ///
12413    /// The chain this build writes always carries the attribution record before
12414    /// the conclusion; a chain that carries only the conclusion is a crash
12415    /// between the two appends, or a chain somebody edited. Either way the
12416    /// record is not trustworthy, and *not trustworthy* is a different bucket
12417    /// from *an operator called unknown did this*.
12418    ///
12419    /// A fabricated actor would be worse than a missing one: indistinguishable
12420    /// from a real operator with that name, and served through the operator API
12421    /// as fact.
12422    #[test]
12423    fn an_ending_with_no_attribution_record_is_not_given_a_name() {
12424        for outcome in ["cancelled", "abandoned", super::BREAK_GLASS_OUTCOME] {
12425            let status =
12426                super::recorded_status(outcome, Some("why"), None, &concluded_only(outcome));
12427            assert!(
12428                matches!(status, RunStatus::Quarantined(_)),
12429                "'{outcome}' with no record of who asked read back as {status:?} — \
12430                 an operator act with nobody on it was given a name"
12431            );
12432        }
12433    }
12434
12435    /// The offline sweep covers every ending, plus the backlog that is not one.
12436    ///
12437    /// Held to `SEALED_OUTCOMES` rather than restated, because the failure this
12438    /// prevents is silent in exactly the artifact nobody re-reads: an ending
12439    /// added to the runtime and not to the export drops those runs from the
12440    /// file an auditor is handed, and the file looks complete.
12441    #[test]
12442    fn the_offline_sweep_covers_every_ending_and_the_quarantine_backlog() {
12443        for outcome in super::SEALED_OUTCOMES {
12444            assert!(
12445                super::OUTCOMES_OF_RECORD.contains(outcome),
12446                "'{outcome}' is an ending this plane can reach and the export's \
12447                 default sweep would silently omit it"
12448            );
12449        }
12450        assert!(
12451            super::OUTCOMES_OF_RECORD.contains(&"quarantined"),
12452            "a quarantine is the plane's outstanding unresolved risk and the one \
12453             thing an auditor is most likely to have come for"
12454        );
12455        for open in ["failed", "exhausted", "suspended"] {
12456            assert!(
12457                !super::OUTCOMES_OF_RECORD.contains(&open),
12458                "'{open}' is work in progress that a resume moves on its own — \
12459                 exporting it is a snapshot of a moving target, not a record"
12460            );
12461        }
12462    }
12463
12464    /// An outcome this build cannot interpret is never permission to resume.
12465    #[test]
12466    fn an_unrecognised_conclusion_fails_closed() {
12467        let verdict = resume_is_closed(&sealed_as("swept"));
12468        assert!(
12469            matches!(verdict, Some(RunStatus::Quarantined(_))),
12470            "an unrecognised conclusion must quarantine rather than resume: {verdict:?}"
12471        );
12472    }
12473
12474    /// A chain carrying the conclusion and **not** the attribution record.
12475    ///
12476    /// The damaged shape, which no healthy write produces: a crash between the
12477    /// two appends, or an edit. Built by taking the real chain and dropping
12478    /// everything before the conclusion, so it cannot drift from what
12479    /// [`sealed_as`] produces.
12480    fn concluded_only(outcome: &str) -> Vec<crate::journal::Record> {
12481        let full = sealed_as(outcome);
12482        full.into_iter()
12483            .filter(|r| matches!(r.kind(), RecordKind::RunConcluded { .. }))
12484            .collect()
12485    }
12486
12487    /// The chain this build actually writes for an ending.
12488    ///
12489    /// **Including the attribution record**, where the ending has one. A
12490    /// conclusion is never the only record of a cancellation, an abandonment or
12491    /// a crossing — the runtime appends who asked first — so a fixture holding
12492    /// the conclusion alone asserts a round trip over a chain this plane does
12493    /// not produce, and would keep passing if the reader began inventing a
12494    /// name for the missing half.
12495    fn sealed_as(outcome: &str) -> Vec<crate::journal::Record> {
12496        use crate::core::{Digest, Epoch};
12497        use crate::journal::{Record, RecordBody};
12498
12499        let run = crate::core::RunId::generate();
12500        let who = || crate::core::Operator::asserted("ops").expect("a name");
12501        let mut kinds: Vec<RecordKind> = match outcome {
12502            "cancelled" => vec![RecordKind::RunCancelled {
12503                actor: who(),
12504                reason: "stop".into(),
12505            }],
12506            "abandoned" => vec![RecordKind::QuarantineDecided {
12507                decider: who(),
12508                reason: "nobody could establish what happened".into(),
12509                decision: crate::core::QuarantineDecision::Abandon,
12510            }],
12511            super::BREAK_GLASS_OUTCOME => vec![RecordKind::BreakGlass {
12512                actor: who(),
12513                roles: vec!["oncall".into()],
12514                reason: "INC-42".into(),
12515            }],
12516            super::HALT_LIFTED_OUTCOME => vec![RecordKind::HaltLifted {
12517                scope: "tenant".into(),
12518                by: who(),
12519                at: crate::core::Timestamp::UNIX_EPOCH,
12520                reason: "INC-42".into(),
12521                thrown_by: who(),
12522                thrown_at: crate::core::Timestamp::UNIX_EPOCH,
12523            }],
12524            super::HOLD_RELEASED_OUTCOME => vec![RecordKind::HoldReleased {
12525                by: who(),
12526                at: crate::core::Timestamp::UNIX_EPOCH,
12527                placed_by: who(),
12528                placed_at: crate::core::Timestamp::UNIX_EPOCH,
12529            }],
12530            super::WITHHELD_OUTCOME => vec![RecordKind::AuthorityWithheld {
12531                subject: "alice".into(),
12532                reason: "credential withdrawn".into(),
12533                by: who(),
12534            }],
12535            _ => Vec::new(),
12536        };
12537        kinds.push(RecordKind::RunConcluded {
12538            outcome: outcome.to_owned(),
12539            chain_head: Digest::of(b""),
12540            reason: None,
12541            exhaustion: None,
12542            live_spend: crate::core::Spend::default(),
12543        });
12544
12545        let mut prev = Digest::of(b"");
12546        let mut out = Vec::new();
12547        for (i, kind) in kinds.into_iter().enumerate() {
12548            let body = RecordBody {
12549                seq: u64::try_from(i + 1).expect("a small fixture"),
12550                run,
12551                case: None,
12552                step: None,
12553                phase: super::Phase::Forward,
12554                epoch: Epoch::default(),
12555                v: 1,
12556                effect_key: None,
12557                kind,
12558            };
12559            let record = Record::seal(body, prev).expect("a sealed record");
12560            prev = record.hash;
12561            out.push(record);
12562        }
12563        out
12564    }
12565
12566    /// One execution pass is authorized and accrued against one period even
12567    /// when it finishes after a billing boundary.
12568    #[test]
12569    fn a_quota_pass_keeps_the_period_it_started_in() {
12570        let quota = crate::quota::TenantQuota {
12571            max_tokens_per_period: Some(1_000),
12572            period: crate::quota::Period::Daily,
12573            ..crate::quota::TenantQuota::default()
12574        };
12575        let before_midnight = crate::core::Timestamp::from_unix_timestamp(86_399).unwrap();
12576        let after_midnight = crate::core::Timestamp::from_unix_timestamp(86_400).unwrap();
12577
12578        let pass = QuotaPass::at(&quota, before_midnight, true);
12579        assert_eq!(
12580            pass.period(),
12581            Some(quota.period.key_for(before_midnight).as_str())
12582        );
12583        assert_ne!(
12584            pass.period(),
12585            Some(quota.period.key_for(after_midnight).as_str()),
12586            "settlement recomputed the period at completion, so this pass was \
12587             authorized against one ledger and charged to another"
12588        );
12589    }
12590}