Skip to main content

agentplane/runtime/
executor.rs

1//! The executor: admission, step dispatch, sealing, and replay.
2//!
3//! M0 executes a single step. The DAG scheduler, plan contract, and topology
4//! checks slot in above this without changing the effect protocol below it —
5//! which is the point of putting the determinism boundary at the effect rather
6//! than at the plan.
7
8#[cfg(feature = "manifest")]
9use std::collections::HashSet;
10use std::collections::{BTreeMap, BTreeSet, HashMap};
11use std::sync::Arc;
12use std::time::Duration;
13
14use serde_json::Value;
15
16use crate::case::{CaseStore, EventStore, TaskStore, TimerStore};
17use crate::core::{
18    ArgSource, Budget, Calendar, Capability, Consumed, CorrelationKey, Delivery, Digest,
19    InboundEvent, Ledger, Outcome, Phase, PlanIR, PlanNode, PolicyBundleIdentity, RunId,
20    RuntimeError, Skill, SkillDescriptor, Spend, StepId, Tainted, WallClock,
21};
22use crate::journal::{Append, JournalStore, Record, RecordKind, ReplayCursor, StepCursor};
23use crate::runtime::BuildError;
24
25use super::ctx::{CaseContext, Mode, Pass, StepCtx};
26use super::metrics;
27use super::telemetry;
28use tracing::Instrument;
29
30#[derive(Debug, Clone)]
31pub(crate) enum CaseBinding {
32    Correlate {
33        kind: String,
34        keys: Vec<CorrelationKey>,
35    },
36    Existing(crate::core::CaseId),
37}
38
39/// What a run is bound to, beyond its target and input: who it acts for,
40/// which matter it belongs to, who asked, which declaration it was reviewed
41/// against. Handed to [`RunRequest::terms`].
42///
43/// The value a front door computes per caller — above all a chain **per run**:
44/// a door that authenticates many callers admits each one's run under the
45/// chain that caller presented. Whether the run happens at most once is not a
46/// term; it is [`RunRequest::once`], because it changes what the answer is.
47///
48/// **Authority is chosen once, by constructor**: [`default`](Self::default)
49/// acts as the plane, [`acting_as`](Self::acting_as) under a caller's chain,
50/// [`served`](Self::served) for a served caller that may have presented none.
51///
52/// ```no_run
53/// # use agentplane::runtime::RunTerms;
54/// # fn f(chain: agentplane::core::Delegation, keys: &[agentplane::core::CorrelationKey]) {
55/// let terms = RunTerms::acting_as(chain).correlated("order", keys);
56/// # }
57/// ```
58///
59/// ```compile_fail
60/// # use agentplane::runtime::RunTerms;
61/// # fn f(chain: agentplane::core::Delegation) {
62/// let terms = RunTerms::served(None).acting_as(chain);
63/// # }
64/// ```
65#[derive(Debug, Clone, Default)]
66pub struct RunTerms {
67    /// Which matter this run belongs to, if any.
68    case: Option<CaseBinding>,
69    /// The admission key to claim. Travels to the `RunAdmitted` record, because
70    /// that is where the store claims it — see
71    /// [`JournalStore::admitted_as`](crate::journal::JournalStore::admitted_as).
72    idempotency_key: Option<String>,
73    /// On whose authority this run acts.
74    acting_as: Acting,
75    /// Who asked for this run, when a person or peer did. Journaled on
76    /// `RunAdmitted`, and barred from deciding the run's own tasks.
77    admitted_by: Option<String>,
78    /// The declaration revision the caller reviewed, when it pinned one.
79    expected_declaration: Option<Digest>,
80    /// Data subjects the embedder names for this run, beside any its
81    /// declaration binds.
82    subjects: Vec<String>,
83}
84
85/// On whose authority a run acts.
86///
87/// Three answers, not two, because "no chain was presented" means different
88/// things at different doors. An in-process embedder that names no chain is
89/// the plane itself, and acts under the plane's
90/// ([`RuntimeBuilder::acting_as`]). A served caller whose credential carried
91/// none is **somebody else** — and falling back to the plane's chain for them
92/// would make the operator's authority an ambient credential every
93/// authenticated peer holds.
94///
95/// Acting under a chain and being bounded by one are separate questions. A
96/// served caller with no chain acts under none, and is still bounded by the
97/// plane's: that chain is the most this plane may do for anybody, so its
98/// scope and its admissibility gate the plan
99/// ([`ceiling`](Self::ceiling)) while nothing of it reaches the policy
100/// context, the journal or a withdrawal lookup.
101#[derive(Debug, Clone, Default)]
102enum Acting {
103    /// The plane's own chain, if it has one: a run the embedder started.
104    #[default]
105    Plane,
106    /// The chain an authenticated caller presented.
107    Caller(crate::core::Delegation),
108    /// A served caller that presented no chain: this run acts under none.
109    Nobody,
110    /// A chain delegated from the plane's own, for a run the plane's run
111    /// commissioned: it acts under that chain, and as the plane, so a peer
112    /// it calls is presented the plane's credential rather than one naming
113    /// the plane's owner as if a person had asked.
114    PlaneDelegate(crate::core::Delegation),
115}
116
117impl Acting {
118    /// The chain this run acts under, given the plane's own.
119    const fn resolve<'a>(
120        &'a self,
121        plane: Option<&'a crate::core::Delegation>,
122    ) -> Option<&'a crate::core::Delegation> {
123        match self {
124            Self::Plane => plane,
125            Self::Caller(chain) | Self::PlaneDelegate(chain) => Some(chain),
126            Self::Nobody => None,
127        }
128    }
129
130    /// The chain that bounds what this run may be admitted for: the one it
131    /// acts under, or — for a served caller that presented none — the
132    /// plane's, which it does not act under.
133    const fn ceiling<'a>(
134        &'a self,
135        plane: Option<&'a crate::core::Delegation>,
136    ) -> Option<&'a crate::core::Delegation> {
137        match self {
138            Self::Plane | Self::Nobody => plane,
139            Self::Caller(chain) | Self::PlaneDelegate(chain) => Some(chain),
140        }
141    }
142}
143
144impl RunTerms {
145    /// Run inside a case that already exists.
146    #[must_use]
147    pub fn in_case(mut self, case: crate::core::CaseId) -> Self {
148        self.case = Some(CaseBinding::Existing(case));
149        self
150    }
151
152    /// Join or open a case by business key before running.
153    #[must_use]
154    pub fn correlated(mut self, kind: &str, keys: &[CorrelationKey]) -> Self {
155        self.case = Some(CaseBinding::Correlate {
156            kind: kind.to_owned(),
157            keys: keys.to_vec(),
158        });
159        self
160    }
161
162    /// Act under this chain rather than the plane's own.
163    ///
164    /// Checked at admission exactly as the plane's chain is: the plan must be
165    /// inside its scope, and [`Delegation::admissible`] must accept this
166    /// plane at this instant. Journaled as the run's `IdentityBound`, so
167    /// "on whose behalf" names the caller and not the plane.
168    ///
169    /// [`Delegation::admissible`]: crate::core::Delegation::admissible
170    #[must_use]
171    pub fn acting_as(chain: crate::core::Delegation) -> Self {
172        Self {
173            acting_as: Acting::Caller(chain),
174            ..Self::default()
175        }
176    }
177
178    /// Act under `chain` as the plane: a commission from a run the plane
179    /// itself admitted, whose sub-run is the plane's work one link down.
180    pub(crate) fn acting_as_plane(mut self, chain: crate::core::Delegation) -> Self {
181        self.acting_as = Acting::PlaneDelegate(chain);
182        self
183    }
184
185    /// Act for a served caller: under the chain its credential carried, or
186    /// under **none** — never the plane's.
187    ///
188    /// The door every surface that admits work for somebody else goes through.
189    /// [`acting_as`](Self::acting_as) alone cannot say "this caller presented
190    /// nothing", and leaving it unset means the plane's own chain.
191    #[must_use]
192    pub fn served(chain: Option<crate::core::Delegation>) -> Self {
193        Self {
194            acting_as: chain.map_or(Acting::Nobody, Acting::Caller),
195            ..Self::default()
196        }
197    }
198
199    /// Record who asked for this run: the authenticated caller a served
200    /// surface admitted it for, or the operator an embedder started it on
201    /// behalf of.
202    ///
203    /// Journaled as `RunAdmitted.admitted_by`, and excluded from deciding
204    /// every task the run opens — the four-eyes control, applied to the one
205    /// actor a declarative agent's approvals could not otherwise name. A
206    /// caller who could approve the call their own request produced would be
207    /// one person on both sides of the review.
208    #[must_use]
209    pub fn admitted_by(mut self, actor: impl Into<String>) -> Self {
210        self.admitted_by = Some(actor.into());
211        self
212    }
213
214    /// Attribute what this run takes in to a data subject.
215    ///
216    /// Recorded in the run's `DataSubjectBound` after any subjects its
217    /// declaration binds, so `agentplane subject` follows the run's input,
218    /// events and case state to the effects they reached. Attribution only:
219    /// no gate reads it.
220    #[must_use]
221    pub fn subject(mut self, subject: impl Into<String>) -> Self {
222        self.subjects.push(subject.into());
223        self
224    }
225
226    /// Admit only if this declaration revision governs the capability.
227    ///
228    /// For a caller that reviewed one revision and schedules runs against a
229    /// plane it does not control: another revision — or no declaration at
230    /// all, including on a build without the `manifest` feature — is refused
231    /// as [`RuntimeError::DeclarationPinMismatch`] before authorization, and
232    /// nothing is recorded.
233    #[must_use]
234    pub const fn expect_declaration(mut self, digest: Digest) -> Self {
235        self.expected_declaration = Some(digest);
236        self
237    }
238
239    fn keyed(mut self, key: &str) -> Self {
240        self.idempotency_key = Some(key.to_owned());
241        self
242    }
243}
244
245/// Default lease duration. A crashed owner's runs become claimable this long
246/// after its last heartbeat.
247///
248/// This bounds how long a *dead* owner's runs are stranded, not how long a run
249/// may take: a live run renews while it executes. Set per plane with
250/// [`RuntimeBuilder::lease_ttl`].
251pub const LEASE_TTL: Duration = Duration::from_secs(30);
252
253/// The shortest lease a live run can actually hold.
254///
255/// Both stores keep expiry in whole seconds and lapse on `expires_at <= now`, so
256/// a one-second lease is expired for part of every second it exists. Two is the
257/// smallest value a renewal can stay ahead of.
258pub const MIN_LEASE_TTL: Duration = Duration::from_secs(2);
259
260/// What a run produced.
261#[derive(Debug, Clone)]
262pub struct RunOutcome {
263    pub run_id: RunId,
264    pub status: RunStatus,
265    /// What this run consumed — steps, effects, spend, elapsed, refusals.
266    ///
267    /// Reported rather than left in the ledger because "what did the settlement
268    /// run cost" has to be answerable per item, and a batch sums its items. A
269    /// figure that only exists inside a dropped `Ledger` is a figure nobody can
270    /// bill against.
271    ///
272    /// The whole tally rather than only the money: the counts are what a
273    /// ceiling is sized against, and an operator raising `max_effects` after a
274    /// run stopped short is asking exactly the question this answers.
275    pub consumed: Consumed,
276    /// Terminal hash of the run's chain — what a signature would cover.
277    pub chain_head: Digest,
278    /// What the run produced, **with its label**.
279    ///
280    /// Labelled rather than bare, and the difference is not cosmetic: a
281    /// caller acting on a run's answer needs to know whether a model, a peer
282    /// or a person wrote it. The label was stripped here until an A2A reply
283    /// projection read a marker key out of an untrusted answer and let a
284    /// remote peer choose the envelope its own reply arrived in — a
285    /// confused-deputy reachable because the one fact that would have refused
286    /// it had been dropped at the boundary.
287    pub output: Option<Tainted<Value>>,
288}
289
290impl RunOutcome {
291    /// What this run spent, in metered units — the money half of
292    /// [`consumed`](Self::consumed).
293    #[must_use]
294    pub const fn spend(&self) -> Spend {
295        self.consumed.spend
296    }
297
298    /// Why this run ended — [`RunStatus::reason`] without the match.
299    ///
300    /// `None` for a success. The accessor is here as well as on the status
301    /// because this is the type an embedder holds after `run(..)`, and a
302    /// summary field that silently stays empty is the shape this exists to
303    /// prevent.
304    #[must_use]
305    pub fn reason(&self) -> Option<std::borrow::Cow<'_, str>> {
306        self.status.reason()
307    }
308
309    /// The output if the run succeeded, or a [`RunFailure`] saying why not.
310    ///
311    /// Every caller that acts on a run's answer asks the same two questions —
312    /// did it work, and what did it say — and pattern-matching
313    /// [`status`](Self::status) to learn the first is how the second gets read
314    /// off a run that quarantined. This folds the pair into one `?`:
315    ///
316    /// ```ignore
317    /// let answer = runtime.start("greet", input).run().await?.success()?;
318    /// ```
319    ///
320    /// `status` stays public for the callers whose branches genuinely differ —
321    /// a suspended run is not a failed one, and an operator surface treats
322    /// them differently. This is for the caller to whom anything short of an
323    /// answer is an error.
324    ///
325    /// # Errors
326    ///
327    /// [`RunFailure`] for every non-succeeded status, carrying the run id and
328    /// the status itself.
329    pub fn success(self) -> Result<Tainted<Value>, RunFailure> {
330        match self.status {
331            // A succeeded run's answer; `Null` for the degenerate run that
332            // finished without producing one, which is still a success.
333            RunStatus::Succeeded => {
334                Ok(self.output.unwrap_or_else(|| Tainted::trusted(Value::Null)))
335            }
336            status => Err(RunFailure {
337                run_id: self.run_id,
338                status,
339            }),
340        }
341    }
342}
343
344/// What an idempotent admission did.
345///
346/// A duplicate comes back as an answer rather than an error: a caller that
347/// retried wants the original run, not a conflict it has to interpret.
348///
349/// Exhaustive, unlike the error types: a key is unused, or held by a run that
350/// has rested, or held by one that has not. There is no fourth state to leave
351/// room for, and `#[non_exhaustive]` would cost every caller a `_` arm that
352/// means nothing.
353#[derive(Debug, Clone)]
354pub enum Admission {
355    /// The key was unused. This run is the one it admitted.
356    Fresh(RunOutcome),
357    /// The key already admitted a run that has reached a resting point —
358    /// concluded, or suspended waiting for something. Its answer, again.
359    ///
360    /// A suspension counts, and that is the case this exists for: a run parked
361    /// on a four-eyes decision has already opened the task, so a redelivery
362    /// that admitted would put a second identical approval in front of a
363    /// reviewer.
364    ///
365    /// Carries no `output`. A step's result is reconstructed by replay rather
366    /// than stored, so what the journal answers is *what happened* — status,
367    /// reason, chain head. Re-derive the value with [`replay`](Runtime::replay)
368    /// under [`Mode::Strict`], which performs no external effect.
369    Replayed(RunOutcome),
370    /// The key already admitted a run that has not rested yet — still
371    /// executing, or abandoned and awaiting the recovery sweep.
372    ///
373    /// The honest answer to an emitter is *accepted, already in progress* — a
374    /// 200 or 202 naming this run, never a retry-provoking failure.
375    InFlight(RunId),
376}
377
378impl Admission {
379    /// The run this key admitted, whichever of the three this is.
380    #[must_use]
381    pub const fn run_id(&self) -> RunId {
382        match self {
383            Self::Fresh(outcome) | Self::Replayed(outcome) => outcome.run_id,
384            Self::InFlight(run) => *run,
385        }
386    }
387
388    /// Whether this call is the one that admitted the run.
389    #[must_use]
390    pub const fn is_fresh(&self) -> bool {
391        matches!(self, Self::Fresh(_))
392    }
393
394    /// What the run produced, if it has produced anything yet.
395    #[must_use]
396    pub const fn outcome(&self) -> Option<&RunOutcome> {
397        match self {
398            Self::Fresh(outcome) | Self::Replayed(outcome) => Some(outcome),
399            Self::InFlight(_) => None,
400        }
401    }
402}
403
404/// What a keyed [`spawn`](RunRequest::spawn) did.
405///
406/// Separate from [`Admission`] because a spawn returns before the work happens,
407/// so it has no outcome to report.
408#[derive(Debug, Clone, Copy, PartialEq, Eq)]
409pub struct Spawned {
410    /// The run holding the key: the one just started, or the one that already
411    /// held it.
412    pub run: RunId,
413    /// False when the key was already held, so nothing new was started.
414    pub fresh: bool,
415}
416
417/// What a run starts from.
418#[derive(Debug, Clone)]
419pub enum Target {
420    /// A capability a registered skill answers — the degenerate one-node plan.
421    Capability(String),
422    /// A plan, validated and frozen before its first step: one that would
423    /// fail at step seven must not begin at step one.
424    Plan(PlanIR),
425}
426
427impl From<&str> for Target {
428    fn from(capability: &str) -> Self {
429        Self::Capability(capability.to_owned())
430    }
431}
432
433impl From<String> for Target {
434    fn from(capability: String) -> Self {
435        Self::Capability(capability)
436    }
437}
438
439impl From<&String> for Target {
440    fn from(capability: &String) -> Self {
441        Self::Capability(capability.clone())
442    }
443}
444
445impl From<PlanIR> for Target {
446    fn from(plan: PlanIR) -> Self {
447        Self::Plan(plan)
448    }
449}
450
451/// A request with no admission key: answered by what the run did.
452#[derive(Debug, Clone, Copy)]
453pub struct Unkeyed;
454
455/// A request under an admission key: answered by what the key admitted.
456#[derive(Debug, Clone)]
457pub struct Keyed(String);
458
459/// A run about to start — see [`Runtime::start`].
460///
461/// The answer's type follows the key. Unkeyed, a run answers with its
462/// [`RunOutcome`] and a spawn with its [`RunId`]; under
463/// [`once`](Self::once), with an [`Admission`] or a [`Spawned`], because a
464/// key already held is answered by the run that holds it, and a caller must
465/// say what it does then.
466///
467/// ```compile_fail
468/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
469/// let _: agentplane::runtime::RunOutcome = rt.start("c", input).once("k").run().await.unwrap();
470/// # }
471/// ```
472///
473/// ```compile_fail
474/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
475/// rt.start("c", input).once("a").once("b");
476/// # }
477/// ```
478///
479/// ```no_run
480/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
481/// let _: agentplane::runtime::RunOutcome = rt.start("c", input.clone()).run().await.unwrap();
482/// let _: agentplane::runtime::Admission = rt.start("c", input).once("k").run().await.unwrap();
483/// # }
484/// ```
485#[must_use = "a run request does nothing until it is run, spawned or settled"]
486#[derive(Debug)]
487pub struct RunRequest<'a, K> {
488    runtime: &'a Runtime,
489    target: Target,
490    input: Tainted<Value>,
491    terms: RunTerms,
492    key: K,
493}
494
495impl<K> RunRequest<'_, K> {
496    /// Bind the run: a case, a caller's chain, who asked, a pinned
497    /// declaration. A second call replaces the first, as one value.
498    pub fn terms(mut self, terms: RunTerms) -> Self {
499        self.terms = terms;
500        self
501    }
502}
503
504impl<'a> RunRequest<'a, Unkeyed> {
505    /// Admit at most once per tenant under this key.
506    ///
507    /// The door for an at-least-once emitter: a bus that retries until it sees
508    /// a 2xx, a webhook fan-out, a queue with redelivery. The store claims the
509    /// key in the same transaction that writes the run's first record, so a
510    /// second request under it cannot admit a second run — not on a retry, not
511    /// across a restart, not from another instance sharing the store.
512    ///
513    /// **The key must carry its producer.** An id is unique only within one
514    /// emitter, so a bare message id lets two counterparties swallow each
515    /// other's messages as apparent retries.
516    /// [`InboundEvent::dedup_key`](crate::core::InboundEvent::dedup_key) is the
517    /// ready-made spelling.
518    ///
519    /// Not the event buffer: an event nobody subscribes to is dead-lettered by
520    /// [`sweep`](Runtime::sweep), so using the buffer as a ledger would spend
521    /// that signal to buy deduplication.
522    pub fn once(self, key: impl Into<String>) -> RunRequest<'a, Keyed> {
523        RunRequest {
524            runtime: self.runtime,
525            target: self.target,
526            input: self.input,
527            terms: self.terms,
528            key: Keyed(key.into()),
529        }
530    }
531
532    /// Run, and wait until it rests.
533    ///
534    /// # Errors
535    ///
536    /// [`RuntimeError::NoProvider`] when no skill provides the capability, and
537    /// whatever admission refuses — policy, quota, a halted tenant, an
538    /// instance draining, a chain this plane may not act under.
539    pub async fn run(self) -> Result<RunOutcome, RuntimeError> {
540        self.runtime
541            .admit_target(self.target, self.input, self.terms)
542            .await
543    }
544
545    /// Admit, and let the run proceed in the background, returning its id.
546    ///
547    /// **Admission happens before this returns.** The policy gate, the lease
548    /// and the admission records are written first, so a refusal is an error
549    /// here and not a task that never appears, and the id can be read at once.
550    /// The run is durable: a process that dies mid-flight leaves a journal
551    /// another instance resumes.
552    ///
553    /// # Panics
554    ///
555    /// Outside a Tokio runtime, as the rest of this crate's timing does.
556    ///
557    /// # Errors
558    ///
559    /// As [`run`](Self::run).
560    pub async fn spawn(self) -> Result<RunId, RuntimeError> {
561        self.runtime
562            .owned()
563            .spawn_target(self.target, self.input, self.terms)
564            .await
565    }
566}
567
568impl RunRequest<'_, Keyed> {
569    /// Run under the key. A key already held comes back as
570    /// [`Admission::Replayed`] or [`Admission::InFlight`], not an error.
571    ///
572    /// # Errors
573    ///
574    /// As [`RunRequest::run`], and [`RuntimeError::PlanContract`] for an empty
575    /// key or one over [`MAX_ADMISSION_KEY_BYTES`].
576    pub async fn run(self) -> Result<Admission, RuntimeError> {
577        let runtime = self.runtime;
578        let terms = self.terms.keyed(&self.key.0);
579        // Boxed: admission is the largest future in the crate, and the claim
580        // would otherwise carry it inline into every keyed caller's future.
581        let admit = Box::pin(runtime.admit_target(self.target, self.input, terms));
582        match runtime.claimed(&self.key.0, admit).await? {
583            Claimed::Fresh(outcome) => Ok(Admission::Fresh(outcome)),
584            Claimed::Held(run) => runtime.answer_with(run).await,
585        }
586    }
587
588    /// Spawn under the key. A key already held names its run, with `fresh`
589    /// false.
590    ///
591    /// # Panics
592    ///
593    /// Outside a Tokio runtime.
594    ///
595    /// # Errors
596    ///
597    /// As [`run`](Self::run).
598    pub async fn spawn(self) -> Result<Spawned, RuntimeError> {
599        let runtime = self.runtime.owned();
600        let terms = self.terms.keyed(&self.key.0);
601        let admit = Box::pin(runtime.spawn_target(self.target, self.input, terms));
602        Ok(match runtime.claimed(&self.key.0, admit).await? {
603            Claimed::Fresh(run) => Spawned { run, fresh: true },
604            Claimed::Held(run) => Spawned { run, fresh: false },
605        })
606    }
607
608    /// Run under the key until the run it admitted rests, whoever admitted it,
609    /// and answer with its outcome.
610    ///
611    /// The shape a process that may have been killed mid-run wants on restart:
612    /// the same request again. An unused key runs. A key whose run has rested
613    /// answers with that run, its value re-derived by strict replay, which
614    /// performs nothing. A key whose run is still working is resumed when
615    /// nobody holds its lease — its holder has gone — and otherwise followed
616    /// on the journal until it rests or its lease lapses.
617    ///
618    /// # Errors
619    ///
620    /// As [`run`](Self::run), and whatever resuming or replaying the held run
621    /// refuses.
622    pub async fn settle(self) -> Result<RunOutcome, RuntimeError> {
623        let runtime = self.runtime;
624        match self.run().await? {
625            Admission::Fresh(outcome) => Ok(outcome),
626            Admission::Replayed(rested) => runtime.with_output(rested).await,
627            Admission::InFlight(run) => runtime.settled(run).await,
628        }
629    }
630}
631
632/// What a keyed claim found.
633enum Claimed<T> {
634    /// The key was unused, and this admitted.
635    Fresh(T),
636    /// The key already admitted this run.
637    Held(RunId),
638}
639
640/// The stores a full plane runs on, as handles rather than as one type.
641///
642/// [`Runtime::builder_on`] is the shorthand for one backend implementing all
643/// of them, and is the wrong shape for a caller that picks its backend at **run
644/// time** — an operator naming either a file or a connection string holds
645/// `Arc<dyn JournalStore>`, and unsizing a generic parameter needs it `Sized`,
646/// so the wiring is a value that `builder_on` builds.
647///
648/// Here rather than in [`store`](crate::store) because that module is the
649/// backends, gated on `redb` or `postgres`, while this exists whether or not
650/// either is compiled in.
651#[derive(Clone)]
652pub struct Stores {
653    pub journal: std::sync::Arc<dyn crate::journal::JournalStore>,
654    pub cases: std::sync::Arc<dyn crate::case::CaseStore>,
655    pub tasks: std::sync::Arc<dyn crate::case::TaskStore>,
656    pub events: std::sync::Arc<dyn crate::case::EventStore>,
657    pub timers: std::sync::Arc<dyn crate::case::TimerStore>,
658    pub memory: std::sync::Arc<dyn crate::memory::MemoryStore>,
659    /// Where halts, subject withdrawals, rate windows and tenant ceilings live.
660    pub quotas: std::sync::Arc<dyn crate::quota::QuotaStore>,
661    /// Where disclosures of a matter are recorded.
662    pub disclosures: std::sync::Arc<dyn crate::disclosure::DisclosureRegister>,
663}
664
665impl std::fmt::Debug for Stores {
666    /// The handles are trait objects with no identity to print, so what a reader
667    /// wants here is *that the set is complete*, which it is by construction.
668    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
669        f.write_str("Stores { journal, cases, tasks, events, timers, memory, quotas, disclosures }")
670    }
671}
672
673impl Stores {
674    /// Every store from one backend that implements all of them.
675    #[must_use]
676    pub fn on<B: FullBackend + 'static>(store: std::sync::Arc<B>) -> Self {
677        Self {
678            journal: std::sync::Arc::clone(&store) as _,
679            cases: std::sync::Arc::clone(&store) as _,
680            tasks: std::sync::Arc::clone(&store) as _,
681            events: std::sync::Arc::clone(&store) as _,
682            timers: std::sync::Arc::clone(&store) as _,
683            memory: std::sync::Arc::clone(&store) as _,
684            quotas: std::sync::Arc::clone(&store) as _,
685            disclosures: store as _,
686        }
687    }
688}
689
690/// How far into a run's history [`Runtime::running_runs`] reads to attribute it.
691///
692/// `RunAdmitted` is seq 1 and `IdentityBound` follows it when the caller
693/// presented a chain, with the quota pass and the frozen plan between them — so
694/// a handful of records carries both and a long-running agent's history is not
695/// scanned to answer *who is this*.
696const ATTRIBUTION_RECORDS: usize = 8;
697
698/// One run this tenant currently holds an admission slot for.
699///
700/// What [`Runtime::running_runs`] answers with. The ids alone would be a listing
701/// an operator reads out; the attribution is what makes it one they can act on.
702#[derive(Debug, Clone)]
703pub struct LiveRun {
704    pub run: RunId,
705    /// The declared agent governing it, when a declared one does.
706    ///
707    /// `None` for a skill registered directly on the plane — a legitimate shape,
708    /// and one an incident about *a bad revision* can therefore rule out.
709    pub governed_by: Option<crate::journal::AgentIdentity>,
710    /// Who this run acts for: the root of the delegation chain bound at
711    /// admission, when the caller presented one.
712    pub subject: Option<String>,
713    /// The slot is held by a run whose lease has lapsed, so no process is
714    /// executing it.
715    ///
716    /// Not a cancellation candidate: the recovery sweep resumes these, and
717    /// cancelling one unwinds work that was about to continue.
718    pub stranded: bool,
719}
720
721/// A run that stopped short of an answer, as an error a caller can `?`.
722///
723/// Carries the [`RunStatus`] whole rather than a flattened string, so a caller
724/// that started with [`RunOutcome::success`] and later needs to tell a
725/// suspension from a quarantine can match on [`status`](Self::status) without
726/// re-plumbing the call site.
727#[derive(Debug, Clone)]
728pub struct RunFailure {
729    pub run_id: RunId,
730    pub status: RunStatus,
731}
732
733/// **Total, including over the status this type is not supposed to hold.**
734///
735/// `status` is public and `RunStatus` carries `Succeeded`, so a
736/// `RunFailure { status: Succeeded }` is a value any caller can build. Asserting
737/// that away would make *formatting an error* the operation that panics, and a
738/// `Display` impl runs wherever an error is logged — disproportionately during
739/// an incident. So it prints the contradiction instead.
740impl std::fmt::Display for RunFailure {
741    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
742        if matches!(self.status, RunStatus::Succeeded) {
743            return write!(
744                f,
745                "run {} is recorded as a failure carrying a success — \
746                 whatever built this value did so wrongly",
747                self.run_id
748            );
749        }
750        write!(f, "run {} did not succeed: ", self.run_id)?;
751        match &self.status {
752            RunStatus::Succeeded => write!(f, "it succeeded"),
753            RunStatus::Failed(reason) => write!(f, "it failed — {reason}"),
754            RunStatus::Suspended(reason) => write!(f, "it is suspended — {reason:?}"),
755            RunStatus::Exhausted(limit) => write!(f, "a budget stopped it — {limit}"),
756            RunStatus::Quarantined(reason) => write!(f, "it is quarantined — {reason}"),
757            RunStatus::Replanning(reason) => write!(f, "it asked to replan — {reason}"),
758            RunStatus::Observed => write!(
759                f,
760                "it is a session this plane observed rather than executed, so it \
761                 neither succeeded nor failed"
762            ),
763            RunStatus::Cancelled { actor, reason } => {
764                write!(f, "{actor} cancelled it — {reason}")
765            }
766            RunStatus::Swept => write!(f, "the plane recorded a pass over its own state"),
767            RunStatus::BrokeGlass { actor, reason } => {
768                write!(f, "{actor} crossed into this tenant — {reason}")
769            }
770            RunStatus::HaltLifted { actor } => write!(f, "{actor} lifted an emergency stop"),
771            RunStatus::HoldReleased { actor } => write!(f, "{actor} released a legal hold"),
772            RunStatus::Abandoned { actor, reason } => {
773                write!(
774                    f,
775                    "{actor} abandoned it, unresolved and not unwound — {reason}"
776                )
777            }
778            RunStatus::Withheld { subject, reason } => {
779                write!(
780                    f,
781                    "the authority it acts for — '{subject}' — was withdrawn: {reason}"
782                )
783            }
784        }
785    }
786}
787
788impl std::error::Error for RunFailure {}
789
790#[derive(Debug, Clone, PartialEq)]
791pub enum RunStatus {
792    /// All terminal work completed.
793    ///
794    /// Structural, never self-reported: a skill saying "done" is not what makes
795    /// a run succeed. Agents confidently announce success on unmet objectives,
796    /// so completion is determined by the runtime, not claimed by the workload.
797    Succeeded,
798    Failed(String),
799    /// Waiting for something that has not happened. The frame is persisted and
800    /// the task is gone: a suspended run costs disk, not a thread.
801    Suspended(crate::core::SuspendReason),
802    /// A limit stopped it. Not a fault — the run did what it was told, and what
803    /// it was told included a ceiling.
804    Exhausted(crate::core::BudgetExceeded),
805    /// Needs human resolution before anything else may happen to it.
806    Quarantined(String),
807    /// A step asked for a different plan.
808    ///
809    /// Never observed by a caller: the executor either produces a successor and
810    /// keeps going, or turns this into a failure with the reason it refused.
811    /// It exists as a status so a step's request travels the same path every
812    /// other outcome does.
813    Replanning(String),
814    /// An operator stopped it.
815    ///
816    /// Distinct from `Failed` on purpose. A failure is the run discovering it
817    /// cannot proceed; a cancellation is a human deciding it should not. They
818    /// call for different responses — one is investigated, the other was
819    /// intended — and an operator scanning for failures should not have to
820    /// mentally subtract their own interventions.
821    ///
822    /// Completed steps are unwound exactly as they are for a failure: stopping a
823    /// run that has moved money and leaving the movement in place is not
824    /// stopping it.
825    Cancelled {
826        actor: crate::core::Operator,
827        reason: String,
828    },
829    /// An operator closed a run whose outcome could never be established.
830    ///
831    /// The end of the road for a quarantine nobody can answer. Distinct from
832    /// `Cancelled` in the one way that matters to whoever reads it next:
833    /// **nothing was unwound**. A cancellation puts the world back; this leaves
834    /// it exactly as the run left it, because compensating around an unknown
835    /// outcome is the single thing quarantine exists to forbid, and an
836    /// operator's patience running out is not new evidence.
837    ///
838    /// So the doubt does not go away when the run does. It stops being a
839    /// *status* — which is a mutable thing a later action can overwrite — and
840    /// becomes an `agentplane audit` finding, which is derived from the journal
841    /// and outlives every status the run will ever have.
842    Abandoned {
843        actor: crate::core::Operator,
844        reason: String,
845    },
846    /// A pass this plane made over its own state, journaled as a run of its own.
847    ///
848    /// The sweep records what it decided — which obligations breached, which
849    /// tasks it escalated, which runs it took over — rather than logging it, so
850    /// *who decided what* is answerable from the chain six weeks later. It is
851    /// not `Succeeded` because nothing asked for it and there is no answer to
852    /// hand back: a sweep is the plane reporting on itself.
853    Swept,
854    /// An operator crossed into this tenant deliberately, and the reason they
855    /// gave for it.
856    ///
857    /// The one exception to tenant isolation, and the exception's entire value
858    /// is this record. Both halves are carried because an incident review asks
859    /// two questions and neither is worth less than the other: `actor` is read
860    /// back from the crossing record, `reason` from the conclusion, and a
861    /// crossing cannot be recorded without one.
862    BrokeGlass {
863        actor: crate::core::Operator,
864        reason: String,
865    },
866    /// An operator lifted an emergency stop, recorded as a run of its own.
867    ///
868    /// `actor` is read back from the run's
869    /// [`HaltLifted`](crate::journal::RecordKind::HaltLifted) record, which
870    /// also holds the stop it ended.
871    HaltLifted {
872        actor: crate::core::Operator,
873    },
874    /// An operator released a legal hold, recorded as a run of its own.
875    ///
876    /// `actor` is read back from the run's
877    /// [`HoldReleased`](crate::journal::RecordKind::HoldReleased) record.
878    HoldReleased {
879        actor: crate::core::Operator,
880    },
881    /// The authority this run acts under was withdrawn while it was running.
882    ///
883    /// **A pause, not a fault**, for the reason exhaustion is one: the run did
884    /// nothing wrong. Its mutations stand and it stays resumable — lift the halt
885    /// and it continues. Unwinding would reverse correct work for a reason
886    /// unrelated to it; cancelling is the verb for that.
887    ///
888    /// Carries the subject, because *whose* authority is what an operator
889    /// matches against the withdrawal they made.
890    Withheld {
891        subject: String,
892        reason: String,
893    },
894    /// A session this plane **observed** rather than executed, closed.
895    ///
896    /// The one status that is not about work this runtime did. A seat beside
897    /// somebody else's agent produces records at the *asserted* rung
898    /// ([`RecordKind::Observed`](crate::journal::RecordKind::Observed)), in a
899    /// run of their own with no admission — the shape a sweep's run already
900    /// takes — so that they share the chain, the Merkle log and the witness
901    /// while sharing no record kind with a dispatched effect.
902    ///
903    /// **It carries nothing, and the absence is the design.** There is no
904    /// reason to give: the session ended because the party running it ended
905    /// it, and this plane is in no position to say why. The session's own
906    /// identifier is on every observation record rather than here, so a plane
907    /// that stopped mid-session still leaves evidence that says which session
908    /// it describes.
909    ///
910    /// Sealed like any other ending, so an external verifier checks it without
911    /// being taught what an observation is. Never resumable: there is nothing
912    /// to resume onto, because nothing here was executing.
913    Observed,
914}
915
916impl RunStatus {
917    /// Why the run ended, whatever kind of ending it was.
918    ///
919    /// `None` only for [`Succeeded`](Self::Succeeded), which has no reason to
920    /// give. Every other conclusion carries one, and the point of gathering
921    /// them here is that an embedder mapping outcomes onto its own wire type
922    /// should not have to match every variant to find the sentence, and an
923    /// embedder that reads only the status ships an empty summary on failure.
924    ///
925    /// Borrowed where the variant already holds a string and formatted where
926    /// it holds a typed value: a suspension names what it waits for and an
927    /// exhaustion names the ceiling it hit, and both are worth more than the
928    /// word "suspended". `Cancelled` yields the operator's reason; the actor
929    /// who gave it stays on the variant, because *why* and *who* are two
930    /// questions and only one of them is the reason.
931    #[must_use]
932    pub fn reason(&self) -> Option<std::borrow::Cow<'_, str>> {
933        use std::borrow::Cow;
934        match self {
935            // Nothing to say, for three different reasons and one answer: a
936            // success has no why, a sweep's records carry theirs, and an
937            // observed session ended because somebody else ended it.
938            // A lift's record carries what it ended; the lifter gave no reason.
939            Self::Succeeded
940            | Self::Swept
941            | Self::Observed
942            | Self::HaltLifted { .. }
943            | Self::HoldReleased { .. } => None,
944            Self::Failed(reason)
945            | Self::Quarantined(reason)
946            | Self::Replanning(reason)
947            | Self::Cancelled { reason, .. }
948            | Self::Abandoned { reason, .. }
949            | Self::BrokeGlass { reason, .. }
950            | Self::Withheld { reason, .. } => Some(Cow::Borrowed(reason.as_str())),
951            Self::Suspended(reason) => Some(Cow::Owned(reason.to_string())),
952            Self::Exhausted(exceeded) => Some(Cow::Owned(exceeded.to_string())),
953        }
954    }
955
956    /// The run's **own** account of why it ended, for the journal.
957    ///
958    /// [`reason`](Self::reason) answers *what should a person be shown*, which
959    /// includes the words an operator typed when they stopped the run. This
960    /// answers the narrower question the conclusion record asks: *is there a
961    /// sentence here that nothing else on the chain already records?*
962    ///
963    /// `None` for every ending a person caused. Those write their own record —
964    /// [`RunCancelled`](crate::journal::RecordKind::RunCancelled),
965    /// [`QuarantineDecided`](crate::journal::RecordKind::QuarantineDecided),
966    /// [`BreakGlass`](crate::journal::RecordKind::BreakGlass),
967    /// [`AuthorityWithheld`](crate::journal::RecordKind::AuthorityWithheld) —
968    /// carrying the actor and the reason together, in the clear. Repeating the
969    /// reason on the conclusion would make one sentence two facts, and seal the
970    /// copy that an erasure then destroys.
971    ///
972    /// `None` for an exhaustion too: the typed ceiling verdict is the record,
973    /// and the prose was a rendering of it.
974    #[must_use]
975    pub fn own_account(&self) -> Option<std::borrow::Cow<'_, str>> {
976        use std::borrow::Cow;
977        match self {
978            // The runtime's own endings: a skill's words, a provider's words,
979            // a divergence this build found. Nothing else on the chain says it.
980            Self::Failed(reason) | Self::Quarantined(reason) | Self::Replanning(reason) => {
981                Some(Cow::Borrowed(reason.as_str()))
982            }
983            // Nothing to add, for two different reasons and one answer. The
984            // first four are somebody's doing and their own record already
985            // carries both halves. The rest either write no conclusion with a
986            // why — a success, a sweep, a suspension — or carry it typed
987            // instead, which is what `exhaustion` is.
988            //
989            // Enumerated rather than left to a wildcard: a status added later
990            // must be made to answer this question, and `_ => None` would
991            // answer it silently and wrongly for anything a person causes.
992            Self::Cancelled { .. }
993            | Self::Abandoned { .. }
994            | Self::BrokeGlass { .. }
995            | Self::HaltLifted { .. }
996            | Self::HoldReleased { .. }
997            | Self::Withheld { .. }
998            | Self::Succeeded
999            | Self::Swept
1000            // The account is the observation records themselves, and this plane
1001            // has none of its own to add: it did not run the session.
1002            | Self::Observed
1003            | Self::Suspended(_)
1004            | Self::Exhausted(_) => None,
1005        }
1006    }
1007
1008    /// Whether the run stopped without reaching a conclusion.
1009    #[must_use]
1010    pub fn is_suspended(&self) -> bool {
1011        matches!(self, Self::Suspended(_))
1012    }
1013
1014    /// Whether the recorded history can no longer be trusted to describe this
1015    /// code, so a human must look before anything else happens.
1016    #[must_use]
1017    pub fn is_quarantined(&self) -> bool {
1018        matches!(self, Self::Quarantined(_))
1019    }
1020
1021    #[must_use]
1022    pub fn as_str(&self) -> &'static str {
1023        match self {
1024            Self::Succeeded => "succeeded",
1025            Self::Failed(_) => "failed",
1026            Self::Suspended(_) => "suspended",
1027            Self::Exhausted(_) => "exhausted",
1028            Self::Quarantined(_) => "quarantined",
1029            Self::Replanning(_) => "replanning",
1030            Self::Cancelled { .. } => "cancelled",
1031            Self::Abandoned { .. } => "abandoned",
1032            Self::Swept => super::sweeper::SWEEP_OUTCOME,
1033            Self::BrokeGlass { .. } => BREAK_GLASS_OUTCOME,
1034            Self::HaltLifted { .. } => HALT_LIFTED_OUTCOME,
1035            Self::HoldReleased { .. } => HOLD_RELEASED_OUTCOME,
1036            Self::Withheld { .. } => WITHHELD_OUTCOME,
1037            Self::Observed => OBSERVED_OUTCOME,
1038        }
1039    }
1040
1041    /// Who brought this run to this state, where a person did.
1042    ///
1043    /// `None` for every ending the runtime reached on its own, which is most of
1044    /// them. One accessor rather than three `matches!` at each surface, because
1045    /// *who* is the question an operator asks immediately after *what* — and a
1046    /// surface that reads the reason and drops the actor answers half of it.
1047    #[must_use]
1048    pub fn actor(&self) -> Option<&crate::core::Operator> {
1049        match self {
1050            Self::Cancelled { actor, .. }
1051            | Self::Abandoned { actor, .. }
1052            | Self::BrokeGlass { actor, .. }
1053            | Self::HaltLifted { actor }
1054            | Self::HoldReleased { actor } => Some(actor),
1055            // A withdrawal names a *subject*, not an actor: the operator who
1056            // threw the halt and the credential it covers are different
1057            // parties, and answering "who" with the subject would report the
1058            // person whose authority was taken as the one who took it.
1059            Self::Succeeded
1060            | Self::Failed(_)
1061            | Self::Suspended(_)
1062            | Self::Exhausted(_)
1063            | Self::Quarantined(_)
1064            | Self::Replanning(_)
1065            | Self::Withheld { .. }
1066            | Self::Swept
1067            // Somebody was at the keyboard, and it was not this plane's
1068            // operator. Naming the observed party's user here would report a
1069            // person this runtime never authenticated as one of its own.
1070            | Self::Observed => None,
1071        }
1072    }
1073
1074    /// Whether an operator stopped this run.
1075    #[must_use]
1076    pub fn is_cancelled(&self) -> bool {
1077        matches!(self, Self::Cancelled { .. })
1078    }
1079
1080    /// Whether this conclusion freezes the journal and enters the Merkle log.
1081    ///
1082    /// Only conclusions that nothing may resume seal. A failed run may be
1083    /// resumed — its completed effects are read back from history rather than
1084    /// performed again, which is the point of having a journal — and an
1085    /// exhausted run may continue once somebody raises the ceiling. Sealing
1086    /// either would commit the Merkle log to a leaf its own resume is then
1087    /// permitted to grow past: the checkpoint would attest a prefix of a
1088    /// history that kept moving, and the store's refusal of appends after a
1089    /// seal would turn every legitimate resume into an error.
1090    ///
1091    /// **A quarantine does not seal**, and that follows from the same rule
1092    /// rather than being an exception to it. A quarantine is the runtime saying
1093    /// it does not know, which is a story that is not over; a Merkle leaf is a
1094    /// claim that a history is complete. Sealing one published a proof of
1095    /// completeness over an admittedly incomplete record, and — the practical
1096    /// half — froze the journal against the one record that answers it, so
1097    /// "a human must resolve it before it can run again" named a resolution the
1098    /// format made impossible. It enters the log when it reaches a real ending,
1099    /// including [`Abandoned`](Self::Abandoned), which is a person deciding
1100    /// there will never be one.
1101    ///
1102    /// This is the **one** implementation of that rule. `resume_is_closed`
1103    /// refuses exactly the outcomes this method seals — a test pins the
1104    /// agreement — because two copies of one rule agree everywhere except the
1105    /// boundary nobody probed.
1106    #[must_use]
1107    pub fn seals(&self) -> bool {
1108        matches!(
1109            self,
1110            Self::Succeeded
1111                | Self::Cancelled { .. }
1112                | Self::Abandoned { .. }
1113                | Self::Swept
1114                | Self::BrokeGlass { .. }
1115                | Self::HaltLifted { .. }
1116                | Self::HoldReleased { .. }
1117                | Self::Observed
1118        )
1119    }
1120}
1121
1122/// Every outcome a sealed run can carry, as the outcome index spells them.
1123///
1124/// The store indexes runs *by* outcome and deliberately has no "all runs"
1125/// query — a backlog listing is a range read rather than a table scan — so any
1126/// caller that wants every sealed run has to name the outcomes one by one. This
1127/// is the list to name them from, and it lives beside [`RunStatus::seals`]
1128/// because it is the same rule in its other spelling: exactly the statuses that
1129/// seal, with no exception clause. A sweep (`swept`) and a break-glass crossing
1130/// (`broke-glass`) are sealed at birth with no goal to have succeeded or failed
1131/// at, and they are statuses all the same — an ending this plane writes and its
1132/// own reader cannot name is an ending it reports as unrecognised.
1133///
1134/// It lived in the CLI first, as string literals — two implementations of one
1135/// rule, where a new sealing outcome would have been exported by nobody and
1136/// the export that silently omitted it would have dropped exactly the runs an
1137/// auditor came for. A test holds this list to `RunStatus::seals` variant by
1138/// variant, which the literals in a binary could never be held to.
1139pub const SEALED_OUTCOMES: &[&str] = &[
1140    "succeeded",
1141    "cancelled",
1142    "abandoned",
1143    super::sweeper::SWEEP_OUTCOME,
1144    BREAK_GLASS_OUTCOME,
1145    HALT_LIFTED_OUTCOME,
1146    HOLD_RELEASED_OUTCOME,
1147    OBSERVED_OUTCOME,
1148];
1149
1150/// What an offline verb sweeps when the operator names no outcome.
1151///
1152/// Every ending — [`SEALED_OUTCOMES`] — **plus `quarantined`**, which is not an
1153/// ending and is exactly why it belongs: a run set aside for a person is the
1154/// plane's outstanding unresolved risk, and an artifact holding everything that
1155/// finished and nothing that did not is not the one an auditor came for. It is
1156/// also the outcome most likely to be the reason they came.
1157///
1158/// Failed and exhausted runs are deliberately absent, and the distinction is
1159/// not sealing but *whether anything is waiting on a person*. Those two are
1160/// work in progress that a resume moves on its own; a quarantine moves only
1161/// when somebody decides, so it can sit for weeks and still be current.
1162///
1163/// A test holds this to [`SEALED_OUTCOMES`] rather than restating it, so a new
1164/// ending is swept here the day it exists.
1165pub const OUTCOMES_OF_RECORD: &[&str] = &[
1166    "succeeded",
1167    "cancelled",
1168    "abandoned",
1169    "quarantined",
1170    super::sweeper::SWEEP_OUTCOME,
1171    BREAK_GLASS_OUTCOME,
1172    HALT_LIFTED_OUTCOME,
1173    HOLD_RELEASED_OUTCOME,
1174    OBSERVED_OUTCOME,
1175];
1176
1177/// How a resume came to hold its lease, which decides what a failure does
1178/// with it.
1179///
1180/// A lease that lapses while still naming an owner is what the recovery sweep
1181/// lists, so keeping one on failure is how a run stays findable; releasing it
1182/// is right only where the run is findable without it.
1183#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1184enum Claim {
1185    /// Claimed by the resume itself, over a run whose journal already lists
1186    /// it — failed, exhausted, withheld, quarantined, waiting.
1187    Standing,
1188    /// Handed over by a wake or a delivery that recorded the awaited result
1189    /// under it, which took the run off the waiting listing.
1190    Unlisted,
1191    /// Claimed by the recovery sweep over a lease its owner died holding.
1192    Recovering,
1193}
1194
1195/// A run that exists but has not run.
1196///
1197/// Produced by admission and consumed by execution, so the two can happen in
1198/// different places — the same request for a blocking call, and a background
1199/// task for a non-blocking one.
1200struct Admitted {
1201    run: RunId,
1202    epoch: crate::core::Epoch,
1203    quota: QuotaPass,
1204    budget: Budget,
1205    agent: String,
1206    plan: PlanIR,
1207    input: Tainted<Value>,
1208    case: Option<CaseContext>,
1209    /// The chain admission bound — the caller's where one was presented,
1210    /// the plane's otherwise — and what every step of this run acts under.
1211    identity: Option<crate::core::Delegation>,
1212    /// The references its `DataSubjectBound` record makes available.
1213    subjects: BTreeSet<crate::core::SubjectRef>,
1214}
1215
1216/// The quota identity of one live execution pass.
1217///
1218/// Captured once at admission or resume and carried to settlement. Computing
1219/// the period again after the work finishes can cross a day or month boundary,
1220/// authorizing against one ledger and charging another.
1221#[derive(Debug, Clone, Default)]
1222struct QuotaPass {
1223    period: Option<String>,
1224    enabled: bool,
1225    release_slot: bool,
1226    /// How many steps this pass may dispatch at once, when the tenant bounds
1227    /// spend: the width the run's reservation was sized for. `None` leaves
1228    /// the run's own budget as the only bound.
1229    width: Option<usize>,
1230}
1231
1232impl QuotaPass {
1233    fn at(
1234        quota: &crate::quota::TenantQuota,
1235        at: crate::core::Timestamp,
1236        release_slot: bool,
1237    ) -> Self {
1238        Self {
1239            period: quota.bounds_spend().then(|| quota.period.key_for(at)),
1240            enabled: true,
1241            release_slot,
1242            width: None,
1243        }
1244    }
1245
1246    const fn disabled() -> Self {
1247        Self {
1248            period: None,
1249            enabled: false,
1250            release_slot: false,
1251            width: None,
1252        }
1253    }
1254
1255    /// Held to the width a spend reservation was sized for, whenever the
1256    /// tenant bounds spend.
1257    fn sized_for(mut self, width: usize) -> Self {
1258        if self.period.is_some() {
1259            self.width = Some(width);
1260        }
1261        self
1262    }
1263
1264    /// How wide this pass may dispatch: the budget's own width, narrowed to
1265    /// the one its reservation was sized for. A replan may widen the plan; it
1266    /// does not widen what the period holds for it.
1267    fn parallelism(&self, budget: &Budget) -> usize {
1268        self.width
1269            .map_or_else(|| budget.parallelism(), |w| budget.parallelism().min(w))
1270    }
1271
1272    fn period(&self) -> Option<&str> {
1273        self.period.as_deref()
1274    }
1275
1276    fn started(&self) -> Option<RecordKind> {
1277        self.enabled.then(|| RecordKind::QuotaPassStarted {
1278            period: self.period.clone(),
1279            release_slot: self.release_slot,
1280        })
1281    }
1282
1283    fn settlement(
1284        &self,
1285        run: RunId,
1286        epoch: crate::core::Epoch,
1287        spend: Spend,
1288        concludes: bool,
1289    ) -> Option<crate::quota::QuotaSettlement> {
1290        self.enabled.then(|| crate::quota::QuotaSettlement {
1291            run,
1292            epoch,
1293            period: self.period.clone(),
1294            spend,
1295            release_slot: self.release_slot,
1296            concludes,
1297        })
1298    }
1299}
1300
1301/// How many steps a run may have in flight: its budget's width, or its
1302/// admitted plan's node count when that is narrower or nothing narrows it.
1303///
1304/// What a spend reservation multiplies the per-call bound by, since each step
1305/// in flight may hold one operation the ledger admitted and has not billed.
1306fn reserved_width(budget: &Budget, plan: &PlanIR) -> usize {
1307    budget.parallelism().min(plan.nodes.len()).max(1)
1308}
1309
1310/// The peers a plane may call, and how it reaches them.
1311///
1312/// One value because the two are one decision: a registry says what each
1313/// peer is granted and which credential reaches it, and the client is the
1314/// transport those grants are spent over. Held by the plane and handed to
1315/// every step, so a skill calling a peer reaches it through the same checked
1316/// registry a declarative agent does — never one it carries itself.
1317#[derive(Debug)]
1318pub(crate) struct PeerWiring {
1319    pub(crate) registry: crate::peers::PeerRegistry,
1320    pub(crate) client: Arc<dyn crate::peers::PeerClient>,
1321}
1322
1323/// Keeps a run's lease alive for as long as it is executing.
1324///
1325/// A lease answers one question — *is this owner dead?* — and it answers by
1326/// expiry. Without renewal it also answers a question it was never asked: a
1327/// healthy run that outlives its TTL looks exactly like a crashed one, and agent
1328/// runs routinely outlive a lease because a single model call can. Another
1329/// instance then takes the run over, bumps the epoch, and the original is fenced
1330/// on its next append: killed mid-flight, having already done real work.
1331///
1332/// Renewing while executing separates the two. The TTL then bounds how long a
1333/// *crashed* owner strands its runs, which is what it is for, and stops bounding
1334/// how long a run may take, which it should never have bounded.
1335///
1336/// Aborted on drop, so the renewal stops the moment execution returns — by any
1337/// path, including a panic unwinding through it. A heartbeat that outlived its
1338/// run would hold a lease nobody is using and strand it for a full TTL after a
1339/// crash, which is the failure this exists to prevent, arriving late.
1340struct Heartbeat(tokio::task::JoinHandle<()>);
1341
1342impl Drop for Heartbeat {
1343    fn drop(&mut self) {
1344        self.0.abort();
1345    }
1346}
1347
1348/// The runtime.
1349#[derive(Debug, Clone)]
1350pub struct Runtime {
1351    store: Arc<dyn JournalStore>,
1352    skills: HashMap<String, Arc<dyn Skill>>,
1353    by_capability: HashMap<Capability, String>,
1354    /// A handle to this plane, for steps that commission other agents on it.
1355    ///
1356    /// Weak, and that is not an optimisation: a strong self-reference would
1357    /// leak every runtime ever built. It exists because commissioning belongs to
1358    /// the *runtime* — a skill holding an `Arc<Runtime>` cannot work, since the
1359    /// runtime needs the skill before the skill can have the runtime.
1360    self_ref: std::sync::Weak<Self>,
1361    /// The declaration governing each skill, by skill name.
1362    ///
1363    /// Per skill rather than per runtime, because a plane runs several agents
1364    /// and a step must be judged against *its own* agent's manifest.
1365    #[cfg(feature = "manifest")]
1366    governed_by: HashMap<String, Arc<crate::manifest::Manifest>>,
1367    /// Which tenant this plane runs as.
1368    ///
1369    /// One plane serves one tenant; a **process** may run several and serve
1370    /// them all, because `api::Planes` resolves a plane from the authenticated
1371    /// caller's tenant. That is what the composite keys on both backends bought:
1372    /// the tenant leads every key, lease, correlation and blob path, so a query
1373    /// that forgets it misses rather than returning somebody else's rows.
1374    tenant: crate::core::TenantId,
1375    /// Who vouched for each declaration, by agent name.
1376    ///
1377    /// Separate from the manifest because it is *not* in it: a document cannot
1378    /// state who signed it. It arrives beside the manifest from a verified
1379    /// registry resolution, and an absent entry means nobody vouched.
1380    #[cfg(feature = "manifest")]
1381    published_by: HashMap<String, crate::core::KeyId>,
1382    /// Model drivers by the name a manifest calls them.
1383    ///
1384    /// Kept after build — not only consumed by it — so a *hand-written* skill
1385    /// can reach the driver its manifest's model role names through
1386    /// [`StepCtx::complete`], instead of smuggling an `Arc<dyn ModelProvider>`
1387    /// field past the declaration.
1388    #[cfg(feature = "manifest")]
1389    providers: HashMap<String, Arc<dyn crate::model::ModelProvider>>,
1390    owner: String,
1391    /// How long a run's lease lasts, and how long a crashed owner's runs stay
1392    /// unclaimable.
1393    lease_ttl: Duration,
1394    /// Where this plane's agents remember things, when a deployment wires one.
1395    memories: Option<Arc<dyn crate::memory::MemoryStore>>,
1396    semantic: Option<Arc<super::SemanticMemory>>,
1397    authorities: Option<Arc<dyn crate::authority::AuthorityStore>>,
1398    /// The peers this plane may call, and the transport that reaches them.
1399    peers: Option<Arc<PeerWiring>>,
1400    /// The catalogue and transport a **skill** reaches tools through.
1401    ///
1402    /// Held by the plane rather than handed to each skill, and that is the
1403    /// point. A skill that built its own [`ToolCatalog`] could declare a tool
1404    /// read-only that its manifest calls mutating — and `ToolSafety::read_only`
1405    /// carries `Recovery::Retry`, so a timed-out money-moving call gets sent
1406    /// again. `check_catalogue_not_laxer_than_grants` refuses exactly that for
1407    /// the plane's catalogue at build; a hand-built one never passed under it.
1408    /// `StepCtx::call_tool` is the path that cannot.
1409    #[cfg(feature = "manifest")]
1410    tools: Option<(
1411        Arc<crate::tools::ToolCatalog>,
1412        Arc<dyn crate::tools::ToolClient>,
1413    )>,
1414    /// The highest sensitivity this plane will write into an append-only
1415    /// record, when it says.
1416    journal_ceiling: Option<crate::core::Sensitivity>,
1417    /// Where this plane's tool transports may connect.
1418    ///
1419    /// Behind `manifest` because the tool path is.
1420    ///
1421    /// Here rather than on each `ToolClient` because it is a **plane**
1422    /// decision: the client owns the connection, the plane owns the
1423    /// destination, exactly as it does for a model driver's base URL. Absent
1424    /// means no egress control on the tool path, spelled the same way an
1425    /// absent policy engine is.
1426    #[cfg(feature = "manifest")]
1427    egress: Option<Arc<crate::core::Egress>>,
1428    #[cfg(feature = "manifest")]
1429    checkers: crate::content::Checkers,
1430    /// Where live model output is forwarded, when anything listens.
1431    streams: Option<Arc<dyn RunStreamObserver>>,
1432    /// How this plane attributes its metrics.
1433    meter: super::metrics::Meter,
1434    /// Durable per-tenant ceilings, when a deployment wires them.
1435    quotas: Option<Arc<dyn crate::quota::QuotaStore>>,
1436    /// The rate ceilings this plane's declarations state, and the quota store
1437    /// that counts them. `None` when no grant declares one.
1438    #[cfg(feature = "manifest")]
1439    rates: Option<Arc<super::ctx::Rates>>,
1440    quota: crate::quota::TenantQuota,
1441    budget: Budget,
1442    /// Whether every plan this runtime admits must carry a verifier.
1443    require_verifier: bool,
1444
1445    cases: Option<Arc<dyn CaseStore>>,
1446    events: Option<Arc<dyn EventStore>>,
1447    tasks: Option<Arc<dyn TaskStore>>,
1448    timers: Option<Arc<dyn TimerStore>>,
1449    blobs: Option<Arc<dyn crate::blob::BlobStore>>,
1450    /// Where disclosures are recorded, read by a retention pass so it names
1451    /// the copies of what it erased.
1452    disclosures: Option<Arc<dyn crate::disclosure::DisclosureRegister>>,
1453    /// Where webhook registrations and their cursors live.
1454    ///
1455    /// Held by the plane rather than only by whoever delivers, because the
1456    /// registrations that stop being delivered are a **backlog**, and a backlog
1457    /// is answered by an operator on the plane's own surface — not by the
1458    /// worker that gave up on it.
1459    #[cfg(feature = "push")]
1460    push: Option<Arc<dyn crate::push::PushStore>>,
1461    /// Where data keys live, when payload bytes are sealed.
1462    #[cfg(feature = "keyring")]
1463    keyring: Option<Arc<dyn crate::keyring::KeyRing>>,
1464    batches: Option<Arc<dyn crate::batch::BatchStore>>,
1465    policy: Option<Arc<dyn crate::core::PolicyEngine>>,
1466    identity: Option<crate::core::Delegation>,
1467    replanner: Option<Arc<dyn crate::plan::Replanner>>,
1468    calendar: Arc<dyn Calendar>,
1469    signer: Option<Arc<dyn crate::core::Signer>>,
1470    /// Where this plane sends its own events, when a deployment configures it.
1471    ///
1472    /// Registered per run **at admission**, and again by every resume that
1473    /// executes: a destination attached after the first record would silently
1474    /// miss it, and "the journal is the outbox" only holds if the cursor starts
1475    /// at sequence one.
1476    #[cfg(feature = "push")]
1477    outbox: Option<Arc<crate::push::Outbox>>,
1478    /// The parties this plane submits its checkpoints to, and how many must
1479    /// answer.
1480    ///
1481    /// The one control here whose whole value is that it is **not** this
1482    /// plane: the chain, the signatures and the Merkle log all verify against
1483    /// inputs the operator supplies, so none of them detects a run removed or
1484    /// two histories shown to two auditors. Absent means the plane's history
1485    /// is anchored nowhere outside itself, which is a deployment decision and
1486    /// is reported as one by `audit`.
1487    witnesses: Option<Arc<Witnessing>>,
1488    /// The runs this instance put on a background task, and the gate that
1489    /// closes admission when it is asked to stop.
1490    ///
1491    /// Process-local on purpose: it answers *what is this instance executing
1492    /// right now*, which no other instance can be told and no store can hold.
1493    /// The durable cousin is [`Runtime::set_halt`], which stops a tenant
1494    /// everywhere and survives a restart — a different question with a
1495    /// different audience.
1496    inflight: Arc<super::drain::InFlight>,
1497}
1498
1499/// The witness set and the bar a submission round is held to.
1500#[derive(Debug)]
1501pub(crate) struct Witnessing {
1502    pub(crate) witnesses: Vec<Arc<dyn crate::journal::Witness>>,
1503    pub(crate) quorum: crate::journal::WitnessQuorum,
1504    /// The size last submitted under a **met** quorum.
1505    ///
1506    /// In memory, deliberately. It exists only to keep a periodic pass from
1507    /// re-submitting a log that has not grown, and the authority on where a
1508    /// witness actually is has always been the witness — which answers `409`
1509    /// with its own size and gets a proof built from there. So a restart
1510    /// re-submits once and the protocol absorbs it; a durable copy would be a
1511    /// second answer to a question the counterparty already answers.
1512    pub(crate) submitted: std::sync::atomic::AtomicU64,
1513    /// The declared maximum time between submissions of an unchanged log;
1514    /// `None` never re-submits one.
1515    pub(crate) interval: Option<std::time::Duration>,
1516    /// The sweep instant of the last round that met quorum, in memory for the
1517    /// reason `submitted` is.
1518    pub(crate) last_met: std::sync::Mutex<Option<crate::core::Timestamp>>,
1519}
1520
1521/// Where a run is entering this plane from.
1522///
1523/// The one thing the drain gate needs to know, and it cannot be derived at the
1524/// gate: by the time admission runs, an agent commissioning another one and a
1525/// peer sending a message look identical. The difference is whose work it is —
1526/// a commission is a step of a run this process is *already* executing and has
1527/// promised to finish, so refusing it would make the drain manufacture the
1528/// undecided effects it exists to prevent.
1529#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1530pub(crate) enum Entry {
1531    /// A caller outside this process: an embedder, an operator, a peer, a batch
1532    /// pass. Refused while draining.
1533    Outside,
1534    /// A run already executing here, ordering work it needs in order to
1535    /// conclude. Never refused for draining — the parent is what the drain is
1536    /// waiting for.
1537    Nested,
1538}
1539
1540/// A backend that implements every store a full plane runs on.
1541///
1542/// Blanket-implemented, so it is a *description* rather than an obligation:
1543/// any type providing the store traits — journal, cases, tasks, events,
1544/// timers, memory, quotas, disclosures — is a `FullBackend` without saying so, and both
1545/// shipped backends are. The quota store is in the set because halts live
1546/// there: a plane without it would admit past every emergency stop. What it buys is [`Runtime::builder_on`], where the
1547/// alternative was a cast of one `Arc` per store that every deployment spelled out
1548/// and none could get differently.
1549pub trait FullBackend:
1550    JournalStore
1551    + CaseStore
1552    + TaskStore
1553    + EventStore
1554    + TimerStore
1555    + crate::memory::MemoryStore
1556    + crate::quota::QuotaStore
1557    + crate::disclosure::DisclosureRegister
1558{
1559}
1560
1561impl<B> FullBackend for B where
1562    B: JournalStore
1563        + CaseStore
1564        + TaskStore
1565        + EventStore
1566        + TimerStore
1567        + crate::memory::MemoryStore
1568        + crate::quota::QuotaStore
1569        + crate::disclosure::DisclosureRegister
1570{
1571}
1572
1573impl Runtime {
1574    /// A builder with the whole case layer wired to one backend.
1575    ///
1576    /// The one-call form of the seven-cast litany: journal, cases, tasks,
1577    /// events, timers, memory and quotas all backed by `store`, which is what a
1578    /// deployment on a single `RedbStore` or `PostgresStore` means anyway. The
1579    /// à-la-carte methods remain — [`cases`](RuntimeBuilder::cases) and
1580    /// friends override an individual store afterwards, and
1581    /// [`builder`](Self::builder) still starts from the journal alone for a
1582    /// plane that wants nothing else.
1583    ///
1584    /// Blob storage is deliberately not included: bytes routinely live in a
1585    /// different system than rows, so [`blobs`](RuntimeBuilder::blobs) stays
1586    /// an explicit decision.
1587    #[must_use]
1588    pub fn builder_on<B: FullBackend + 'static>(store: Arc<B>) -> RuntimeBuilder {
1589        Self::builder_with(Stores::on(store))
1590    }
1591
1592    /// A builder with the whole case layer wired from handles.
1593    ///
1594    /// What [`builder_on`](Self::builder_on) is in terms of: the seven casts live
1595    /// in one place, so a deployment that chose its backend at run time — an
1596    /// operator naming either a file or a connection string — wires exactly what
1597    /// a deployment on one concrete type wires, rather than a hand-written
1598    /// approximation of it that drifts the day an eighth store is added.
1599    #[must_use]
1600    pub fn builder_with(stores: Stores) -> RuntimeBuilder {
1601        Self::builder(stores.journal)
1602            .cases(stores.cases)
1603            .tasks(stores.tasks)
1604            .events(stores.events)
1605            .timers(stores.timers)
1606            .memory(stores.memory)
1607            .quota(stores.quotas, crate::quota::TenantQuota::default())
1608            .disclosures(stores.disclosures)
1609    }
1610
1611    #[must_use]
1612    pub fn builder(store: Arc<dyn JournalStore>) -> RuntimeBuilder {
1613        RuntimeBuilder {
1614            store,
1615            signer: None,
1616            skills: Vec::new(),
1617            owner: None,
1618            lease_ttl: LEASE_TTL,
1619            memories: None,
1620            semantic: None,
1621            authorities: None,
1622            peers: None,
1623            tenant_label: super::telemetry::TenantLabel::default(),
1624            quotas: None,
1625            quota: crate::quota::TenantQuota::default(),
1626            budget: Budget::unlimited(),
1627            require_verifier: false,
1628            cases: None,
1629            events: None,
1630            tasks: None,
1631            timers: None,
1632            blobs: None,
1633            disclosures: None,
1634            #[cfg(feature = "push")]
1635            push: None,
1636            #[cfg(feature = "keyring")]
1637            keyring: None,
1638            #[cfg(feature = "manifest")]
1639            tools: None,
1640            journal_ceiling: None,
1641            #[cfg(feature = "manifest")]
1642            egress: None,
1643            #[cfg(feature = "manifest")]
1644            checkers: crate::content::Checkers::default(),
1645            streams: None,
1646            tenant: crate::core::TenantId::default(),
1647            batches: None,
1648            policy: None,
1649            identity: None,
1650            replanner: None,
1651            calendar: None,
1652            #[cfg(feature = "manifest")]
1653            toolbox: None,
1654            #[cfg(feature = "manifest")]
1655            tool_servers: Vec::new(),
1656            #[cfg(feature = "manifest")]
1657            agents: Vec::new(),
1658            #[cfg(feature = "manifest")]
1659            providers: HashMap::new(),
1660            #[cfg(feature = "push")]
1661            outbox: None,
1662            witnesses: Vec::new(),
1663            quorum: None,
1664            witness_interval: None,
1665        }
1666    }
1667
1668    /// The driver registered under a manifest's name for it, if any.
1669    #[cfg(feature = "manifest")]
1670    pub(crate) fn model_provider(
1671        &self,
1672        name: &str,
1673    ) -> Option<Arc<dyn crate::model::ModelProvider>> {
1674        self.providers.get(name).map(Arc::clone)
1675    }
1676
1677    /// The case store, if this runtime has one.
1678    #[must_use]
1679    pub fn cases(&self) -> Option<&Arc<dyn CaseStore>> {
1680        self.cases.as_ref()
1681    }
1682
1683    /// The worklist, if this runtime has one.
1684    #[must_use]
1685    pub fn tasks(&self) -> Option<&Arc<dyn TaskStore>> {
1686        self.tasks.as_ref()
1687    }
1688
1689    /// The inbound-event store, if this runtime has one.
1690    #[must_use]
1691    pub fn events(&self) -> Option<&Arc<dyn EventStore>> {
1692        self.events.as_ref()
1693    }
1694
1695    /// The ceilings every run under this runtime starts with.
1696    ///
1697    /// Readable because "what is this plane allowed to spend" is a question an
1698    /// operator asks of a running system, and answering it by re-reading the
1699    /// config that *should* have been applied is how a misapplied budget stays
1700    /// invisible.
1701    #[must_use]
1702    pub const fn budget(&self) -> &Budget {
1703        &self.budget
1704    }
1705
1706    /// The blob store, if this runtime has one.
1707    #[must_use]
1708    pub fn blobs(&self) -> Option<&Arc<dyn crate::blob::BlobStore>> {
1709        self.blobs.as_ref()
1710    }
1711
1712    /// Run the live half of the case-layer drill with this plane's own stores.
1713    ///
1714    /// The wiring is the point of this method existing: the drill's value is
1715    /// holding each case's references against the stores the plane *actually
1716    /// runs with*, and an embedder assembling `drill::Stores` by hand could
1717    /// hand it a different blob store than the one their runs write to —
1718    /// a drill that passes over the wrong bucket. Stores this runtime does
1719    /// not have are reported as unchecked by the drill itself, which is a
1720    /// different and better answer than an error: a plane with no blob store
1721    /// has no bytes to lose.
1722    ///
1723    /// **The clock is the caller's**, the same decision
1724    /// [`attention`](Self::attention) and the push sweep make. A rehearsal's
1725    /// instant is a fact about the outside world rather than a journaled
1726    /// observation, and taking it here would be a fourth ambient-clock escape
1727    /// inside the deterministic zone for a value the caller already has. A
1728    /// fixed instant also makes the recorded verdict testable.
1729    ///
1730    /// # Errors
1731    ///
1732    /// If this runtime has no case store — the drill walks cases, so there is
1733    /// nothing to drill — or if the case layer cannot be enumerated.
1734    pub async fn drill(
1735        &self,
1736        at: crate::core::Timestamp,
1737    ) -> Result<crate::drill::DrillReport, RuntimeError> {
1738        let cases = self.cases().ok_or_else(|| {
1739            RuntimeError::PlanContract(
1740                "this runtime has no case store — the drill walks cases, so there is \
1741                 nothing to drill; build it with `.cases(store)`"
1742                    .into(),
1743            )
1744        })?;
1745        let stores = crate::drill::Stores {
1746            cases,
1747            blobs: self.blobs(),
1748            #[cfg(feature = "keyring")]
1749            keys: self.keyring.as_ref(),
1750            tenant: &self.tenant,
1751        };
1752        let report = crate::drill::drill(&stores)
1753            .await
1754            .map_err(RuntimeError::from_store)?;
1755
1756        // **The verdict is recorded, not only returned.** What an audit asks
1757        // is *when did you last rehearse, and did it pass* — and left to the
1758        // caller that answer lives in whatever ran the verb, which is a CI log
1759        // or a wiki page. The checkpoint pins which store it ran against, so a
1760        // drill over a restored copy cannot be read as one over production.
1761        let checkpoint = self
1762            .store
1763            .checkpoint()
1764            .await
1765            .map_err(RuntimeError::from_store)?;
1766        cases
1767            .record_drill(&crate::case::DrillRecord {
1768                at,
1769                sound: report.is_sound(),
1770                cases: report.cases as u64,
1771                findings: report.findings.len() as u64,
1772                not_checked: report.not_checked.len() as u64,
1773                origin: checkpoint.origin.clone(),
1774                size: checkpoint.size,
1775            })
1776            .await
1777            .map_err(RuntimeError::from_store)?;
1778        Ok(report)
1779    }
1780
1781    /// The last recovery rehearsal this plane ran, or `None` if it never has.
1782    ///
1783    /// The half that makes `drill` evidence rather than a command: an audit
1784    /// asks *when did you last rehearse and did it pass*, and that answer has
1785    /// to come from the plane rather than from the thing that scheduled it.
1786    ///
1787    /// # Errors
1788    ///
1789    /// [`RuntimeError::PlanContract`] when this plane has no case store — the
1790    /// same condition that makes a drill impossible in the first place — and
1791    /// [`RuntimeError`] for a store failure.
1792    pub async fn last_drill(&self) -> Result<Option<crate::case::DrillRecord>, RuntimeError> {
1793        self.cases()
1794            .ok_or_else(|| {
1795                RuntimeError::PlanContract(
1796                    "this runtime has no case store, so it has never drilled one".into(),
1797                )
1798            })?
1799            .last_drill()
1800            .await
1801            .map_err(RuntimeError::from_store)
1802    }
1803
1804    /// Erase every closed case opened before `older_than`. The retention verb.
1805    ///
1806    /// The wiring is the point, exactly as it is for [`drill`](Self::drill):
1807    /// erasure derives blob addresses and key scopes from the tenant and the
1808    /// case, so a caller assembling the stores by hand could tombstone
1809    /// addresses no run ever wrote and report a clean pass over nothing.
1810    ///
1811    /// `reason` lands on every tombstone and on each key destruction, so a
1812    /// later read says *expired, on this date, for this reason* rather than
1813    /// *missing* — the distinction the drill's three-way verdict turns on.
1814    ///
1815    /// `at` is a parameter rather than a clock read, for the reason the
1816    /// sweeper's is: a pass that read the clock could not be tested against a
1817    /// year of ageing cases.
1818    ///
1819    /// Read [`RetentionReport::not_erasable`](crate::retention::RetentionReport::not_erasable)
1820    /// before concluding anything. Without a key ring this expires blob
1821    /// tombstones in the live store and **journal payloads stay verbatim**,
1822    /// which is the residual the erasure page names and the one an erasure
1823    /// request actually turns on.
1824    ///
1825    /// # Errors
1826    ///
1827    /// If this runtime has no case store, or the case layer cannot be walked.
1828    /// A store that fails on one case is a report entry, not an error.
1829    pub async fn retain(
1830        &self,
1831        older_than: crate::core::Timestamp,
1832        at: crate::core::Timestamp,
1833        reason: &str,
1834    ) -> Result<crate::retention::RetentionReport, RuntimeError> {
1835        let cases = self.cases().ok_or_else(|| {
1836            RuntimeError::PlanContract(
1837                "this runtime has no case store — retention walks cases, so there is \
1838                 nothing to retain; build it with `.cases(store)`"
1839                    .into(),
1840            )
1841        })?;
1842        let stores = crate::retention::Stores {
1843            cases,
1844            blobs: self.blobs(),
1845            #[cfg(feature = "keyring")]
1846            keys: self.keyring.as_ref(),
1847            tenant: &self.tenant,
1848            disclosures: self.disclosures.as_ref(),
1849        };
1850        crate::retention::retain(&stores, self.store.as_ref(), older_than, at, reason)
1851            .await
1852            .map_err(RuntimeError::from_store)
1853    }
1854
1855    /// The timer store, if this runtime has one.
1856    #[must_use]
1857    pub fn timers(&self) -> Option<&Arc<dyn TimerStore>> {
1858        self.timers.as_ref()
1859    }
1860
1861    #[must_use]
1862    pub fn batches(&self) -> Option<&Arc<dyn crate::batch::BatchStore>> {
1863        self.batches.as_ref()
1864    }
1865
1866    #[cfg(feature = "push")]
1867    #[must_use]
1868    pub fn push(&self) -> Option<&Arc<dyn crate::push::PushStore>> {
1869        self.push.as_ref()
1870    }
1871
1872    /// Re-arm a webhook registration a delivery worker gave up on.
1873    ///
1874    /// A parked registration is not delivered and not deleted: it keeps its
1875    /// cursor, so re-arming resumes at the first record the receiver never
1876    /// acknowledged rather than at the head of the run. That is the whole
1877    /// reason parking exists, and the reason this verb is on the plane — a
1878    /// backlog nobody can empty is a warning line in yesterday's logs.
1879    ///
1880    /// The clock is read here rather than taken from the caller, unlike the
1881    /// delivery worker's: a worker's schedule has to be assertable in a test,
1882    /// and an operator asking for *now* means now.
1883    ///
1884    /// Returns whether a parked registration was found. `false` is an answer —
1885    /// already live, or never existed — and is told rather than swallowed,
1886    /// because an operator left waiting for a sweep with nothing to do has no
1887    /// way to tell the difference.
1888    ///
1889    /// # Errors
1890    ///
1891    /// [`RuntimeError::PlanContract`] if this plane has no push store, and
1892    /// [`RuntimeError::Store`] if it is unreachable.
1893    #[cfg(feature = "push")]
1894    pub async fn rearm_push(&self, task: RunId, id: &str) -> Result<bool, RuntimeError> {
1895        let push = self
1896            .push
1897            .as_ref()
1898            .ok_or_else(|| RuntimeError::PlanContract("this plane has no push store".into()))?;
1899        let at = now_for_admission().unix_timestamp().max(0).unsigned_abs();
1900        push.unpark(task, id, at)
1901            .await
1902            .map_err(RuntimeError::from_store)
1903    }
1904
1905    /// Ask a run to stop, and drive the stop if nothing else will.
1906    ///
1907    /// The request is durable before this returns, so an operator who gets an
1908    /// acknowledgement has one whether or not the run was reachable. What
1909    /// happens next depends on where the run is:
1910    ///
1911    /// * **Suspended** — nothing is executing, so this resumes the run itself.
1912    ///   It observes the request at its first step boundary and unwinds.
1913    /// * **Running here or elsewhere** — the owner observes the request at its
1914    ///   next step boundary. Nothing is interrupted mid-effect, deliberately:
1915    ///   stopping between "announced" and "recorded" manufactures the in-doubt
1916    ///   case the effect protocol exists to avoid.
1917    /// * **Quarantined** — refused with [`RuntimeError::CannotUnwind`]: the
1918    ///   unwind a stop promises is what a quarantine forbids, and reopening or
1919    ///   abandoning it are the verbs that apply.
1920    /// * **Concluded and closed** (succeeded, abandoned, cancelled) — refused
1921    ///   with [`RuntimeError::AlreadyConcluded`] and nothing is recorded; a run
1922    ///   somebody already stopped answers `false`. A failed or exhausted run is
1923    ///   resumable, and stopping it unwinds its work.
1924    /// * **Held by no provider on this plane** — an operator's terminal: the
1925    ///   request is recorded and not driven, and the plane that holds the run's
1926    ///   agent observes it on its next resume.
1927    ///
1928    /// Returns whether *this* call recorded the request. A second caller gets
1929    /// `false`: the first asker stays on the record, because "who intervened"
1930    /// must not be rewritten by a retry.
1931    ///
1932    /// # Errors
1933    ///
1934    /// [`RuntimeError::PlanContract`] for a blank reason,
1935    /// [`RuntimeError::CannotUnwind`] for a quarantined run,
1936    /// [`RuntimeError::AlreadyConcluded`] for a closed one, and
1937    /// [`RuntimeError`] if the store is unreachable or resuming a suspended run
1938    /// fails.
1939    pub async fn request_cancel(
1940        &self,
1941        run: RunId,
1942        actor: &crate::core::Operator,
1943        reason: &str,
1944    ) -> Result<bool, RuntimeError> {
1945        // An intervention with no stated reason documents nobody's judgement,
1946        // and it is the one field of the record the requester writes.
1947        if reason.trim().is_empty() {
1948            return Err(RuntimeError::PlanContract(
1949                "stopping a run needs a reason — what the person stopping it saw".to_owned(),
1950            ));
1951        }
1952        // Checked before recording. Writing first and failing afterwards leaves a
1953        // request standing against an id that does not exist, and the operator's
1954        // retry then comes back "somebody else already asked" — which is a
1955        // confusing way to say "you mistyped".
1956        let records = self
1957            .store
1958            .read(run, 1)
1959            .await
1960            .map_err(RuntimeError::from_store)?;
1961        if records.is_empty() {
1962            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
1963                run.to_string(),
1964            )));
1965        }
1966
1967        // A quarantine is refused here rather than recorded and ignored. The
1968        // stop this verb promises is *unwind and put the world back*, and that
1969        // is the one thing a quarantined run may not do — so the request landed
1970        // durably, answered the operator `recorded: true`, and the resume then
1971        // walked straight past it. An operator who believes they stopped a run
1972        // holding an unresolved payment is worse off than one who was told no.
1973        //
1974        // Read from the recorded conclusion, not from a live status: a run that
1975        // quarantines *after* this call is a genuine race the standing request
1976        // loses, and that is correct — the request was made about a run nobody
1977        // had given up on.
1978        if recorded_conclusion(&records).as_deref() == Some("quarantined") {
1979            return Err(RuntimeError::CannotUnwind {
1980                run: run.to_string(),
1981            });
1982        }
1983        // Refused only where the conclusion closes the run. A failed or an
1984        // exhausted run is resumable and its work stands, so stopping it is the
1985        // operator's remedy — the unwind it asks for is exactly what those
1986        // conclusions left undone. Read from the latest conclusion, not the
1987        // last record: an answer to a doubt may follow a conclusion, and the
1988        // run is no less closed for it. A run already stopped answers `false`
1989        // rather than an error, so a second asker is told the intervention of
1990        // record is somebody else's.
1991        if let Some(status) = resume_is_closed(&records) {
1992            if self.cancellation(run).await?.is_some() {
1993                return Ok(false);
1994            }
1995            return Err(RuntimeError::AlreadyConcluded {
1996                run: run.to_string(),
1997                outcome: recorded_conclusion(&records)
1998                    .unwrap_or_else(|| status.as_str().to_owned()),
1999            });
2000        }
2001
2002        let fresh = self
2003            .store
2004            .request_cancel(run, actor, reason)
2005            .await
2006            .map_err(RuntimeError::from_store)?;
2007
2008        // Drive it. A suspended run has no thread to notice anything, so an
2009        // operator's stop would sit unobserved until the deadline swept it —
2010        // which for a run waiting on a six-week obligation is not a stop.
2011        //
2012        // Resuming a run that has already concluded is a no-op inside `replay`,
2013        // which reads the recorded status back rather than re-executing — so
2014        // there is no "is it finished?" check to race with here.
2015        //
2016        // Driven **only when the run is genuinely idle**: `replay` claims the
2017        // lease before it reads, and a `LeaseHeld` refusal means somebody is
2018        // executing the run right now — here or on another instance. That is
2019        // not a failure of this call. The request is already durable, and the
2020        // live owner checks for it at every step boundary (the cancellation
2021        // read at the top of the executor's ready-set loop), so the stop
2022        // arrives without a second executor ever touching the run. Resuming
2023        // anyway would, on the owner's own instance, hand the resume the *same
2024        // epoch* the live execution is writing under — two executors on one
2025        // chain that fencing, by construction, cannot tell apart.
2026        //
2027        // A plane holding no provider, case store or key ring for the run — an
2028        // operator's terminal, a library plane — records the request and
2029        // leaves the driving to the plane that does: that plane's next resume
2030        // finds the request standing.
2031        if fresh {
2032            match self.replay(run, Mode::Resume).await {
2033                Ok(_)
2034                | Err(
2035                    RuntimeError::LeaseHeld { .. }
2036                    | RuntimeError::NoProvider { .. }
2037                    | RuntimeError::NoCaseStore { .. }
2038                    | RuntimeError::PayloadsSealed { .. },
2039                ) => {}
2040                Err(e) => return Err(e),
2041            }
2042        }
2043        Ok(fresh)
2044    }
2045
2046    /// Answer a quarantine, and let the runtime decide the run again.
2047    ///
2048    /// A quarantine is this runtime saying it does not know, and the one
2049    /// conclusion it cannot reach past on its own. Everything else that stops a
2050    /// run has a party who can move it — a resume for a crash, a raised ceiling
2051    /// for an exhaustion, a reviewer for a suspension. This is the party for
2052    /// this one.
2053    ///
2054    /// # What this does and does not decide
2055    ///
2056    /// The person decides *whether the run may be judged again*, or that it
2057    /// never will be. They do not decide the outcome. A
2058    /// [`Reopen`](crate::core::QuarantineDecision::Reopen) hands the run back
2059    /// to the executor, which re-derives its verdict from a history that now
2060    /// holds whatever the person established with
2061    /// [`reconcile_effect`](Self::reconcile_effect) — and quarantines it again,
2062    /// on the record, if the answer was wrong or incomplete. An
2063    /// [`Abandon`](crate::core::QuarantineDecision::Abandon) closes it where it
2064    /// stands, unwinding nothing.
2065    ///
2066    /// The decision is journaled **before** it is acted on, so a crash between
2067    /// the two leaves the instruction standing rather than lost: the next
2068    /// resume finds it and finishes the job.
2069    ///
2070    /// Returns the outcome of the pass this drove — so an operator who reopens
2071    /// a run learns immediately whether it got any further, rather than having
2072    /// to go and look.
2073    ///
2074    /// # Errors
2075    ///
2076    /// [`RuntimeError::NotQuarantined`] if the run's last conclusion is
2077    /// anything else, [`RuntimeError::LeaseHeld`] if somebody is executing it,
2078    /// and [`RuntimeError`] for a store failure or a refused resume.
2079    ///
2080    /// # Panics
2081    ///
2082    /// Never: the `expect` covers a lease this function has just acquired.
2083    pub async fn decide_quarantine(
2084        &self,
2085        run: RunId,
2086        decider: &crate::core::Operator,
2087        reason: &str,
2088        decision: crate::core::QuarantineDecision,
2089    ) -> Result<RunOutcome, RuntimeError> {
2090        // The whole weight of this record is that a named person took
2091        // responsibility for a fact the runtime could not establish, and
2092        // "unknown decided this for no stated reason" documents nothing while
2093        // looking like process. Only the reason is checked here: `Operator` has
2094        // no empty value, so the decider is guaranteed by the type on every
2095        // path that builds one rather than by a check on this one.
2096        if reason.trim().is_empty() {
2097            return Err(RuntimeError::PlanContract(
2098                "answering a quarantine needs a decider and a reason — who looked, and what they \
2099                 found"
2100                    .to_owned(),
2101            ));
2102        }
2103
2104        let lease = self
2105            .store
2106            .acquire(run, &self.owner, self.lease_ttl)
2107            .await
2108            .map_err(RuntimeError::from_store)?;
2109        let appended = self
2110            .record_decision(run, &lease, decider, reason, decision)
2111            .await;
2112        if let Err(e) = appended {
2113            // The lease was claimed by this call and never handed to a resume,
2114            // so it is this call's to give back. Leaving it would strand the
2115            // run for a TTL and hand it to the recovery sweep, which would
2116            // "recover" a run nobody was running.
2117            let _ = self.store.release_lease(run, lease.epoch).await;
2118            return Err(e);
2119        }
2120        // Handed over rather than released and re-taken: the same window a wake
2121        // path closes. Both decisions travel this path — an abandonment is
2122        // concluded inside the resume, from the record just written. The run
2123        // still stands quarantined on the record, so it keeps its listing
2124        // whatever this pass does.
2125        self.replay_releasing(run, Pass::Resume, Some(lease), Claim::Standing)
2126            .await
2127    }
2128
2129    /// Answer a quarantine **without driving the run**.
2130    ///
2131    /// [`decide_quarantine`](Self::decide_quarantine) records the decision and
2132    /// then hands the run to the executor, which is what an operator on a
2133    /// running plane wants: they learn immediately whether a reopened run got
2134    /// any further.
2135    ///
2136    /// A terminal holding a store cannot do the second half. It has the
2137    /// journal and not the agent — a declarative run's program is a manifest
2138    /// this process was not given — so driving the run would mean refusing the
2139    /// verb for want of an argument the operator may not have during an
2140    /// incident.
2141    ///
2142    /// **That is a supported state rather than a lesser one.** The decision is
2143    /// journaled before it is acted on precisely so a crash between the two
2144    /// leaves the instruction standing, and this is the same position reached
2145    /// deliberately: the next resume finds the record and finishes the job —
2146    /// so what the caller must not do is report the run as moved.
2147    ///
2148    /// # Errors
2149    ///
2150    /// As [`decide_quarantine`](Self::decide_quarantine), minus everything the
2151    /// resume could have raised.
2152    pub async fn record_quarantine_decision(
2153        &self,
2154        run: RunId,
2155        decider: &crate::core::Operator,
2156        reason: &str,
2157        decision: crate::core::QuarantineDecision,
2158    ) -> Result<(), RuntimeError> {
2159        if reason.trim().is_empty() {
2160            return Err(RuntimeError::PlanContract(
2161                "answering a quarantine needs a decider and a reason — who looked, and what they \
2162                 found"
2163                    .to_owned(),
2164            ));
2165        }
2166        let lease = self
2167            .store
2168            .acquire(run, &self.owner, self.lease_ttl)
2169            .await
2170            .map_err(RuntimeError::from_store)?;
2171        let appended = self
2172            .record_decision(run, &lease, decider, reason, decision)
2173            .await;
2174        // Always given back: this verb never executes the run, so the lease is
2175        // held only for the width of one append — the rule
2176        // `reconcile_effect` follows.
2177        let _ = self.store.release_lease(run, lease.epoch).await;
2178        appended
2179    }
2180
2181    /// Append the decision, once the run is known to be asking for one.
2182    async fn record_decision(
2183        &self,
2184        run: RunId,
2185        lease: &crate::journal::Lease,
2186        decider: &crate::core::Operator,
2187        reason: &str,
2188        decision: crate::core::QuarantineDecision,
2189    ) -> Result<(), RuntimeError> {
2190        let records = self
2191            .store
2192            .read(run, 1)
2193            .await
2194            .map_err(RuntimeError::from_store)?;
2195        if records.is_empty() {
2196            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2197                run.to_string(),
2198            )));
2199        }
2200        let status = recorded_conclusion(&records);
2201        // A run whose payloads were erased is asking the same question a
2202        // quarantine asks — *the runtime cannot go on, and a person must
2203        // decide* — so it is answered by the same verb whatever outcome it was
2204        // left under. Without this it has no verb at all: it cannot be
2205        // resumed, and a cancellation promises an unwind that erased arguments
2206        // make impossible. Abandonment is the honest ending and this is the
2207        // door to it.
2208        if status.as_deref() != Some("quarantined") && !payloads_erased(&records) {
2209            return Err(RuntimeError::NotQuarantined {
2210                run: run.to_string(),
2211                status: status.unwrap_or_else(|| "still running".to_owned()),
2212            });
2213        }
2214        let mut append = Append::new(
2215            run,
2216            RecordKind::QuarantineDecided {
2217                decider: decider.clone(),
2218                reason: reason.to_owned(),
2219                decision,
2220            },
2221        );
2222        if let Some(case) = records.iter().find_map(|r| r.body.case) {
2223            append = append.case(case);
2224        }
2225        self.store
2226            .append(lease.epoch, vec![append])
2227            .await
2228            .map_err(RuntimeError::from_store)?;
2229        Ok(())
2230    }
2231
2232    /// Supply, out of band, the outcome of an effect the journal cannot decide.
2233    ///
2234    /// The manual twin of [`Effect::reconcile`](crate::core::Effect::reconcile),
2235    /// and it produces the same record for the same reason: an operator who
2236    /// looks the charge up in the provider's console is answering exactly the
2237    /// question a probe asks, so the answer belongs in the place a probe's
2238    /// answer goes, where replay reads it back and nothing re-derives it.
2239    ///
2240    /// What separates the two is on the record too. A probe's verdict is
2241    /// attributed to the run; this one names the person, because "the provider
2242    /// told us" and "somebody asserted it" are different evidence.
2243    ///
2244    /// **It supplies a missing fact and never replaces a recorded one.** Only
2245    /// an effect [`undecided_effects`](crate::journal::undecided_effects) names
2246    /// may be answered. Without that rule an operator could talk a run out of
2247    /// compensating work that is standing in the world, and the journal would
2248    /// show an orderly reconciliation while it happened.
2249    ///
2250    /// This changes no status by itself: the run stays quarantined until
2251    /// [`decide_quarantine`](Self::decide_quarantine) hands it back. Several
2252    /// doubts can be answered before the run is judged again, and answering one
2253    /// of three is not a decision.
2254    ///
2255    /// # Errors
2256    ///
2257    /// [`RuntimeError::NotUndecided`] if that effect's outcome is already on
2258    /// the record, [`RuntimeError::LeaseHeld`] if somebody is executing the
2259    /// run, and [`RuntimeError`] for a store failure.
2260    pub async fn reconcile_effect(
2261        &self,
2262        run: RunId,
2263        effect: crate::core::EffectKey,
2264        assertion: crate::core::Assertion,
2265        asserted_by: &crate::core::Operator,
2266        note: &str,
2267    ) -> Result<(), RuntimeError> {
2268        // `asserted_by` needs no emptiness check: `Operator` has none to fail.
2269        if note.trim().is_empty() {
2270            return Err(RuntimeError::PlanContract(
2271                "an assertion about an effect needs a name and a note — who established this, and \
2272                 how"
2273                .to_owned(),
2274            ));
2275        }
2276        let lease = self
2277            .store
2278            .acquire(run, &self.owner, self.lease_ttl)
2279            .await
2280            .map_err(RuntimeError::from_store)?;
2281        let result = self
2282            .record_assertion(run, &lease, effect, assertion, asserted_by, note)
2283            .await;
2284        // Always given back: this verb never executes the run, so the lease is
2285        // held only for the width of one append.
2286        let _ = self.store.release_lease(run, lease.epoch).await;
2287        result
2288    }
2289
2290    async fn record_assertion(
2291        &self,
2292        run: RunId,
2293        lease: &crate::journal::Lease,
2294        effect: crate::core::EffectKey,
2295        assertion: crate::core::Assertion,
2296        asserted_by: &crate::core::Operator,
2297        note: &str,
2298    ) -> Result<(), RuntimeError> {
2299        let records = self
2300            .store
2301            .read(run, 1)
2302            .await
2303            .map_err(RuntimeError::from_store)?;
2304        if records.is_empty() {
2305            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2306                run.to_string(),
2307            )));
2308        }
2309        let Some(undecided) = crate::journal::undecided_effects(&records)
2310            .into_iter()
2311            .find(|u| u.effect == effect)
2312        else {
2313            return Err(RuntimeError::NotUndecided {
2314                run: run.to_string(),
2315                effect: effect.to_string(),
2316            });
2317        };
2318
2319        let output = match &assertion {
2320            crate::core::Assertion::Landed(value) => Some(value.clone()),
2321            crate::core::Assertion::DidNotHappen => None,
2322        };
2323        let mut append = Append::new(
2324            run,
2325            RecordKind::EffectReconciled {
2326                disposition: assertion.disposition(),
2327                // Stated on the record rather than left to a reader's default.
2328                // A durable format that does not say what it means is one a
2329                // later build guesses about, and the guess here is the label on
2330                // a value the run is about to act on.
2331                declared: output
2332                    .is_some()
2333                    .then(crate::core::DeclaredOutput::untrusted),
2334                output,
2335                // A person's assertion costs nothing. What the call itself
2336                // consumed, if it consumed anything, is already on the record
2337                // of the attempt this answers.
2338                spend: Spend::ZERO,
2339                // No probe ran: this is a person's assertion, so the
2340                // provider-facing field stays empty and their account goes in
2341                // the field that is read without a key.
2342                detail: None,
2343                note: Some(note.to_owned()),
2344                asserted_by: Some(asserted_by.clone()),
2345            },
2346        )
2347        .step(undecided.step)
2348        .phase(undecided.phase)
2349        .effect(effect);
2350        if let Some(case) = records.iter().find_map(|r| r.body.case) {
2351            append = append.case(case);
2352        }
2353        self.store
2354            .append(lease.epoch, vec![append])
2355            .await
2356            .map_err(RuntimeError::from_store)?;
2357        tracing::info!(
2358            target: telemetry::RECONCILED,
2359            %run,
2360            step = %undecided.step,
2361            verdict = ?assertion.disposition(),
2362            actor = %asserted_by,
2363        );
2364        self.meter
2365            .count(metrics::RECONCILIATIONS, assertion.disposition().as_str());
2366        Ok(())
2367    }
2368
2369    /// Every mutating effect this run's journal cannot decide the outcome of.
2370    ///
2371    /// What an operator holding a quarantine has to go and look up, and the
2372    /// keys they quote back to [`reconcile_effect`](Self::reconcile_effect).
2373    ///
2374    /// # Errors
2375    ///
2376    /// If the store is unreachable.
2377    pub async fn undecided(&self, run: RunId) -> Result<Vec<crate::core::Undecided>, RuntimeError> {
2378        let records = self
2379            .store
2380            .read(run, 1)
2381            .await
2382            .map_err(RuntimeError::from_store)?;
2383        Ok(crate::journal::undecided_effects(&records))
2384    }
2385
2386    /// Stop this tenant from starting new work, or let it start again.
2387    ///
2388    /// The emergency stop. `Some(reason)` halts, `None` lifts, and the reason is
2389    /// required because the next person to look will be somebody else, possibly
2390    /// at three in the morning, and *why* is the whole question.
2391    ///
2392    /// **What it stops, precisely.** New admissions, across every instance,
2393    /// because the flag is in the store rather than in this process — a switch
2394    /// that stops only the instance it was thrown on is the in-process-counter
2395    /// failure arriving during an incident. Refusals are their own error, not a
2396    /// ceiling: a ceiling means *not right now* and invites a retry, which is
2397    /// exactly what somebody pulling this switch is trying to stop.
2398    ///
2399    /// **What a workload-scoped halt does not stop, deliberately.** Runs
2400    /// already executing, and suspended runs resuming. Those are existing work,
2401    /// and refusing to let them continue would strand them mid-saga with
2402    /// reversals unrun — turning an incident into a second one. To stop work in
2403    /// flight, cancel it: that unwinds what it did and records who asked. This
2404    /// is the front door, not a power cut, and saying so is the difference
2405    /// between a control an operator can reason about and one they discover the
2406    /// shape of during an outage.
2407    ///
2408    /// **How wide it reaches** is the [`HaltScope`](crate::quota::HaltScope),
2409    /// and the scopes divide on *which axis they name* rather than on how
2410    /// broad they are. Three name a workload — the tenant, every revision of
2411    /// one declared agent, or one exact reviewed digest — and stop admission.
2412    /// The fourth names an **authority**, any principal on the delegation
2413    /// chain a run was admitted under, and is the one that reaches work
2414    /// already running:
2415    /// there the incident *is* the credential, so a run carrying on under it is
2416    /// the harm rather than a side effect, and it pauses at its next step
2417    /// boundary with its completed work standing
2418    /// ([`withdraws_from`](crate::quota::HaltScope::withdraws_from) is the
2419    /// predicate, so the distinction is checked rather than remembered).
2420    /// Scopes are independent rows — lifting a narrow halt leaves a broad one
2421    /// standing. An ungoverned run, with no manifest, is stopped only by a
2422    /// tenant halt; there is nothing narrower to key it on.
2423    ///
2424    /// `at` is the caller's, never a clock read here: every lifecycle instant in
2425    /// this crate is supplied for the reason a hold's is — a pass that reads its
2426    /// own clock cannot be tested against an ageing plane, and [`I1`] forbids the
2427    /// runtime an ambient clock outside the effect protocol.
2428    ///
2429    /// Requires a quota store; without one there is nowhere durable to keep the
2430    /// flag, and an emergency stop that a restart forgets is not one.
2431    ///
2432    /// [`I1`]: crate::journal
2433    ///
2434    /// # Errors
2435    ///
2436    /// If no quota store is wired, or the store is unreachable.
2437    pub async fn set_halt(
2438        &self,
2439        scope: &crate::quota::HaltScope,
2440        by: &crate::core::Operator,
2441        at: crate::core::Timestamp,
2442        reason: &str,
2443    ) -> Result<(), RuntimeError> {
2444        let quotas = self.quota_store()?;
2445        quotas
2446            .set_halt(scope, by, at, reason)
2447            .await
2448            .map_err(RuntimeError::Store)?;
2449        self.forget_credentials(scope);
2450        Ok(())
2451    }
2452
2453    /// Drop the peer credentials held for a subject a halt names, when it is
2454    /// thrown and when it is lifted — not only when a hop next meets it, which
2455    /// may never happen while the halt stands.
2456    fn forget_credentials(&self, scope: &crate::quota::HaltScope) {
2457        if let (crate::quota::HaltScope::Subject { id }, Some(wiring)) =
2458            (scope, self.peers.as_ref())
2459        {
2460            wiring.registry.forget(id);
2461        }
2462    }
2463
2464    /// Lift one, and let work at that scope start again — recording who did.
2465    ///
2466    /// Answers the run holding the lift record and whether this call removed
2467    /// the row, or `None` when nothing was standing at `scope`: an operator who
2468    /// lifts a scope nobody halted is told so rather than told *done*, and
2469    /// nothing is written.
2470    ///
2471    /// **The record comes first, then the row goes.** The lift is written as
2472    /// the sole [`HaltLifted`](crate::journal::RecordKind::HaltLifted) record
2473    /// of a sealed run of its own, carrying `by`, `at` and the whole stop it
2474    /// ended; only then is the register's row removed. The journal and the
2475    /// register share no transaction, so a lift that cannot be recorded does
2476    /// not happen, and a removal that fails after the record is written is an
2477    /// error naming the record's run — the halt still stands, the register is
2478    /// the answer to what is in force, and lifting again writes a second
2479    /// record. The record attests the instruction and the row as read, not
2480    /// that the removal succeeded.
2481    ///
2482    /// **Only the row the record names goes.** The removal is
2483    /// [`QuotaStore::lift_halt_if`](crate::quota::QuotaStore::lift_halt_if)
2484    /// with that row, so a halt re-thrown between the read and the removal is
2485    /// left standing and the call fails naming the record's run. A row another
2486    /// lift already removed answers `removed: false`.
2487    ///
2488    /// [`QuotaStore::lift_halt`](crate::quota::QuotaStore::lift_halt) is the
2489    /// raw register verb; a direct call to it is unrecorded.
2490    ///
2491    /// # Errors
2492    ///
2493    /// If no quota store is wired, the store is unreachable, or the record
2494    /// cannot be written; [`RuntimeError::ControlStands`] if the row cannot be
2495    /// removed after the record was written, or was re-thrown in between.
2496    pub async fn lift_halt(
2497        &self,
2498        scope: &crate::quota::HaltScope,
2499        by: &crate::core::Operator,
2500        at: crate::core::Timestamp,
2501    ) -> Result<Option<ControlLifted>, RuntimeError> {
2502        let quotas = self.quota_store()?;
2503        let standing = quotas.halts().await.map_err(RuntimeError::Store)?;
2504        let Some(halt) = standing.into_iter().find(|h| &h.scope == scope) else {
2505            return Ok(None);
2506        };
2507        let run = self
2508            .seal_operator_record(
2509                RecordKind::HaltLifted {
2510                    scope: scope.key(),
2511                    by: by.clone(),
2512                    at,
2513                    reason: halt.reason.clone(),
2514                    thrown_by: halt.by.clone(),
2515                    thrown_at: halt.at,
2516                },
2517                None,
2518                HALT_LIFTED_OUTCOME,
2519                None,
2520            )
2521            .await?;
2522        let removed = quotas
2523            .lift_halt_if(&halt)
2524            .await
2525            .map_err(|e| removal_failed(run, &e))?;
2526        self.forget_credentials(scope);
2527        if !removed {
2528            let now = quotas.halts().await.map_err(|e| removal_failed(run, &e))?;
2529            if now.iter().any(|h| &h.scope == scope) {
2530                return Err(RuntimeError::ControlStands {
2531                    run: run.to_string(),
2532                    detail: format!(
2533                        "the halt on '{}' was re-thrown after this lift read it, and the new \
2534                         halt still stands",
2535                        scope.key()
2536                    ),
2537                });
2538            }
2539        }
2540        Ok(Some(ControlLifted {
2541            record: run,
2542            removed,
2543        }))
2544    }
2545
2546    /// Release the legal hold on `case` — recording who did.
2547    ///
2548    /// The hold's counterpart of [`lift_halt`](Self::lift_halt), in the same
2549    /// order for the same reason: the
2550    /// [`HoldReleased`](crate::journal::RecordKind::HoldReleased) record is
2551    /// written, stamped with `case`, before the row goes. It names the releaser
2552    /// and the hold's placing operator and instant, and not the hold's reason,
2553    /// so the erasure the release permits leaves its authorization readable.
2554    ///
2555    /// `None` when the matter is not held, with nothing written; `removed:
2556    /// false` when another release removed the row first.
2557    ///
2558    /// **Only the hold the record names goes.** The removal is
2559    /// [`CaseStore::release_hold_if`](crate::case::CaseStore::release_hold_if)
2560    /// with that hold, so a hold placed again between the read and the removal
2561    /// is left standing and the call fails naming the record's run.
2562    /// [`CaseStore::release_hold`](crate::case::CaseStore::release_hold) is the
2563    /// raw register verb; a direct call to it is unrecorded.
2564    ///
2565    /// # Errors
2566    ///
2567    /// [`StoreError::NotFound`](crate::core::StoreError::NotFound) if the case
2568    /// does not exist; [`RuntimeError::ControlStands`] if the row cannot be
2569    /// removed after the record was written, or was placed again in between;
2570    /// otherwise if no case store is
2571    /// wired, the store is unreachable, or the record cannot be written.
2572    pub async fn release_hold(
2573        &self,
2574        case: crate::core::CaseId,
2575        by: &crate::core::Operator,
2576        at: crate::core::Timestamp,
2577    ) -> Result<Option<ControlLifted>, RuntimeError> {
2578        let cases = self.cases.as_ref().ok_or_else(|| {
2579            RuntimeError::PlanContract("releasing a hold needs a case store".to_owned())
2580        })?;
2581        let Some(standing) = cases.hold(case).await.map_err(RuntimeError::Store)? else {
2582            return match cases.case(case).await.map_err(RuntimeError::Store)? {
2583                Some(_) => Ok(None),
2584                None => Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2585                    format!("case {case}"),
2586                ))),
2587            };
2588        };
2589        let run = self
2590            .seal_operator_record(
2591                RecordKind::HoldReleased {
2592                    by: by.clone(),
2593                    at,
2594                    placed_by: standing.by.clone(),
2595                    placed_at: standing.placed_at,
2596                },
2597                Some(case),
2598                HOLD_RELEASED_OUTCOME,
2599                None,
2600            )
2601            .await?;
2602        let removed = cases
2603            .release_hold_if(case, &standing)
2604            .await
2605            .map_err(|e| removal_failed(run, &e))?;
2606        if !removed
2607            && cases
2608                .hold(case)
2609                .await
2610                .map_err(|e| removal_failed(run, &e))?
2611                .is_some()
2612        {
2613            return Err(RuntimeError::ControlStands {
2614                run: run.to_string(),
2615                detail: format!(
2616                    "the hold on case {case} was placed again after this release read it, \
2617                     and the new hold still stands"
2618                ),
2619            });
2620        }
2621        Ok(Some(ControlLifted {
2622            record: run,
2623            removed,
2624        }))
2625    }
2626
2627    /// Every recorded halt lift, newest first: the one record of each run.
2628    ///
2629    /// # Errors
2630    ///
2631    /// If the journal is unreachable, or a lift run does not hold a lift record.
2632    pub async fn lifted_halts(&self, limit: usize) -> Result<Vec<Record>, RuntimeError> {
2633        self.operator_records(HALT_LIFTED_OUTCOME, limit).await
2634    }
2635
2636    /// Every recorded hold release, newest first: the one record of each run.
2637    ///
2638    /// # Errors
2639    ///
2640    /// If the journal is unreachable, or a release run does not hold a release
2641    /// record.
2642    pub async fn released_holds(&self, limit: usize) -> Result<Vec<Record>, RuntimeError> {
2643        self.operator_records(HOLD_RELEASED_OUTCOME, limit).await
2644    }
2645
2646    /// The first record of each run sealed under `outcome`, in the index's
2647    /// newest-first order. A run whose first record is not the act its outcome
2648    /// names is a store error, never skipped: a history that silently thins is
2649    /// one that reads as complete.
2650    async fn operator_records(
2651        &self,
2652        outcome: &str,
2653        limit: usize,
2654    ) -> Result<Vec<Record>, RuntimeError> {
2655        let runs = self
2656            .store
2657            .runs_by_outcome(outcome, limit)
2658            .await
2659            .map_err(RuntimeError::from_store)?;
2660        let mut found = Vec::with_capacity(runs.len());
2661        for run in runs {
2662            let first = self
2663                .store
2664                .read_page(run, 1, 1)
2665                .await
2666                .map_err(RuntimeError::from_store)?
2667                .into_iter()
2668                .next();
2669            match first {
2670                Some(record)
2671                    if matches!(
2672                        (outcome, record.kind()),
2673                        (HALT_LIFTED_OUTCOME, RecordKind::HaltLifted { .. })
2674                            | (HOLD_RELEASED_OUTCOME, RecordKind::HoldReleased { .. })
2675                    ) =>
2676                {
2677                    found.push(record);
2678                }
2679                _ => {
2680                    return Err(RuntimeError::Store(crate::core::StoreError::Corrupt {
2681                        seq: 1,
2682                        detail: format!(
2683                            "run {run} is sealed as '{outcome}' but holds no such record"
2684                        ),
2685                    }));
2686                }
2687            }
2688        }
2689        Ok(found)
2690    }
2691
2692    /// The quota store, when one is wired.
2693    pub(crate) fn quota_store_if_wired(&self) -> Option<&Arc<dyn crate::quota::QuotaStore>> {
2694        self.quotas.as_ref()
2695    }
2696
2697    /// Every run holding spend against this tenant's periods, and how much.
2698    ///
2699    /// The runs a [`QuotaError::SpentOut`](crate::quota::QuotaError::SpentOut)
2700    /// refusal counts as *reserved*. A run holds its reservation until it
2701    /// concludes, so the ones that stopped waiting on a person — quarantined,
2702    /// exhausted, withheld, failed — hold it until that person acts;
2703    /// [`attention`](Self::attention) names those.
2704    ///
2705    /// # Errors
2706    ///
2707    /// If no quota store is wired, or it is unreachable.
2708    pub async fn reservations(
2709        &self,
2710        limit: usize,
2711    ) -> Result<Vec<crate::quota::Held>, RuntimeError> {
2712        let quotas = self.quotas.as_ref().ok_or_else(|| {
2713            RuntimeError::Store(crate::core::StoreError::Backend(
2714                "no quota store is wired, so no run holds spend against a period".to_owned(),
2715            ))
2716        })?;
2717        quotas
2718            .reservations(limit)
2719            .await
2720            .map_err(RuntimeError::Store)
2721    }
2722
2723    /// The quota store, or the refusal that says why an emergency stop needs one.
2724    fn quota_store(&self) -> Result<&std::sync::Arc<dyn crate::quota::QuotaStore>, RuntimeError> {
2725        self.quotas.as_ref().ok_or_else(|| {
2726            RuntimeError::Store(crate::core::StoreError::Backend(
2727                "an emergency stop needs a quota store to keep the flag in — an \
2728                 in-process one is forgotten by a restart and never seen by a \
2729                 second instance"
2730                    .to_owned(),
2731            ))
2732        })
2733    }
2734
2735    /// Every standing emergency stop on this tenant, and why.
2736    ///
2737    /// The operator's question — *what is stopped right now?* — which a
2738    /// per-scope lookup cannot answer without already knowing what to ask
2739    /// about.
2740    ///
2741    /// # Errors
2742    ///
2743    /// If no quota store is wired, or the store is unreachable.
2744    pub async fn halts(&self) -> Result<Vec<crate::quota::Halt>, RuntimeError> {
2745        let quotas = self.quotas.as_ref().ok_or_else(|| {
2746            RuntimeError::Store(crate::core::StoreError::Backend(
2747                "no quota store is wired, so no emergency stop can be set or read".to_owned(),
2748            ))
2749        })?;
2750        quotas.halts().await.map_err(RuntimeError::Store)
2751    }
2752
2753    /// Which runs are executing **right now**, and under whose authority.
2754    ///
2755    /// [`runs_by_outcome`](crate::journal::JournalStore::runs_by_outcome)
2756    /// answers from the journal's index, which only *concluded* runs are in; the
2757    /// live set is the tenant's admission slots.
2758    ///
2759    /// **The join is the point, not the ids.** An incident is rarely *cancel
2760    /// this run* but a revoked credential or a bad revision, so each entry
2761    /// carries the declared agent and the delegation subject it was admitted
2762    /// under.
2763    ///
2764    /// A slot whose lease has lapsed is marked `stranded` rather than omitted:
2765    /// the recovery sweep resumes those, and cancelling one undoes work that was
2766    /// about to continue.
2767    ///
2768    /// Bounded by `limit`, and reads a page of each run's header records to
2769    /// attribute it — an operator question rather than a hot path.
2770    ///
2771    /// # Errors
2772    ///
2773    /// If no quota store is wired, or either store is unreachable.
2774    pub async fn running_runs(&self, limit: usize) -> Result<Vec<LiveRun>, RuntimeError> {
2775        let quotas = self.quotas.as_ref().ok_or_else(|| {
2776            RuntimeError::Store(crate::core::StoreError::Backend(
2777                "no quota store is wired, so nothing tracks which runs hold this \
2778                 tenant's admission slots"
2779                    .to_owned(),
2780            ))
2781        })?;
2782        let live = quotas
2783            .running_runs(limit)
2784            .await
2785            .map_err(RuntimeError::Store)?;
2786        // One read, not one per run: the stranded set is small and the whole of
2787        // it is needed to answer any entry.
2788        let stranded: std::collections::BTreeSet<RunId> = self
2789            .store
2790            .abandoned_runs(limit)
2791            .await
2792            .map_err(RuntimeError::Store)?
2793            .into_iter()
2794            .collect();
2795
2796        let mut out = Vec::with_capacity(live.len());
2797        for run in live {
2798            // The header records only. `RunAdmitted` is the first and
2799            // `IdentityBound` follows it when the caller presented a chain, so a
2800            // short page carries both without reading a run's whole history —
2801            // which on a long-running agent is the difference between an
2802            // operator listing and a scan.
2803            let head = self
2804                .store
2805                .read_page(run, 1, ATTRIBUTION_RECORDS)
2806                .await
2807                .map_err(RuntimeError::Store)?;
2808            let governed_by = head.iter().find_map(|r| match r.kind() {
2809                crate::journal::RecordKind::RunAdmitted { governed_by, .. } => {
2810                    governed_by.as_ref().map(|id| (**id).clone())
2811                }
2812                _ => None,
2813            });
2814            let subject = head.iter().find_map(|r| match r.kind() {
2815                crate::journal::RecordKind::IdentityBound { chain } => {
2816                    chain.first().map(|p| p.id.clone())
2817                }
2818                _ => None,
2819            });
2820            out.push(LiveRun {
2821                stranded: stranded.contains(&run),
2822                run,
2823                governed_by,
2824                subject,
2825            });
2826        }
2827        Ok(out)
2828    }
2829
2830    /// The runs that are waiting, and what each one waits for.
2831    ///
2832    /// The third listing, beside concluded runs and running ones: a suspended
2833    /// run has no outcome and holds no admission slot, so neither of the others
2834    /// names it. Re-arming these is the recovery runbook's last step, and until
2835    /// this existed that step named a verb with no argument an operator could
2836    /// obtain.
2837    ///
2838    /// Soonest due first, bounded by `limit`. Straight through to the store's
2839    /// maintained index — no page of records is read, because what a run waits
2840    /// for is on the suspension itself.
2841    ///
2842    /// # Errors
2843    ///
2844    /// If the store is unreachable.
2845    pub async fn waiting_runs(
2846        &self,
2847        limit: usize,
2848    ) -> Result<Vec<crate::journal::WaitingRun>, RuntimeError> {
2849        self.store
2850            .waiting_runs(limit)
2851            .await
2852            .map_err(RuntimeError::from_store)
2853    }
2854
2855    /// Stop this instance taking on work, then wait for what it is already
2856    /// doing.
2857    ///
2858    /// A process killed mid-step leaves an announced effect with no terminal
2859    /// record, so its declared [`Recovery`](crate::core::Recovery) has to
2860    /// decide whether it happened — the right answer to an accident, and an
2861    /// expensive one to schedule on every deploy.
2862    ///
2863    /// Admission closes **first**: every later [`start`](Self::start) is
2864    /// refused
2865    /// [`RuntimeError::Draining`](crate::core::RuntimeError::Draining), leaving
2866    /// no lease, no slot and no journal. Then this waits up to `grace` for the
2867    /// runs it put on a background task to reach a journaled resting point.
2868    /// Closing first is what makes the answer worth having — a wait alone
2869    /// reports about an instant — and it is also the only signal a long-lived
2870    /// read gets, so an open A2A subscription ends rather than holding a
2871    /// graceful shutdown for as long as the run it watches.
2872    ///
2873    /// **Run it beside a server's own graceful shutdown, not after it** —
2874    /// `tokio::join!` of the two. Serialising deadlocks one on the other in
2875    /// whichever order you pick: a run awaited inside a request handler is
2876    /// finished by the server, and an open subscription is ended by this.
2877    ///
2878    /// Three things it deliberately does not do. It does not wait for runs on
2879    /// somebody else's task; those belong to that caller's own shutdown. It
2880    /// does not gate resumes or an agent commissioning another one — both
2881    /// continue work this plane already owns, and a commission is a step of a
2882    /// run this is waiting for. And it does not hand back the leases of runs it
2883    /// did not finish: a release says takeover is safe, and for a run still
2884    /// inside `perform` the next owner would re-perform the call beside the one
2885    /// in flight. Left to expire, the lease is the signal the recovery sweep
2886    /// reads as *an instance died holding this run* — which is what a
2887    /// drained-out run is.
2888    ///
2889    /// Idempotent: draining an instance that is already draining waits again.
2890    pub async fn drain(&self, grace: std::time::Duration) -> super::DrainReport {
2891        self.inflight.drain(grace).await
2892    }
2893
2894    /// Whether this instance has stopped admitting runs.
2895    ///
2896    /// For an embedder whose server keeps its listener open while it drains and
2897    /// therefore needs something to answer a readiness probe with. The shipped
2898    /// server needs none: it closes its listeners first, so a probe that
2899    /// connects already fails — which is why this crate's HTTP surface has no
2900    /// `/health` route to flip. See [`crate::api`] on why it has none at all.
2901    #[must_use]
2902    pub fn is_draining(&self) -> bool {
2903        self.inflight.is_draining()
2904    }
2905
2906    /// The cancellation recorded against a run, if one was.
2907    ///
2908    /// Reads the journal rather than a flag: a cancellation is a record, so
2909    /// this answers the same thing after a restart and on another instance.
2910    ///
2911    /// # Errors
2912    ///
2913    /// If the store is unreachable.
2914    pub async fn cancellation(
2915        &self,
2916        run: RunId,
2917    ) -> Result<Option<crate::journal::Cancellation>, RuntimeError> {
2918        self.store
2919            .cancellation(run)
2920            .await
2921            .map_err(RuntimeError::from_store)
2922    }
2923
2924    /// The policy engine, if this runtime has one.
2925    ///
2926    /// Exposed because a surface that faces strangers has to be able to ask
2927    /// whether one exists at all. Inside the process the caller is the
2928    /// embedder's own code and an absent engine is a choice; on a socket it is
2929    /// a hole, and the HTTP surface refuses to start without one.
2930    #[must_use]
2931    pub fn policy(&self) -> Option<&Arc<dyn crate::core::PolicyEngine>> {
2932        self.policy.as_ref()
2933    }
2934
2935    /// Which tenant this plane runs as.
2936    #[must_use]
2937    pub fn tenant(&self) -> &crate::core::TenantId {
2938        &self.tenant
2939    }
2940
2941    /// What this plane's policy engine cannot evaluate for a served caller
2942    /// that presented no chain.
2943    ///
2944    /// The build probes the chain shapes this plane's own runs take: its own
2945    /// chain where it has one, and both shapes where it has none. A served
2946    /// surface adds the one the build cannot know about — a caller acting
2947    /// under no chain on a plane that has one — so every surface that admits
2948    /// work for a stranger asks this before it starts.
2949    #[cfg(any(feature = "a2a-server", feature = "mcp-server"))]
2950    pub(crate) fn served_policy_problems(&self) -> Vec<String> {
2951        match (&self.policy, &self.identity) {
2952            (Some(engine), Some(_)) => runtime_policy_problems(engine.as_ref(), None),
2953            _ => Vec::new(),
2954        }
2955    }
2956
2957    /// How far from its owner the plane's own chain is, or zero without one.
2958    ///
2959    /// The depth a run bound by the plane's chain but acting under none starts
2960    /// from: acting under no chain does not put a served caller closer to the
2961    /// owner than the plane that serves it.
2962    pub(crate) fn own_depth(&self) -> usize {
2963        self.identity
2964            .as_ref()
2965            .map_or(0, crate::core::Delegation::depth)
2966    }
2967
2968    /// The journal, under the name the batch driver reads it by.
2969    #[must_use]
2970    pub(crate) fn meter(&self) -> &super::metrics::Meter {
2971        &self.meter
2972    }
2973
2974    /// Write an operator act as the sole record of a sealed run of its own.
2975    ///
2976    /// The act, then its conclusion, then the seal — so it carries the chain,
2977    /// the signature, Merkle inclusion and the witness, and an offline reader
2978    /// checks it without being taught what the act was. `case` stamps the
2979    /// record onto a matter's history; `reason` is the conclusion's, for the
2980    /// one act whose reason lives there.
2981    async fn seal_operator_record(
2982        &self,
2983        kind: RecordKind,
2984        case: Option<crate::core::CaseId>,
2985        outcome: &str,
2986        reason: Option<&str>,
2987    ) -> Result<RunId, RuntimeError> {
2988        let run = RunId::generate();
2989        let epoch = BREAK_GLASS_EPOCH;
2990        let mut act = Append::new(run, kind);
2991        if let Some(case) = case {
2992            act = act.case(case);
2993        }
2994        self.store
2995            .append(epoch, vec![act])
2996            .await
2997            .map_err(RuntimeError::from_store)?;
2998        let head = self
2999            .store
3000            .head(run)
3001            .await
3002            .map_err(RuntimeError::from_store)?;
3003        self.store
3004            .append(
3005                epoch,
3006                vec![Append::new(
3007                    run,
3008                    RecordKind::RunConcluded {
3009                        outcome: outcome.to_owned(),
3010                        reason: reason.map(str::to_owned),
3011                        exhaustion: None,
3012                        live_spend: Spend::default(),
3013                        chain_head: head.hash,
3014                    },
3015                )],
3016            )
3017            .await
3018            .map_err(RuntimeError::from_store)?;
3019        self.store
3020            .seal(run, epoch, outcome)
3021            .await
3022            .map_err(RuntimeError::from_store)?;
3023        Ok(run)
3024    }
3025
3026    /// Record an operator deliberately crossing into this tenant, and return
3027    /// the run that holds the record.
3028    ///
3029    /// Every other tenancy control keeps a cross-tenant read from being reached
3030    /// by accident. This is the designed exception, and the rule for it is that an
3031    /// exception without a record is indistinguishable from the breach it is
3032    /// meant to be. So the access is written into **this** tenant's journal —
3033    /// the one whose data is about to be reached — in a sealed run of its own,
3034    /// exactly as a sweep writes its decisions. It therefore inherits the hash
3035    /// chain, the per-record signature and the Merkle inclusion, and the
3036    /// offline audit tool reports it without being taught that break-glass
3037    /// exists.
3038    ///
3039    /// **The record is written before any data is served, and a failure to
3040    /// write it is a failure to access.** That direction is the whole control:
3041    /// the alternative — serve first, record best-effort — is a break-glass
3042    /// that works exactly as well when its own evidence is lost.
3043    ///
3044    /// # Errors
3045    ///
3046    /// If `reason` is blank, or if the record cannot be written. An
3047    /// unexplained exception is the thing this record exists to prevent, so it
3048    /// is refused rather than stored empty.
3049    pub async fn record_break_glass(
3050        &self,
3051        actor: &crate::core::Operator,
3052        roles: &[String],
3053        reason: &str,
3054    ) -> Result<RunId, RuntimeError> {
3055        if reason.trim().is_empty() {
3056            return Err(RuntimeError::PlanContract(
3057                "break-glass needs a reason: an unexplained crossing of the tenant \
3058                 boundary is what this record exists to prevent"
3059                    .to_owned(),
3060            ));
3061        }
3062        let run = self
3063            .seal_operator_record(
3064                RecordKind::BreakGlass {
3065                    actor: actor.clone(),
3066                    roles: roles.to_vec(),
3067                    reason: reason.to_owned(),
3068                },
3069                None,
3070                BREAK_GLASS_OUTCOME,
3071                Some(reason),
3072            )
3073            .await?;
3074        tracing::warn!(
3075            tenant = %self.tenant(),
3076            %actor,
3077            %run,
3078            reason,
3079            "break-glass: an operator crossed the tenant boundary"
3080        );
3081        Ok(run)
3082    }
3083
3084    /// The journal, for reads the runtime does not mediate.
3085    ///
3086    /// This is also the embedder's **resumable output stream**:
3087    /// [`JournalStore::read`] is a seq-cursored read over a run's records, so
3088    /// polling from the last sequence seen is a durable, reconnect-safe
3089    /// progress feed that any instance can serve — the exact mechanism the
3090    /// A2A streaming surface is built on. There is deliberately no curated
3091    /// event type between an embedder and the records: a third vocabulary
3092    /// beside the journal's and the wire's would be a second truth wearing
3093    /// ergonomics, and the records are already the history the stream must
3094    /// agree with.
3095    pub fn journal(&self) -> &Arc<dyn JournalStore> {
3096        &self.store
3097    }
3098
3099    /// Which case a run belongs to, or `None` if it belongs to none.
3100    ///
3101    /// Read from the **journal**, not from a column beside it. The binding is
3102    /// stamped on the run's own records at admission, so answering from there
3103    /// is answering from the plan of record — a case-store column saying the
3104    /// same thing would be a second copy of one fact, and the two could
3105    /// disagree about a run the case layer never saw.
3106    ///
3107    /// It is the first question an operator surface asks, which is why it is a
3108    /// method rather than a documented one-liner: every caller was otherwise
3109    /// going to reach for the first record and read `body.case` off it, and a
3110    /// caller who reached for the *last* one instead would still be right today
3111    /// and wrong the moment a run is admitted before its case is known.
3112    ///
3113    /// # Errors
3114    ///
3115    /// If the journal cannot be read. An unknown run is `Ok(None)` rather than
3116    /// an error: *no such run* and *a run in no case* are both honest answers
3117    /// to this question, and neither is a fault.
3118    pub async fn case_of(&self, run: RunId) -> Result<Option<crate::core::CaseId>, RuntimeError> {
3119        Ok(self
3120            .store
3121            .read(run, 1)
3122            .await
3123            .map_err(RuntimeError::from_store)?
3124            .first()
3125            .and_then(|record| record.body.case))
3126    }
3127
3128    #[must_use]
3129    /// The witness set and quorum this plane was built with, if any.
3130    pub(crate) fn witnessing(&self) -> Option<&Arc<Witnessing>> {
3131        self.witnesses.as_ref()
3132    }
3133
3134    pub fn store(&self) -> &Arc<dyn JournalStore> {
3135        &self.store
3136    }
3137
3138    /// This **process instance's** identity, as it appears in run leases.
3139    ///
3140    /// Not the agent's name: several instances of one agent are normal. See
3141    /// [`RuntimeBuilder::owner`].
3142    ///
3143    /// Public because "which instance holds this run" is a question an operator
3144    /// asks of a stuck system, and the answer is otherwise only in a store row.
3145    #[must_use]
3146    pub fn owner_id(&self) -> &str {
3147        &self.owner
3148    }
3149
3150    /// The input as the declared admission rules leave it: refused with no
3151    /// record, as an admission policy denial is, or with its sensitivity
3152    /// raised to every matched classification — which `RunAdmitted` then
3153    /// records as the input's label.
3154    #[cfg(feature = "manifest")]
3155    fn admit_content(
3156        &self,
3157        agent: &str,
3158        input: Tainted<Value>,
3159    ) -> Result<Tainted<Value>, RuntimeError> {
3160        let Some(manifest) = self
3161            .resolve(agent)
3162            .ok()
3163            .and_then(|skill| self.governing(skill.as_ref()))
3164        else {
3165            return Ok(input);
3166        };
3167        let outcome = super::ctx::judged(
3168            manifest.spec.security.content.as_ref(),
3169            crate::content::At::Admission,
3170            input.peek(),
3171        )
3172        .map_err(RuntimeError::PolicyDenied)?;
3173        if let Some(hit) = outcome.refused.first() {
3174            return Err(RuntimeError::PolicyDenied(
3175                crate::core::PolicyError::Content {
3176                    rule: hit.rule.clone(),
3177                    pointer: hit.pointer.clone(),
3178                },
3179            ));
3180        }
3181        let raised = outcome.raise(input.label().sensitivity);
3182        if raised == input.label().sensitivity {
3183            return Ok(input);
3184        }
3185        Ok(input.raised_to(raised))
3186    }
3187
3188    #[cfg(not(feature = "manifest"))]
3189    #[allow(clippy::unused_self, clippy::unnecessary_wraps)]
3190    fn admit_content(
3191        &self,
3192        _agent: &str,
3193        input: Tainted<Value>,
3194    ) -> Result<Tainted<Value>, RuntimeError> {
3195        Ok(input)
3196    }
3197
3198    /// The content rules' verdict on an event delivered to a suspended run,
3199    /// under the declaration governing the run's skill; `None` where no rule
3200    /// matched or none is declared.
3201    #[cfg(feature = "manifest")]
3202    fn delivered_verdict(
3203        &self,
3204        history: &[crate::journal::Record],
3205        payload: &serde_json::Value,
3206    ) -> Option<crate::core::ContentVerdict> {
3207        let capability = history.iter().find_map(|r| match r.kind() {
3208            RecordKind::RunAdmitted { capability, .. } => Some(capability.clone()),
3209            _ => None,
3210        })?;
3211        let manifest = self.governing(self.resolve(&capability).ok()?.as_ref())?;
3212        super::ctx::verdict(super::ctx::judged(
3213            manifest.spec.security.content.as_ref(),
3214            crate::content::At::Source(super::ctx::AWAIT_KIND),
3215            payload,
3216        ))
3217    }
3218
3219    #[cfg(not(feature = "manifest"))]
3220    #[allow(clippy::unused_self)]
3221    fn delivered_verdict(
3222        &self,
3223        _history: &[crate::journal::Record],
3224        _payload: &serde_json::Value,
3225    ) -> Option<crate::core::ContentVerdict> {
3226        None
3227    }
3228
3229    /// The declaration governing a skill, if its agent has one.
3230    ///
3231    /// Per skill, not per plane: a runtime runs several agents, and a step must
3232    /// be judged against the manifest of the agent whose skill it is. Looking up
3233    /// one plane-wide manifest would apply another agent's ceilings.
3234    #[cfg(feature = "manifest")]
3235    fn governing(&self, skill: &dyn Skill) -> Option<Arc<crate::manifest::Manifest>> {
3236        self.governed_by.get(&skill.descriptor().name).cloned()
3237    }
3238
3239    /// Renew this run's lease until the returned guard is dropped.
3240    ///
3241    /// Requires a Tokio runtime, as the rest of this crate's timing does.
3242    fn heartbeat(&self, run: RunId, epoch: crate::core::Epoch) -> Heartbeat {
3243        let store = Arc::clone(&self.store);
3244        let owner = self.owner.clone();
3245        let ttl = self.lease_ttl;
3246        // A third of the TTL, so two renewals can be lost to a slow store before
3247        // the lease lapses. Renewing *at* the TTL would mean any hesitation
3248        // leaves it expired, and an expired lease is one anybody may take —
3249        // including, per `acquire`, this caller, which would fence the run with
3250        // its own heartbeat.
3251        let period = ttl / 3;
3252        Heartbeat(tokio::spawn(async move {
3253            loop {
3254                tokio::time::sleep(period).await;
3255                // `renew`, never `acquire`. A renewal extends only a lease
3256                // still held by exactly this `(owner, epoch)`; it cannot
3257                // claim. That distinction is load-bearing here: this task
3258                // races the run's own conclusion, and an acquire arriving
3259                // just after `conclude` released the lease would re-take it —
3260                // a live, never-released lease over a concluded run, which
3261                // the recovery sweep then "recovers" forever.
3262                //
3263                // Stop only when the store says the lease is gone — released
3264                // by the conclusion, lapsed and reclaimed, or taken over: the
3265                // run's next append will be fenced, which is the correct
3266                // outcome, and claiming now would only resurrect ownership
3267                // this instance no longer has. Any other failure is the store
3268                // not answering, and giving up then would let a run that is
3269                // still executing lapse into somebody else's recovery.
3270                match store.renew(run, &owner, epoch, ttl).await {
3271                    Ok(_) => {}
3272                    Err(crate::core::StoreError::LeaseNotHeld { .. }) => return,
3273                    Err(error) => tracing::warn!(
3274                        target: "agentplane::lease",
3275                        run = %run,
3276                        error = %error,
3277                        "a lease renewal failed; the next one retries"
3278                    ),
3279                }
3280            }
3281        }))
3282    }
3283
3284    /// The standing withdrawal that reaches `identity`, if one does.
3285    ///
3286    /// The one predicate the step boundary and a subject-bound hop both ask,
3287    /// so the two cannot disagree about whose authority is withdrawn. Read on
3288    /// a live pass only — a halt thrown today must not decide what a run did
3289    /// last year — and an unreachable store propagates, because carrying on
3290    /// would leave a withdrawal unenforced exactly while accounting is down.
3291    pub(crate) async fn withdrawal_against(
3292        &self,
3293        identity: Option<&crate::core::Delegation>,
3294    ) -> Result<Option<Withdrawal>, crate::core::StoreError> {
3295        let (Some(quotas), Some(chain)) = (self.quotas.as_ref(), identity) else {
3296            return Ok(None);
3297        };
3298        let halts = quotas.halts().await?;
3299        Ok(halts.into_iter().find_map(|halt| {
3300            let subject = halt.scope.withdraws_from(chain)?.to_owned();
3301            Some(Withdrawal {
3302                subject,
3303                reason: halt.reason,
3304                by: halt.by,
3305            })
3306        }))
3307    }
3308
3309    /// Stop a run because the authority it acts under was withdrawn.
3310    ///
3311    /// A **pause**: `stop` is reached with a status `maybe_unwind` passes
3312    /// straight through, so the run's mutations stand and it stays resumable.
3313    /// That is the treatment exhaustion gets and for the same reason — the run
3314    /// did nothing wrong, and reversing correct work because a credential lapsed
3315    /// would be a second incident.
3316    ///
3317    /// `record` is false on a resume that finds the withdrawal still standing:
3318    /// the withholding already on the chain says so, and a second one per resume
3319    /// attempt would turn a queue of retries into a queue of records.
3320    #[allow(clippy::too_many_arguments)]
3321    async fn withhold(
3322        &self,
3323        cx: Unwind<'_>,
3324        withdrawal: Withdrawal,
3325        record: bool,
3326        completed: &[(StepId, Capability)],
3327        outputs: &BTreeMap<StepId, Tainted<Value>>,
3328        cursor: &mut ReplayCursor,
3329        case_id: Option<crate::core::CaseId>,
3330        quota: &QuotaPass,
3331    ) -> Result<RunOutcome, RuntimeError> {
3332        let Withdrawal {
3333            subject,
3334            reason,
3335            by,
3336        } = withdrawal;
3337        if record {
3338            let stamped = (cx.stamp)(Append::new(
3339                cx.run,
3340                RecordKind::AuthorityWithheld {
3341                    subject: subject.clone(),
3342                    reason: reason.clone(),
3343                    by: by.clone(),
3344                },
3345            ));
3346            self.store
3347                .append(cx.epoch, vec![stamped])
3348                .await
3349                .map_err(RuntimeError::from_store)?;
3350        }
3351        self.stop(
3352            cx,
3353            RunStatus::Withheld { subject, reason },
3354            completed,
3355            outputs,
3356            cursor,
3357            case_id,
3358            quota,
3359        )
3360        .await
3361    }
3362
3363    /// Refuse a run whose tenant is at a ceiling.
3364    ///
3365    /// Fails **closed**: an unreachable quota store refuses rather than admits,
3366    /// because a ceiling that yields when its accounting is down is a ceiling an
3367    /// attacker removes by taking the accounting down.
3368    ///
3369    /// Live admission only, and replay and resume are skipped for **different**
3370    /// reasons. Re-checking during replay would let a run that happened produce
3371    /// a different history when it is re-read: a ceiling crossed since admission
3372    /// would rewrite the past into a refusal. Resume is not that — it decides
3373    /// only what happens next — and it is ungated because refusing it strands a
3374    /// run mid-saga with its reversals unrun. What that costs the ceiling is
3375    /// stated where the ceiling is defined ([`crate::quota`]); it is a real cost
3376    /// rather than an absent one.
3377    async fn check_quota(
3378        &self,
3379        run: RunId,
3380        governed_by: Option<&crate::journal::AgentIdentity>,
3381        chain: Option<&crate::core::Delegation>,
3382        at: crate::core::Timestamp,
3383        (budget, width): (&Budget, usize),
3384    ) -> Result<QuotaPass, RuntimeError> {
3385        let Some(quotas) = self.quotas.as_ref() else {
3386            return Ok(QuotaPass::disabled());
3387        };
3388        let pass = QuotaPass::at(&self.quota, at, true).sized_for(width);
3389
3390        // The halt is checked **before** the unlimited shortcut, because an
3391        // emergency stop is not a ceiling and a tenant with no ceilings is
3392        // exactly the one an operator is most likely to need to stop. Reading it
3393        // fails closed for the same reason the ceilings do: a switch that yields
3394        // when its store is unreachable is a switch an attacker throws by taking
3395        // the store down.
3396        //
3397        // Every standing halt in one read, and the **narrowest match wins the
3398        // message**: an operator who stopped one agent and whoever is refused
3399        // are looking for different sentences, and "the tenant is halted" told
3400        // to the caller of agent 12 sends them to the wrong incident. Ordering
3401        // the scopes puts `Subject` before `Revision` before `Agent` before
3402        // `Tenant`, so the reason reported is the most specific one that
3403        // actually covers this run — and a withdrawn authority outranks every
3404        // workload scope, because it is the answer that tells the caller their
3405        // credential is the problem rather than the agent.
3406        match quotas.halts().await {
3407            Ok(halts) => {
3408                if let Some(halt) = halts
3409                    .into_iter()
3410                    .filter(|halt| halt.scope.covers(governed_by, chain))
3411                    .max_by(|a, b| a.scope.cmp(&b.scope))
3412                {
3413                    return Err(RuntimeError::QuotaExceeded(
3414                        crate::quota::QuotaError::Halted {
3415                            tenant: self.tenant.as_str().to_owned(),
3416                            scope: halt.scope,
3417                            reason: halt.reason,
3418                        },
3419                    ));
3420                }
3421            }
3422            Err(e) => {
3423                return Err(RuntimeError::QuotaExceeded(
3424                    crate::quota::QuotaError::Unavailable(e.to_string()),
3425                ));
3426            }
3427        }
3428
3429        // What the run can cost at most, held against the period in the same
3430        // store transaction that checks the ceiling and takes the slot. A run
3431        // that leaves a bounded unit unbounded is refused here: admitted, it
3432        // would hold nothing against a period it can spend without limit.
3433        let hold = match crate::quota::reservation(&self.quota, budget, width as u64) {
3434            Ok(amount) => {
3435                amount
3436                    .zip(pass.period())
3437                    .map(|(amount, period)| crate::quota::SpendHold {
3438                        period: period.to_owned(),
3439                        amount,
3440                    })
3441            }
3442            Err(field) => {
3443                return Err(RuntimeError::QuotaExceeded(
3444                    crate::quota::QuotaError::Unbounded {
3445                        tenant: self.tenant.as_str().to_owned(),
3446                        unit: if field.contains("minor") {
3447                            "money"
3448                        } else {
3449                            "tokens"
3450                        },
3451                        field,
3452                    },
3453                ));
3454            }
3455        };
3456
3457        quotas
3458            .reserve(run, &self.quota, hold.as_ref(), at)
3459            .await
3460            .map_err(RuntimeError::QuotaExceeded)?;
3461        Ok(pass)
3462    }
3463
3464    /// Atomically settle this pass's spend and admission slot.
3465    ///
3466    /// Required before physical sealing and lease release. A failure keeps the
3467    /// lease owned so the abandonment sweep retries from journal evidence; a
3468    /// best-effort write here would turn a transient store outage into permanent
3469    /// under-accounting. The period is the one captured when this pass started,
3470    /// never recomputed here.
3471    async fn settle_quota(
3472        &self,
3473        run: RunId,
3474        epoch: crate::core::Epoch,
3475        spend: Spend,
3476        pass: &QuotaPass,
3477        concludes: bool,
3478    ) -> Result<(), RuntimeError> {
3479        let Some(quotas) = self.quotas.as_ref() else {
3480            return Ok(());
3481        };
3482        let pending = |error: crate::core::StoreError| RuntimeError::QuotaSettlementPending {
3483            run: run.to_string(),
3484            epoch,
3485            detail: error.to_string(),
3486        };
3487        if !pass.enabled {
3488            // A conclusion no pass opened — an abandonment decided while the
3489            // run was stopped — still ends the run, and a run that has ended
3490            // holds nothing against the period.
3491            if concludes {
3492                quotas.release(run).await.map_err(pending)?;
3493            }
3494            return Ok(());
3495        }
3496        let settlement = pass
3497            .settlement(run, epoch, spend, concludes)
3498            .expect("an enabled quota pass has a settlement");
3499        quotas.settle(&settlement).await.map_err(pending)
3500    }
3501
3502    /// Release a reservation for an admission whose journal never existed.
3503    pub(crate) async fn release_empty_quota_reservation(
3504        &self,
3505        run: RunId,
3506    ) -> Result<(), crate::core::StoreError> {
3507        match self.quotas.as_ref() {
3508            Some(quotas) => quotas.release(run).await,
3509            None => Ok(()),
3510        }
3511    }
3512
3513    /// Settle and release the admission slots sealed runs still hold.
3514    ///
3515    /// A plane with no quota store may seal a run whose admission holds a
3516    /// slot in this ledger, and no resume or recovery selects a sealed run
3517    /// again. Settlement is idempotent, so a slot another path already settled
3518    /// is not billed twice. Returns how many slots were given back.
3519    pub(crate) async fn release_sealed_slots(&self, limit: usize) -> Result<usize, RuntimeError> {
3520        let Some(quotas) = self.quotas.as_ref() else {
3521            return Ok(0);
3522        };
3523        let held = quotas
3524            .running_runs(limit)
3525            .await
3526            .map_err(RuntimeError::Store)?;
3527        let positions = self
3528            .store
3529            .log_positions(&held)
3530            .await
3531            .map_err(RuntimeError::from_store)?;
3532        let mut released = 0;
3533        for (run, sealed) in held.into_iter().zip(positions) {
3534            if sealed.is_none() {
3535                continue;
3536            }
3537            let records = self
3538                .store
3539                .read(run, 1)
3540                .await
3541                .map_err(RuntimeError::from_store)?;
3542            match self.settle_recorded_quota_passes(run, &records).await {
3543                Ok(()) => released += 1,
3544                Err(error) => tracing::warn!(
3545                    %run,
3546                    %error,
3547                    "a sealed run's admission slot could not be released; the next sweep retries"
3548                ),
3549            }
3550        }
3551        Ok(released)
3552    }
3553
3554    /// Seal the lease-free runs whose sealing conclusion is written and whose
3555    /// seal is not.
3556    ///
3557    /// Each [`LEASE_FREE_OUTCOMES`] writer appends its conclusion and then
3558    /// seals in a second call, and holds no lease, so recovery never selects
3559    /// a run whose writer died between the two. The conclusion alone already
3560    /// lists the run under its outcome, so a history reads it as done while
3561    /// the audit finds it in no log. The conclusion is what was decided; the
3562    /// seal adds nothing to it but the log position, so finishing it here
3563    /// writes no decision the writer did not make.
3564    ///
3565    /// Looks at the newest `limit` runs of each outcome. Returns how many it
3566    /// sealed.
3567    pub(crate) async fn seal_unsealed_conclusions(
3568        &self,
3569        limit: usize,
3570    ) -> Result<usize, RuntimeError> {
3571        let mut sealed = 0;
3572        for &outcome in LEASE_FREE_OUTCOMES {
3573            let runs = self
3574                .store
3575                .runs_by_outcome(outcome, limit)
3576                .await
3577                .map_err(RuntimeError::from_store)?;
3578            let positions = self
3579                .store
3580                .log_positions(&runs)
3581                .await
3582                .map_err(RuntimeError::from_store)?;
3583            for (run, position) in runs.into_iter().zip(positions) {
3584                if position.is_some() {
3585                    continue;
3586                }
3587                match self.store.seal(run, LEASE_FREE_EPOCH, outcome).await {
3588                    Ok(_) => sealed += 1,
3589                    Err(error) => tracing::warn!(
3590                        %run,
3591                        %error,
3592                        "a concluded run's seal could not be finished; the next sweep retries"
3593                    ),
3594                }
3595            }
3596        }
3597        Ok(sealed)
3598    }
3599
3600    /// Settle every quota pass present in history, idempotently.
3601    ///
3602    /// Reached before a resume can dispatch. A normally settled pass is a cheap
3603    /// receipt lookup; a pass whose process died is accrued here. Conclusions
3604    /// carry the exact in-memory live total, including a known outcome whose
3605    /// terminal effect record failed before a later append recovered. An
3606    /// unconcluded crash has only durable terminal effect records to sum.
3607    async fn settle_recorded_quota_passes(
3608        &self,
3609        run: RunId,
3610        records: &[Record],
3611    ) -> Result<(), RuntimeError> {
3612        // A plane with no ledger resumes a run only when no pass of it accrued
3613        // spend to a period, which would otherwise go unbilled. It cannot give
3614        // back the concurrency slot of an admission whose process died
3615        // mid-pass; the ledger's own plane does, on its next sweep once the
3616        // run is sealed.
3617        let Some(quotas) = self.quotas.as_ref() else {
3618            if records.iter().any(|record| {
3619                matches!(
3620                    record.kind(),
3621                    RecordKind::QuotaPassStarted {
3622                        period: Some(_),
3623                        ..
3624                    }
3625                )
3626            }) {
3627                return Err(RuntimeError::PlanContract(
3628                    "this run has passes that accrued spend to a quota period, and the resuming \
3629                     plane has no quota store to settle them"
3630                        .to_owned(),
3631                ));
3632            }
3633            return Ok(());
3634        };
3635
3636        for record in records {
3637            let RecordKind::QuotaPassStarted {
3638                period,
3639                release_slot,
3640            } = record.kind()
3641            else {
3642                continue;
3643            };
3644            let epoch = record.body.epoch;
3645            let spend = records
3646                .iter()
3647                .rev()
3648                .find_map(|candidate| (candidate.body.epoch == epoch).then_some(candidate.kind()))
3649                .and_then(|kind| match kind {
3650                    RecordKind::RunConcluded { live_spend, .. } => Some(*live_spend),
3651                    _ => None,
3652                })
3653                .unwrap_or_else(|| spend_recorded_in_epoch(records, epoch));
3654            quotas
3655                .settle(&crate::quota::QuotaSettlement {
3656                    run,
3657                    epoch,
3658                    period: period.clone(),
3659                    spend,
3660                    release_slot: *release_slot,
3661                    concludes: concluded_in(records, epoch),
3662                })
3663                .await
3664                .map_err(|error| RuntimeError::QuotaSettlementPending {
3665                    run: run.to_string(),
3666                    epoch,
3667                    detail: error.to_string(),
3668                })?;
3669        }
3670        // A run whose history has ended holds nothing against the period,
3671        // whichever path ended it — including one no pass marker opened.
3672        if records.iter().any(is_sealing_conclusion) {
3673            quotas
3674                .release(run)
3675                .await
3676                .map_err(|error| RuntimeError::QuotaSettlementPending {
3677                    run: run.to_string(),
3678                    epoch: records.last().map_or(0, |r| r.body.epoch),
3679                    detail: error.to_string(),
3680                })?;
3681        }
3682        Ok(())
3683    }
3684
3685    /// Repair quota receipts, and finish a conclusion an owner died after.
3686    ///
3687    /// Under recovery, a run whose last record is a conclusion lost its owner
3688    /// between concluding and handing the lease back. What remains is the seal and the release, never the business
3689    /// work: recovery is not permission to retry a failure, with or without a
3690    /// quota store. A run that crashed before concluding continues through
3691    /// replay.
3692    async fn recover_quota_before_resume(
3693        &self,
3694        run: RunId,
3695        records: &[Record],
3696        lease: &crate::journal::Lease,
3697        recovering: bool,
3698    ) -> Result<Option<RunOutcome>, RuntimeError> {
3699        self.settle_recorded_quota_passes(run, records).await?;
3700        if !recovering {
3701            return Ok(None);
3702        }
3703        let Some(last) = records.last() else {
3704            return Ok(None);
3705        };
3706        let RecordKind::RunConcluded {
3707            outcome,
3708            reason,
3709            exhaustion,
3710            ..
3711        } = last.kind()
3712        else {
3713            return Ok(None);
3714        };
3715
3716        let status = recorded_status(outcome, reason.as_deref(), exhaustion.as_ref(), records);
3717        let chain_head = if SEALED_OUTCOMES.contains(&outcome.as_str()) {
3718            let head = self
3719                .store
3720                .seal(run, lease.epoch, outcome)
3721                .await
3722                .map_err(RuntimeError::from_store)?;
3723            // A repaired seal is a conclusion like any other.
3724            self.retire_waits(run).await;
3725            head
3726        } else {
3727            self.store
3728                .head(run)
3729                .await
3730                .map_err(RuntimeError::from_store)?
3731                .hash
3732        };
3733        Ok(Some(RunOutcome {
3734            run_id: run,
3735            status,
3736            chain_head,
3737            consumed: Consumed::default(),
3738            output: None,
3739        }))
3740    }
3741
3742    /// Return a closed run without continuing it, repairing a missing physical
3743    /// seal when a crash landed after the conclusion append.
3744    async fn closed_resume_outcome(
3745        &self,
3746        run: RunId,
3747        records: &[Record],
3748        lease: &crate::journal::Lease,
3749    ) -> Result<Option<RunOutcome>, RuntimeError> {
3750        let Some(status) = resume_is_closed(records) else {
3751            return Ok(None);
3752        };
3753        let outcome = recorded_conclusion(records).unwrap_or_else(|| status.as_str().to_owned());
3754        let chain_head = if SEALED_OUTCOMES.contains(&outcome.as_str()) {
3755            let head = self
3756                .store
3757                .seal(run, lease.epoch, &outcome)
3758                .await
3759                .map_err(RuntimeError::from_store)?;
3760            // The crash this repairs may have landed before the conclusion
3761            // retired the run's waits; retiring twice is harmless.
3762            self.retire_waits(run).await;
3763            head
3764        } else {
3765            self.store
3766                .head(run)
3767                .await
3768                .map_err(RuntimeError::from_store)?
3769                .hash
3770        };
3771        Ok(Some(RunOutcome {
3772            run_id: run,
3773            status,
3774            chain_head,
3775            output: None,
3776            consumed: Consumed::default(),
3777        }))
3778    }
3779
3780    /// Finish an abandonment a decision asked for and a crash interrupted.
3781    ///
3782    /// The instruction is journaled before it is acted on, so this is what
3783    /// makes the ending arrive whatever happened in between — and it is the
3784    /// same path the verb itself takes, rather than a repair beside it.
3785    ///
3786    /// Nothing is unwound and no step runs: the run seals holding whatever it
3787    /// left in the world, which is the whole content of the decision.
3788    async fn abandon_if_decided(
3789        &self,
3790        run: RunId,
3791        records: &[Record],
3792        lease: &crate::journal::Lease,
3793    ) -> Result<Option<RunOutcome>, RuntimeError> {
3794        let Some(status) = pending_abandonment(records) else {
3795            return Ok(None);
3796        };
3797        self.conclude(
3798            run,
3799            lease.epoch,
3800            status,
3801            None,
3802            true,
3803            recorded_case_id(records),
3804            Consumed::default(),
3805            Spend::ZERO,
3806            &QuotaPass::disabled(),
3807        )
3808        .await
3809        .map(Some)
3810    }
3811
3812    /// This plane, writing through a journal that puts `quota`'s marker on
3813    /// the pass's first append — so a pass that writes nothing leaves no
3814    /// marker. `None` for a pass with no marker.
3815    fn marking_pass(&self, run: RunId, quota: &QuotaPass) -> Option<Self> {
3816        let marker = quota.started()?;
3817        Some(Self {
3818            store: Arc::new(super::pass_journal::PassJournal::new(
3819                Arc::clone(&self.store),
3820                run,
3821                marker,
3822            )),
3823            ..self.clone()
3824        })
3825    }
3826
3827    /// Open the quota identity of the live tail a resume is about to execute.
3828    ///
3829    /// Its marker is not written here: it rides the pass's first append (see
3830    /// [`super::pass_journal`]), so a resume that writes nothing leaves none.
3831    async fn start_replay_quota_pass(
3832        &self,
3833        run: RunId,
3834        mode: Pass,
3835        width: usize,
3836    ) -> Result<QuotaPass, RuntimeError> {
3837        let (Pass::Resume, Some(quotas)) = (mode, self.quotas.as_ref()) else {
3838            return Ok(QuotaPass::disabled());
3839        };
3840        let pass = QuotaPass::at(&self.quota, now_for_admission(), false).sized_for(width);
3841        // The run spends in the period it resumes in, so what it still holds
3842        // moves there — never refused, because a resume is not.
3843        if let Some(period) = pass.period() {
3844            quotas
3845                .carry(run, period)
3846                .await
3847                .map_err(RuntimeError::Store)?;
3848        }
3849        Ok(pass)
3850    }
3851
3852    /// The ceilings a run gets: its agent's, or the plane's if it has no agent.
3853    ///
3854    /// Per agent, because that is who declared them. A plane-wide budget would
3855    /// let one agent's generosity bound another's runs, which is the whole
3856    /// reason a declaration belongs to an identity rather than to a process.
3857    fn budget_for(&self, target: &str) -> Budget {
3858        #[cfg(feature = "manifest")]
3859        if let Ok(skill) = self.resolve(target)
3860            && let Some(m) = self.governing(skill.as_ref())
3861        {
3862            return m.budget();
3863        }
3864        let _ = target;
3865        self.budget
3866    }
3867
3868    /// Which declaration governs runs of this capability, if a declared agent
3869    /// does.
3870    ///
3871    /// Resolved the same way the budget is, and for the same reason: a run is
3872    /// governed by the agent that answers its entry capability. A commissioned
3873    /// sub-run opens its own run and records its own governor, so every run in a
3874    /// room has exactly one — there is no case where this has to pick.
3875    ///
3876    /// A manifest that cannot produce a digest is one that could not have been
3877    /// published; recording `None` would claim the run was ungoverned, so the
3878    /// failure is surfaced as an absent identity rather than a false one.
3879    #[cfg(feature = "manifest")]
3880    fn identity_for(&self, target: &str) -> Option<crate::journal::AgentIdentity> {
3881        let skill = self.resolve(target).ok()?;
3882        self.identity_of(self.governing(skill.as_ref())?.as_ref())
3883    }
3884
3885    /// What the agent answering `capability` may do, as its declaration says.
3886    ///
3887    /// `None` when nothing on this plane answers it; a reach with no
3888    /// declaration when something does and no declaration governs it.
3889    #[cfg(feature = "manifest")]
3890    pub(crate) fn reach_of(&self, capability: &str) -> Option<crate::core::Reach> {
3891        let skill = self.resolve(capability).ok()?;
3892        let declaration = self.governing(skill.as_ref()).and_then(|manifest| {
3893            let digest = manifest.digest().ok()?;
3894            Some(super::reach::declared(&manifest, digest, |server| {
3895                self.peers.as_ref().is_some_and(|wiring| {
3896                    wiring
3897                        .registry
3898                        .grant(&crate::peers::PeerId::new(server))
3899                        .is_some()
3900                })
3901            }))
3902        });
3903        Some(crate::core::Reach {
3904            capability: capability.to_owned(),
3905            declaration,
3906        })
3907    }
3908
3909    /// The declaration's identity, as every gate reports it.
3910    ///
3911    /// One construction, because admission and each effect gate must name the
3912    /// same revision the same way: two spellings of *which agent is acting*
3913    /// would let a rule permitted at the door mean something else at the sink.
3914    #[cfg(feature = "manifest")]
3915    fn identity_of(&self, m: &crate::manifest::Manifest) -> Option<crate::journal::AgentIdentity> {
3916        Some(crate::journal::AgentIdentity {
3917            name: m.metadata.name.clone(),
3918            version: m.metadata.version.clone(),
3919            digest: m.digest().ok()?,
3920            publisher: self.published_by.get(&m.metadata.name).cloned(),
3921        })
3922    }
3923
3924    /// Whether anything on this plane answers `target`.
3925    ///
3926    /// For a catalogue built before the first call: a surface that advertises a
3927    /// capability nothing provides offers a caller — often a model — a verb
3928    /// that fails at admission every time, and the failure names the plane
3929    /// rather than the catalogue that lied about it.
3930    ///
3931    /// Gated on the feature that builds a catalogue, so a build without one
3932    /// answers *this question is not asked here* rather than carrying a method
3933    /// nothing reaches.
3934    #[cfg(feature = "mcp-server")]
3935    pub(crate) fn provides(&self, target: &str) -> bool {
3936        self.resolve(target).is_ok()
3937    }
3938
3939    /// Whether a grant's server component names a registered A2A peer — a
3940    /// call the run may wait on.
3941    #[cfg(feature = "mcp-server")]
3942    pub(crate) fn wires_peer(&self, server: &str) -> bool {
3943        self.peers.as_ref().is_some_and(|wiring| {
3944            wiring
3945                .registry
3946                .grant(&crate::peers::PeerId::new(server))
3947                .is_some()
3948        })
3949    }
3950
3951    fn resolve(&self, target: &str) -> Result<Arc<dyn Skill>, RuntimeError> {
3952        if let Some(s) = self.skills.get(target) {
3953            return Ok(Arc::clone(s));
3954        }
3955        let cap = Capability::new(target);
3956        if let Some(name) = self.by_capability.get(&cap)
3957            && let Some(s) = self.skills.get(name)
3958        {
3959            return Ok(Arc::clone(s));
3960        }
3961        // Capabilities rather than skill names: `run` is documented to take the
3962        // capability, and listing the names would answer a question nobody
3963        // asked with vocabulary that does not work in the call that failed.
3964        let mut available: Vec<String> = self
3965            .by_capability
3966            .keys()
3967            .map(std::string::ToString::to_string)
3968            .collect();
3969        available.sort();
3970        Err(RuntimeError::NoProvider {
3971            target: target.to_owned(),
3972            available,
3973        })
3974    }
3975
3976    /// Start a run: a capability a skill answers, or a plan.
3977    ///
3978    /// The one door. What the run is bound to — a case, a caller's chain, a
3979    /// pinned declaration — goes in [`RunRequest::terms`]; whether it runs at
3980    /// most once goes in [`RunRequest::once`], which changes what the answer
3981    /// is; and the terminal says whether to wait for it.
3982    ///
3983    /// ```no_run
3984    /// # use std::sync::Arc;
3985    /// # use agentplane::core::Tainted;
3986    /// # use agentplane::runtime::{Runtime, RunTerms};
3987    /// # async fn f(rt: Arc<Runtime>, input: Tainted<serde_json::Value>) -> Result<(), agentplane::RuntimeError> {
3988    /// let outcome = rt.start("order.settle", input.clone()).run().await?;
3989    /// let admission = rt
3990    ///     .start("order.settle", input)
3991    ///     .terms(RunTerms::default().correlated("order", &[]))
3992    ///     .once("billing:msg-7")
3993    ///     .run()
3994    ///     .await?;
3995    /// # Ok(()) }
3996    /// ```
3997    ///
3998    /// The input arrives **labelled**, and that is the whole of the decision.
3999    /// [`Tainted::trusted`](crate::core::Tainted::trusted) is what an operator
4000    /// writes for a literal, a constant or a configuration value they vouch for;
4001    /// anything that came from outside — an inbound event, a queue message, a
4002    /// counterparty's payload — keeps the label it arrived with. A bare value
4003    /// admitted as trusted would satisfy `require_trusted` fields with
4004    /// attacker-chosen data and give the egress ceiling nothing to join with.
4005    pub fn start(
4006        &self,
4007        target: impl Into<Target>,
4008        input: Tainted<Value>,
4009    ) -> RunRequest<'_, Unkeyed> {
4010        RunRequest {
4011            runtime: self,
4012            target: target.into(),
4013            input,
4014            terms: RunTerms::default(),
4015            key: Unkeyed,
4016        }
4017    }
4018
4019    /// This plane as the handle a background run keeps.
4020    ///
4021    /// Every `Runtime` is built inside an `Arc` that holds `self_ref`, and a
4022    /// borrow of it is a borrow of that `Arc`, so the upgrade cannot fail
4023    /// while `&self` exists.
4024    fn owned(&self) -> Arc<Self> {
4025        self.self_ref
4026            .upgrade()
4027            .expect("a runtime is only reachable through the Arc its builder made")
4028    }
4029
4030    /// Admit and execute, blocking until the run rests.
4031    async fn admit_target(
4032        &self,
4033        target: Target,
4034        input: Tainted<Value>,
4035        terms: RunTerms,
4036    ) -> Result<RunOutcome, RuntimeError> {
4037        match target {
4038            Target::Capability(capability) => self.admit(&capability, input, terms).await,
4039            Target::Plan(plan) => self.admit_plan(plan, input, terms).await,
4040        }
4041    }
4042
4043    /// Admit, and let the run proceed in the background.
4044    async fn spawn_target(
4045        self: &Arc<Self>,
4046        target: Target,
4047        input: Tainted<Value>,
4048        terms: RunTerms,
4049    ) -> Result<RunId, RuntimeError> {
4050        // Resolved before the id is minted: an unknown capability is the
4051        // caller's mistake and must be an error, not a run that exists and
4052        // immediately fails. A plan is validated inside admission, which also
4053        // precedes the id becoming visible.
4054        let plan = match target {
4055            Target::Capability(capability) => {
4056                let skill = self.resolve(&capability)?;
4057                PlanIR::single(first_capability(&skill.descriptor()))
4058            }
4059            Target::Plan(plan) => plan,
4060        };
4061
4062        let run = RunId::generate();
4063        // Registered before admission, not after. Admission is several store
4064        // writes, and a drain beginning inside them would take its snapshot
4065        // without this run, report a complete stop, and let the process exit
4066        // while the run was still being admitted. Registered first, both orders
4067        // work out: a drain that starts after this waits for the run, and one
4068        // that starts before it is seen by the gate inside `admit_only`, which
4069        // refuses and drops the ticket on the way out.
4070        let ticket = self.inflight.enter(run);
4071        let admitted = self
4072            .admit_only(run, plan, input, terms, Entry::Outside)
4073            .await?;
4074
4075        let plane = Arc::clone(self);
4076        // In the caller's span, as a blocking run is: a background run's
4077        // trace is still the trace of whatever started it.
4078        tokio::spawn(
4079            async move {
4080                let _ticket = ticket;
4081                plane.execute_admitted(admitted).await
4082            }
4083            .in_current_span(),
4084        );
4085        Ok(run)
4086    }
4087
4088    /// Admission under a key: the one copy, for blocking and background,
4089    /// capability and plan.
4090    ///
4091    /// Two gates, and only the second arbitrates. The read below is an
4092    /// optimisation — it saves a redelivery the cost of a policy evaluation, a
4093    /// quota reservation, a lease and a correlation lookup — and settles
4094    /// nothing, since two instances racing both see nothing here. The
4095    /// constraint inside `append` decides, on the same division
4096    /// `CaseStore::correlate_or_open` runs on.
4097    async fn claimed<T, F>(&self, key: &str, admit: F) -> Result<Claimed<T>, RuntimeError>
4098    where
4099        F: Future<Output = Result<T, RuntimeError>>,
4100    {
4101        validate_admission_key(key)?;
4102        if let Some(held) = self.holder_of(key).await? {
4103            return Ok(Claimed::Held(held));
4104        }
4105        match admit.await {
4106            Ok(fresh) => Ok(Claimed::Fresh(fresh)),
4107            Err(RuntimeError::Store(crate::core::StoreError::DuplicateAdmission {
4108                run, ..
4109            })) => Ok(Claimed::Held(parse_holder(&run)?)),
4110            Err(e) => Err(e),
4111        }
4112    }
4113
4114    /// Wait until `run` rests and answer with its outcome — the value
4115    /// re-derived by strict replay when it concluded.
4116    ///
4117    /// A run whose lease is free is resumed here: its holder has gone. One
4118    /// whose lease is held is followed on the journal until it rests or the
4119    /// lease lapses, so a holder that dies mid-wait is taken over rather than
4120    /// waited for forever.
4121    async fn settled(&self, run: RunId) -> Result<RunOutcome, RuntimeError> {
4122        loop {
4123            if let Some(rested) = self.recorded_outcome(run).await? {
4124                return self.with_output(rested).await;
4125            }
4126            // Subscribed before the attempt, so an append between the two
4127            // still wakes the wait below.
4128            let mut follower = self.store.watch(run).await;
4129            match self.replay(run, Mode::Resume).await {
4130                Err(RuntimeError::LeaseHeld { remaining_secs, .. }) => {
4131                    let lapse = Duration::from_secs(remaining_secs.max(1));
4132                    match follower.as_mut() {
4133                        Some(follower) => {
4134                            let _ = tokio::time::timeout(lapse, follower.changed()).await;
4135                        }
4136                        None => tokio::time::sleep(lapse).await,
4137                    }
4138                }
4139                resumed => return resumed,
4140            }
4141        }
4142    }
4143
4144    /// A rested run's outcome with its value: strict replay re-derives what a
4145    /// concluded run answered, performing nothing. A suspension has no answer
4146    /// yet, and is returned as recorded.
4147    async fn with_output(&self, rested: RunOutcome) -> Result<RunOutcome, RuntimeError> {
4148        if rested.status == RunStatus::Succeeded {
4149            self.replay(rested.run_id, Mode::Strict).await
4150        } else {
4151            Ok(rested)
4152        }
4153    }
4154
4155    /// Which run holds this admission key, if any.
4156    async fn holder_of(&self, key: &str) -> Result<Option<RunId>, RuntimeError> {
4157        self.store
4158            .admitted_as(key)
4159            .await
4160            .map_err(RuntimeError::from_store)
4161    }
4162
4163    /// Answer a duplicate with what the original run has on the record.
4164    async fn answer_with(&self, held: RunId) -> Result<Admission, RuntimeError> {
4165        Ok(match self.recorded_outcome(held).await? {
4166            Some(outcome) => Admission::Replayed(outcome),
4167            None => Admission::InFlight(held),
4168        })
4169    }
4170
4171    /// What a run has recorded, **without resuming it**.
4172    ///
4173    /// `None` while the run is still working — it has records but has reached
4174    /// neither a conclusion nor a suspension.
4175    ///
4176    /// No lease is taken and nothing is appended. The original is frequently
4177    /// live when its redelivery lands, and a read that acquired a lease would
4178    /// fence a healthy run mid-step in order to report on it.
4179    ///
4180    /// The **last** record decides, never any record: a run that suspended, was
4181    /// resumed and carried on has a suspension in its history and is not
4182    /// suspended now. Same rule as the A2A status projection, so the two
4183    /// surfaces cannot disagree about one run.
4184    ///
4185    /// # Errors
4186    ///
4187    /// If the journal cannot be read.
4188    pub async fn recorded_outcome(&self, run: RunId) -> Result<Option<RunOutcome>, RuntimeError> {
4189        let records = self
4190            .store
4191            .read(run, 1)
4192            .await
4193            .map_err(RuntimeError::from_store)?;
4194        let Some(last) = records.last() else {
4195            return Ok(None);
4196        };
4197        let Some(status) = observed_status(&records) else {
4198            return Ok(None);
4199        };
4200        Ok(Some(RunOutcome {
4201            run_id: run,
4202            status,
4203            chain_head: last.hash,
4204            // Reading a record performs nothing, so it consumes nothing — the
4205            // same answer the closed-run path gives, and for the same reason:
4206            // the spend belongs to the pass that did the work.
4207            consumed: Consumed::default(),
4208            // A step's result is reconstructed by replay rather than stored, so
4209            // there is none to hand back here. See `Admission::Replayed`.
4210            output: None,
4211        }))
4212    }
4213
4214    /// Rebuild a run's case binding **from its history**, never by re-deriving.
4215    ///
4216    /// Two facts are read back rather than recomputed, and each has its own
4217    /// failure if it is not.
4218    ///
4219    /// The **case id**, because re-correlating could land on a different case if
4220    /// the keys were since released — silently rewriting which business fact the
4221    /// run belongs to.
4222    ///
4223    /// The **business keys**, because a case accumulates them over months. A run
4224    /// that resolved `memory.formation.subject: $correlation/meter` would
4225    /// otherwise resolve it against a set the live run never saw, write a second
4226    /// memory under a second subject, and produce a history that disagrees with
4227    /// itself with nothing on the record explaining why.
4228    fn recorded_case(&self, records: &[Record]) -> Option<CaseContext> {
4229        let (case_id, correlation) = records.iter().find_map(|r| match r.kind() {
4230            RecordKind::CaseBound { correlation, .. } => {
4231                r.body.case.map(|case_id| (case_id, correlation.clone()))
4232            }
4233            _ => None,
4234        })?;
4235        self.cases.as_ref().map(|cases| CaseContext {
4236            cases: Arc::clone(cases),
4237            tasks: self.tasks.clone(),
4238            events: self.events.clone(),
4239            calendar: Arc::clone(&self.calendar),
4240            case_id,
4241            correlation,
4242        })
4243    }
4244
4245    /// The contract this runtime enforces on every plan.
4246    pub(crate) fn contract(&self) -> crate::plan::Contract {
4247        let contract = crate::plan::Contract::new(self.by_capability.keys().cloned());
4248        if self.require_verifier {
4249            contract.require_verifier()
4250        } else {
4251            contract
4252        }
4253    }
4254
4255    async fn admit(
4256        &self,
4257        target: &str,
4258        input: Tainted<Value>,
4259        terms: RunTerms,
4260    ) -> Result<RunOutcome, RuntimeError> {
4261        // A bare target is the degenerate plan: one node, terminal.
4262        let skill = self.resolve(target)?;
4263        let capability = first_capability(&skill.descriptor());
4264        self.admit_plan(PlanIR::single(capability), input, terms)
4265            .await
4266    }
4267
4268    async fn admit_plan(
4269        &self,
4270        plan: PlanIR,
4271        input: Tainted<Value>,
4272        terms: RunTerms,
4273    ) -> Result<RunOutcome, RuntimeError> {
4274        self.admit_plan_as(RunId::generate(), plan, input, terms, Entry::Outside)
4275            .await
4276    }
4277
4278    /// Admit the run one already-executing run ordered from another agent.
4279    ///
4280    /// Apart from every other entry point for one reason: a commission is a
4281    /// *step* of a run this process is executing, so the drain gate must not
4282    /// see it. A drain that refused here would fail the commissioning step of
4283    /// exactly the runs it is waiting for — and that step's failure is
4284    /// `Interrupted`, the in-doubt classification, because a commissioning
4285    /// caller cannot know what the agent it ordered from managed to do. The
4286    /// drain would manufacture the state it exists to prevent.
4287    pub(crate) async fn commission_run(
4288        &self,
4289        target: &str,
4290        input: Tainted<Value>,
4291        terms: RunTerms,
4292    ) -> Result<RunOutcome, RuntimeError> {
4293        let skill = self.resolve(target)?;
4294        let capability = first_capability(&skill.descriptor());
4295        self.admit_plan_as(
4296            RunId::generate(),
4297            PlanIR::single(capability),
4298            input,
4299            terms,
4300            Entry::Nested,
4301        )
4302        .await
4303    }
4304
4305    /// Record the chain this run acts under, beside the plan it authorizes.
4306    ///
4307    /// The two are read back together: a chain without its plan says nothing
4308    /// about what it was allowed to do, and a plan without its chain says
4309    /// nothing about who was allowed to run it.
4310    fn bind_identity(
4311        run: RunId,
4312        chain: Option<&crate::core::Delegation>,
4313        records: &mut Vec<Append>,
4314    ) {
4315        if let Some(chain) = chain {
4316            records.push(Append::new(
4317                run,
4318                RecordKind::IdentityBound {
4319                    chain: chain.links().cloned().collect(),
4320                },
4321            ));
4322        }
4323    }
4324
4325    /// Check the plan against the delegation chain's authority.
4326    ///
4327    /// The plan is the authorization graph, so this is where authority belongs:
4328    /// a plan that names a capability outside the chain's scope must never
4329    /// start, rather than failing at whichever step happens to reach it first.
4330    /// Checking here also makes the refusal deterministic — it depends only on
4331    /// the frozen plan and the chain, both of which are recorded.
4332    ///
4333    /// The chain's other two bounds are checked beside it, and only here: a
4334    /// chain issued for another plane or past its validity is refused before
4335    /// the run exists, and a run admitted under a live chain is never re-judged
4336    /// — replay reads the recorded chain back.
4337    ///
4338    /// `acts_under` is whether the run acts under `ceiling` or is only bounded
4339    /// by it. A refusal names the chain's subject only in the first case: a
4340    /// caller bounded by the plane's chain is not its subject, and a denial
4341    /// naming the operator would say the operator was refused.
4342    fn authorize_scope(
4343        &self,
4344        plan: &PlanIR,
4345        ceiling: Option<&crate::core::Delegation>,
4346        acts_under: bool,
4347    ) -> Result<(), RuntimeError> {
4348        let Some(chain) = ceiling else {
4349            return Ok(());
4350        };
4351        chain.admissible(self.tenant.as_str(), now_for_admission())?;
4352        let scope = chain.effective_scope();
4353        for node in &plan.nodes {
4354            if !scope.permits(&node.capability) {
4355                return Err(RuntimeError::PolicyDenied(
4356                    crate::core::PolicyError::Denied {
4357                        principal: if acts_under {
4358                            chain.subject().id.clone()
4359                        } else {
4360                            node.capability.to_string()
4361                        },
4362                        action: crate::core::ACTION_ADMIT.to_owned(),
4363                        resource: node.capability.to_string(),
4364                    },
4365                ));
4366            }
4367        }
4368        Ok(())
4369    }
4370
4371    /// Authorize starting a run, before the run exists.
4372    ///
4373    /// A denial here leaves no journal at all, which is correct: nothing
4374    /// happened, and a run record for something that was never allowed to start
4375    /// would be a run nobody can explain.
4376    fn authorize_admission(
4377        &self,
4378        capability: &str,
4379        governed_by: Option<&crate::journal::AgentIdentity>,
4380        chain: Option<&crate::core::Delegation>,
4381        input: &Value,
4382    ) -> Result<(), RuntimeError> {
4383        let Some(engine) = self.policy.as_ref() else {
4384            return Ok(());
4385        };
4386        // Who is acting, and what is being asked for. Passing one string as both
4387        // made every admission rule a tautology — `principal == resource` cannot
4388        // express "this agent may not run that capability", which is the whole
4389        // question at admission on a plane hosting several agents.
4390        //
4391        // The principal is an **authenticated** identity or it is nothing. The
4392        // scope check a few lines above already denies under the delegation
4393        // subject, so anything else here would give one refused run two answers
4394        // to "who was refused" in the same error type.
4395        //
4396        // The agent's `metadata.name` is deliberately *not* used, tempting as it
4397        // is. A name is self-asserted — a manifest is a file, and its name is
4398        // whatever the author typed — so a rule granting authority to a name
4399        // grants it to any file claiming that name. Worse, a fallback would be
4400        // *silent*: `principal == X` would mean an authenticated identity in a
4401        // deployment with a delegation chain and a self-asserted label in one
4402        // without, so the same rule would change meaning with the wiring.
4403        //
4404        // Rules that need to bind to the agent bind to `context.agent.digest`,
4405        // which is content-addressed and pins what the declaration actually
4406        // said — so "this exact declaration may admit, and an edited one may
4407        // not" is expressible. The capability is the fallback principal because
4408        // it claims nothing: it is what was asked for, not who asked.
4409        //
4410        // Built by the function `policy check` also calls, from the values
4411        // `RunAdmitted` and `IdentityBound` record.
4412        let request = crate::policy::requests::admission(
4413            &crate::policy::requests::Acting {
4414                tenant: self.tenant.as_str(),
4415                capability,
4416                agent: governed_by,
4417                chain,
4418            },
4419            input,
4420        );
4421        // Every refusal, not only a rule's. `let … else` over `Deny` alone
4422        // admits the run for every other decision, so a policy set that cannot
4423        // be evaluated would open the door it exists to hold — the fail-open a
4424        // widened vocabulary creates for free unless the permit is the arm that
4425        // has to be named.
4426        let decision = engine.authorize(&request.as_request());
4427        let malformed = decision.is_malformed();
4428        let Some(reason) = decision.reason().map(ToOwned::to_owned) else {
4429            return Ok(());
4430        };
4431
4432        tracing::error!(
4433            target: telemetry::POLICY_DENIED,
4434            action = crate::core::ACTION_ADMIT,
4435            resource = %capability,
4436            policy_error = malformed,
4437            %reason,
4438        );
4439        self.meter
4440            .count(metrics::POLICY_DENIALS, crate::core::ACTION_ADMIT);
4441        Err(RuntimeError::PolicyDenied(
4442            crate::core::PolicyError::Denied {
4443                principal: request.principal,
4444                action: crate::core::ACTION_ADMIT.to_owned(),
4445                resource: capability.to_owned(),
4446            },
4447        ))
4448    }
4449
4450    /// The record that opens a run.
4451    ///
4452    /// Its own function because the label matters: journaled rather than
4453    /// recomputed, so a replay reaches the same verdict at every taint gate as
4454    /// the run it reproduces.
4455    fn admission(
4456        &self,
4457        capability: &str,
4458        governed_by: Option<crate::journal::AgentIdentity>,
4459        input: &Tainted<Value>,
4460        idempotency_key: Option<String>,
4461        admitted_by: Option<String>,
4462        acting_as: &Acting,
4463    ) -> RecordKind {
4464        RecordKind::RunAdmitted {
4465            capability: capability.to_owned(),
4466            governed_by: governed_by.map(Box::new),
4467            input: input.peek().clone(),
4468            input_label: input.label().clone(),
4469            input_field_labels: input
4470                .field_labels()
4471                .map(|(path, label)| (path.to_owned(), label.clone()))
4472                .collect(),
4473            policy_bundle: self.policy.as_ref().map(|p| Box::new(p.bundle())),
4474            canon: crate::core::canon::VERSION,
4475            idempotency_key,
4476            admitted_by,
4477            served_unchained: matches!(acting_as, Acting::Nobody),
4478            plane_chain: match acting_as {
4479                Acting::Plane => self.identity.is_some(),
4480                Acting::PlaneDelegate(_) => true,
4481                Acting::Caller(_) | Acting::Nobody => false,
4482            },
4483        }
4484    }
4485
4486    /// Everything up to and including the admission records, and no work.
4487    ///
4488    /// The seam a non-blocking submit needs. Admission is what makes a run
4489    /// *exist*: the policy gate, the lease, and the records that say what is
4490    /// about to happen. Splitting there means a caller can be told the run was
4491    /// accepted — or refused — before any of it runs, and a refusal stays an
4492    /// immediate answer rather than becoming a task that silently never appears.
4493    #[allow(clippy::too_many_lines)]
4494    async fn admit_only(
4495        &self,
4496        run: RunId,
4497        plan: PlanIR,
4498        input: Tainted<Value>,
4499        terms: RunTerms,
4500        entry: Entry,
4501    ) -> Result<Admitted, RuntimeError> {
4502        // First, and before the plan is even read: a draining instance is going
4503        // away, and everything below this line takes on work it may not be able
4504        // to finish. Refusing here leaves no lease, no quota slot and no
4505        // journal — the same nothing a policy denial leaves.
4506        if entry == Entry::Outside && self.inflight.is_draining() {
4507            return Err(RuntimeError::Draining);
4508        }
4509        crate::plan::validate(&plan, &self.contract())
4510            .map_err(|e| RuntimeError::PlanContract(e.to_string()))?;
4511
4512        let agent = plan
4513            .nodes
4514            .first()
4515            .map_or_else(|| "plan".to_owned(), |n| n.capability.to_string());
4516
4517        // Resolved once: the gate and the record must agree about which
4518        // declaration governs this run, and computing it twice is how they
4519        // start disagreeing.
4520        #[cfg(feature = "manifest")]
4521        let governed_by = self.identity_for(&agent);
4522        #[cfg(not(feature = "manifest"))]
4523        let governed_by: Option<crate::journal::AgentIdentity> = None;
4524
4525        if let Some(expected) = terms.expected_declaration {
4526            let found = governed_by.as_ref().map(|id| id.digest);
4527            if found != Some(expected) {
4528                return Err(RuntimeError::DeclarationPinMismatch {
4529                    agent,
4530                    expected,
4531                    found,
4532                });
4533            }
4534        }
4535
4536        // The caller's chain where one was presented, the plane's otherwise —
4537        // one choice, made once, so the gate, the policy context and the
4538        // record cannot disagree about whom this run acts for.
4539        let chain = terms.acting_as.resolve(self.identity.as_ref());
4540        self.authorize_scope(
4541            &plan,
4542            terms.acting_as.ceiling(self.identity.as_ref()),
4543            chain.is_some(),
4544        )?;
4545        self.authorize_admission(&agent, governed_by.as_ref(), chain, input.peek())?;
4546        let input = self.admit_content(&agent, input)?;
4547
4548        self.admit_reserved(run, plan, input, terms, agent, governed_by)
4549            .await
4550    }
4551
4552    /// Admission past the quota reservation: the lease, the records, the case
4553    /// binding, the outbox.
4554    ///
4555    /// Every error path hands the lease back. A lease left standing over a
4556    /// failed admission is not idle bookkeeping: it expires unreleased, which
4557    /// is the exact signature the recovery sweep reads as "an instance died
4558    /// holding this run" — so a half-admitted run (records appended, outbox
4559    /// registration failed) would be *executed* by the sweeper a TTL later,
4560    /// having never been admitted. The one error that concludes rather than
4561    /// merely unwinding is the post-append one, where records already exist:
4562    /// the run is concluded `failed` in its own chain first, so nothing later
4563    /// reads an open journal as work in progress.
4564    async fn admit_reserved(
4565        &self,
4566        run: RunId,
4567        plan: PlanIR,
4568        input: Tainted<Value>,
4569        terms: RunTerms,
4570        agent: String,
4571        governed_by: Option<crate::journal::AgentIdentity>,
4572    ) -> Result<Admitted, RuntimeError> {
4573        // Ownership precedes reservation. A process can die between any two
4574        // store calls; with this order, a slot reserved before the admission
4575        // batch always has an owned lease beside it, and the abandonment sweep
4576        // can release both. Reserving first could strand a slot over no run and
4577        // no lease — invisible to every recovery query.
4578        let lease = self
4579            .store
4580            .acquire(run, &self.owner, self.lease_ttl)
4581            .await
4582            .map_err(RuntimeError::from_store)?;
4583
4584        let budget = self.budget_for(&agent);
4585        // Boxed: the quota decision holds its store round trips across awaits,
4586        // and inlined it pushes every admission future past the size clippy
4587        // allows on the served surfaces that await one.
4588        let quota = match Box::pin(self.check_quota(
4589            run,
4590            governed_by.as_ref(),
4591            // **The chain this run acts under, not the plane's.** A served
4592            // surface admits each run under its caller's chain, so reading
4593            // the plane's here would check a withdrawal against the
4594            // operator's own subject and never match the caller's — a
4595            // refusal that does not happen, which leaves no trace anywhere.
4596            terms.acting_as.resolve(self.identity.as_ref()),
4597            now_for_admission(),
4598            (&budget, reserved_width(&budget, &plan)),
4599        ))
4600        .await
4601        {
4602            Ok(pass) => pass,
4603            Err(error) => {
4604                let _ = self.store.release_lease(run, lease.epoch).await;
4605                return Err(error);
4606            }
4607        };
4608
4609        let out = self
4610            .admit_under_lease(run, &lease, plan, input, terms, &agent, governed_by, quota)
4611            .await;
4612        if out.is_err() {
4613            if let Some(quotas) = self.quotas.as_ref()
4614                && let Err(error) = quotas.release(run).await
4615            {
4616                tracing::debug!(%run, %error, "could not release the quota slot of a failed admission");
4617            }
4618            if let Err(error) = self.store.release_lease(run, lease.epoch).await {
4619                tracing::debug!(
4620                    %run,
4621                    %error,
4622                    "could not release the lease of a failed admission; it will expire on its own"
4623                );
4624            }
4625        }
4626        out
4627    }
4628
4629    #[allow(clippy::too_many_arguments, clippy::too_many_lines)]
4630    async fn admit_under_lease(
4631        &self,
4632        run: RunId,
4633        lease: &crate::journal::Lease,
4634        plan: PlanIR,
4635        input: Tainted<Value>,
4636        terms: RunTerms,
4637        agent: &str,
4638        governed_by: Option<crate::journal::AgentIdentity>,
4639        quota: QuotaPass,
4640    ) -> Result<Admitted, RuntimeError> {
4641        let RunTerms {
4642            case,
4643            idempotency_key,
4644            acting_as,
4645            admitted_by,
4646            expected_declaration: _,
4647            subjects,
4648        } = terms;
4649        let chain = acting_as.resolve(self.identity.as_ref());
4650        let mut records = vec![
4651            Append::new(
4652                run,
4653                self.admission(
4654                    agent,
4655                    governed_by,
4656                    &input,
4657                    idempotency_key,
4658                    admitted_by,
4659                    &acting_as,
4660                ),
4661            ),
4662            // From here the plan is an authorization graph: compiled from
4663            // trusted input, frozen before anything untrusted was read, and
4664            // recorded so the journal that follows can be checked against it.
4665            Append::new(
4666                run,
4667                RecordKind::PlanFrozen {
4668                    steps: plan
4669                        .nodes
4670                        .iter()
4671                        .map(|n| n.capability.to_string())
4672                        .collect(),
4673                    plan: serde_json::to_value(&plan)?,
4674                },
4675            ),
4676        ];
4677
4678        Self::bind_identity(run, chain, &mut records);
4679        if let Some(started) = quota.started() {
4680            records.push(Append::new(run, started));
4681        }
4682
4683        #[cfg(feature = "manifest")]
4684        let declared: Vec<crate::journal::SubjectBinding> = self
4685            .resolve(agent)
4686            .ok()
4687            .and_then(|skill| self.governing(skill.as_ref()))
4688            .map(|m| {
4689                m.spec
4690                    .data_subjects
4691                    .iter()
4692                    .map(crate::manifest::DataSubject::binding)
4693                    .collect()
4694            })
4695            .unwrap_or_default();
4696        #[cfg(not(feature = "manifest"))]
4697        let declared: Vec<crate::journal::SubjectBinding> = Vec::new();
4698        // Before correlation, because `correlate_or_open` commits a case: a
4699        // binding that refuses the run must refuse it before anything outside
4700        // the journal names it. `$case` alone needs the case, and the case is
4701        // bound whenever the run asked for one, so it cannot refuse later.
4702        let bound = bind_subjects(&declared, subjects, &input, case.is_some())?;
4703
4704        // Correlation is deterministic and runs before planning: which case a
4705        // message belongs to is a matter of fact, settled by a lookup.
4706        let case_ctx = match (case, self.cases.as_ref()) {
4707            (Some(CaseBinding::Correlate { kind, keys }), Some(cases)) => {
4708                let correlation = cases
4709                    .correlate_or_open(&kind, &keys, now_for_admission())
4710                    .await
4711                    .map_err(RuntimeError::from_store)?;
4712                let case_id = correlation.case_id();
4713                // The case's own keys, not the ones this message carried. A run
4714                // that *joined* an existing case may have been admitted with one
4715                // key while the case holds three, and a binding naming any of
4716                // them must resolve — which is the difference between "the keys
4717                // in this request" and "the business facts this matter is
4718                // identified by".
4719                let bound_keys = case_correlation(cases.as_ref(), case_id, &keys).await?;
4720                // Stamp the case on the records already queued as well: every
4721                // record of a case-bound run carries its case, which is what
4722                // makes "show me everything about this matter" one range scan.
4723                for r in &mut records {
4724                    r.case = Some(case_id);
4725                }
4726                records.push(
4727                    Append::new(
4728                        run,
4729                        RecordKind::CaseBound {
4730                            case_kind: kind,
4731                            opened: correlation.is_new(),
4732                            correlation: bound_keys.clone(),
4733                        },
4734                    )
4735                    .case(case_id),
4736                );
4737                Some(CaseContext {
4738                    cases: Arc::clone(cases),
4739                    tasks: self.tasks.clone(),
4740                    events: self.events.clone(),
4741                    calendar: Arc::clone(&self.calendar),
4742                    case_id,
4743                    correlation: bound_keys,
4744                })
4745            }
4746            (Some(CaseBinding::Existing(case_id)), Some(cases)) => {
4747                let existing = cases
4748                    .case(case_id)
4749                    .await
4750                    .map_err(RuntimeError::from_store)?
4751                    .ok_or_else(|| {
4752                        RuntimeError::PlanContract(format!("no such case: {case_id}"))
4753                    })?;
4754                if existing.status.is_closed() {
4755                    return Err(RuntimeError::PlanContract(format!(
4756                        "case '{case_id}' is closed and cannot accept another run"
4757                    )));
4758                }
4759                for record in &mut records {
4760                    record.case = Some(case_id);
4761                }
4762                records.push(
4763                    Append::new(
4764                        run,
4765                        RecordKind::CaseBound {
4766                            case_kind: existing.kind,
4767                            opened: false,
4768                            correlation: existing.correlation.clone(),
4769                        },
4770                    )
4771                    .case(case_id),
4772                );
4773                Some(CaseContext {
4774                    cases: Arc::clone(cases),
4775                    tasks: self.tasks.clone(),
4776                    events: self.events.clone(),
4777                    calendar: Arc::clone(&self.calendar),
4778                    case_id,
4779                    correlation: existing.correlation,
4780                })
4781            }
4782            (Some(_), None) => {
4783                return Err(RuntimeError::PlanContract(
4784                    "this run was admitted with correlation keys but the runtime has no case \
4785                     store — build it with `.cases(store)`"
4786                        .into(),
4787                ));
4788            }
4789            (None, _) => None,
4790        };
4791
4792        let bound = fill_case_subjects(bound, case_ctx.as_ref());
4793        let subject_refs = subject_refs(run, &bound);
4794        if !bound.is_empty() {
4795            let mut append = Append::new(run, RecordKind::DataSubjectBound { bindings: bound });
4796            append.case = case_ctx.as_ref().map(CaseContext::id);
4797            records.push(append);
4798        }
4799
4800        // The append claims the admission key, so a key already held is refused
4801        // here, before anything outside the journal names the run.
4802        self.store
4803            .append(lease.epoch, records)
4804            .await
4805            .map_err(RuntimeError::from_store)?;
4806
4807        // Registered before the run is handed back, and its failure fails
4808        // admission: a case that does not list the run, or destinations that
4809        // never see its history.
4810        //
4811        // After the append rather than before: a registration written for a
4812        // run whose admission then failed — or whose process died — would name
4813        // a run no journal holds, and nothing could tell which case to take it
4814        // off. The price of that order is paid here instead: the admission
4815        // records above are already durable, so a registration failure must
4816        // **conclude** the run rather than merely return — an open journal
4817        // under a lease that then lapses is exactly what the recovery sweep
4818        // reads as a crashed run, and it would execute a run whose admission
4819        // failed. Concluded `failed`, in the chain, with the reason beside it;
4820        // the caller's cleanup then releases the lease, and a resume registers
4821        // again before it executes.
4822        if let Err(open_failed) = self
4823            .register(run, case_ctx.as_ref().map(CaseContext::id))
4824            .await
4825        {
4826            let head = self
4827                .store
4828                .head(run)
4829                .await
4830                .map_err(RuntimeError::from_store)?;
4831            self.store
4832                .append(
4833                    lease.epoch,
4834                    vec![Append::new(
4835                        run,
4836                        RecordKind::RunConcluded {
4837                            outcome: RunStatus::Failed(String::new()).as_str().to_owned(),
4838                            reason: Some(format!(
4839                                "the run's registration was refused ({open_failed}), so this \
4840                                 run was concluded without ever executing"
4841                            )),
4842                            exhaustion: None,
4843                            live_spend: Spend::default(),
4844                            chain_head: head.hash,
4845                        },
4846                    )],
4847                )
4848                .await
4849                .map_err(RuntimeError::from_store)?;
4850            return Err(RuntimeError::from_store(open_failed));
4851        }
4852
4853        Ok(Admitted {
4854            run,
4855            epoch: lease.epoch,
4856            quota,
4857            budget: self.budget_for(agent),
4858            agent: agent.to_owned(),
4859            plan,
4860            input,
4861            case: case_ctx,
4862            identity: chain.cloned(),
4863            subjects: subject_refs,
4864        })
4865    }
4866
4867    /// Execute a run that has already been admitted.
4868    async fn execute_admitted(&self, a: Admitted) -> Result<RunOutcome, RuntimeError> {
4869        let mut cursor = ReplayCursor::default();
4870        // Named, not `_`: `let _ = ` drops immediately, which would renew
4871        // nothing at all while looking exactly like this.
4872        let _heartbeat = self.heartbeat(a.run, a.epoch);
4873        self.execute(
4874            Execution {
4875                run: a.run,
4876                epoch: a.epoch,
4877                quota: a.quota,
4878                plan: &a.plan,
4879                input: a.input,
4880                mode: Pass::Live,
4881                case: a.case,
4882                budget: a.budget,
4883                agent: a.agent,
4884                identity: a.identity,
4885                subjects: a.subjects,
4886                refusal: None,
4887                withheld_subject: None,
4888                successors: Vec::new(),
4889                started: BTreeSet::new(),
4890                finished: BTreeSet::new(),
4891                succeeded: BTreeSet::new(),
4892                recorded_groups: BTreeMap::new(),
4893            },
4894            &mut cursor,
4895        )
4896        .await
4897    }
4898
4899    /// Admit and execute, which is what every blocking entry point does.
4900    ///
4901    /// The run id is the caller's rather than minted here, for batches: an
4902    /// item's run id is written to the batch store *before* the run starts, so
4903    /// that a crash leaves a reservation pointing at a journal that can be
4904    /// replayed rather than an item that must be guessed about.
4905    pub(crate) async fn admit_plan_as(
4906        &self,
4907        run: RunId,
4908        plan: PlanIR,
4909        input: Tainted<Value>,
4910        terms: RunTerms,
4911        entry: Entry,
4912    ) -> Result<RunOutcome, RuntimeError> {
4913        let admitted = self.admit_only(run, plan, input, terms, entry).await?;
4914        self.execute_admitted(admitted).await
4915    }
4916
4917    /// Why an open run may not cross its history frontier on this plane, if it
4918    /// may not.
4919    ///
4920    /// Two facts recorded at admission are compared with what this plane holds:
4921    /// the policy bundle (who may authorize an effect) and, for a declarative
4922    /// agent, the declaration (what the agent *is*). A resume under either one
4923    /// changed would continue one run's journal under another's rules. The
4924    /// answer is a refusal to journal rather than an error to raise; the caller
4925    /// decides where it goes.
4926    fn resume_refusal(&self, records: &[Record]) -> Result<Option<RuntimeError>, RuntimeError> {
4927        if let Some(refusal) = self.policy_bundle_refusal(records)? {
4928            return Ok(Some(refusal));
4929        }
4930        #[cfg(feature = "manifest")]
4931        if let Some(refusal) = self.declaration_refusal(records)? {
4932            return Ok(Some(refusal));
4933        }
4934        Ok(None)
4935    }
4936
4937    /// [`RuntimeError::NoProvider`] when a capability any of the run's plans
4938    /// names resolves to nothing on this plane.
4939    ///
4940    /// Every plan the run froze is checked, not only the one in force: an
4941    /// unwind resolves the steps that ran, and those may belong to a plan a
4942    /// replan replaced.
4943    fn refuse_undrivable<'p>(
4944        &self,
4945        plans: impl IntoIterator<Item = &'p PlanIR>,
4946    ) -> Result<(), RuntimeError> {
4947        for node in plans.into_iter().flat_map(|p| &p.nodes) {
4948            self.resolve(&node.capability.0)?;
4949        }
4950        Ok(())
4951    }
4952
4953    /// Name the run where it is watched from outside its journal: on its case,
4954    /// and to every outbox destination.
4955    ///
4956    /// Only after the admission's append, and idempotent — an existing case
4957    /// row and an existing delivery cursor are kept — so every writing resume
4958    /// repeats it.
4959    async fn register(
4960        &self,
4961        run: RunId,
4962        case: Option<crate::core::CaseId>,
4963    ) -> Result<(), crate::core::StoreError> {
4964        if let (Some(case), Some(cases)) = (case, self.cases.as_ref()) {
4965            cases.attach_run(case, run).await?;
4966        }
4967        #[cfg(feature = "push")]
4968        if let Some(outbox) = &self.outbox {
4969            outbox.open(run).await?;
4970        }
4971        Ok(())
4972    }
4973
4974    /// [`RuntimeError::NoCaseStore`] when the run is bound to a case and this
4975    /// plane has no case store.
4976    ///
4977    /// Everything a resume writes belongs to the case, and without the store
4978    /// it would land under the run alone, where erasing the case misses it.
4979    /// About this plane rather than the run, so another plane may resume it.
4980    fn refuse_caseless(&self, run: RunId, records: &[Record]) -> Result<(), RuntimeError> {
4981        match recorded_case_id(records) {
4982            Some(case) if self.cases.is_none() => Err(RuntimeError::NoCaseStore {
4983                run: run.to_string(),
4984                case: case.to_string(),
4985            }),
4986            _ => Ok(()),
4987        }
4988    }
4989
4990    /// Journal a resume refusal as the run's quarantine, naming what moved.
4991    async fn quarantine_refused(
4992        &self,
4993        run: RunId,
4994        records: &[Record],
4995        lease: &crate::journal::Lease,
4996        refusal: &RuntimeError,
4997    ) -> Result<RunOutcome, RuntimeError> {
4998        self.conclude(
4999            run,
5000            lease.epoch,
5001            RunStatus::Quarantined(refusal.to_string()),
5002            None,
5003            true,
5004            recorded_case_id(records),
5005            Consumed::default(),
5006            Spend::ZERO,
5007            &QuotaPass::disabled(),
5008        )
5009        .await
5010    }
5011
5012    /// The bundle half of [`resume_refusal`](Self::resume_refusal).
5013    ///
5014    /// Resume can dispatch new effects after it reaches the end of history.
5015    /// They must be judged by the same complete bundle recorded at admission,
5016    /// or one run would claim one policy while later effects were authorized
5017    /// by another.
5018    fn policy_bundle_refusal(
5019        &self,
5020        records: &[Record],
5021    ) -> Result<Option<RuntimeError>, RuntimeError> {
5022        let recorded = records
5023            .iter()
5024            .find_map(|record| match record.kind() {
5025                RecordKind::RunAdmitted { policy_bundle, .. } => {
5026                    Some(policy_bundle.as_deref().cloned())
5027                }
5028                _ => None,
5029            })
5030            .ok_or_else(|| {
5031                RuntimeError::PlanContract("journal has no RunAdmitted record".into())
5032            })?;
5033        let configured = self.policy.as_ref().map(|policy| policy.bundle());
5034        if recorded != configured {
5035            return Ok(Some(RuntimeError::PolicyBundleChanged {
5036                recorded: recorded.as_ref().map(PolicyBundleIdentity::digest),
5037                configured: configured.as_ref().map(PolicyBundleIdentity::digest),
5038            }));
5039        }
5040        Ok(None)
5041    }
5042
5043    /// The declaration half of [`resume_refusal`](Self::resume_refusal).
5044    ///
5045    /// For a declarative agent the manifest is the whole behaviour, and a resume
5046    /// under an edited one runs a different program over the first one's
5047    /// journal — which replay would eventually catch as a key mismatch, several
5048    /// effects in and without being able to say what changed. The digest is
5049    /// already on the record at admission, so refusing here costs a comparison
5050    /// and turns a late diagnosis into a named remedy.
5051    ///
5052    /// **Silent in two cases, and deliberately.** A run admitted with no
5053    /// declaration is a coded skill, whose behaviour is the embedder's binary:
5054    /// this crate cannot identify that and will not pretend to. And a plane that
5055    /// holds no manifest under the recorded name cannot run the agent
5056    /// declaratively at all, so the refusal it needs is the one plan resolution
5057    /// already gives.
5058    #[cfg(feature = "manifest")]
5059    fn declaration_refusal(
5060        &self,
5061        records: &[Record],
5062    ) -> Result<Option<RuntimeError>, RuntimeError> {
5063        let Some(recorded) = records.iter().find_map(|record| match record.kind() {
5064            RecordKind::RunAdmitted { governed_by, .. } => governed_by.as_deref().cloned(),
5065            _ => None,
5066        }) else {
5067            return Ok(None);
5068        };
5069        let Some(current) = self
5070            .governed_by
5071            .values()
5072            .find(|m| m.metadata.name == recorded.name)
5073        else {
5074            return Ok(None);
5075        };
5076        let configured = current
5077            .digest()
5078            .map_err(|e| RuntimeError::PlanContract(e.to_string()))?;
5079        if configured != recorded.digest {
5080            return Ok(Some(RuntimeError::DeclarationChanged {
5081                agent: recorded.name,
5082                recorded: recorded.digest,
5083                configured,
5084            }));
5085        }
5086        Ok(None)
5087    }
5088
5089    /// A plan left sealed is erased only where a key ring could have opened
5090    /// it.
5091    ///
5092    /// A wired ring opens every payload whose key still exists and fails the
5093    /// read when it cannot be reached, so a payload still sealed behind one was
5094    /// erased. Behind no ring it says nothing about the key at all: the data
5095    /// is intact, and telling a reader it is gone sends them to the wrong
5096    /// remedy.
5097    fn sealed_or_erased(&self, error: RuntimeError) -> RuntimeError {
5098        match error {
5099            RuntimeError::PayloadsErased { run } if !self.holds_key_ring() => {
5100                RuntimeError::PayloadsSealed { run }
5101            }
5102            other => other,
5103        }
5104    }
5105
5106    /// Whether this plane was handed a key ring to open sealed payloads with.
5107    #[cfg_attr(not(feature = "keyring"), allow(clippy::unused_self))]
5108    const fn holds_key_ring(&self) -> bool {
5109        #[cfg(feature = "keyring")]
5110        {
5111            self.keyring.is_some()
5112        }
5113        #[cfg(not(feature = "keyring"))]
5114        {
5115            false
5116        }
5117    }
5118
5119    /// Re-execute a recorded run from its journal.
5120    ///
5121    /// * [`Mode::Strict`] verifies determinism: every effect must match, and the
5122    ///   run must not want any effect the journal lacks.
5123    /// * [`Mode::Resume`] recovers a crashed run: history is replayed, then
5124    ///   execution continues live from wherever the record ends.
5125    ///
5126    /// Either way no external effect is performed for anything already in the
5127    /// journal. That is the whole point — a resumed run does not re-issue the
5128    /// invoice it already issued.
5129    pub async fn replay(&self, run: RunId, mode: Mode) -> Result<RunOutcome, RuntimeError> {
5130        let mode = Pass::from(mode);
5131        // ── Ownership before observation ───────────────────────────────────
5132        //
5133        // Resume writes, so it takes the lease *before* it reads. The old
5134        // order — snapshot the records, then acquire — left a window in which
5135        // a live owner could append past the snapshot; the resume then
5136        // replayed a prefix of the true history and continued live from a
5137        // frontier that was not the frontier, re-deriving keys the owner had
5138        // already written. Claiming first means the snapshot below is taken
5139        // under this instance's own fence. Strict verifies and never writes,
5140        // so it needs no lease — and must not take one, or every regression
5141        // check would fence a run somebody may be legitimately running.
5142        let lease = if mode == Pass::Resume {
5143            Some(
5144                self.store
5145                    .acquire(run, &self.owner, self.lease_ttl)
5146                    .await
5147                    .map_err(RuntimeError::from_store)?,
5148            )
5149        } else {
5150            None
5151        };
5152
5153        self.replay_releasing(run, mode, lease, Claim::Standing)
5154            .await
5155    }
5156
5157    /// Replay a recorded run strictly and say what it found.
5158    ///
5159    /// [`replay`](Self::replay) with [`Mode::Strict`], answered as a
5160    /// [`Verdict`](super::Verdict) rather than a status: *verified* when every
5161    /// recorded effect was asked for again and the run reached the ending it
5162    /// recorded, whatever that ending was; *diverged* at the first effect this
5163    /// build asked for differently; *cannot replay* when the run cannot be
5164    /// re-derived here, for a reason that is not the declaration. Both
5165    /// revisions are named on every answer — the one the run was admitted
5166    /// under and the one this plane holds by that name.
5167    ///
5168    /// Writes nothing, takes no lease and dispatches nothing, as a strict
5169    /// replay never does. It is the question an author asks of an edited
5170    /// manifest: replayed under the edit, this run's history either reaches
5171    /// nothing it changed or names the first effect it did.
5172    ///
5173    /// # Errors
5174    ///
5175    /// If the run does not exist or its journal does not verify, or on a store
5176    /// failure — never for a divergence or a reason the verdict names.
5177    pub async fn verify(&self, run: RunId) -> Result<super::Verdict, RuntimeError> {
5178        use super::{CannotReplay, Finding};
5179
5180        let records = self
5181            .store
5182            .read(run, 1)
5183            .await
5184            .map_err(RuntimeError::from_store)?;
5185        if records.is_empty() {
5186            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
5187                run.to_string(),
5188            )));
5189        }
5190        let recorded = records.iter().find_map(|r| match r.kind() {
5191            RecordKind::RunAdmitted { governed_by, .. } => governed_by.as_deref().cloned(),
5192            _ => None,
5193        });
5194        let candidate = self.candidate_revision(recorded.as_ref());
5195        let verdict = |finding| super::Verdict {
5196            run,
5197            recorded: recorded.clone(),
5198            candidate: candidate.clone(),
5199            finding,
5200        };
5201
5202        // A property of the history rather than of how it ended, so it is
5203        // named whatever the run's state: every key moves under another rule.
5204        if let Err(RuntimeError::CanonicalizationChanged {
5205            recorded,
5206            implemented,
5207        }) = ensure_replayable_canon(&records)
5208        {
5209            return Ok(verdict(Finding::CannotReplay(
5210                CannotReplay::CanonicalizationChanged {
5211                    recorded,
5212                    implemented,
5213                },
5214            )));
5215        }
5216        let Some(ending) = recorded_ending(&records) else {
5217            return Ok(verdict(Finding::CannotReplay(CannotReplay::NotConcluded)));
5218        };
5219        if ended_by_an_act(&ending) {
5220            return Ok(verdict(Finding::CannotReplay(CannotReplay::EndedByAnAct {
5221                outcome: ending,
5222            })));
5223        }
5224
5225        let mut dispatched = false;
5226        let mut divergence = None;
5227        let replayed = self
5228            .replay_under(
5229                run,
5230                Pass::Strict,
5231                None,
5232                Claim::Standing,
5233                &mut dispatched,
5234                &mut divergence,
5235            )
5236            .await;
5237        let finding = match (replayed, divergence) {
5238            (Ok(_), Some(d)) => Finding::Diverged(d),
5239            (Ok(outcome), None) if outcome.status.as_str() == ending => {
5240                Finding::Verified { outcome: ending }
5241            }
5242            (Ok(outcome), None) => Finding::OutcomeDiffers {
5243                recorded: ending,
5244                replayed: outcome.status.as_str().to_owned(),
5245            },
5246            (Err(RuntimeError::PayloadsErased { .. }), _) => {
5247                Finding::CannotReplay(CannotReplay::Erased)
5248            }
5249            (Err(RuntimeError::PayloadsSealed { .. }), _) => {
5250                Finding::CannotReplay(CannotReplay::KeyAbsent)
5251            }
5252            (Err(RuntimeError::NoProvider { target, .. }), _) => {
5253                Finding::CannotReplay(CannotReplay::EntryPointRemoved { capability: target })
5254            }
5255            (Err(other), _) => return Err(other),
5256        };
5257        Ok(verdict(finding))
5258    }
5259
5260    /// The declaration this plane holds under the recorded agent's name.
5261    #[cfg(feature = "manifest")]
5262    fn candidate_revision(
5263        &self,
5264        recorded: Option<&crate::journal::AgentIdentity>,
5265    ) -> Option<crate::journal::AgentIdentity> {
5266        let name = &recorded?.name;
5267        let held = self
5268            .governed_by
5269            .values()
5270            .find(|m| &m.metadata.name == name)?;
5271        self.identity_of(held)
5272    }
5273
5274    /// Without the manifest there are no declarations to name.
5275    #[cfg(not(feature = "manifest"))]
5276    #[allow(clippy::unused_self)]
5277    const fn candidate_revision(
5278        &self,
5279        _recorded: Option<&crate::journal::AgentIdentity>,
5280    ) -> Option<crate::journal::AgentIdentity> {
5281        None
5282    }
5283
5284    /// Resume a run under a lease the caller already holds.
5285    ///
5286    /// The wake path's handover: a timer or event delivery records the wake
5287    /// under its own lease and then resumes the run. Releasing that lease and
5288    /// letting the resume acquire a fresh one opened a window — a crash
5289    /// between the two left a *released* lease over a run whose timer was
5290    /// disarmed and whose subscription was retired, which no queue in the
5291    /// system names: the abandonment sweep looks only for leases that expired
5292    /// while still holding an owner. Handing the lease over closes the window;
5293    /// the run is owned continuously from wake to conclusion, and a crash
5294    /// anywhere in between leaves an expired *owned* lease the sweep drains.
5295    ///
5296    /// A resume that fails does the same thing a crash does, and for the same
5297    /// reason: the recorded wake took the run off the waiting listing, so the
5298    /// lease is the only thing left that names it.
5299    pub(crate) async fn resume_holding(
5300        &self,
5301        run: RunId,
5302        lease: crate::journal::Lease,
5303    ) -> Result<RunOutcome, RuntimeError> {
5304        self.replay_releasing(run, Pass::Resume, Some(lease), Claim::Unlisted)
5305            .await
5306    }
5307
5308    /// Recover an owner-abandoned run without turning settlement repair into
5309    /// an operator-requested retry of a recorded conclusion.
5310    ///
5311    /// A recovery that fails for a reason no retry changes — the journal, not
5312    /// the moment, is what refuses — quarantines the run under the lease the
5313    /// failed pass kept, naming the reason. Left alone it would lapse owned
5314    /// and be recovered, and refused, every lease period for ever; a
5315    /// quarantine is listed, counted, and answered by a person.
5316    pub(crate) async fn recover_abandoned_run(
5317        &self,
5318        run: RunId,
5319    ) -> Result<RunOutcome, RuntimeError> {
5320        let lease = self
5321            .store
5322            .acquire(run, &self.owner, self.lease_ttl)
5323            .await
5324            .map_err(RuntimeError::from_store)?;
5325        match self
5326            .replay_releasing(run, Pass::Resume, Some(lease.clone()), Claim::Recovering)
5327            .await
5328        {
5329            Err(error) if recurs_on_every_resume(&error) => {
5330                let records = self
5331                    .store
5332                    .read(run, 1)
5333                    .await
5334                    .map_err(RuntimeError::from_store)?;
5335                let refusal = RuntimeError::PlanContract(format!(
5336                    "recovery refused, and would be refused the same way on every attempt: \
5337                     {error}"
5338                ));
5339                self.quarantine_refused(run, &records, &lease, &refusal)
5340                    .await
5341            }
5342            other => other,
5343        }
5344    }
5345
5346    async fn replay_releasing(
5347        &self,
5348        run: RunId,
5349        mode: Pass,
5350        lease: Option<crate::journal::Lease>,
5351        claim: Claim,
5352    ) -> Result<RunOutcome, RuntimeError> {
5353        let mut dispatched = false;
5354        let outcome = self
5355            .replay_under(run, mode, lease.as_ref(), claim, &mut dispatched, &mut None)
5356            .await;
5357
5358        // Idempotent: a resume that reached a conclusion already handed its
5359        // lease back in `conclude`, and re-releasing the same epoch changes
5360        // nothing. Released on success, and on a failure only where the run
5361        // is still listed somewhere else: the caller claimed a run whose
5362        // journal already says where it stands, and the refusal came before
5363        // anything past the frontier was written. Every other failure keeps
5364        // the lease so it lapses owned, which is what puts the run on the
5365        // abandonment listing the recovery sweep drains. A wake or a delivery
5366        // has already recorded the awaited result, so the run is off the
5367        // waiting listing; a recovery holds a run nothing else lists; and a
5368        // pass that failed mid-flight wrote records past a conclusion that no
5369        // longer describes it. Releasing any of those strands the run with no
5370        // driver and on no listing.
5371        let release = match &outcome {
5372            Ok(_) => true,
5373            // Settlement is retried by recovery, which finds the run through
5374            // this lease.
5375            Err(RuntimeError::QuotaSettlementPending { .. }) => false,
5376            Err(_) => claim == Claim::Standing && !dispatched,
5377        };
5378        if release
5379            && let Some(lease) = &lease
5380            && let Err(e) = self.store.release_lease(run, lease.epoch).await
5381        {
5382            tracing::debug!(
5383                %run,
5384                error = %e,
5385                "could not hand back the resume lease; it will expire on its own"
5386            );
5387        }
5388        outcome
5389    }
5390
5391    #[allow(clippy::too_many_lines)]
5392    async fn replay_under(
5393        &self,
5394        run: RunId,
5395        mode: Pass,
5396        lease: Option<&crate::journal::Lease>,
5397        claim: Claim,
5398        dispatched: &mut bool,
5399        divergence: &mut Option<crate::journal::Divergence>,
5400    ) -> Result<RunOutcome, RuntimeError> {
5401        let records = self
5402            .store
5403            .read(run, 1)
5404            .await
5405            .map_err(RuntimeError::from_store)?;
5406        if records.is_empty() {
5407            return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
5408                run.to_string(),
5409            )));
5410        }
5411
5412        // Never trust a journal that does not verify. A tampered or truncated
5413        // history would let replay "confirm" something that never happened.
5414        //
5415        // Per-record hashes are already checked on read, so a single altered
5416        // record fails before we get here; this catches the structural attacks
5417        // that survive individually-valid records — deletion, reordering, and
5418        // splicing history from another run.
5419        Record::verify_chain(&records, Digest::ZERO).map_err(RuntimeError::from_store)?;
5420
5421        ensure_replayable_canon(&records)?;
5422
5423        if mode == Pass::Resume
5424            && let Some(outcome) = self
5425                .recover_quota_before_resume(
5426                    run,
5427                    &records,
5428                    lease.expect("resume holds a lease"),
5429                    claim == Claim::Recovering,
5430                )
5431                .await?
5432        {
5433            return Ok(outcome);
5434        }
5435
5436        if mode == Pass::Resume
5437            && let Some(outcome) = self
5438                .closed_resume_outcome(run, &records, lease.expect("resume holds a lease"))
5439                .await?
5440        {
5441            return Ok(outcome);
5442        }
5443
5444        // Placed *before* the compensation guard below on purpose: a run that
5445        // quarantined partway through an unwind is exactly the one an operator
5446        // has no other way to close, and it must not be the one this refuses to
5447        // close.
5448        if mode == Pass::Resume
5449            && let Some(outcome) = self
5450                .abandon_if_decided(run, &records, lease.expect("resume holds a lease"))
5451                .await?
5452        {
5453            return Ok(outcome);
5454        }
5455
5456        if mode == Pass::Resume {
5457            refuse_resume_over_undone_work(run, &records)?;
5458        }
5459
5460        // **Read after the conclusions above, and that ordering is the rule.**
5461        // A plan is what a run needs to *execute*; concluding one needs no
5462        // plan, and a cancellation and an abandonment are recorded in the
5463        // clear. Reading here rather than at the top of this function is what
5464        // leaves an operator a verb when the plan can no longer be read at all
5465        // — which is an ordinary state, because erasing a case's key erases
5466        // the plan of every run in it.
5467        let input = records
5468            .iter()
5469            .find_map(Record::admitted_input)
5470            .ok_or_else(|| {
5471                RuntimeError::PlanContract("journal has no RunAdmitted record".into())
5472            })?;
5473        let (plan, successors) =
5474            frozen_plans(run, &records).map_err(|e| self.sealed_or_erased(e))?;
5475
5476        // Only a plane that can drive the run judges it. A plane holding no
5477        // provider for the run's steps — an operator's terminal, a fleet
5478        // member serving other agents — answers `NoProvider` before comparing
5479        // its policy or declarations with the run's: a mismatch there says
5480        // nothing about the run, and quarantining it would stop a run this
5481        // plane could never have continued.
5482        if mode == Pass::Resume {
5483            self.refuse_undrivable(std::iter::once(&plan).chain(&successors))?;
5484            self.refuse_caseless(run, &records)?;
5485        }
5486
5487        // Resume can dispatch new effects past the end of history, so it is
5488        // held to the bundle and the declaration recorded at admission. Strict
5489        // replay performs no effects and remains usable as an offline verifier
5490        // without loading the historical engine.
5491        //
5492        // A refusal here is journaled as a quarantine rather than returned. The
5493        // callers that reach this unattended — a wake, a delivery, the recovery
5494        // sweep — have already recorded the result that took the run off its
5495        // waiting listing, and an error there reaches nobody: the run would
5496        // sit on no listing, or be retried every tick into the same refusal.
5497        // A quarantine is the state that already means *the runtime cannot go
5498        // on and a person must decide*, it is listed and counted, and both of
5499        // its answers work here — reopen on a plane holding the recorded
5500        // revision, or abandon.
5501        if mode == Pass::Resume
5502            && let Some(refusal) = self.resume_refusal(&records)?
5503        {
5504            let lease = lease.expect("resume holds a lease");
5505            return self
5506                .quarantine_refused(run, &records, lease, &refusal)
5507                .await;
5508        }
5509
5510        // A resume executes the run, so it registers it first: what it adds is
5511        // whatever an admission that failed or died partway never wrote.
5512        if mode == Pass::Resume {
5513            self.register(run, recorded_case_id(&records))
5514                .await
5515                .map_err(RuntimeError::from_store)?;
5516        }
5517
5518        *dispatched = true;
5519        // The agent recorded at admission, so a replay is bounded by the
5520        // ceilings the run actually had — and dispatched no wider than the
5521        // width its reservation was sized for.
5522        let budget = self.budget_for(&recorded_agent(&records));
5523        let quota = self
5524            .start_replay_quota_pass(run, mode, reserved_width(&budget, &plan))
5525            .await?;
5526        let through = self.marking_pass(run, &quota);
5527        let this = through.as_ref().unwrap_or(self);
5528
5529        let case_ctx = self.recorded_case(&records);
5530
5531        let mut cursor = ReplayCursor::from_records(&records);
5532
5533        // Strict verification must not write, and it does not: it holds no
5534        // lease, and appends happen only past the end of history, which
5535        // Strict mode refuses to reach. Resume writes under the lease it
5536        // claimed before reading.
5537        let epoch = lease.map_or_else(|| records.last().map_or(1, |r| r.body.epoch), |l| l.epoch);
5538
5539        // Strict verification never writes, so it holds no lease to renew.
5540        let _heartbeat = lease.map(|l| self.heartbeat(run, l.epoch));
5541        this.execute(
5542            Execution {
5543                run,
5544                epoch,
5545                quota,
5546                plan: &plan,
5547                input,
5548                mode,
5549                case: case_ctx,
5550                budget,
5551                agent: recorded_agent(&records),
5552                identity: recorded_chain(&records)?,
5553                subjects: recorded_subjects(run, &records),
5554                // A step-level refusal has no effect key, so it cannot ride the
5555                // replay cursor like an effect's does. It is lifted here from
5556                // the records `replay` has already read.
5557                refusal: recorded_step_refusal(&records),
5558                withheld_subject: standing_withholding(&records),
5559                successors,
5560                started: recorded_started_steps(&records),
5561                finished: recorded_finished_steps(&records),
5562                succeeded: recorded_succeeded_steps(&records),
5563                recorded_groups: recorded_groups(&records),
5564            },
5565            &mut cursor,
5566        )
5567        .await
5568        .inspect(|_| *divergence = cursor.divergence().cloned())
5569    }
5570
5571    /// The trace root. Every span below is a child, so "what did this run do"
5572    /// is one query rather than a correlation exercise across logs.
5573    ///
5574    /// Instrumented rather than entered: an `Entered` guard held across an
5575    /// `.await` belongs to the thread, and with concurrent dispatch that
5576    /// reparents whatever runs next onto this span.
5577    async fn execute(
5578        &self,
5579        plan: Execution<'_>,
5580        cursor: &mut ReplayCursor,
5581    ) -> Result<RunOutcome, RuntimeError> {
5582        let span = tracing::info_span!(
5583            telemetry::RUN_SPAN,
5584            { telemetry::GEN_AI_OPERATION } = telemetry::GEN_AI_INVOKE_AGENT,
5585            { telemetry::RUN_ID } = tracing::field::display(plan.run),
5586            // Which declaration this run is executing. A plane serving several
5587            // agents cannot otherwise separate them, and separating them is
5588            // the first thing an operator does.
5589            { telemetry::GEN_AI_AGENT_NAME } = plan.agent.as_str(),
5590            { telemetry::MODE } = telemetry::mode_str(plan.mode),
5591            { telemetry::GEN_AI_CONVERSATION_ID } = plan
5592                .case
5593                .as_ref()
5594                .map(|c| super::ctx::CaseContext::id(c).to_string()),
5595            // Empty under the default policy, which is what `Omitted` means.
5596            // Recorded rather than declared so the attribute is absent rather
5597            // than blank — a collector cannot read *not disclosed* as *no
5598            // tenant* if there is no field at all.
5599            { telemetry::TENANT } = tracing::field::Empty,
5600            { telemetry::OUTCOME } = tracing::field::Empty,
5601            { telemetry::SEMCONV } = telemetry::SEMCONV_VERSION,
5602            { telemetry::OTEL_NAME } = format!("{} {}", telemetry::GEN_AI_INVOKE_AGENT, plan.agent),
5603            { telemetry::OTEL_KIND } = "internal",
5604        );
5605        if !self.meter.tenant().is_empty() {
5606            span.record(telemetry::TENANT, self.meter.tenant());
5607        }
5608        self.execute_inner(plan, cursor).instrument(span).await
5609    }
5610
5611    /// The driver loop: ready set, admit, dispatch, apply, repeat.
5612    ///
5613    /// Long by line count and deliberately not split further. Every *step* is
5614    /// already its own method — `admit_ready`, `dispatch`, `collect`, `apply`,
5615    /// `adopt_successor`, `stop`. What is left is the order they happen in, and
5616    /// that order is the algorithm. Breaking it up again would scatter one
5617    /// readable sequence across functions that exist only to satisfy a line
5618    /// count, which is the opposite of the thing the lint is protecting.
5619    #[allow(clippy::too_many_lines)]
5620    async fn execute_inner(
5621        &self,
5622        plan: Execution<'_>,
5623        cursor: &mut ReplayCursor,
5624    ) -> Result<RunOutcome, RuntimeError> {
5625        let Execution {
5626            run,
5627            epoch,
5628            quota,
5629            plan: ir,
5630            input,
5631            mode,
5632            case,
5633            budget,
5634            agent,
5635            identity,
5636            subjects,
5637            refusal: recorded_refusal,
5638            withheld_subject,
5639            successors,
5640            started,
5641            finished,
5642            succeeded,
5643            recorded_groups,
5644        } = plan;
5645        // Every value derived from what the run took in carries its subjects.
5646        let input = input.attributed(&subjects);
5647        let writing = !matches!(mode, Pass::Strict);
5648        let case_id = case.as_ref().map(super::ctx::CaseContext::id);
5649        let stamp = |a: Append| match case_id {
5650            Some(c) => a.case(c),
5651            None => a,
5652        };
5653
5654        // The plan in force. Owned rather than borrowed, because a replan
5655        // replaces it and a reference into the version list could not
5656        // survive that. `recorded_successors` is empty on a live run and seeded
5657        // from the journal on a replay, so a successor is read back rather than
5658        // re-synthesised.
5659        let mut current: PlanIR = ir.clone();
5660        let mut replans: u32 = 0;
5661        // A withholding the recorded history left standing, if it did. Seeded
5662        // from the records for the same reason the step refusal is: it is a fact
5663        // about *this* run that a resume has to answer, and asking the journal
5664        // again at every boundary would be a read per ready set for a question
5665        // whose answer only this loop can change.
5666        let mut withheld: Option<String> = withheld_subject;
5667        let recorded_successors = successors;
5668
5669        // One ledger for the run. A step never gets its own allowance to blow.
5670        let ledger = Arc::new(std::sync::Mutex::new(Ledger::new(budget)));
5671
5672        // Steps already completed, and what they produced. Rebuilt from the
5673        // journal on replay so a resumed run knows where it got to.
5674        let mut done: BTreeSet<StepId> = BTreeSet::new();
5675
5676        // The same steps as `done`, in the order they finished, **with the
5677        // capability that actually ran**. Unwinding needs both: a set has no
5678        // order to reverse, and after a replan the current plan may have
5679        // different work — or nothing at all — at a completed step's id.
5680        // Resolving the compensation from the live plan then undoes something
5681        // that never ran, which is a refund for a charge nobody made.
5682        let mut completed: Vec<(StepId, Capability)> = Vec::new();
5683        // The node each completed step ran as. A successor may carry a
5684        // completed step over only as this exact node: `done` is by id, so a
5685        // redeclared node would count as done on the strength of other work.
5686        let mut ran: BTreeMap<StepId, PlanNode> = BTreeMap::new();
5687        let mut outputs: BTreeMap<StepId, Tainted<Value>> = BTreeMap::new();
5688
5689        // ── Ready-set scheduling ───────────────────────────────────────────
5690        //
5691        // Dispatch order is a deterministic total order (ascending step id), so
5692        // replay reproduces it exactly. A plan with parallelism that
5693        // dispatched in completion order would replay differently every time.
5694        // The new work of a ready set whose recorded successes replay first.
5695        let mut deferred: Option<Vec<StepId>> = None;
5696        loop {
5697            let mut ready = current.ready(&done);
5698            if let Some(rest) = deferred.take() {
5699                let held: Vec<StepId> =
5700                    ready.iter().copied().filter(|s| rest.contains(s)).collect();
5701                if !held.is_empty() {
5702                    ready = held;
5703                }
5704            }
5705            // A resume's ready set can hold a step the record shows succeeding
5706            // beside one it does not — parallel work cut off mid-flight. The
5707            // recorded steps replay as a set of their own first, so the checks
5708            // below meet the frontier with their outputs already restored; the
5709            // rest is the next set, with nothing their replay made ready
5710            // joining it.
5711            if writing
5712                && ready.iter().any(|s| succeeded.contains(s))
5713                && ready.iter().any(|s| !succeeded.contains(s))
5714            {
5715                let (recorded, rest) = ready.into_iter().partition(|s| succeeded.contains(s));
5716                ready = recorded;
5717                deferred = Some(rest);
5718            }
5719            // ── The stop check, at a step boundary and nowhere else ────────
5720            //
5721            // Between boundaries an effect may be announced and not yet
5722            // recorded, and interrupting there manufactures the in-doubt case
5723            // the whole protocol exists to avoid. Checking here costs one
5724            // store read per ready set and buys a cancellation that can never
5725            // strand an effect.
5726            //
5727            // Skipped while replaying: a recorded run's history already
5728            // contains whatever stop it received, and re-reading the live
5729            // request would let a cancellation arriving *today* rewrite what a
5730            // run did last year.
5731            //
5732            // A resume asks only at its frontier — once the ready set holds a
5733            // step the record does not show succeeding. Before that, the
5734            // replayed steps have not yet refilled `completed` and `outputs`,
5735            // and an unwind from there would hand each compensation nothing
5736            // and walk the steps in id order rather than in reverse of the
5737            // order they finished.
5738            let at_frontier = ready.is_empty() || ready.iter().any(|s| !succeeded.contains(s));
5739            if writing
5740                && at_frontier
5741                && let Some(c) = self
5742                    .store
5743                    .cancellation(run)
5744                    .await
5745                    .map_err(RuntimeError::from_store)?
5746            {
5747                let status = RunStatus::Cancelled {
5748                    actor: c.actor.clone(),
5749                    reason: c.reason.clone(),
5750                };
5751                // Journaled *before* unwinding, so the reason the run stopped is
5752                // in the chain even if compensation then fails and quarantines
5753                // it. An operator reading a half-unwound run must be able to see
5754                // that somebody asked for this.
5755                self.store
5756                    .append(
5757                        epoch,
5758                        vec![stamp(Append::new(
5759                            run,
5760                            RecordKind::RunCancelled {
5761                                actor: c.actor,
5762                                reason: c.reason,
5763                            },
5764                        ))],
5765                    )
5766                    .await
5767                    .map_err(RuntimeError::from_store)?;
5768                // A step that paused with its group open holds work only that
5769                // step can take back: its reversals are built from the members'
5770                // outputs as the step runs. Re-run it once to abandon the group
5771                // — members read back, the pause aborts instead of waiting —
5772                // and only then unwind the steps before it.
5773                let abandon: Vec<StepId> = ready
5774                    .iter()
5775                    .copied()
5776                    .filter(|&s| {
5777                        recorded_groups.iter().any(|((step, phase, _), g)| {
5778                            *step == s && *phase == Phase::Forward && g.opened > g.settled
5779                        })
5780                    })
5781                    .collect();
5782                if !abandon.is_empty() {
5783                    let dispatched = self
5784                        .dispatch(
5785                            &abandon,
5786                            cursor,
5787                            Batch {
5788                                agent: &agent,
5789                                identity: identity.as_ref(),
5790                                subjects: &subjects,
5791                                run,
5792                                epoch,
5793                                ir: &current,
5794                                mode,
5795                                case: &case,
5796                                ledger: &ledger,
5797                                writing,
5798                                stamp: &stamp,
5799                                input: &input,
5800                                outputs: &outputs,
5801                                started: &started,
5802                                finished: &finished,
5803                                recorded_groups: &recorded_groups,
5804                                parallelism: 1,
5805                                abandoning: true,
5806                            },
5807                        )
5808                        .await;
5809                    collect(dispatched, &abandon, cursor)?;
5810                }
5811                return self
5812                    .stop(
5813                        Unwind {
5814                            agent: &agent,
5815                            identity: identity.as_ref(),
5816                            subjects: &subjects,
5817                            run,
5818                            epoch,
5819                            ir: &current,
5820                            mode,
5821                            case: case.clone(),
5822                            ledger: &ledger,
5823                            writing,
5824                            stamp: &stamp,
5825                        },
5826                        status,
5827                        &completed,
5828                        &outputs,
5829                        cursor,
5830                        case_id,
5831                        &quota,
5832                    )
5833                    .await;
5834            }
5835
5836            // ── The withdrawal check, at the same boundary ─────────────────
5837            //
5838            // Beside the stop check and for its reasons. Unlike a cancellation
5839            // this is a **pause**: `stop` is reached with a status
5840            // `maybe_unwind` passes straight through, as exhaustion is.
5841            if writing && at_frontier {
5842                let standing = self
5843                    .withdrawal_against(identity.as_ref())
5844                    .await
5845                    .map_err(RuntimeError::Store)?;
5846                match (standing, withheld.take()) {
5847                    // Lifted since the run was withheld: that decision is a
5848                    // fact about the run and goes on the record, beside the
5849                    // withholding it supersedes rather than instead of it.
5850                    // Without it a strict replay would stop at the old pause and
5851                    // report `withheld` about a run whose own later records show
5852                    // it finishing.
5853                    (None, Some(prior)) => {
5854                        self.store
5855                            .append(
5856                                epoch,
5857                                vec![stamp(Append::new(
5858                                    run,
5859                                    RecordKind::AuthorityRestored { subject: prior },
5860                                ))],
5861                            )
5862                            .await
5863                            .map_err(RuntimeError::from_store)?;
5864                    }
5865                    // Still withdrawn. The standing record already says so — no
5866                    // second one, for the reason a re-exhausted run writes no
5867                    // second refusal.
5868                    // The prior is dropped rather than restored: this pass
5869                    // returns, and the *records* are what a later resume reads
5870                    // its standing state from.
5871                    (Some(withdrawal), Some(_prior)) => {
5872                        return self
5873                            .withhold(
5874                                Unwind {
5875                                    agent: &agent,
5876                                    identity: identity.as_ref(),
5877                                    subjects: &subjects,
5878                                    run,
5879                                    epoch,
5880                                    ir: &current,
5881                                    mode,
5882                                    case: case.clone(),
5883                                    ledger: &ledger,
5884                                    writing,
5885                                    stamp: &stamp,
5886                                },
5887                                withdrawal,
5888                                false,
5889                                &completed,
5890                                &outputs,
5891                                cursor,
5892                                case_id,
5893                                &quota,
5894                            )
5895                            .await;
5896                    }
5897                    (Some(withdrawal), None) => {
5898                        return self
5899                            .withhold(
5900                                Unwind {
5901                                    agent: &agent,
5902                                    identity: identity.as_ref(),
5903                                    subjects: &subjects,
5904                                    run,
5905                                    epoch,
5906                                    ir: &current,
5907                                    mode,
5908                                    case: case.clone(),
5909                                    ledger: &ledger,
5910                                    writing,
5911                                    stamp: &stamp,
5912                                },
5913                                withdrawal,
5914                                true,
5915                                &completed,
5916                                &outputs,
5917                                cursor,
5918                                case_id,
5919                                &quota,
5920                            )
5921                            .await;
5922                    }
5923                    (None, None) => {}
5924                }
5925            }
5926            if ready.is_empty() {
5927                break;
5928            }
5929
5930            // Admission first, and deliberately not concurrent: which step a
5931            // ceiling refuses must be a property of the plan, not of which
5932            // future happened to poll first.
5933            let (admitted, refused) = self
5934                .admit_ready(
5935                    &ready,
5936                    &ledger,
5937                    mode,
5938                    recorded_refusal.as_ref(),
5939                    cursor,
5940                    Journalling {
5941                        run,
5942                        epoch,
5943                        writing,
5944                        stamp: &stamp,
5945                    },
5946                )
5947                .await?;
5948
5949            let dispatched = self
5950                .dispatch(
5951                    &admitted,
5952                    cursor,
5953                    Batch {
5954                        agent: &agent,
5955                        identity: identity.as_ref(),
5956                        subjects: &subjects,
5957                        run,
5958                        epoch,
5959                        ir: &current,
5960                        mode,
5961                        case: &case,
5962                        ledger: &ledger,
5963                        writing,
5964                        stamp: &stamp,
5965                        input: &input,
5966                        outputs: &outputs,
5967                        started: &started,
5968                        finished: &finished,
5969                        recorded_groups: &recorded_groups,
5970                        parallelism: quota.parallelism(&budget),
5971                        abandoning: false,
5972                    },
5973                )
5974                .await;
5975            let outcomes = collect(dispatched, &ready, cursor)?;
5976
5977            // ── Strict verification consumes everything, per step ──────────
5978            //
5979            // Divergence has two directions, and the ordered key comparison
5980            // only sees one of them. A build that asks for a *different*
5981            // effect fails at the mismatch; a build that asks for **fewer**
5982            // effects than the record sails past the remainder and would
5983            // report the step verified — a false confirmation about exactly
5984            // the history a verification pass exists to check. So a step that
5985            // finished with journaled effects still unread is the same
5986            // quarantine-grade finding as a key mismatch, named by the first
5987            // effect nothing requested.
5988            if mode == Pass::Strict {
5989                for &(step, ref status, _) in &outcomes {
5990                    // A step that stopped — suspended, failed, refused — ended
5991                    // where its history says it ended, and the stop itself is
5992                    // the verified finding. Only a step that claims to have
5993                    // *finished* owes a fully consumed slice.
5994                    if !matches!(status, RunStatus::Succeeded) {
5995                        continue;
5996                    }
5997                    if let Some(missing) = cursor.unconsumed_in(step, Phase::Forward) {
5998                        cursor.note_divergence(crate::journal::Divergence::unconsumed(
5999                            step,
6000                            Phase::Forward,
6001                            &missing,
6002                        ));
6003                        self.meter.count(metrics::DIVERGENCES, "");
6004                        tracing::error!(
6005                            target: telemetry::NONDETERMINISM,
6006                            %step, key = %missing.key, unconsumed = true,
6007                        );
6008                        return self
6009                            .conclude(
6010                                run,
6011                                epoch,
6012                                RunStatus::Quarantined(format!(
6013                                    "strict replay verified less than the run recorded: \
6014                                     step {step} finished with journaled effect {missing} \
6015                                     never requested — this build performs fewer effects than the \
6016                                     recorded one"
6017                                )),
6018                                None,
6019                                writing,
6020                                case_id,
6021                                Consumed::default(),
6022                                Spend::default(),
6023                                &quota,
6024                            )
6025                            .await;
6026                    }
6027                }
6028            }
6029
6030            // A step that stops for any reason stops the run: whatever remains
6031            // either depended on it, or will be dispatched when it resumes.
6032            // Anything already done may need undoing first.
6033            let mut stopped = apply(
6034                &current,
6035                outcomes,
6036                &mut done,
6037                &mut completed,
6038                &mut ran,
6039                &mut outputs,
6040            )
6041            .or_else(|| refused.map(RunStatus::Exhausted));
6042
6043            if let Some(RunStatus::Replanning(reason)) = &stopped {
6044                let recorded = recorded_successors.get(replans as usize);
6045                let journal = Journalling {
6046                    run,
6047                    epoch,
6048                    writing: writing && recorded.is_none(),
6049                    stamp: &stamp,
6050                };
6051                let cx = Replan {
6052                    run,
6053                    current: &current,
6054                    ran: &ran,
6055                    reason,
6056                    already_replanned: replans,
6057                    max_replans: budget.max_replans,
6058                    recorded,
6059                };
6060                match self
6061                    .adopt_successor(cx, journal, &outputs, &completed)
6062                    .await?
6063                {
6064                    Ok(next) => {
6065                        replans += 1;
6066                        current = next;
6067                        continue;
6068                    }
6069                    Err(refusal) => stopped = Some(refusal),
6070                }
6071            }
6072
6073            if let Some(status) = stopped {
6074                return self
6075                    .stop(
6076                        Unwind {
6077                            agent: &agent,
6078                            identity: identity.as_ref(),
6079                            subjects: &subjects,
6080                            run,
6081                            epoch,
6082                            ir: &current,
6083                            mode,
6084                            case: case.clone(),
6085                            ledger: &ledger,
6086                            writing,
6087                            stamp: &stamp,
6088                        },
6089                        status,
6090                        &completed,
6091                        &outputs,
6092                        cursor,
6093                        case_id,
6094                        &quota,
6095                    )
6096                    .await;
6097            }
6098        }
6099
6100        // ── Strict verification consumes everything, at the end ────────────
6101        //
6102        // The per-batch check above covers steps this build dispatched; this
6103        // covers the ones it never asked for at all — a step a changed plan no
6104        // longer reaches, a compensation slice nothing requested. Either way the
6105        // record holds work this build cannot account for, and "verified" must
6106        // not be the answer.
6107        if mode == Pass::Strict
6108            && let Some((step, phase, missing)) = cursor.first_unconsumed()
6109        {
6110            cursor.note_divergence(crate::journal::Divergence::unconsumed(
6111                step, phase, &missing,
6112            ));
6113            self.meter.count(metrics::DIVERGENCES, "");
6114            tracing::error!(
6115                target: telemetry::NONDETERMINISM,
6116                %step, key = %missing.key, unconsumed = true,
6117            );
6118            return self
6119                .conclude(
6120                    run,
6121                    epoch,
6122                    RunStatus::Quarantined(format!(
6123                        "strict replay verified less than the run recorded: journaled \
6124                         effect {missing} at step {step}, {phase:?} phase, was never \
6125                         requested — this build performs fewer effects than the recorded one"
6126                    )),
6127                    None,
6128                    writing,
6129                    case_id,
6130                    Consumed::default(),
6131                    Spend::default(),
6132                    &quota,
6133                )
6134                .await;
6135        }
6136
6137        // Read into a value first. A `MutexGuard` built inline as an argument
6138        // lives until the end of the full expression — which here is *after*
6139        // the await — so the lock would be held across a suspension. That is
6140        // the same shape as the `Span::enter()` bug: a guard whose
6141        // scope is wider than it looks, and invisible until something else
6142        // needs the lock.
6143        let (consumed, live_spend) = {
6144            let l = ledger.lock().expect("budget mutex");
6145            (l.consumed(), l.live_spend())
6146        };
6147        self.conclude(
6148            run,
6149            epoch,
6150            completion(&current, &done),
6151            run_output(&current, &outputs),
6152            writing,
6153            case_id,
6154            consumed,
6155            live_spend,
6156            &quota,
6157        )
6158        .await
6159    }
6160
6161    /// Read the clock into the ledger, when a wall-clock ceiling is declared.
6162    ///
6163    /// Separate so the guard's scope is two statements that cannot span the
6164    /// await between them, and so the condition is asked once rather than
6165    /// spelled at the caller.
6166    async fn observe_clock(
6167        cx: &mut StepCtx<'_>,
6168        ledger: &Arc<std::sync::Mutex<Ledger>>,
6169    ) -> Result<(), crate::core::StepError> {
6170        if !ledger.lock().expect("budget mutex").tracks_wallclock() {
6171            return Ok(());
6172        }
6173        let at = cx.now().await?;
6174        ledger.lock().expect("budget mutex").observe_clock(at);
6175        Ok(())
6176    }
6177
6178    /// Run a ready set concurrently, at most `max_parallel_steps` at a time.
6179    ///
6180    /// The ready set is every node whose predecessors are done, so nothing in
6181    /// it depends on anything else in it — running them one at a time is a
6182    /// choice, and the wrong one when steps are waiting on models and networks.
6183    ///
6184    /// Each step takes its own slice of history, which is what makes this sound:
6185    /// a step touches only its own effects, so no shared mutable state is left
6186    /// between them, and the per-step replay cursor verifies each one's order
6187    /// independently of how the journal happened to interleave them.
6188    ///
6189    /// The width is bounded because it is not otherwise bounded by anything:
6190    /// a fan-out is as wide as its author wrote it, and every step in flight
6191    /// holds a connection, a provider's rate limit and one operation's worth
6192    /// of spend the metered ceilings have checked and not yet billed. Ordered
6193    /// rather than completion-ordered — `buffered`, not `buffer_unordered` —
6194    /// so the outcomes still apply in ready order and a plan's dispatch is a
6195    /// property of the plan rather than of which future finished first.
6196    async fn dispatch(
6197        &self,
6198        admitted: &[StepId],
6199        cursor: &mut ReplayCursor,
6200        batch: Batch<'_>,
6201    ) -> Vec<Dispatched> {
6202        use futures_util::StreamExt;
6203
6204        let slices: Vec<(StepId, StepCursor)> = admitted
6205            .iter()
6206            .map(|&s| (s, cursor.take(s, Phase::Forward)))
6207            .collect();
6208        // Floored at one because a width of zero would poll nothing forever.
6209        // A zero ceiling is refused at parse and at build, so this is the
6210        // second half of a rule rather than the only half — and the failure it
6211        // guards against is a hang, which is the one shape a later reader
6212        // cannot diagnose from the outside.
6213        let width = batch.parallelism.max(1);
6214
6215        futures_util::stream::iter(slices.into_iter().map(|(step, slice)| async move {
6216            let node = batch
6217                .ir
6218                .node(step)
6219                .ok_or_else(|| RuntimeError::PlanContract(format!("no node {step}")))?;
6220            let (status, out, slice) = self
6221                .run_step(
6222                    StepRun {
6223                        agent: batch.agent,
6224                        identity: batch.identity,
6225                        subjects: batch.subjects,
6226                        run: batch.run,
6227                        epoch: batch.epoch,
6228                        node,
6229                        phase: Phase::Forward,
6230                        mode: batch.mode,
6231                        case: batch.case.clone(),
6232                        ledger: batch.ledger,
6233                        writing: batch.writing,
6234                        stamp: batch.stamp,
6235                        already_started: batch.started.contains(&step),
6236                        already_finished: batch.finished.contains(&step),
6237                        recorded_groups: batch
6238                            .recorded_groups
6239                            .iter()
6240                            .filter(|((s, p, _), _)| *s == step && *p == Phase::Forward)
6241                            .map(|((_, _, name), n)| (name.clone(), *n))
6242                            .collect(),
6243                        abandoning: batch.abandoning,
6244                    },
6245                    batch.input,
6246                    batch.outputs,
6247                    slice,
6248                )
6249                .await?;
6250            Ok((step, status, out, slice))
6251        }))
6252        .buffered(width)
6253        .collect()
6254        .await
6255    }
6256
6257    /// Take the successor a step asked for: check it, record it, announce it.
6258    ///
6259    /// The nested result separates two failures. The outer `RuntimeError` is the
6260    /// runtime itself failing — the journal would not accept the plan — and is
6261    /// never recoverable. The inner `RunStatus` is the *request* being refused,
6262    /// which is an ordinary outcome the run reports.
6263    async fn adopt_successor(
6264        &self,
6265        cx: Replan<'_>,
6266        journal: Journalling<'_>,
6267        outputs: &BTreeMap<StepId, Tainted<Value>>,
6268        completed: &[(StepId, Capability)],
6269    ) -> Result<Result<PlanIR, RunStatus>, RuntimeError> {
6270        // What each step that started and journaled an effect ran as, read
6271        // from the journal only when a successor is about to be checked — a
6272        // replay reads the successor back instead.
6273        let started = if cx.recorded.is_none() {
6274            let records = self
6275                .store
6276                .read(cx.run, 1)
6277                .await
6278                .map_err(RuntimeError::from_store)?;
6279            let effectful: BTreeSet<StepId> = records
6280                .iter()
6281                .filter(|r| r.body.phase.is_forward())
6282                .filter_map(|r| match r.kind() {
6283                    RecordKind::EffectStarted { .. } => r.body.step,
6284                    _ => None,
6285                })
6286                .collect();
6287            let mut started = started_skills(&records);
6288            started.retain(|step, _| effectful.contains(step));
6289            started
6290        } else {
6291            BTreeMap::new()
6292        };
6293        let next = match self.successor(cx, outputs, completed, &started).await {
6294            Ok(next) => next,
6295            Err(refusal) => return Ok(Err(refusal)),
6296        };
6297        if journal.writing {
6298            self.freeze(journal.run, journal.epoch, &next, journal.stamp)
6299                .await?;
6300        }
6301        announce_replan(
6302            &self.meter,
6303            journal.run,
6304            &next,
6305            next.reason.as_deref().unwrap_or(""),
6306        );
6307        Ok(Ok(next))
6308    }
6309
6310    /// Record a plan version in the journal.
6311    ///
6312    /// A successor is frozen exactly as a first plan is, so replay reads it back
6313    /// rather than asking a planner that may since have changed its mind.
6314    async fn freeze(
6315        &self,
6316        run: RunId,
6317        epoch: u64,
6318        plan: &PlanIR,
6319        stamp: &(dyn Fn(Append) -> Append + Send + Sync),
6320    ) -> Result<(), RuntimeError> {
6321        self.store
6322            .append(
6323                epoch,
6324                vec![stamp(Append::new(
6325                    run,
6326                    RecordKind::PlanFrozen {
6327                        steps: plan.nodes.iter().map(|n| n.capability.0.clone()).collect(),
6328                        plan: serde_json::to_value(plan)?,
6329                    },
6330                ))],
6331            )
6332            .await
6333            .map_err(RuntimeError::from_store)?;
6334        Ok(())
6335    }
6336
6337    /// Produce the successor plan a step asked for, or say why not.
6338    ///
6339    /// Three gates, and the first is not negotiable.
6340    ///
6341    /// **Provenance.** The frozen plan is an authorization graph compiled from
6342    /// trusted input only. A replan *changes that graph*, so once any
6343    /// untrusted value has reached working memory, anything shaping the new plan
6344    /// may be attacker-chosen — and choosing the authorization graph is the
6345    /// whole game. `plan-then-execute` is enforced here, structurally. A run
6346    /// that wants a different plan after reading untrusted input is describing
6347    /// exactly the attack.
6348    ///
6349    /// **Budget.** A run that replans without bound has stopped making progress
6350    /// and started thrashing.
6351    ///
6352    /// **On replay, the successor is read back, never re-synthesised.** A
6353    /// planner asked twice can answer differently — a changed router, a
6354    /// different model — and replay would then verify the run against a plan
6355    /// that never governed it. Same rule as the first plan, for the same
6356    /// reason.
6357    async fn successor(
6358        &self,
6359        cx: Replan<'_>,
6360        outputs: &BTreeMap<StepId, Tainted<Value>>,
6361        completed: &[(StepId, Capability)],
6362        started: &BTreeMap<StepId, String>,
6363    ) -> Result<PlanIR, RunStatus> {
6364        if let Some(source) = untrusted_in(outputs) {
6365            return Err(RunStatus::Failed(format!(
6366                "replanning refused: untrusted data from {source} is already in working memory, and the plan is an authorization graph — letting it change now would let that data choose what runs next ({})",
6367                cx.reason
6368            )));
6369        }
6370
6371        let spent = cx.already_replanned;
6372        if let Some(max) = cx.max_replans
6373            && spent >= max
6374        {
6375            return Err(RunStatus::Exhausted(crate::core::BudgetExceeded::Replans {
6376                allowed: max,
6377            }));
6378        }
6379
6380        // Replay: the successor is in the journal, at the position this replan
6381        // reached. Reading it back is what keeps a re-planned run replayable.
6382        if let Some(recorded) = cx.recorded {
6383            return Ok(recorded.clone());
6384        }
6385
6386        let replanner = self.replanner.as_ref().ok_or_else(|| {
6387            RunStatus::Failed(format!(
6388                "a step asked to replan and this runtime has no planner — build it with `.replanner(..)` ({})",
6389                cx.reason
6390            ))
6391        })?;
6392
6393        let next = replanner
6394            .replan(cx.current, cx.reason, completed)
6395            .await
6396            .map_err(|e| RunStatus::Failed(format!("replanning failed: {e}")))?;
6397
6398        // A successor faces the same contract a first plan does. One that fails
6399        // validation stops the run rather than half-applying.
6400        crate::plan::validate(&next, &self.contract())
6401            .map_err(|e| RunStatus::Failed(format!("the successor plan is invalid: {e}")))?;
6402
6403        // A completed step carries over only as the node it ran as, or not at
6404        // all. Other work at its id breaks replay — effect keys derive from the
6405        // step id — and has the saga undo something that never ran. The same
6406        // capability redeclared is refused too: `done` is by id, so a node with
6407        // new arguments, dependencies or flags would count as done on the
6408        // strength of work done before any of them held — a terminal verifier
6409        // satisfied without ever seeing what it verifies.
6410        for (step, node_ran) in cx.ran {
6411            let Some(node) = next.node(*step) else {
6412                continue;
6413            };
6414            if node.capability != node_ran.capability {
6415                return Err(RunStatus::Failed(format!(
6416                    "the successor plan reuses step {step} — which already ran \
6417                     as '{}' — for '{}'. Keep a completed step's node or leave \
6418                     the step out; effect keys are derived from the step id, so \
6419                     new work at a used id cannot be replayed",
6420                    node_ran.capability.0, node.capability.0
6421                )));
6422            }
6423            if node != node_ran {
6424                return Err(RunStatus::Failed(format!(
6425                    "the successor plan redeclares completed step {step} ('{}') \
6426                     with different arguments, dependencies or flags. A \
6427                     completed step counts as done by its id, so the new node \
6428                     would be satisfied by work done under the old one — carry \
6429                     it over unchanged, or give the new work a fresh id",
6430                    node.capability.0
6431                )));
6432            }
6433        }
6434
6435        // A step that started, journaled an effect and did not complete holds
6436        // keys derived from its id just the same — it may have charged before
6437        // asking to replan — so its id may not be reused for work another
6438        // skill would do. One that journaled nothing, the step that asked for
6439        // a fallback before touching anything, leaves its id free.
6440        for (step, skill) in started {
6441            if cx.ran.contains_key(step) {
6442                continue;
6443            }
6444            let Some(node) = next.node(*step) else {
6445                continue;
6446            };
6447            let resolved = self
6448                .resolve(&node.capability.0)
6449                .map(|s| s.descriptor().name)
6450                .ok();
6451            if resolved.as_deref() != Some(skill.as_str()) {
6452                return Err(RunStatus::Failed(format!(
6453                    "the successor plan reuses step {step} — which already \
6454                     started as '{skill}' and journaled effects — for '{}'. Keep the step's \
6455                     capability or leave the step out; effect keys are derived \
6456                     from the step id, and the unwind undoes a started step as \
6457                     the skill that ran",
6458                    node.capability.0
6459                )));
6460            }
6461        }
6462
6463        if next.derived_from != Some(cx.current.digest()) {
6464            return Err(RunStatus::Failed(
6465                "the successor plan does not name its predecessor — use `PlanIR::succeed_with`, or the audit trail has a hole where the lineage should be"
6466                    .into(),
6467            ));
6468        }
6469
6470        // The third leg of the lineage `succeed_with` sets. A successor with
6471        // no reason would freeze into the journal as a plan that replaced
6472        // another with nothing on the record saying why — the half of an
6473        // incident (*what changed the run's mind*) that versioned replanning
6474        // exists to keep.
6475        if next.reason.as_deref().is_none_or(str::is_empty) {
6476            return Err(RunStatus::Failed(
6477                "the successor plan carries no reason — use `PlanIR::succeed_with`, or the \
6478                 journal freezes a plan that replaced another with nothing saying why"
6479                    .into(),
6480            ));
6481        }
6482
6483        Ok(next)
6484    }
6485
6486    /// Decide which of a ready set may start, in ready order.
6487    ///
6488    /// Three regimes, one per mode, and the middle one is the subtle case:
6489    ///
6490    /// * **Strict** consumes recorded verdicts and recomputes none. A run
6491    ///   replayed under a larger budget stopped where it stopped; recomputing
6492    ///   made a step-limited run replay as *succeeded* — a false audit result.
6493    /// * **Resume** takes each verdict from wherever the truth lives. A step
6494    ///   with unconsumed history was admitted by the run that recorded it,
6495    ///   and the history *is* the verdict. A step at the frontier is about to
6496    ///   run **live**, and every live step is metered — keying this on "is
6497    ///   the mode a replay" instead of "will this step dispatch live" let a
6498    ///   resumed run cross `max_steps` unmetered for as long as it kept
6499    ///   running. A *recorded refusal* is re-evaluated against the ledger now
6500    ///   in force: exhaustion is a pause, and raising the ceiling and
6501    ///   resuming is how it un-pauses.
6502    /// * **Live** asks the ledger, which is the ordinary case.
6503    async fn admit_ready(
6504        &self,
6505        ready: &[StepId],
6506        ledger: &Arc<std::sync::Mutex<Ledger>>,
6507        mode: Pass,
6508        recorded: Option<&(StepId, String, String)>,
6509        cursor: &ReplayCursor,
6510        journal: Journalling<'_>,
6511    ) -> Result<(Vec<StepId>, Option<crate::core::BudgetExceeded>), RuntimeError> {
6512        let mut admitted = Vec::new();
6513
6514        for &step in ready {
6515            // ── A recorded step refusal at this step ───────────────────────
6516            if let Some((at, limit, used)) = recorded
6517                && *at == step
6518            {
6519                if mode == Pass::Strict {
6520                    // Verbatim: the recorded run stopped here, and a
6521                    // verification pass reaches the same conclusion without
6522                    // consulting today's ledger.
6523                    return Ok((
6524                        admitted,
6525                        Some(crate::core::BudgetExceeded::Recorded {
6526                            limit: limit.clone(),
6527                            used: used.clone(),
6528                        }),
6529                    ));
6530                }
6531                // Resume: the replayed prefix has already billed the journaled
6532                // historical usage into this ledger, so the question is "does
6533                // the budget now in force admit one more step over what the
6534                // run already spent". Still no → the run concludes exhausted
6535                // again, and the standing refusal already says so — no second
6536                // record. Yes → the re-admission is its own record, beside
6537                // the refusal it supersedes, so a strict replay of this
6538                // history reads refusal-then-continuation as a decision
6539                // somebody made rather than as divergence.
6540                if let Err(exceeded) = ledger
6541                    .lock()
6542                    .expect("budget mutex")
6543                    .admit_step(admitted.len())
6544                {
6545                    return Ok((admitted, Some(exceeded)));
6546                }
6547                if journal.writing {
6548                    self.store
6549                        .append(
6550                            journal.epoch,
6551                            vec![(journal.stamp)(
6552                                Append::new(
6553                                    journal.run,
6554                                    RecordKind::BudgetReadmitted {
6555                                        limit: limit.clone(),
6556                                    },
6557                                )
6558                                .step(step),
6559                            )],
6560                        )
6561                        .await
6562                        .map_err(RuntimeError::from_store)?;
6563                }
6564                admitted.push(step);
6565                continue;
6566            }
6567
6568            let verdict = match mode {
6569                Pass::Strict => Ok(()),
6570                Pass::Resume if !cursor.exhausted(step, Phase::Forward) => Ok(()),
6571                Pass::Live | Pass::Resume => ledger
6572                    .lock()
6573                    .expect("budget mutex")
6574                    .admit_step(admitted.len()),
6575            };
6576
6577            let Err(exceeded) = verdict else {
6578                admitted.push(step);
6579                continue;
6580            };
6581
6582            if journal.writing {
6583                // The refusal goes in the journal under the step it refused, so
6584                // replay reads the verdict rather than recomputing it.
6585                let used = format!("{:?}", ledger.lock().expect("budget mutex").consumed());
6586                self.store
6587                    .append(
6588                        journal.epoch,
6589                        vec![(journal.stamp)(
6590                            Append::new(
6591                                journal.run,
6592                                RecordKind::BudgetRefused {
6593                                    limit: exceeded.to_string(),
6594                                    used,
6595                                },
6596                            )
6597                            .step(step),
6598                        )],
6599                    )
6600                    .await
6601                    .map_err(RuntimeError::from_store)?;
6602            }
6603            return Ok((admitted, Some(exceeded)));
6604        }
6605
6606        Ok((admitted, None))
6607    }
6608
6609    /// End a run that stopped early: undo what warrants undoing, then seal.
6610    ///
6611    /// Both places a run can stop short go through here, so the unwind can never
6612    /// be attached to one of them and forgotten on the other.
6613    #[allow(clippy::too_many_arguments)]
6614    async fn stop(
6615        &self,
6616        cx: Unwind<'_>,
6617        status: RunStatus,
6618        completed: &[(StepId, Capability)],
6619        outputs: &BTreeMap<StepId, Tainted<Value>>,
6620        cursor: &mut ReplayCursor,
6621        case_id: Option<crate::core::CaseId>,
6622        quota: &QuotaPass,
6623    ) -> Result<RunOutcome, RuntimeError> {
6624        let (run, epoch, writing, ir) = (cx.run, cx.epoch, cx.writing, cx.ir);
6625        let output = run_output(ir, outputs);
6626        // Cloned before `maybe_unwind` consumes `cx`; read *after* it, because an
6627        // item's cost is what the whole attempt consumed, compensation included.
6628        let ledger = cx.ledger.clone();
6629        let unwound = self
6630            .maybe_unwind(cx, status, completed, outputs, cursor)
6631            .await?;
6632        // A strict pass that stops early is still a verification, and its
6633        // verdict must not outrun what it consumed. The ready loop's
6634        // unconsumed-effects check runs only when the loop drains normally,
6635        // and every early conclusion — a failure, an unwind, a recorded
6636        // suspension — comes through here instead; without this check a build
6637        // that performs fewer effects than the record (a compensation that
6638        // shrank, a step that now suspends earlier) would return the recorded
6639        // status as "verified" over effects it never requested. Checked after
6640        // `maybe_unwind` because a strict pass consumes recorded compensation
6641        // slices there, and what remains after that is genuinely unclaimed.
6642        //
6643        // A run that already quarantined keeps its own reason. Divergence
6644        // leaves history unconsumed as a matter of course — the step stopped
6645        // where it disagreed — so reporting the leftovers here would replace
6646        // the precise finding ("expected this effect, recomputed that one")
6647        // with a vaguer restatement of its consequence, and an operator would
6648        // be told the build performs fewer effects when what it actually did
6649        // was perform a different one.
6650        if !writing
6651            && !matches!(unwound, RunStatus::Quarantined(_))
6652            && let Some((step, phase, missing)) = cursor.first_unconsumed()
6653        {
6654            cursor.note_divergence(crate::journal::Divergence::unconsumed(
6655                step, phase, &missing,
6656            ));
6657            self.meter.count(metrics::DIVERGENCES, "");
6658            tracing::error!(
6659                target: telemetry::NONDETERMINISM,
6660                %step, key = %missing.key, unconsumed = true,
6661            );
6662            return self
6663                .conclude(
6664                    run,
6665                    epoch,
6666                    RunStatus::Quarantined(format!(
6667                        "strict replay verified less than the run recorded: journaled \
6668                         effect {missing} at step {step}, {phase:?} phase, was never \
6669                         requested — this build performs fewer effects than the recorded one"
6670                    )),
6671                    None,
6672                    writing,
6673                    case_id,
6674                    Consumed::default(),
6675                    Spend::default(),
6676                    quota,
6677                )
6678                .await;
6679        }
6680        // Scoped before the await — see `execute_inner` on why an inline guard
6681        // outlives the call it is passed to.
6682        let (consumed, live_spend) = {
6683            let l = ledger.lock().expect("budget mutex");
6684            (l.consumed(), l.live_spend())
6685        };
6686        self.conclude(
6687            run, epoch, unwound, output, writing, case_id, consumed, live_spend, quota,
6688        )
6689        .await
6690    }
6691
6692    /// Undo the completed steps, if the way the run stopped calls for it.
6693    ///
6694    /// # When a run does *not* unwind
6695    ///
6696    /// * **Quarantined.** The run holds an effect whose outcome is unknown, and
6697    ///   you cannot safely undo around one: compensating a payment that may
6698    ///   never have gone out creates a refund for money nobody took. Everything
6699    ///   stays exactly where it is until a person supplies the fact the journal
6700    ///   lacks — [`Runtime::reconcile_effect`](super::Runtime::reconcile_effect)
6701    ///   and [`decide_quarantine`](super::Runtime::decide_quarantine) — and
6702    ///   giving up
6703    ///   ([`Abandon`](crate::core::QuarantineDecision::Abandon)) still unwinds
6704    ///   nothing, because impatience is not evidence. This is the rule that
6705    ///   separates a saga that is honest about distributed systems from one that
6706    ///   tidies up and hopes.
6707    /// * **Suspended.** The run is healthy and waiting. Nothing has failed.
6708    ///
6709    /// A failure *after the pivot* also stops the unwind at the pivot: once the
6710    /// business has committed, reversing the decisions leading up to it would
6711    /// contradict something the outside world has already acted on.
6712    ///
6713    /// # Two rules every unwind takes
6714    ///
6715    /// **It must not unwind around an unknown outcome.** The same rule
6716    /// quarantine already enforces, applied at the unwind's own door. For a
6717    /// cancelled run the check is unconditional — cancellation gets no second
6718    /// pass, so an unresolved outcome stays unresolved forever. For a failed
6719    /// run it is conditional on the unwind list: an orphan under a failure is
6720    /// resolvable, because the announcement is journaled, the effect declared
6721    /// a `Recovery`, and resuming resolves it — but only while nothing was
6722    /// compensated, since compensation closes the run to resume. A failure
6723    /// that would compensate around doubt therefore quarantines instead; a
6724    /// failure with nothing to unwind stays a failure and keeps its resume.
6725    ///
6726    /// **It must undo the steps it interrupted.** Compensation walks
6727    /// *completed* steps, which covers the step that failed only if it never
6728    /// mutated. A stop arrives from outside while a step is typically
6729    /// suspended — holding effects it performed and never completing — and a
6730    /// failure in one concurrent sibling interrupts the others the same way:
6731    /// severity ordering can conclude the run while a suspended sibling holds
6732    /// a landed mutation. Unwinding only completed steps would leave exactly
6733    /// that work standing — a run that posted to a ledger and then suspended
6734    /// for approval would conclude with the posting still in the world and,
6735    /// once anything else was compensated, no resume left to finish it.
6736    async fn maybe_unwind(
6737        &self,
6738        cx: Unwind<'_>,
6739        status: RunStatus,
6740        completed: &[(StepId, Capability)],
6741        outputs: &BTreeMap<StepId, Tainted<Value>>,
6742        cursor: &mut ReplayCursor,
6743    ) -> Result<RunStatus, RuntimeError> {
6744        match status {
6745            // A cancelled run unwinds exactly as a failed one does. Stopping a
6746            // run that has already moved money and leaving the movement in
6747            // place is not stopping it — and the operator who asked is entitled
6748            // to assume "stop" means the world is put back, not that the
6749            // process merely exited.
6750            //
6751            // **Exhausted is deliberately not here.** Exhaustion is a pause,
6752            // not a fault: the run did what it was told, and what it was told
6753            // included a ceiling. Its mutations stand, because the operator's
6754            // two honest options both need them standing — raise the ceiling
6755            // and resume (which continues *over* the completed work), or
6756            // cancel (which unwinds through this same function under a status
6757            // that does). Unwinding on exhaustion would make the three ends of
6758            // an exhausted run contradict each other: the work reversed, the
6759            // run still resumable, and the resume reporting success over a
6760            // world where the work no longer stands.
6761            RunStatus::Failed(_) | RunStatus::Cancelled { .. } => {}
6762            other => return Ok(other),
6763        }
6764
6765        let (list, evidence) = match self.gated_unwind_list(&cx, &status, completed).await? {
6766            Ok(scope) => scope,
6767            Err(quarantine) => return Ok(quarantine),
6768        };
6769        let UnwindEvidence {
6770            mutated,
6771            started: _,
6772            undone: already_undone,
6773            recorded_groups,
6774            failed: _,
6775        } = evidence;
6776        let completed = &list[..];
6777
6778        for (step, capability) in completed.iter().rev().cloned() {
6779            // Resolved from what ran, not from the plan in force. After a replan
6780            // the two can differ, and undoing whatever now occupies that slot is
6781            // how a saga compensates work that never happened.
6782            let skill = self.resolve(&capability.0)?;
6783            let declared = skill.compensation();
6784
6785            match declared {
6786                // The point of no return. Everything from here back stays.
6787                crate::core::Compensation::Pivot => break,
6788                crate::core::Compensation::Unnecessary => continue,
6789                crate::core::Compensation::Undeclared => {
6790                    if !mutated.contains(&step) {
6791                        // Nothing to undo, and the journal proves it.
6792                        continue;
6793                    }
6794                    return Ok(RunStatus::Quarantined(format!(
6795                        "step {step} ('{}') changed external state and declares no \
6796                         compensation, so the run cannot be safely unwound — \
6797                         declare Compensation on it, or resolve this by hand",
6798                        capability.0
6799                    )));
6800                }
6801                crate::core::Compensation::Compensatable => {}
6802            }
6803
6804            let result = self
6805                .run_compensation(
6806                    &cx,
6807                    step,
6808                    skill.as_ref(),
6809                    outputs,
6810                    cursor,
6811                    recorded_groups
6812                        .iter()
6813                        .filter(|((s, p, _), _)| *s == step && *p == Phase::Compensating)
6814                        .map(|((_, _, name), n)| (name.clone(), *n))
6815                        .collect(),
6816                )
6817                .await;
6818
6819            // A compensation may legitimately need to wait — a refund that needs
6820            // four eyes is still a refund. Suspension is not failure: the run is
6821            // healthy, its frame is durable, and it will finish unwinding when
6822            // the answer arrives. Reporting it as a failure would quarantine a
6823            // run doing exactly the right thing and tell the operator the
6824            // compensation had broken.
6825            if let Err(crate::core::SkillError::Step(crate::core::StepError::Suspended(reason))) =
6826                &result
6827            {
6828                return Ok(RunStatus::Suspended(reason.clone()));
6829            }
6830
6831            let outcome = match &result {
6832                Ok(()) => "compensated".to_owned(),
6833                Err(e) => e.to_string(),
6834            };
6835
6836            // Re-run but do not re-record. On resume the compensation executes
6837            // again with every effect served from the journal, which is what
6838            // keeps strict verification meaningful — but a second
6839            // `StepCompensated` would report one compensation as two.
6840            if result.is_ok() {
6841                tracing::info!(target: telemetry::COMPENSATED, run = %cx.run, %step);
6842                self.meter.count(metrics::COMPENSATIONS, "done");
6843            }
6844            if cx.writing && !already_undone.contains(&step) {
6845                self.store
6846                    .append(
6847                        cx.epoch,
6848                        vec![(cx.stamp)(
6849                            Append::new(
6850                                cx.run,
6851                                RecordKind::StepCompensated {
6852                                    compensation: declared,
6853                                    outcome: outcome.clone(),
6854                                },
6855                            )
6856                            .step(step)
6857                            .phase(Phase::Compensating),
6858                        )],
6859                    )
6860                    .await
6861                    .map_err(RuntimeError::from_store)?;
6862            }
6863
6864            if let Err(fault) = &result {
6865                tracing::error!(
6866                    target: telemetry::COMPENSATION_FAILED,
6867                    run = %cx.run,
6868                    %step,
6869                    error_type = fault.class(),
6870                    reason_digest = %telemetry::reason_digest(&outcome),
6871                );
6872                self.meter.count(metrics::COMPENSATIONS, "failed");
6873                // Not a problem more compensation solves. Unwinding further
6874                // would undo steps *before* one that is now in an unknown
6875                // state, which is strictly worse than stopping and saying so.
6876                return Ok(RunStatus::Quarantined(format!(
6877                    "compensation failed for step {step} ('{}'): {outcome} — the run is \
6878                     partially unwound and needs an operator",
6879                    capability.0
6880                )));
6881            }
6882        }
6883
6884        Ok(status)
6885    }
6886
6887    /// Whether `run`'s journal holds a landed mutation no compensation undid.
6888    ///
6889    /// The attention roll-up's question about a failed run: `failed` alone may
6890    /// be cleared by a resume, but one standing on landed work leaves that
6891    /// work in the world until somebody finishes or unwinds it.
6892    pub(super) async fn holds_landed_work(&self, run: RunId) -> Result<bool, RuntimeError> {
6893        let evidence = self.unwind_evidence(run).await?;
6894        Ok(!evidence.mutated.is_subset(&evidence.undone))
6895    }
6896
6897    /// What the journal knows about an unwind before it starts:
6898    /// `(steps that changed something, steps already compensated)`.
6899    ///
6900    /// Evidence, not bookkeeping. The journal already knows both, it knows the
6901    /// same thing on replay, and nothing has to be threaded through the executor
6902    /// to keep a parallel copy honest.
6903    ///
6904    /// **A mutating effect that provably did not happen does not mark its
6905    /// step.** The announcement alone is not the evidence: an effect whose
6906    /// terminal record classifies it `DidNotHappen` — a peer that refused
6907    /// before acting, a call that never left — changed nothing, and
6908    /// compensating it is a refund for money nobody took. Everything else
6909    /// counts, and the defaults lean that way: an announcement with no
6910    /// terminal record is an orphan and therefore in doubt, and each new
6911    /// attempt starts from doubt again, so a retry that lands after a refusal
6912    /// is still evidence the step touched the world.
6913    ///
6914    /// **Nor does a member of a group that settled `Aborted`.** That settlement
6915    /// is written only once every landed member was reversed, so the group's
6916    /// effects — members and reversals alike — left nothing standing, and
6917    /// counting them marked a cleanly aborted step as mutated: a quarantine for
6918    /// a step declaring no compensation, a second undo for one that did.
6919    ///
6920    /// What this does **not** establish is that a `DidNotHappen` classification
6921    /// is true — that is the driver's claim, and a driver that mislabels a
6922    /// landed call leaves a mutation nothing here will undo.
6923    async fn unwind_evidence(&self, run: RunId) -> Result<UnwindEvidence, RuntimeError> {
6924        let records = self
6925            .store
6926            .read(run, 1)
6927            .await
6928            .map_err(RuntimeError::from_store)?;
6929
6930        // Per mutating forward effect: the step it belongs to, and whether the
6931        // record leaves it capable of having touched the world.
6932        let mut touching: BTreeMap<crate::core::EffectKey, (StepId, bool)> = BTreeMap::new();
6933        // The mutating effects of the group each step has open. A step holds
6934        // at most one open group, so the step is the key.
6935        let mut grouped: BTreeMap<StepId, Vec<crate::core::EffectKey>> = BTreeMap::new();
6936        let mut undone = BTreeSet::new();
6937        let mut failed = BTreeSet::new();
6938        for r in &records {
6939            let Some(step) = r.body.step else { continue };
6940            let key = r.effect_key();
6941            match r.kind() {
6942                RecordKind::StepFinished { outcome } if r.body.phase.is_forward() => {
6943                    if outcome == RunStatus::Succeeded.as_str() {
6944                        failed.remove(&step);
6945                    } else {
6946                        failed.insert(step);
6947                    }
6948                }
6949                RecordKind::EffectStarted { mutates: true, .. } if r.body.phase.is_forward() => {
6950                    if let Some(key) = key {
6951                        // A fresh attempt is undecided until its own terminal
6952                        // record says otherwise, so it supersedes whatever the
6953                        // previous attempt concluded.
6954                        touching.insert(key, (step, true));
6955                        if let Some(members) = grouped.get_mut(&step) {
6956                            members.push(key);
6957                        }
6958                    }
6959                }
6960                RecordKind::GroupOpened { .. } if r.body.phase.is_forward() => {
6961                    grouped.insert(step, Vec::new());
6962                }
6963                RecordKind::GroupSettled { outcome, .. } if r.body.phase.is_forward() => {
6964                    let members = grouped.remove(&step).unwrap_or_default();
6965                    if *outcome == crate::core::GroupOutcome::Aborted {
6966                        for member in members {
6967                            touching.remove(&member);
6968                        }
6969                    }
6970                }
6971                RecordKind::EffectFailed { disposition, .. }
6972                | RecordKind::EffectReconciled { disposition, .. } => {
6973                    if let Some(entry) = key.and_then(|k| touching.get_mut(&k)) {
6974                        entry.1 = *disposition != crate::core::Disposition::DidNotHappen;
6975                    }
6976                }
6977                RecordKind::StepCompensated { .. } => {
6978                    undone.insert(step);
6979                }
6980                _ => {}
6981            }
6982        }
6983        let mutated = touching
6984            .into_values()
6985            .filter_map(|(step, touched)| touched.then_some(step))
6986            .collect();
6987        Ok(UnwindEvidence {
6988            mutated,
6989            started: started_skills(&records),
6990            undone,
6991            recorded_groups: recorded_groups(&records),
6992            failed,
6993        })
6994    }
6995
6996    /// The half of a step's frame that belongs to the **plane** rather than to
6997    /// the step.
6998    ///
6999    /// One place, for both a forward step and a compensation: a plane
7000    /// capability wired into one of the two and not the other is a control
7001    /// that applies going forward and not while unwinding.
7002    fn plane_frame(&self, step: StepFrame) -> super::ctx::Frame {
7003        let StepFrame {
7004            run,
7005            epoch,
7006            step,
7007            phase,
7008            mode,
7009            case,
7010            ledger,
7011            identity,
7012            subjects,
7013            agent,
7014            #[cfg(feature = "manifest")]
7015            manifest,
7016            recorded_groups,
7017        } = step;
7018        super::ctx::Frame {
7019            run,
7020            epoch,
7021            step,
7022            phase,
7023            mode,
7024            case,
7025            timers: self.timers.clone(),
7026            blobs: self.blobs.clone(),
7027            memories: self.memories.clone(),
7028            semantic: self.semantic.clone(),
7029            authorities: self.authorities.clone(),
7030            peers: self.peers.clone(),
7031            #[cfg(feature = "manifest")]
7032            tools: self.tools.clone(),
7033            journal_ceiling: self.journal_ceiling,
7034            #[cfg(feature = "manifest")]
7035            egress: self.egress.clone(),
7036            #[cfg(feature = "manifest")]
7037            checkers: self.checkers.clone(),
7038            streams: self.streams.clone(),
7039            #[cfg(feature = "manifest")]
7040            rates: self.rates.clone(),
7041            meter: self.meter.clone(),
7042            #[cfg(feature = "keyring")]
7043            keyring: self.keyring.clone(),
7044            tenant: self.tenant.clone(),
7045            ledger,
7046            policy: self.policy.clone(),
7047            identity,
7048            subjects,
7049            agent,
7050            plane: self.self_ref.clone(),
7051            #[cfg(feature = "manifest")]
7052            declaration: manifest.as_deref().and_then(|m| self.identity_of(m)),
7053            #[cfg(feature = "manifest")]
7054            manifest,
7055            signer: self.signer.clone(),
7056            recorded_groups,
7057        }
7058    }
7059
7060    /// Run one step's `compensate`, in its own phase and cursor slice.
7061    ///
7062    /// Split out so the unwind reads as the policy it is. The step gets a full
7063    /// `StepCtx` on purpose: compensating effects are journaled, retried,
7064    /// reconciled and replayed exactly like forward ones, and may suspend for a
7065    /// human — a refund that needs four eyes is still a refund.
7066    async fn run_compensation(
7067        &self,
7068        cx: &Unwind<'_>,
7069        step: StepId,
7070        skill: &dyn Skill,
7071        outputs: &BTreeMap<StepId, Tainted<Value>>,
7072        cursor: &mut ReplayCursor,
7073        recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
7074    ) -> Result<(), crate::core::SkillError> {
7075        // A step with no recorded output still gets compensated: the absence of
7076        // a result says nothing about whether it changed anything, and the
7077        // compensation is what knows.
7078        let output = outputs
7079            .get(&step)
7080            .cloned()
7081            .unwrap_or_else(|| Tainted::trusted(Value::Null));
7082
7083        let mut ctx = StepCtx::new(
7084            &self.store,
7085            cursor.take(step, Phase::Compensating),
7086            self.plane_frame(StepFrame {
7087                run: cx.run,
7088                epoch: cx.epoch,
7089                step,
7090                phase: Phase::Compensating,
7091                mode: cx.mode,
7092                case: cx.case.clone(),
7093                ledger: Arc::clone(cx.ledger),
7094                identity: cx.identity.cloned(),
7095                subjects: cx.subjects.clone(),
7096                agent: cx.agent.to_owned(),
7097                #[cfg(feature = "manifest")]
7098                manifest: self.governing(skill),
7099                recorded_groups,
7100            }),
7101        );
7102
7103        let result = skill.compensate(&mut ctx, &output).await;
7104        let result = moved_blob_wins(&ctx, result);
7105        let mut slice = ctx.into_cursor();
7106        if let Err(crate::core::SkillError::Step(error)) = &result
7107            && let Some(divergence) =
7108                crate::journal::Divergence::of(step, Phase::Compensating, error, &slice)
7109        {
7110            slice.note_divergence(divergence);
7111        }
7112        cursor.restore(step, Phase::Compensating, slice);
7113        result
7114    }
7115
7116    /// What the unwind walks, gated — or the quarantine that replaces it.
7117    ///
7118    /// Reads the journal's evidence and applies the stop gates. The gates run
7119    /// exactly when this conclusion closes the run to resume: always for a
7120    /// cancellation (it gets no second pass), and for a failure precisely
7121    /// when the unwind will compensate something — compensation is what
7122    /// closes resume, and a failure that compensates nothing stays open, so
7123    /// its orphans resolve and its suspended siblings re-register on the
7124    /// resume instead of being unwound around.
7125    async fn gated_unwind_list(
7126        &self,
7127        cx: &Unwind<'_>,
7128        status: &RunStatus,
7129        completed: &[(StepId, Capability)],
7130    ) -> Result<Result<(Vec<(StepId, Capability)>, UnwindEvidence), RunStatus>, RuntimeError> {
7131        // Which steps actually changed something outside. Read from the
7132        // journal rather than tracked in memory: it is the same evidence live
7133        // and on replay, and it is what lets an *undeclared* step be judged
7134        // on what it did instead of on what nobody said about it.
7135        let evidence = self.unwind_evidence(cx.run).await?;
7136        // The question is asked of the interrupted siblings too: a sibling
7137        // holding landed work the failure stranded is work the unwind must
7138        // reach, and asked of completed steps alone, a failure with nothing
7139        // completed to compensate left it standing. The failing step itself
7140        // is not asked — its own landed work leaves the run resumable.
7141        let candidates = Self::with_interrupted_siblings(completed, &evidence);
7142        let list = if status.is_cancelled() || self.will_compensate(&candidates, &evidence.mutated)
7143        {
7144            match self
7145                .stop_list(cx.run, completed, &evidence.started, &evidence.mutated)
7146                .await?
7147            {
7148                Ok(list) => list,
7149                Err(quarantine) => return Ok(Err(quarantine)),
7150            }
7151        } else {
7152            completed.to_vec()
7153        };
7154        Ok(Ok((list, evidence)))
7155    }
7156
7157    /// The unwind list for a stop, or the quarantine that replaces it.
7158    ///
7159    /// Both rules in `maybe_unwind`'s doc comment, applied in order: refuse to
7160    /// unwind while an outcome is unknown, and extend the list with the steps
7161    /// the stop interrupted. Called only for a conclusion that closes the run
7162    /// to resume — a cancellation, or a failure whose unwind will compensate
7163    /// — so doubt quarantines unconditionally here: there is no later pass on
7164    /// which an unresolved outcome could still resolve, and compensating
7165    /// around the one call nobody can account for would destroy the evidence
7166    /// an operator needs to resolve it by hand.
7167    async fn stop_list(
7168        &self,
7169        run: RunId,
7170        completed: &[(StepId, Capability)],
7171        started: &BTreeMap<StepId, String>,
7172        mutated: &BTreeSet<StepId>,
7173    ) -> Result<Result<Vec<(StepId, Capability)>, RunStatus>, RuntimeError> {
7174        if let Some(step) = self.undecided_effect(run).await? {
7175            return Ok(Err(RunStatus::Quarantined(format!(
7176                "step {step} holds a mutating effect whose outcome is unknown — it \
7177                 was announced and never concluded, or concluded in doubt with no \
7178                 reconciliation — so the run cannot be unwound: compensating around \
7179                 it would undo everything except the one thing nobody can account for"
7180            ))));
7181        }
7182        Ok(Ok(Self::with_interrupted_steps(
7183            completed, started, mutated,
7184        )))
7185    }
7186
7187    /// Whether an unwind over these completed steps would compensate anything.
7188    ///
7189    /// The question that decides whether a failure closes its run: a declared
7190    /// `Compensatable` step will be compensated, and an `Undeclared` step
7191    /// that provably mutated quarantines inside the loop — either way the
7192    /// conclusion is not one a resume can reopen. `Pivot` and `Unnecessary`
7193    /// steps compensate nothing. Deliberately conservative on a capability
7194    /// that no longer resolves: an unresolvable skill is treated as
7195    /// compensating, so the unwind path — which will surface the resolution
7196    /// error loudly — is taken rather than silently skipped.
7197    fn will_compensate(
7198        &self,
7199        completed: &[(StepId, Capability)],
7200        mutated: &BTreeSet<StepId>,
7201    ) -> bool {
7202        completed.iter().any(|(step, capability)| {
7203            self.resolve(&capability.0)
7204                .map_or(true, |skill| match skill.compensation() {
7205                    crate::core::Compensation::Compensatable => true,
7206                    crate::core::Compensation::Undeclared => mutated.contains(step),
7207                    crate::core::Compensation::Pivot | crate::core::Compensation::Unnecessary => {
7208                        false
7209                    }
7210                })
7211        })
7212    }
7213
7214    /// The completed steps, plus every sibling a failure interrupted holding
7215    /// landed work: a step that mutated, never completed, and did not itself
7216    /// fail. Undone as the skill its `StepStarted` names, as
7217    /// [`with_interrupted_steps`](Self::with_interrupted_steps) undoes it.
7218    fn with_interrupted_siblings(
7219        completed: &[(StepId, Capability)],
7220        evidence: &UnwindEvidence,
7221    ) -> Vec<(StepId, Capability)> {
7222        let mut out = completed.to_vec();
7223        let done: BTreeSet<StepId> = out.iter().map(|(s, _)| *s).collect();
7224        for step in evidence
7225            .mutated
7226            .iter()
7227            .filter(|s| !done.contains(s) && !evidence.failed.contains(s))
7228        {
7229            if let Some(skill) = evidence.started.get(step) {
7230                out.push((*step, Capability::new(skill.as_str())));
7231            }
7232        }
7233        out
7234    }
7235
7236    /// The unwind list for a stop: completed steps, plus any step that mutated
7237    /// without completing — the one a cancellation interrupted, or the sibling
7238    /// a failure stranded mid-suspension. The interrupted steps go last, so
7239    /// the caller's reverse walk undoes them first.
7240    ///
7241    /// An interrupted step is undone as the skill its `StepStarted` names —
7242    /// the capability that ran — never as whatever the plan in force has at
7243    /// its id. A step that charged and then asked to replan is interrupted
7244    /// under a successor that may drop it or put other work at its id, and
7245    /// either reading would leave the charge standing or refund it through a
7246    /// skill that never ran.
7247    ///
7248    /// A step whose compensation is already on the record stays in the list,
7249    /// exactly as a completed step does. The walk re-runs its compensation
7250    /// with every effect served from the journal and the `StepCompensated`
7251    /// dedup keeps the record single — while filtering it out here made a
7252    /// *replayed* unwind walk fewer steps than the recorded one, so a strict
7253    /// pass over an honestly-unwound run left the interrupted step's
7254    /// compensation slice unconsumed and quarantined a history that was
7255    /// telling the truth.
7256    fn with_interrupted_steps(
7257        completed: &[(StepId, Capability)],
7258        started: &BTreeMap<StepId, String>,
7259        mutated: &BTreeSet<StepId>,
7260    ) -> Vec<(StepId, Capability)> {
7261        let mut out = completed.to_vec();
7262        let done: BTreeSet<StepId> = out.iter().map(|(s, _)| *s).collect();
7263        for step in mutated.iter().filter(|s| !done.contains(s)) {
7264            if let Some(skill) = started.get(step) {
7265                out.push((*step, Capability::new(skill.as_str())));
7266            }
7267        }
7268        out
7269    }
7270
7271    /// The lowest step holding an effect this journal cannot decide, if any.
7272    ///
7273    /// Ordinarily the run is already `Quarantined` when this is true, and a
7274    /// quarantined run never unwinds. Cancellation opens a second door into the
7275    /// unwind, and it has to be shut the same way — otherwise an operator's
7276    /// stop compensates every step *around* the one nobody can account for,
7277    /// which is precisely the refund for money nobody took, arriving through a
7278    /// control meant to make things safer.
7279    ///
7280    /// The rule itself is [`undecided_effects`](crate::journal::undecided_effects),
7281    /// shared with the operator API and the offline audit. This adds only the
7282    /// question the unwind asks of it: *how far back does the doubt reach*.
7283    async fn undecided_effect(&self, run: RunId) -> Result<Option<StepId>, RuntimeError> {
7284        let records = self
7285            .store
7286            .read(run, 1)
7287            .await
7288            .map_err(RuntimeError::from_store)?;
7289        Ok(crate::journal::undecided_effects(&records)
7290            .into_iter()
7291            .map(|u| u.step)
7292            .min())
7293    }
7294
7295    /// Execute one plan node.
7296    async fn run_step(
7297        &self,
7298        ctx: StepRun<'_>,
7299        run_input: &Tainted<Value>,
7300        outputs: &BTreeMap<StepId, Tainted<Value>>,
7301        cursor: crate::journal::StepCursor,
7302    ) -> Result<
7303        (
7304            RunStatus,
7305            Option<Tainted<Value>>,
7306            crate::journal::StepCursor,
7307        ),
7308        RuntimeError,
7309    > {
7310        let span = tracing::info_span!(
7311            telemetry::STEP_SPAN,
7312            { telemetry::STEP } = tracing::field::display(ctx.node.id),
7313            { telemetry::CAPABILITY } = tracing::field::display(&ctx.node.capability.0),
7314            { telemetry::PHASE } = ctx.phase.as_str(),
7315            { telemetry::MODE } = telemetry::mode_str(ctx.mode),
7316            { telemetry::OUTCOME } = tracing::field::Empty,
7317        );
7318        self.run_step_inner(ctx, run_input, outputs, cursor)
7319            .instrument(span)
7320            .await
7321    }
7322
7323    #[allow(clippy::too_many_lines)]
7324    async fn run_step_inner(
7325        &self,
7326        ctx: StepRun<'_>,
7327        run_input: &Tainted<Value>,
7328        outputs: &BTreeMap<StepId, Tainted<Value>>,
7329        cursor: crate::journal::StepCursor,
7330    ) -> Result<
7331        (
7332            RunStatus,
7333            Option<Tainted<Value>>,
7334            crate::journal::StepCursor,
7335        ),
7336        RuntimeError,
7337    > {
7338        let StepRun {
7339            identity,
7340            subjects,
7341            run,
7342            epoch,
7343            node,
7344            phase,
7345            mode,
7346            case,
7347            ledger,
7348            writing,
7349            stamp,
7350            agent,
7351            already_started,
7352            already_finished,
7353            recorded_groups,
7354            abandoning,
7355        } = ctx;
7356        let step = node.id;
7357        let skill = self.resolve(&node.capability.0)?;
7358
7359        // Assemble this step's input from its declared sources. Labels join, so
7360        // provenance flows through the graph without anyone threading it by hand.
7361        let step_input = assemble(node, run_input, outputs)?;
7362
7363        // Announce a step doing new work, once: always in `Live`, and on a
7364        // resume only for a step the journal never announced — a frontier
7365        // step running for the first time. Keying this on the mode alone got
7366        // both directions wrong: resumed frontier steps went unannounced, and
7367        // the `!Strict` finish gate below duplicated `StepFinished` for every
7368        // fully replayed step on every resume.
7369        let announce = writes_step_record(mode, !already_started);
7370        if announce {
7371            self.store
7372                .append(
7373                    epoch,
7374                    vec![stamp(
7375                        Append::new(
7376                            run,
7377                            RecordKind::StepStarted {
7378                                skill: skill.descriptor().name,
7379                            },
7380                        )
7381                        .step(step),
7382                    )],
7383                )
7384                .await
7385                .map_err(RuntimeError::from_store)?;
7386        }
7387
7388        let mut cx = StepCtx::new(
7389            &self.store,
7390            cursor,
7391            self.plane_frame(StepFrame {
7392                run,
7393                epoch,
7394                step,
7395                phase,
7396                mode,
7397                case,
7398                ledger: Arc::clone(ledger),
7399                identity: identity.cloned(),
7400                subjects: subjects.clone(),
7401                agent: agent.to_owned(),
7402                #[cfg(feature = "manifest")]
7403                manifest: self.governing(skill.as_ref()),
7404                recorded_groups,
7405            }),
7406        );
7407        cx.abandoning = abandoning;
7408        // The one place elapsed time is measured, and it is measured with a
7409        // journaled clock read like every other observation — a wall-clock
7410        // ceiling decided from an ambient clock would give a replayed run a
7411        // different verdict every time it was looked at.
7412        //
7413        // Paid only by a run that asked for the ceiling, which is why the
7414        // step's first effect is this one when a wall-clock budget is declared
7415        // and the skill's own when it is not. The consequence is worth stating:
7416        // that history can be replayed under a *raised* wall-clock ceiling, and
7417        // not under a build that removed it, because the recorded run performs
7418        // a read this one would not request.
7419        let result = match Self::observe_clock(&mut cx, ledger).await {
7420            Ok(()) => skill.invoke(&mut cx, step_input).await,
7421            Err(e) => Err(crate::core::SkillError::Step(e)),
7422        };
7423        let result = settle_abandoned_group(&mut cx, result).await;
7424        let result = moved_blob_wins(&cx, result);
7425        let wrote = cx.wrote_records();
7426        let mut cursor = cx.into_cursor();
7427        if let Err(crate::core::SkillError::Step(error)) = &result
7428            && let Some(divergence) = crate::journal::Divergence::of(step, phase, error, &cursor)
7429        {
7430            cursor.note_divergence(divergence);
7431        }
7432        ledger.lock().expect("budget mutex").record_step();
7433
7434        let fault = result.as_ref().err().map(crate::core::SkillError::class);
7435        let (status, output) = classify(&self.meter, result);
7436        tracing::Span::current().record(telemetry::OUTCOME, status.as_str());
7437        if let RunStatus::Quarantined(why) = &status {
7438            tracing::error!(
7439                target: telemetry::QUARANTINED,
7440                %run,
7441                %step,
7442                error_type = fault.unwrap_or("quarantined"),
7443                reason_digest = %telemetry::reason_digest(why),
7444            );
7445        }
7446
7447        // Record the ending only where the journal does not already hold it:
7448        // every live step; a resumed step that did new work; and a resumed
7449        // step reaching an ending the record never captured — how a suspended
7450        // step that finishes purely from replayed history (its awaited event
7451        // recorded by the delivery) still gets its one `StepFinished`. What
7452        // this rules out is a duplicate ending for every fully replayed
7453        // completed step, appended on every resume of a run doing nothing new.
7454        let record_ending = writes_step_record(mode, announce || wrote || !already_finished);
7455        if writing && record_ending {
7456            // A suspended step has not finished, so it records why it stopped
7457            // rather than claiming an outcome.
7458            let record = match &status {
7459                RunStatus::Suspended(reason) => RecordKind::RunSuspended {
7460                    reason: reason.clone(),
7461                },
7462                other => RecordKind::StepFinished {
7463                    outcome: other.as_str().to_owned(),
7464                },
7465            };
7466            self.store
7467                .append(epoch, vec![stamp(Append::new(run, record).step(step))])
7468                .await
7469                .map_err(RuntimeError::from_store)?;
7470        }
7471
7472        Ok((status, output, cursor))
7473    }
7474
7475    /// Whether the chain's last record is already this conclusion. `head` is
7476    /// the chain's last sequence.
7477    async fn last_is_conclusion(
7478        &self,
7479        run: RunId,
7480        head: crate::core::Seq,
7481        status: &RunStatus,
7482    ) -> Result<bool, RuntimeError> {
7483        if head < 1 {
7484            return Ok(false);
7485        }
7486        let page = self
7487            .store
7488            .read_page(run, head, 1)
7489            .await
7490            .map_err(RuntimeError::from_store)?;
7491        Ok(matches!(
7492            page.first().map(Record::kind),
7493            Some(RecordKind::RunConcluded { outcome, .. }) if outcome == status.as_str()
7494        ))
7495    }
7496
7497    /// Seal the run and report.
7498    /// Eight arguments, and each is a distinct fact about how the run ended
7499    /// that the caller already holds. Bundling them into a struct would move the
7500    /// same fields one indirection away without removing a single one.
7501    #[allow(clippy::too_many_arguments)]
7502    async fn conclude(
7503        &self,
7504        run: RunId,
7505        epoch: u64,
7506        status: RunStatus,
7507        output: Option<Tainted<Value>>,
7508        writing: bool,
7509        case: Option<crate::core::CaseId>,
7510        consumed: Consumed,
7511        live_spend: Spend,
7512        quota: &QuotaPass,
7513    ) -> Result<RunOutcome, RuntimeError> {
7514        // Loud toward the operator, ordinary toward the caller. A failed run is
7515        // a conclusion a resume can honestly answer, so it is not an incident —
7516        // but without this event nothing says why one failed except the
7517        // journal and an index that needs the HTTP surface mounted.
7518        if let RunStatus::Failed(reason) = &status {
7519            tracing::warn!(
7520                target: telemetry::RUN_FAILED,
7521                %run,
7522                error_type = "failed",
7523                reason_digest = %telemetry::reason_digest(reason),
7524            );
7525        }
7526
7527        // A suspended run is not sealed: its chain is going to be extended the
7528        // moment whatever it waits for arrives.
7529        let chain_head = if writing && !status.is_suspended() {
7530            // The conclusion goes *in* the chain — before the chain is closed
7531            // over it, where the conclusion is one that closes it. Two things
7532            // follow, and both were missing while the outcome lived only in a
7533            // side table: tamper detection covers how the run ended, and a
7534            // resumed run can read that fact from the same history it verifies
7535            // rather than inferring it from the last step that happened to
7536            // finish. The stores also derive the outcome index from this
7537            // record, last conclusion wins — which is what keeps a
7538            // failed-then-resumed-then-succeeded run from being listed as
7539            // failed forever.
7540            let before = self
7541                .store
7542                .head(run)
7543                .await
7544                .map_err(RuntimeError::from_store)?;
7545            // A resume that did nothing new reaches the same open conclusion
7546            // the record already carries, and appending it again would grow
7547            // the journal by one identical conclusion per resume. The chain's
7548            // own last record is the authority: only when it already *is*
7549            // this conclusion is the append skipped. A resume that worked and then concluded the same way
7550            // has its work between the two conclusions, so both are kept.
7551            // Scoped to open conclusions, because a sealing one is
7552            // unreachable twice: the seal refuses further appends and a
7553            // sealed run's replay is a no-op long before here.
7554            let repeated =
7555                !status.seals() && self.last_is_conclusion(run, before.seq, &status).await?;
7556            if repeated {
7557                before.hash
7558            } else {
7559                let mut sealed = Append::new(
7560                    run,
7561                    RecordKind::RunConcluded {
7562                        outcome: status.as_str().to_owned(),
7563                        reason: status.own_account().map(std::borrow::Cow::into_owned),
7564                        exhaustion: match &status {
7565                            RunStatus::Exhausted(limit) => Some(limit.clone()),
7566                            _ => None,
7567                        },
7568                        live_spend,
7569                        chain_head: before.hash,
7570                    },
7571                );
7572                if let Some(c) = case {
7573                    sealed = sealed.case(c);
7574                }
7575                let concluded = self
7576                    .store
7577                    .append(epoch, vec![sealed])
7578                    .await
7579                    .map_err(RuntimeError::from_store)?;
7580
7581                concluded.last().map_or(before.hash, |r| r.hash)
7582            }
7583        } else {
7584            self.store
7585                .head(run)
7586                .await
7587                .map_err(RuntimeError::from_store)?
7588                .hash
7589        };
7590
7591        // Hand the lease back rather than letting it time out.
7592        //
7593        // Whatever the outcome — sealed, suspended, exhausted — this instance is
7594        // finished with the run. Holding the lease until expiry would make every
7595        // failover wait out the TTL for nothing.
7596        //
7597        // Best-effort on purpose. A release that fails costs a TTL of patience;
7598        // turning it into a run failure would convert a tidiness problem into a
7599        // correctness one, after the work is already done and journaled.
7600        //
7601        // Only when this pass held a lease at all. A strict pass acquires
7602        // nothing — it is a read — and both stores release on epoch match
7603        // alone, so a strict verification of a run whose owner is live under
7604        // that same epoch would mark the *owner's* lease released: its
7605        // heartbeat stops renewing and any delivery or sweep may fence a
7606        // healthy run mid-step. The same guard keeps a read-only pass out of
7607        // the quota ledger and the metrics: a verification is not work the
7608        // tenant did today, and accruing a historical run's spend into the
7609        // current period on every audit would bill the past once per look.
7610        if writing {
7611            // Settlement precedes both physical sealing and lease release. If
7612            // it fails, the conclusion remains open under an owned lease; once
7613            // that lease expires, recovery derives this pass from the journal
7614            // and retries the idempotent receipt.
7615            //
7616            // Whatever the outcome this instance is finished with the run, so
7617            // a **suspended** run gives its slot back too — it costs a row,
7618            // not a thread, and holding the slot would mean a tenant waiting
7619            // on a hundred approvals could start nothing. The figure accrued
7620            // is what *this pass* dispatched, not the run's cumulative spend,
7621            // which would bill the replayed prefix once per resume.
7622            self.settle_quota(run, epoch, live_spend, quota, status.seals())
7623                .await?;
7624
7625            let chain_head = if status.seals() {
7626                let head = self
7627                    .store
7628                    .seal(run, epoch, status.as_str())
7629                    .await
7630                    .map_err(RuntimeError::from_store)?;
7631                self.retire_waits(run).await;
7632                head
7633            } else {
7634                chain_head
7635            };
7636
7637            if let Err(e) = self.store.release_lease(run, epoch).await {
7638                tracing::debug!(
7639                    %run,
7640                    error = %e,
7641                    "could not hand back the lease; it will expire on its own"
7642                );
7643            }
7644
7645            announce(&self.meter, run, &status);
7646
7647            return Ok(RunOutcome {
7648                run_id: run,
7649                status,
7650                chain_head,
7651                consumed,
7652                output,
7653            });
7654        }
7655
7656        Ok(RunOutcome {
7657            run_id: run,
7658            status,
7659            chain_head,
7660            consumed,
7661            // The caller is outside the lattice, so the label is dropped at the
7662            // boundary rather than inside the graph.
7663            output,
7664        })
7665    }
7666}
7667
7668/// The run's result: the terminal step's output.
7669///
7670/// Not "whichever step finished last". That coincides with the terminal step
7671/// only while dispatch is sequential, and stops being well-defined the moment
7672/// two steps run at once. Lowest id wins when a plan has several terminals, so
7673/// the answer is a property of the plan rather than of the schedule.
7674fn run_output(ir: &PlanIR, outputs: &BTreeMap<StepId, Tainted<Value>>) -> Option<Tainted<Value>> {
7675    ir.nodes
7676        .iter()
7677        .filter(|n| n.terminal)
7678        .map(|n| n.id)
7679        .min()
7680        .and_then(|id| outputs.get(&id).cloned())
7681        // A run that stopped before any terminal step still has something to
7682        // report: the furthest output it did produce.
7683        .or_else(|| outputs.iter().next_back().map(|(_, v)| v.clone()))
7684}
7685
7686/// Effects a run announced and never recorded an outcome for — the waits a
7687/// closed run leaves behind.
7688fn unanswered_waits(records: &[Record]) -> Vec<crate::core::EffectKey> {
7689    let settled: BTreeSet<_> = records
7690        .iter()
7691        .filter(|r| {
7692            matches!(
7693                r.kind(),
7694                RecordKind::EffectDone { .. }
7695                    | RecordKind::EffectFailed { .. }
7696                    | RecordKind::EffectReconciled { .. }
7697            )
7698        })
7699        .filter_map(Record::effect_key)
7700        .collect();
7701    records
7702        .iter()
7703        .filter(|r| matches!(r.kind(), RecordKind::EffectStarted { .. }))
7704        .filter_map(Record::effect_key)
7705        .filter(|k| !settled.contains(k))
7706        .collect()
7707}
7708
7709/// Whether `epoch` holds the conclusion that sealed the run — the pass whose
7710/// settlement releases what the run still held.
7711fn concluded_in(records: &[Record], epoch: crate::core::Epoch) -> bool {
7712    records
7713        .iter()
7714        .any(|r| r.body.epoch == epoch && is_sealing_conclusion(r))
7715}
7716
7717fn is_sealing_conclusion(record: &Record) -> bool {
7718    matches!(
7719        record.kind(),
7720        RecordKind::RunConcluded { outcome, .. } if SEALED_OUTCOMES.contains(&outcome.as_str())
7721    )
7722}
7723
7724/// Spend durably recorded by effects written under one live pass epoch.
7725fn spend_recorded_in_epoch(records: &[Record], epoch: crate::core::Epoch) -> Spend {
7726    records
7727        .iter()
7728        .filter(|record| record.body.epoch == epoch)
7729        .fold(Spend::default(), |total, record| {
7730            let spend = match record.kind() {
7731                RecordKind::EffectDone { spend, .. }
7732                | RecordKind::EffectFailed { spend, .. }
7733                | RecordKind::EffectReconciled { spend, .. } => *spend,
7734                _ => Spend::default(),
7735            };
7736            total.plus(spend)
7737        })
7738}
7739
7740/// Whether the plan actually finished.
7741///
7742/// Structural, never self-reported: a workload asserting it is done is not
7743/// evidence, so the runtime checks that every terminal node ran.
7744fn completion(ir: &PlanIR, done: &BTreeSet<StepId>) -> RunStatus {
7745    if ir.is_complete(done) {
7746        return RunStatus::Succeeded;
7747    }
7748    let missing: Vec<String> = ir
7749        .nodes
7750        .iter()
7751        .filter(|n| n.terminal && !done.contains(&n.id))
7752        .map(|n| n.id.to_string())
7753        .collect();
7754    RunStatus::Failed(format!(
7755        "plan did not complete: terminal step(s) {} never ran",
7756        missing.join(", ")
7757    ))
7758}
7759
7760/// Say that a run changed its plan, and what it changed from.
7761fn announce_replan(meter: &super::metrics::Meter, run: RunId, next: &PlanIR, reason: &str) {
7762    tracing::info!(
7763        target: telemetry::REPLANNED,
7764        %run,
7765        from = next.derived_from.map(Digest::to_hex),
7766        version = next.version,
7767        reason_digest = %telemetry::reason_digest(reason),
7768    );
7769    meter.count(metrics::REPLANS, "");
7770}
7771
7772/// Say how a run ended, on the run span and — for the loud ones — as an event.
7773fn announce(meter: &super::metrics::Meter, run: RunId, status: &RunStatus) {
7774    // Counted here and nowhere else. A step that quarantines also fails its run,
7775    // so counting at both levels would report one incident as two — and the
7776    // terminal status is the fact an operator is counting.
7777    meter.count(metrics::RUNS, status.as_str());
7778    match status {
7779        RunStatus::Quarantined(why) => {
7780            tracing::error!(
7781                target: telemetry::QUARANTINED,
7782                %run,
7783                error_type = "quarantined",
7784                reason_digest = %telemetry::reason_digest(why),
7785            );
7786            meter.count(metrics::QUARANTINES, "");
7787        }
7788        // Loud, though a person asked for it. The operator who decided already
7789        // knows; this reaches whoever answers for what the run left standing.
7790        RunStatus::Abandoned { actor, reason } => {
7791            tracing::error!(
7792                target: telemetry::ABANDONED,
7793                %run,
7794                %actor,
7795                reason_digest = %telemetry::reason_digest(reason),
7796            );
7797        }
7798        RunStatus::Exhausted(limit) => {
7799            tracing::warn!(target: telemetry::BUDGET_REFUSED, %run, %limit);
7800        }
7801        _ => {}
7802    }
7803    tracing::Span::current().record(telemetry::OUTCOME, status.as_str());
7804}
7805
7806/// Record a batch's successes, then report the first step that stopped.
7807///
7808/// **Every** success is recorded, including those of siblings dispatched
7809/// alongside the one that stopped. Returning early on the first failure loses
7810/// them — and a sibling that already performed a mutating effect would then
7811/// never be compensated, because `completed` is what the unwind reverses. The
7812/// work happened; the saga has to know about it.
7813///
7814/// When siblings stop for different reasons, **severity wins over ready order**.
7815/// A suspension is the run working; a failure is the run over. Letting one
7816/// sibling's wait mask another's failure would defer the unwind until an event
7817/// that may never arrive — leaving the failed sibling's mutations in place
7818/// indefinitely. Within one severity, ready order decides, so the choice stays a
7819/// property of the plan rather than of the schedule.
7820fn apply(
7821    plan: &PlanIR,
7822    outcomes: Vec<StepOutcome>,
7823    done: &mut BTreeSet<StepId>,
7824    completed: &mut Vec<(StepId, Capability)>,
7825    ran: &mut BTreeMap<StepId, PlanNode>,
7826    outputs: &mut BTreeMap<StepId, Tainted<Value>>,
7827) -> Option<RunStatus> {
7828    let mut stopped: Option<RunStatus> = None;
7829    for (step, status, output) in outcomes {
7830        let RunStatus::Succeeded = status else {
7831            if stopped
7832                .as_ref()
7833                .is_none_or(|held| severity(&status) > severity(held))
7834            {
7835                stopped = Some(status);
7836            }
7837            continue;
7838        };
7839        if let Some(v) = output {
7840            outputs.insert(step, v);
7841        }
7842        done.insert(step);
7843        if let Some(node) = plan.node(step) {
7844            completed.push((step, node.capability.clone()));
7845            ran.insert(step, node.clone());
7846        }
7847    }
7848    stopped
7849}
7850
7851/// How much a stop reason dominates a competing one.
7852///
7853/// `Quarantined` is highest because it is the only one that must *not* unwind:
7854/// something is undecidable, and compensating around it can make the damage
7855/// worse. `Suspended` is lowest because it is not a stop at all — the run is
7856/// healthy and waiting.
7857fn severity(status: &RunStatus) -> u8 {
7858    match status {
7859        RunStatus::Quarantined(_) => 3,
7860        // A stop ranks with a failure, not above it: both end the run, both
7861        // unwind, and when they arrive together the run is over either way.
7862        // Abandonment is not produced by a step and never competes here; it is
7863        // named so a new stop reason cannot be given a rank by omission.
7864        RunStatus::Cancelled { .. } | RunStatus::Abandoned { .. } | RunStatus::Failed(_) => 2,
7865        // A withdrawal ranks with exhaustion for the same reason: both pause a
7866        // healthy run, neither unwinds, and a sibling that actually failed has
7867        // decided more about the run than either.
7868        RunStatus::Exhausted(_) | RunStatus::Withheld { .. } => 1,
7869        // A replan request is the weakest signal in a batch: a sibling that
7870        // failed outright has already decided the run, and re-planning around a
7871        // failure is not what the requesting step was asking for.
7872        // Neither is reachable here at all — a sweep, a crossing and a lift are
7873        // sealed at birth with no steps to compete — and both are named for the reason
7874        // abandonment is: a rank given by omission is a decision nobody made.
7875        RunStatus::Replanning(_)
7876        | RunStatus::Suspended(_)
7877        | RunStatus::Succeeded
7878        | RunStatus::Swept
7879        // Unreachable for the same reason, and named rather than left to a
7880        // wildcard: an observed session has no steps of this plane's to
7881        // compete, because this plane ran none of it.
7882        | RunStatus::Observed
7883        | RunStatus::BrokeGlass { .. }
7884        | RunStatus::HaltLifted { .. }
7885        | RunStatus::HoldReleased { .. } => 0,
7886    }
7887}
7888
7889/// Gather a dispatched batch back into ready order.
7890///
7891/// Not completion order. `completed` is what the unwind reverses, and a saga
7892/// whose compensation order depended on which future finished first would undo
7893/// a plan differently on every run.
7894type Dispatched = Result<(StepId, RunStatus, Option<Tainted<Value>>, StepCursor), RuntimeError>;
7895
7896/// One step's result, once its history has been handed back.
7897type StepOutcome = (StepId, RunStatus, Option<Tainted<Value>>);
7898
7899fn collect(
7900    dispatched: Vec<Dispatched>,
7901    ready: &[StepId],
7902    cursor: &mut ReplayCursor,
7903) -> Result<Vec<StepOutcome>, RuntimeError> {
7904    let mut outcomes = Vec::with_capacity(dispatched.len());
7905    for result in dispatched {
7906        let (step, status, out, slice) = result?;
7907        cursor.restore(step, Phase::Forward, slice);
7908        outcomes.push((step, status, out));
7909    }
7910    // The key is an `Option`, and `None` would sort *first* — so the invariant
7911    // that makes this total is worth naming rather than trusting: every
7912    // dispatched step came from `admit_ready(&ready, ..)`, which returns a
7913    // subset of `ready`. A future that dispatched a step outside the ready set
7914    // would silently re-order this batch rather than fail, and the order is what
7915    // an unwind reverses.
7916    outcomes.sort_by_key(|(step, _, _)| ready.iter().position(|r| r == step));
7917    Ok(outcomes)
7918}
7919
7920/// What one ready set's dispatch needs.
7921struct Batch<'a> {
7922    agent: &'a str,
7923    identity: Option<&'a crate::core::Delegation>,
7924    subjects: &'a BTreeSet<crate::core::SubjectRef>,
7925    run: RunId,
7926    epoch: u64,
7927    ir: &'a PlanIR,
7928    mode: Pass,
7929    case: &'a Option<CaseContext>,
7930    ledger: &'a Arc<std::sync::Mutex<Ledger>>,
7931    writing: bool,
7932    stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
7933    input: &'a Tainted<Value>,
7934    outputs: &'a BTreeMap<StepId, Tainted<Value>>,
7935    /// Steps whose `StepStarted` the journal already holds. Empty on a live
7936    /// run.
7937    started: &'a BTreeSet<StepId>,
7938    /// Steps whose `StepFinished` the journal already holds. Empty on a live
7939    /// run.
7940    finished: &'a BTreeSet<StepId>,
7941    /// Group records the journal already holds, by writing step and phase.
7942    /// Empty on a live run.
7943    recorded_groups: &'a BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
7944    /// How many of this ready set may be in flight at once.
7945    parallelism: usize,
7946    /// Re-running steps only to abandon groups a pause left open.
7947    abandoning: bool,
7948}
7949
7950/// A standing withdrawal, as the step boundary meets it.
7951///
7952/// Carries the operator from the halt rather than only its reason: the
7953/// withholding record is the one durable trace of this act on the run it
7954/// stopped, and a pause nobody is named on cannot be asked about afterwards.
7955pub(crate) struct Withdrawal {
7956    pub(crate) subject: String,
7957    pub(crate) reason: String,
7958    pub(crate) by: crate::core::Operator,
7959}
7960
7961/// What the journal proves about a run an unwind is deciding over.
7962struct UnwindEvidence {
7963    /// Steps that announced a mutating forward effect.
7964    mutated: BTreeSet<StepId>,
7965    /// The skill each step that started ran as, from its `StepStarted`.
7966    started: BTreeMap<StepId, String>,
7967    /// Steps whose compensation is already on the record.
7968    undone: BTreeSet<StepId>,
7969    /// Group records already on the journal, by writing step and phase.
7970    recorded_groups: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
7971    /// Steps whose last forward ending is not a success: the failing step,
7972    /// which stays resumable, as opposed to a sibling the failure interrupted.
7973    failed: BTreeSet<StepId>,
7974}
7975
7976/// What producing a successor plan needs.
7977struct Replan<'a> {
7978    run: RunId,
7979    current: &'a PlanIR,
7980    /// The node each completed step ran as.
7981    ran: &'a BTreeMap<StepId, PlanNode>,
7982    reason: &'a str,
7983    already_replanned: u32,
7984    max_replans: Option<u32>,
7985    /// The successor this run produced when it first ran, if this is a replay.
7986    recorded: Option<&'a PlanIR>,
7987}
7988
7989/// The first untrusted value in working memory, if any.
7990///
7991/// Returns the source so the refusal can name it: "replanning refused" without
7992/// saying *what* made it unsafe sends an operator looking through the whole run.
7993fn untrusted_in(outputs: &BTreeMap<StepId, Tainted<Value>>) -> Option<String> {
7994    outputs.values().find_map(|v| {
7995        let label = v.label();
7996        label.is_untrusted().then(|| {
7997            label
7998                .provenance
7999                .first()
8000                .map_or_else(|| "an untrusted source".to_owned(), ToString::to_string)
8001        })
8002    })
8003}
8004
8005/// Where a refusal is recorded, when one is.
8006struct Journalling<'a> {
8007    run: RunId,
8008    epoch: u64,
8009    writing: bool,
8010    stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8011}
8012
8013/// What unwinding a run needs.
8014struct Unwind<'a> {
8015    identity: Option<&'a crate::core::Delegation>,
8016    subjects: &'a BTreeSet<crate::core::SubjectRef>,
8017    run: RunId,
8018    epoch: u64,
8019    ir: &'a PlanIR,
8020    mode: Pass,
8021    case: Option<CaseContext>,
8022    ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8023    writing: bool,
8024    stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8025    agent: &'a str,
8026}
8027
8028/// What one step's execution needs.
8029// Each flag is an independent fact about this dispatch; no combination is
8030// invalid, so an enum over them would satisfy the lint at the reader's expense.
8031#[allow(clippy::struct_excessive_bools)]
8032struct StepRun<'a> {
8033    identity: Option<&'a crate::core::Delegation>,
8034    subjects: &'a BTreeSet<crate::core::SubjectRef>,
8035    run: RunId,
8036    epoch: u64,
8037    node: &'a PlanNode,
8038    phase: Phase,
8039    mode: Pass,
8040    case: Option<CaseContext>,
8041    ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8042    writing: bool,
8043    stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8044    agent: &'a str,
8045    /// Whether the journal already announced this step. Always `false` live.
8046    already_started: bool,
8047    /// Whether the journal already records this step finishing. Always
8048    /// `false` live.
8049    already_finished: bool,
8050    /// Group records this step and phase already wrote. Always empty live.
8051    recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
8052    /// Whether this step is re-run only to abandon a group a pause left
8053    /// open, because the run was cancelled while paused.
8054    abandoning: bool,
8055}
8056
8057/// The half of a `StepCtx`'s frame that belongs to the **step**.
8058///
8059/// A struct rather than a dozen positional arguments, for the reason
8060/// [`StepRun`] is one: the other half comes from the plane, and a caller who
8061/// transposed `phase` and `mode` positionally would compile.
8062struct StepFrame {
8063    run: RunId,
8064    epoch: crate::core::Epoch,
8065    step: StepId,
8066    phase: Phase,
8067    mode: Pass,
8068    case: Option<CaseContext>,
8069    ledger: Arc<std::sync::Mutex<Ledger>>,
8070    identity: Option<crate::core::Delegation>,
8071    subjects: BTreeSet<crate::core::SubjectRef>,
8072    agent: String,
8073    #[cfg(feature = "manifest")]
8074    manifest: Option<Arc<crate::manifest::Manifest>>,
8075    recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
8076}
8077
8078/// Where the recorded run was refused by a *step* limit, if the refusal still
8079/// stands.
8080///
8081/// A step-level refusal has no effect key, so it cannot ride the replay cursor
8082/// the way an effect's does; it is lifted from the records instead.
8083///
8084/// **The last word wins.** An exhausted run may be resumed under a raised
8085/// ceiling; when the resume re-admits the refused step it journals a
8086/// `BudgetReadmitted` beside the old refusal, and from then on the refusal is
8087/// history that was *superseded*, not a verdict to re-serve. Without this, a
8088/// strict replay of the resumed history would stop at the old refusal and
8089/// report `Exhausted` about a run whose own later records show it finishing —
8090/// the refusal followed by the continuation would read as divergence.
8091fn recorded_step_refusal(records: &[Record]) -> Option<(StepId, String, String)> {
8092    let mut standing: Option<(StepId, String, String)> = None;
8093    for r in records {
8094        match r.kind() {
8095            RecordKind::BudgetRefused { limit, used } if r.effect_key().is_none() => {
8096                if let Some(step) = r.body.step {
8097                    standing = Some((step, limit.clone(), used.clone()));
8098                }
8099            }
8100            RecordKind::BudgetReadmitted { .. }
8101                if standing
8102                    .as_ref()
8103                    .is_some_and(|(s, _, _)| Some(*s) == r.body.step) =>
8104            {
8105                standing = None;
8106            }
8107            _ => {}
8108        }
8109    }
8110    standing
8111}
8112
8113/// The subject of a withholding the records left standing, if one stands.
8114///
8115/// **The last word wins**, exactly as it does for a step refusal: a withheld run
8116/// resumes once somebody lifts the halt, and the resume journals an
8117/// `AuthorityRestored` beside the withholding. From then on the withholding is
8118/// history that was *superseded* rather than a verdict to re-serve — without
8119/// this, a resumed run would be treated as still withheld and would conclude
8120/// that way forever, with each resume adding nothing.
8121fn standing_withholding(records: &[Record]) -> Option<String> {
8122    let mut standing: Option<String> = None;
8123    for r in records {
8124        match r.kind() {
8125            RecordKind::AuthorityWithheld { subject, .. } => standing = Some(subject.clone()),
8126            RecordKind::AuthorityRestored { .. } => standing = None,
8127            _ => {}
8128        }
8129    }
8130    standing
8131}
8132
8133/// The run's most recent recorded conclusion, if it has one.
8134///
8135/// `None` for a run that has only ever suspended — a suspension is not a
8136/// conclusion, and `conclude` writes no `RunSealed` for it.
8137fn recorded_conclusion(records: &[Record]) -> Option<String> {
8138    records.iter().rev().find_map(|r| match r.kind() {
8139        RecordKind::RunConcluded { outcome, .. } => Some(outcome.clone()),
8140        _ => None,
8141    })
8142}
8143
8144/// Whether this pass writes a step-level record: always live, never under
8145/// strict verification, and on a resume only when `new_fact` — the record is
8146/// not already in the journal, or the step did new work.
8147const fn writes_step_record(mode: Pass, new_fact: bool) -> bool {
8148    match mode {
8149        Pass::Live => true,
8150        Pass::Resume => new_fact,
8151        Pass::Strict => false,
8152    }
8153}
8154
8155/// The steps whose start is already on the record.
8156///
8157/// Read back so a resume neither re-announces a step history already announced
8158/// nor forgets to announce one it is starting for the first time.
8159fn recorded_started_steps(records: &[Record]) -> BTreeSet<StepId> {
8160    records
8161        .iter()
8162        .filter_map(|r| match r.kind() {
8163            RecordKind::StepStarted { .. } => r.body.step,
8164            _ => None,
8165        })
8166        .collect()
8167}
8168
8169/// The skill each step started as, from its forward `StepStarted`.
8170///
8171/// The capability that actually ran, which after a replan the plan in force
8172/// may no longer name at that id — or may name as other work.
8173fn started_skills(records: &[Record]) -> BTreeMap<StepId, String> {
8174    records
8175        .iter()
8176        .filter(|r| r.body.phase.is_forward())
8177        .filter_map(|r| match (r.kind(), r.body.step) {
8178            (RecordKind::StepStarted { skill }, Some(step)) => Some((step, skill.clone())),
8179            _ => None,
8180        })
8181        .collect()
8182}
8183
8184/// The steps whose ending is already on the record.
8185///
8186/// Read back so a resume records a `StepFinished` exactly for the steps that
8187/// reach an ending the journal does not yet hold: a fully replayed completed
8188/// step keeps its one record, and a suspended step that finally finishes —
8189/// even purely from replayed history, its awaited event having been recorded
8190/// by the delivery — gets the one it never had.
8191fn recorded_finished_steps(records: &[Record]) -> BTreeSet<StepId> {
8192    records
8193        .iter()
8194        .filter_map(|r| match r.kind() {
8195            RecordKind::StepFinished { .. } => r.body.step,
8196            _ => None,
8197        })
8198        .collect()
8199}
8200
8201/// The case the run is bound to, read from its `CaseBound` record.
8202fn recorded_case_id(records: &[Record]) -> Option<crate::core::CaseId> {
8203    records.iter().find_map(|r| match r.kind() {
8204        RecordKind::CaseBound { .. } => r.body.case,
8205        _ => None,
8206    })
8207}
8208
8209/// The steps whose latest forward ending on the record is a success.
8210fn recorded_succeeded_steps(records: &[Record]) -> BTreeSet<StepId> {
8211    let mut succeeded = BTreeSet::new();
8212    for r in records.iter().filter(|r| r.body.phase.is_forward()) {
8213        if let (RecordKind::StepFinished { outcome }, Some(step)) = (r.kind(), r.body.step) {
8214            if outcome == RunStatus::Succeeded.as_str() {
8215                succeeded.insert(step);
8216            } else {
8217                succeeded.remove(&step);
8218            }
8219        }
8220    }
8221    succeeded
8222}
8223
8224/// The group records already on the journal, by writing step and phase.
8225///
8226/// Read back for the same reason the step sets are: group records are not
8227/// effects, so the cursor cannot dedup them, and a resumed step that re-opens
8228/// its group or reaches its end at the frontier — cursor exhausted, writes
8229/// enabled — would otherwise write the same record a second time.
8230fn recorded_groups(
8231    records: &[Record],
8232) -> BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup> {
8233    let mut recorded: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup> =
8234        BTreeMap::new();
8235    for r in records {
8236        let (group, opened) = match r.kind() {
8237            RecordKind::GroupOpened { group, .. } => (group, true),
8238            RecordKind::GroupSettled { group, .. } => (group, false),
8239            _ => continue,
8240        };
8241        let Some(step) = r.body.step else { continue };
8242        let entry = recorded
8243            .entry((step, r.body.phase, group.clone()))
8244            .or_default();
8245        if opened {
8246            entry.opened += 1;
8247        } else {
8248            entry.settled += 1;
8249        }
8250    }
8251    recorded
8252}
8253
8254/// Every capability an agent advertises is provided by one of **its own**
8255/// skills.
8256///
8257/// An agent advertising a capability none of its skills provide is a card that
8258/// lies, and the caller who believed it finds out at dispatch — in production —
8259/// rather than here at startup.
8260///
8261/// This checked a plane-wide map, and the difference is not pedantry. A skill
8262/// registered on the *builder* rather than on the agent — `.agent(Agent::new(&m))`
8263/// followed by `.skill(s)` — satisfied a plane-wide check while being
8264/// **ungoverned**: `governed_by` is keyed from the agent's own skills, so that
8265/// skill gets no manifest. It runs under the plane's default budget instead of
8266/// the declared one, and `StepCtx::gate` never refuses a model or tool the file
8267/// did not list, because there is no file.
8268///
8269/// The plane built cleanly and the assertion's own message said "its skills",
8270/// so the only signal was a `None` from `cx.manifest()` that a skill has no
8271/// reason to check. That is a declaration reading as a control while governing
8272/// nothing, which is the one shape this codebase refuses everywhere.
8273#[cfg(feature = "manifest")]
8274fn check_declaration_matches_skills(
8275    m: &crate::manifest::Manifest,
8276    mine: &HashSet<Capability>,
8277) -> Result<(), BuildError> {
8278    let missing: Vec<String> = m
8279        .spec
8280        .capabilities
8281        .provides
8282        .iter()
8283        .filter(|c| !mine.contains(&Capability::new(c.as_str())))
8284        .cloned()
8285        .collect();
8286    if !missing.is_empty() {
8287        return Err(BuildError::AdvertisesWhatItCannotProvide {
8288            agent: m.metadata.name.clone(),
8289            missing,
8290        });
8291    }
8292
8293    // And the other direction, which is the one that leaves no trace. A skill
8294    // registered under a manifest is *governed* by it — the manifest's budget,
8295    // model grants, egress ceiling and policy identity all apply — so a
8296    // capability the skill answers and the declaration never names is a
8297    // reviewed surface with an unreviewed door in it. The declaration is the
8298    // artifact that gets read, digested and pinned, and the A2A card is built
8299    // from it, so the extra capability is served and advertised nowhere.
8300    //
8301    // It is the same argument the manifest already makes about prompts: a
8302    // system prompt composed in the deployer's code has no version, changes in
8303    // a deploy, and nothing connects the change to the runs it affected. A
8304    // capability added in code does the same to the agent's surface.
8305    let undeclared: Vec<String> = {
8306        let declared: HashSet<Capability> = m
8307            .spec
8308            .capabilities
8309            .provides
8310            .iter()
8311            .map(|c| Capability::new(c.as_str()))
8312            .collect();
8313        let mut extra: Vec<String> = mine
8314            .iter()
8315            .filter(|c| !declared.contains(c))
8316            .map(|c| c.0.clone())
8317            .collect();
8318        // Deterministic, so the message does not depend on hash order.
8319        extra.sort();
8320        extra
8321    };
8322    if !undeclared.is_empty() {
8323        return Err(BuildError::ProvidesWhatItDoesNotAdvertise {
8324            agent: m.metadata.name.clone(),
8325            undeclared,
8326        });
8327    }
8328    Ok(())
8329}
8330
8331/// Add one skill to the plane's two lookup tables, refusing a collision.
8332///
8333/// Both maps are plane-wide, and a bare `insert` would take a second
8334/// registration silently. That was tolerable when a plane was one agent and is
8335/// not now: dispatch resolves a capability to a skill *and to the manifest
8336/// governing it*, so a silent overwrite does not merely shadow the loser — it
8337/// moves work the loser still advertises out from under the loser's budget,
8338/// model grants and egress ceiling. Nothing in the journal would show it,
8339/// because the winner looks like the only claimant that ever existed.
8340///
8341/// Returns the skill's name, which is the key governance is recorded under.
8342///
8343/// # Errors
8344///
8345/// If another skill already holds this name, or another skill already claims one
8346/// of its capabilities.
8347fn register_skill(
8348    skill: Arc<dyn Skill>,
8349    caps: &mut HashMap<Capability, String>,
8350    skills: &mut HashMap<String, Arc<dyn Skill>>,
8351) -> Result<String, BuildError> {
8352    let d = skill.descriptor();
8353    if let Some(existing) = skills.get(&d.name)
8354        // Registering the *same* `Arc` twice is idempotent rather than a
8355        // mistake; two distinct skills under one name is the collision.
8356        && !Arc::ptr_eq(existing, &skill)
8357    {
8358        return Err(BuildError::DuplicateSkillName { name: d.name });
8359    }
8360    // `capabilities()`, not the raw field: a descriptor that declared nothing
8361    // answers its own name, and registering that here is what makes
8362    // `run("greet")` and an `agent/greet` grant resolve without a hello-world
8363    // program inventing a second name for its one skill.
8364    for cap in d.capabilities() {
8365        if let Some(first) = caps.get(&cap)
8366            && first != &d.name
8367        {
8368            return Err(BuildError::CapabilityClaimedTwice {
8369                capability: cap.0,
8370                first: first.clone(),
8371                second: d.name,
8372            });
8373        }
8374        caps.insert(cap, d.name.clone());
8375    }
8376    skills.insert(d.name.clone(), skill);
8377    Ok(d.name)
8378}
8379
8380/// The capability a bare `run(target, ..)` admits: the first declared, or the
8381/// skill's own name — [`SkillDescriptor::capabilities`] is never empty.
8382fn first_capability(descriptor: &SkillDescriptor) -> Capability {
8383    descriptor
8384        .capabilities()
8385        .into_iter()
8386        .next()
8387        .unwrap_or_else(|| Capability::new(descriptor.name.clone()))
8388}
8389
8390/// The business keys a case is identified by, for the binding record.
8391///
8392/// Read from the case rather than taken from the request, because a run joining
8393/// an existing case is admitted with whichever key its message carried while the
8394/// case may be identified by several — and a manifest binding naming any of them
8395/// must resolve.
8396///
8397/// Falls back to the admitted keys if the case cannot be read back. That is not
8398/// defensive padding: `correlate_or_open` has already committed the binding, so
8399/// a failure here is a read that lost a race with nothing, and the keys the run
8400/// was admitted with are a true subset of the case's. Recording them is strictly
8401/// better than recording none, and *nothing* is what would silently make a
8402/// binding unresolvable for a run whose case is perfectly well identified.
8403async fn case_correlation(
8404    cases: &dyn CaseStore,
8405    case_id: crate::core::CaseId,
8406    admitted: &[CorrelationKey],
8407) -> Result<Vec<CorrelationKey>, RuntimeError> {
8408    let stored = cases
8409        .case(case_id)
8410        .await
8411        .map_err(RuntimeError::from_store)?
8412        .map(|case| case.correlation)
8413        .unwrap_or_default();
8414    let mut keys = if stored.is_empty() {
8415        admitted.to_vec()
8416    } else {
8417        stored
8418    };
8419    // Canonical order, so the record is byte-stable for a given set and two
8420    // stores that answer in different orders produce the same journal.
8421    keys.sort();
8422    keys.dedup();
8423    Ok(keys)
8424}
8425
8426/// A lease owner that no other process will accidentally share.
8427///
8428/// The previous default was the constant `"agentplane"`, which every replica and
8429/// every restart used. Two consequences, both silent:
8430///
8431/// * Two replicas each saw the other's lease as their own and renewed it
8432///   without bumping the epoch — two writers on one run, which is the exact
8433///   situation fencing exists to make impossible.
8434/// * A process restarting after a crash "renewed" the dead process's lease
8435///   instead of waiting for expiry and fencing it, so a zombie still holding a
8436///   socket could keep writing under the same epoch as its replacement.
8437///
8438/// A per-process random identity turns both into the correct behaviour: a
8439/// different owner cannot renew, so it waits for expiry and takes over with
8440/// `epoch + 1`.
8441///
8442/// Not derived from a hostname or PID: containers reuse both. Randomness is the
8443/// property that matters; readability is what the `owner` override is for, and a
8444/// deployment with a real instance identity — a pod name — should pass it.
8445fn default_owner() -> String {
8446    use std::collections::hash_map::RandomState;
8447    use std::hash::{BuildHasher, Hasher};
8448    use std::sync::atomic::{AtomicU64, Ordering};
8449
8450    // OS entropy, and deliberately *not* the clock: this crate forbids reading
8451    // the wall clock outside a journaled effect, and rightly — a lease owner is
8452    // a poor reason to make an exception to a rule that keeps replay honest.
8453    // `RandomState` is seeded by the operating system, so two processes differ
8454    // even where a container has reused a PID.
8455    static SEED: std::sync::OnceLock<u64> = std::sync::OnceLock::new();
8456    // A counter beside it, so two runtimes built in one process — which tests do
8457    // constantly — never alias each other either.
8458    static SEQ: AtomicU64 = AtomicU64::new(0);
8459
8460    let seed = *SEED.get_or_init(|| RandomState::new().build_hasher().finish());
8461    let n = SEQ.fetch_add(1, Ordering::Relaxed);
8462    format!("agentplane-{seed:016x}-{n}")
8463}
8464
8465/// Resolve a run's data-subject bindings against its input, before the run
8466/// is bound to a case.
8467///
8468/// Declared bindings first, in declaration order, then the subjects the
8469/// embedder named; each entry's position is the index every reference to it
8470/// names. A binding that resolves to nothing refuses the run rather than
8471/// attributing its data to nobody. A `$case` binding is refused when the run
8472/// asked for no case, and otherwise left for [`fill_case_subjects`].
8473fn bind_subjects(
8474    declared: &[crate::journal::SubjectBinding],
8475    named: Vec<String>,
8476    input: &Tainted<Value>,
8477    in_case: bool,
8478) -> Result<Vec<crate::journal::BoundSubject>, RuntimeError> {
8479    use crate::journal::SubjectBinding;
8480    let refuse = |binding: &SubjectBinding, reason: &str| RuntimeError::SubjectUnbound {
8481        binding: binding.to_string(),
8482        reason: reason.to_owned(),
8483    };
8484    let mut resolved = Vec::new();
8485    let named = named
8486        .into_iter()
8487        .map(|subject| (SubjectBinding::Named, Some(subject)));
8488    for (position, (binding, given)) in declared
8489        .iter()
8490        .cloned()
8491        .map(|binding| (binding, None))
8492        .chain(named)
8493        .enumerate()
8494    {
8495        let (subject, trusted) = match (&binding, given) {
8496            (_, Some(subject)) => (subject, true),
8497            (SubjectBinding::Input { pointer }, None) => {
8498                let Some(selected) = input.project_pointer(pointer) else {
8499                    return Err(refuse(&binding, "it selects nothing in the run's input"));
8500                };
8501                let subject = match selected.peek() {
8502                    Value::String(text) => text.clone(),
8503                    Value::Number(number) => number.to_string(),
8504                    _ => {
8505                        return Err(refuse(&binding, "it selects neither a string nor a number"));
8506                    }
8507                };
8508                (
8509                    subject,
8510                    selected.label().trust == crate::core::Trust::Trusted,
8511                )
8512            }
8513            (SubjectBinding::Case, None) => {
8514                if !in_case {
8515                    return Err(refuse(&binding, "the run belongs to no case"));
8516                }
8517                (CASE_PENDING.to_owned(), true)
8518            }
8519            (SubjectBinding::Named, None) => {
8520                return Err(refuse(&binding, "no subject was named"));
8521            }
8522        };
8523        if subject.trim().is_empty() {
8524            return Err(refuse(&binding, "it resolves to an empty subject"));
8525        }
8526        let index = u16::try_from(position)
8527            .map_err(|_| refuse(&binding, "a run binds at most 65536 subjects"))?;
8528        resolved.push(crate::journal::BoundSubject {
8529            index,
8530            binding,
8531            trusted,
8532            subject,
8533        });
8534    }
8535    Ok(resolved)
8536}
8537
8538/// What a `$case` binding holds between [`bind_subjects`] and
8539/// [`fill_case_subjects`]: never journaled.
8540const CASE_PENDING: &str = "$case";
8541
8542/// Give each `$case` binding the id of the case the run was bound to.
8543fn fill_case_subjects(
8544    mut bound: Vec<crate::journal::BoundSubject>,
8545    case: Option<&CaseContext>,
8546) -> Vec<crate::journal::BoundSubject> {
8547    if let Some(case) = case {
8548        for entry in &mut bound {
8549            if entry.binding == crate::journal::SubjectBinding::Case {
8550                entry.subject = case.case_id.to_string();
8551            }
8552        }
8553    }
8554    bound
8555}
8556
8557/// The references a run's `DataSubjectBound` record makes available to every
8558/// value it reads.
8559fn subject_refs(
8560    run: RunId,
8561    bound: &[crate::journal::BoundSubject],
8562) -> BTreeSet<crate::core::SubjectRef> {
8563    bound
8564        .iter()
8565        .map(|b| crate::core::SubjectRef {
8566            run,
8567            index: b.index,
8568        })
8569        .collect()
8570}
8571
8572/// The references a recorded run's `DataSubjectBound` names, read back so a
8573/// resume attributes exactly as the live run did.
8574fn recorded_subjects(run: RunId, records: &[Record]) -> BTreeSet<crate::core::SubjectRef> {
8575    records
8576        .iter()
8577        .find_map(|r| match r.kind() {
8578            RecordKind::DataSubjectBound { bindings } => Some(subject_refs(run, bindings)),
8579            _ => None,
8580        })
8581        .unwrap_or_default()
8582}
8583
8584/// Refuse a history this build cannot re-derive.
8585///
8586/// Before recomputing anything, because every effect key a replay derives comes
8587/// out of the canonicalizer: a run written under another rule recomputes
8588/// different keys and would be quarantined as *non-determinism* — the most
8589/// serious conclusion this runtime reaches, reported for a healthy run because
8590/// the rule moved underneath it.
8591///
8592/// The chain itself is fine and always was: it hashes the bytes it stored rather
8593/// than re-canonicalizing them. What moved is everything *derived*, which is
8594/// exactly the class replay compares.
8595fn ensure_replayable_canon(records: &[Record]) -> Result<(), RuntimeError> {
8596    if let Some(recorded) = records.iter().find_map(recorded_canon)
8597        && recorded != crate::core::canon::VERSION
8598    {
8599        return Err(RuntimeError::CanonicalizationChanged {
8600            recorded,
8601            implemented: crate::core::canon::VERSION,
8602        });
8603    }
8604    Ok(())
8605}
8606
8607/// Which canonicalization rule wrote this run's derived digests.
8608fn recorded_canon(r: &Record) -> Option<u16> {
8609    match r.kind() {
8610        RecordKind::RunAdmitted { canon, .. } => Some(*canon),
8611        _ => None,
8612    }
8613}
8614
8615/// The principal a run was admitted as.
8616///
8617/// Read back rather than recomputed, for the same reason the plan is: the
8618/// principal a run was authorized as is a fact *about that run*, and deriving it
8619/// again from a plan that may since have been edited would silently re-attribute
8620/// history.
8621/// The chain a recorded run acted under, re-checked structurally on the way
8622/// back — never re-verified as a credential, for the reason `core::identity`
8623/// gives. `None` for a run admitted without one.
8624fn recorded_chain(records: &[Record]) -> Result<Option<crate::core::Delegation>, RuntimeError> {
8625    records
8626        .iter()
8627        .find_map(|r| match r.kind() {
8628            RecordKind::IdentityBound { chain } => Some(chain.clone()),
8629            _ => None,
8630        })
8631        .map(crate::core::Delegation::rehydrate)
8632        .transpose()
8633        .map_err(RuntimeError::Delegation)
8634}
8635
8636/// Whether a resume refused with `error` would be refused the same way on
8637/// every attempt: the journal it read is what refuses, not the store, a lease,
8638/// a quota or this plane's configuration.
8639///
8640/// Named rather than negated: a failure not listed here is retried, so a
8641/// transient one added later costs a retry loop rather than a quarantine.
8642/// `PlanContract`, `NoProvider` and `NoCaseStore` are not listed because they
8643/// are about this plane — a quota store, an agent or a case store it lacks —
8644/// and another plane may settle what this one cannot.
8645const fn recurs_on_every_resume(error: &RuntimeError) -> bool {
8646    matches!(
8647        error,
8648        RuntimeError::Encoding(_)
8649            | RuntimeError::ChainBroken { .. }
8650            | RuntimeError::CanonicalizationChanged { .. }
8651            | RuntimeError::Delegation(_)
8652    )
8653}
8654
8655fn recorded_agent(records: &[Record]) -> String {
8656    records
8657        .iter()
8658        .find_map(|r| match r.kind() {
8659            RecordKind::RunAdmitted { capability, .. } => Some(capability.clone()),
8660            _ => None,
8661        })
8662        .unwrap_or_default()
8663}
8664
8665/// Build a step's input from its declared argument sources.
8666///
8667/// Labels join across sources, so a step reading anything untrusted produces an
8668/// untrusted input without the plan author having to say so.
8669fn assemble(
8670    node: &PlanNode,
8671    run_input: &Tainted<Value>,
8672    outputs: &BTreeMap<StepId, Tainted<Value>>,
8673) -> Result<Tainted<Value>, RuntimeError> {
8674    // The common case — a single argument — passes the value through rather than
8675    // wrapping it in a one-key object, so simple plans stay legible.
8676    if node.args.len() == 1
8677        && let Some((_, only)) = node.args.iter().next()
8678    {
8679        return resolve_arg(node, only, run_input, outputs);
8680    }
8681
8682    let mut fields = Vec::with_capacity(node.args.len());
8683    for (name, source) in &node.args {
8684        let v = resolve_arg(node, source, run_input, outputs)?;
8685        fields.push((name.clone(), v));
8686    }
8687    Ok(Tainted::object(fields))
8688}
8689
8690fn resolve_arg(
8691    node: &PlanNode,
8692    source: &ArgSource,
8693    run_input: &Tainted<Value>,
8694    outputs: &BTreeMap<StepId, Tainted<Value>>,
8695) -> Result<Tainted<Value>, RuntimeError> {
8696    Ok(match source {
8697        // Projected as a step's output is: a field keeps the label it was
8698        // admitted with, and one admitted without its own inherits the whole
8699        // value's — the parts of an untrusted document are untrusted.
8700        ArgSource::RunInput { field } => project(run_input, field.as_deref()),
8701        ArgSource::Const { value } => Tainted::trusted(value.clone()),
8702        ArgSource::Node { step, field } => {
8703            // The contract already proved this is upstream, so a miss here means
8704            // the scheduler dispatched out of order — a bug worth naming rather
8705            // than papering over with a null.
8706            let upstream = outputs.get(step).ok_or_else(|| {
8707                RuntimeError::PlanContract(format!(
8708                    "step {} read step {step}, which has not produced a value",
8709                    node.id
8710                ))
8711            })?;
8712            project(upstream, field.as_deref())
8713        }
8714    })
8715}
8716
8717/// A value, or one of its top-level fields with its descendants' labels; an
8718/// absent field is a null under the whole value's label.
8719fn project(value: &Tainted<Value>, field: Option<&str>) -> Tainted<Value> {
8720    match field {
8721        Some(field) => value
8722            .project_field(field)
8723            .unwrap_or_else(|| Tainted::with_label(Value::Null, value.label().clone())),
8724        None => value.clone(),
8725    }
8726}
8727
8728/// A step that read a blob rewritten under its address ends unreproducible,
8729/// whatever the skill returned: its history cannot be reproduced, whether the
8730/// skill propagated the error, retried it or swallowed it.
8731fn moved_blob_wins<T>(
8732    cx: &StepCtx<'_>,
8733    result: Result<T, crate::core::SkillError>,
8734) -> Result<T, crate::core::SkillError> {
8735    match cx.moved_blob() {
8736        Some(what) => Err(crate::core::SkillError::Step(
8737            crate::core::StepError::Unreproducible {
8738                what: what.to_owned(),
8739                detail: "a pinned read returned other bytes under its address".to_owned(),
8740            },
8741        )),
8742        None => result,
8743    }
8744}
8745
8746/// Settle a group the skill left open, because `Drop` cannot.
8747///
8748/// A skill that fails with `?` never reaches `commit` or `abort`, so the handle
8749/// is dropped with members standing. Reversing them is async and `Drop` is not,
8750/// which is why the group lives on the context and the executor finishes what
8751/// the handle abandoned — the same relationship the executor already has with a
8752/// step's compensation.
8753///
8754/// Three situations, and they are not the same:
8755///
8756/// * **suspended** — the step has not ended. Its frame is persisted and it will
8757///   re-run from the top, rebuilding the group from the journal as it replays
8758///   the members. Reversing here would undo a run that is merely waiting.
8759/// * **failed** — abort, unless the failure leaves the world in doubt. Doubt is
8760///   the one condition under which nothing may be reversed.
8761/// * **succeeded with a group still open** — an author bug, and the safe
8762///   reading is that the group was never meant to take. It is reversed and the
8763///   step fails loudly, because a group that commits by being forgotten is
8764///   worse than one that does not commit at all.
8765async fn settle_abandoned_group(
8766    cx: &mut StepCtx<'_>,
8767    result: Result<Outcome, crate::core::SkillError>,
8768) -> Result<Outcome, crate::core::SkillError> {
8769    use crate::core::{SkillError, StepError};
8770
8771    let Some(name) = cx.open_group().map(|g| g.name.clone()) else {
8772        return result;
8773    };
8774    // A pause leaves the group open for the resumed step to settle — unless
8775    // the run was cancelled while paused and this pass exists to abandon it.
8776    if !cx.abandoning
8777        && matches!(&result, Err(SkillError::Step(e)) if crate::runtime::group::is_pause(e))
8778    {
8779        return result;
8780    }
8781
8782    // A member whose failure may have reached the world travels no further.
8783    // Reversing around a call that may have — or did — happen leaves the world
8784    // holding a write no `Aborted` settlement can honestly claim to have undone.
8785    let doubt = match &result {
8786        Err(SkillError::Step(e)) => crate::runtime::group::may_have_externalised(e),
8787        _ => false,
8788    };
8789    if doubt {
8790        let detail = match &result {
8791            Err(e) => e.to_string(),
8792            Ok(_) => String::new(),
8793        };
8794        let settled = cx
8795            .settle_open_group(crate::core::GroupOutcome::Quarantined, Some(&detail))
8796            .await;
8797        return match settled {
8798            Ok(()) => result,
8799            Err(e) => Err(SkillError::Step(e)),
8800        };
8801    }
8802
8803    match cx
8804        .abort_open_group("the step ended without settling the group")
8805        .await
8806    {
8807        // The abort itself could not be completed. That outranks whatever the
8808        // step was reporting: a partly unwound group is the more dangerous fact.
8809        Err(e) => Err(SkillError::Step(e)),
8810        Ok(()) => match result {
8811            // A reported failure keeps its own reason. `Outcome::Fail` is an
8812            // `Ok` at the type level and a failure in fact: leaving the group
8813            // to the runtime is the ordinary path there, not an author bug, and
8814            // overwriting the reason would tell an operator the step "returned
8815            // successfully" while hiding why it actually stopped.
8816            Err(e) => Err(e),
8817            failed @ Ok(Outcome::Fail { .. }) => failed,
8818            // Anything else claimed to make progress while leaving a group
8819            // unsettled.
8820            Ok(_) => Err(SkillError::Step(StepError::GroupAborted {
8821                what: format!(
8822                    "step made progress with group '{name}' still open — it was \
8823                     reversed, because a group that commits by being forgotten is worse \
8824                     than one that does not commit at all"
8825                ),
8826            })),
8827        },
8828    }
8829}
8830
8831/// Turn a step's result into a run status.
8832///
8833/// The distinction that matters is between an ordinary failure and a run whose
8834/// *history can no longer be trusted*. Divergence and orphaned effects are the
8835/// latter: they mean the journal no longer describes what this code does, so a
8836/// human has to look before anything else happens. Folding them into `Failed`
8837/// would put them in the same bucket as "the invoice was rejected", and they
8838/// would be retried like one.
8839fn classify(
8840    meter: &super::metrics::Meter,
8841    result: Result<Outcome, crate::core::SkillError>,
8842) -> (RunStatus, Option<Tainted<Value>>) {
8843    use crate::core::{SkillError, StepError};
8844
8845    match result {
8846        // The label travels with the value. Stripping it here would silently
8847        // launder provenance at every step boundary: a downstream step reading
8848        // an untrusted upstream output would receive it marked trusted, and the
8849        // taint gates further on would have nothing to act on.
8850        Ok(Outcome::Done(v)) => (RunStatus::Succeeded, Some(v)),
8851        Ok(Outcome::Fail { reason }) => (RunStatus::Failed(reason), None),
8852        // Not a failure: the executor decides whether a new plan is allowed,
8853        // because the answer depends on the run's provenance and budget, which
8854        // a step cannot see.
8855        Ok(Outcome::Replan { reason }) => (RunStatus::Replanning(reason), None),
8856        // Suspension is not a failure: the run is healthy and waiting. It
8857        // reaches here as an error only because that is how control leaves a
8858        // skill.
8859        Err(SkillError::Step(StepError::Suspended(reason))) => (RunStatus::Suspended(reason), None),
8860        // The ceiling did its job. Reporting this as a failure would have
8861        // operators debugging a system that behaved exactly as instructed.
8862        Err(SkillError::Step(StepError::Budget(exceeded))) => {
8863            (RunStatus::Exhausted(exceeded), None)
8864        }
8865        // A withdrawal found at a hop is the boundary's pause, reached
8866        // mid-step: the hop recorded the withholding before refusing.
8867        Err(SkillError::Step(StepError::Withheld { subject, reason })) => {
8868            (RunStatus::Withheld { subject, reason }, None)
8869        }
8870        Err(e) => {
8871            let msg = e.to_string();
8872            // Matched structurally, never on the message: a match on the word
8873            // "quarantined" would let a reworded error silently downgrade a run
8874            // to `Failed` — history kept, the flag that said not to trust it
8875            // lost.
8876            // Each of these is a failure P7 exists to make loud, and each gets
8877            // its own event so "did this happen" is a query rather than a grep.
8878            match &e {
8879                SkillError::Step(StepError::NonDeterminism {
8880                    seq,
8881                    expected,
8882                    actual,
8883                    detail,
8884                }) => {
8885                    tracing::error!(
8886                        target: telemetry::NONDETERMINISM,
8887                        %seq, %expected, %actual,
8888                        error_type = "nondeterminism",
8889                        reason_digest = %telemetry::reason_digest(detail),
8890                    );
8891                    meter.count(metrics::DIVERGENCES, "");
8892                }
8893                SkillError::Step(StepError::ReplayOverrun { actual, kind }) => {
8894                    tracing::error!(
8895                        target: telemetry::NONDETERMINISM,
8896                        %actual, %kind, overrun = true,
8897                    );
8898                    meter.count(metrics::DIVERGENCES, "");
8899                }
8900                SkillError::Step(StepError::Undecidable { key, detail, .. }) => {
8901                    tracing::error!(
8902                        target: telemetry::UNDECIDABLE,
8903                        %key,
8904                        error_type = "undecidable",
8905                        reason_digest = %telemetry::reason_digest(detail),
8906                    );
8907                    meter.count(metrics::UNDECIDABLE, "");
8908                }
8909                SkillError::Step(StepError::Unreproducible { what, detail }) => {
8910                    tracing::error!(
8911                        target: telemetry::UNREPRODUCIBLE,
8912                        error_type = "unreproducible",
8913                        reason_digest = %telemetry::reason_digest(&format!("{what}: {detail}")),
8914                    );
8915                    meter.count(metrics::UNREPRODUCIBLE, "");
8916                }
8917                _ => {}
8918            }
8919
8920            let untrustworthy = matches!(
8921                e,
8922                SkillError::Step(
8923                    StepError::NonDeterminism { .. }
8924                        | StepError::ReplayOverrun { .. }
8925                        | StepError::Undecidable { .. }
8926                        | StepError::GroupUnsettled { .. }
8927                        | StepError::Unreproducible { .. }
8928                )
8929            );
8930            if untrustworthy {
8931                (RunStatus::Quarantined(msg), None)
8932            } else {
8933                (RunStatus::Failed(msg), None)
8934            }
8935        }
8936    }
8937}
8938
8939/// Everything one execution needs, gathered so the executor is not called with
8940/// eight positional arguments — two of which are `u64`-shaped and would swap
8941/// silently.
8942struct Execution<'a> {
8943    /// Where the recorded run was refused by a step limit, if it was. `None`
8944    /// for a live run, which has no history to consult.
8945    refusal: Option<(StepId, String, String)>,
8946    /// The subject of a withholding the recorded history left standing. `None`
8947    /// for a live run, and for a resumed one whose withdrawal was already
8948    /// lifted and recorded.
8949    withheld_subject: Option<String>,
8950    /// Successor plans the recorded run produced, oldest first. Empty on a live
8951    /// run. Read back rather than re-synthesised, because a planner asked twice
8952    /// can answer differently.
8953    successors: Vec<PlanIR>,
8954    /// Steps whose `StepStarted` is already on the record. Empty on a live
8955    /// run. Read back so a resume neither re-announces a step nor forgets to
8956    /// announce one it starts for the first time.
8957    started: BTreeSet<StepId>,
8958    /// Steps whose `StepFinished` is already on the record. Empty on a live
8959    /// run.
8960    finished: BTreeSet<StepId>,
8961    /// Steps whose latest forward ending on the record is a success. Empty on
8962    /// a live run. A resume replays these before it asks whether to stop, so
8963    /// an unwind sees their outputs and the order they finished in.
8964    succeeded: BTreeSet<StepId>,
8965    /// Group records already on the journal, keyed by the step and phase
8966    /// that wrote them. Empty on a live run. Read back for the same reason
8967    /// `finished` is: group records are not effects, so the cursor cannot
8968    /// dedup them, and a resumed step that re-opens or re-settles its group
8969    /// at the frontier — where its cursor is exhausted and writes are
8970    /// enabled — would report one group as two.
8971    recorded_groups: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
8972    run: RunId,
8973    epoch: u64,
8974    quota: QuotaPass,
8975    plan: &'a PlanIR,
8976    input: Tainted<Value>,
8977    mode: Pass,
8978    case: Option<CaseContext>,
8979    budget: Budget,
8980    /// The run's data-subject references: from the `DataSubjectBound` just
8981    /// written on a live run, read back from it on a replay.
8982    subjects: BTreeSet<crate::core::SubjectRef>,
8983    /// Who is acting, for the policy principal. Read back from `RunAdmitted` on
8984    /// a replay rather than recomputed, for the same reason the plan is: the
8985    /// principal a run was authorized as is a fact about that run.
8986    agent: String,
8987    /// On whose behalf, for the policy context and for what this run
8988    /// commissions. Read back from `IdentityBound` on a replay for the same
8989    /// reason: a resumed run acting under the plane's chain instead of the one
8990    /// recorded for it would reach different verdicts than the live run did.
8991    identity: Option<crate::core::Delegation>,
8992}
8993
8994/// What a conclusion says when nothing on the chain says who ended the run.
8995///
8996/// Quarantine rather than a placeholder name, for the reason the exhausted arm
8997/// beside it quarantines when the typed ceiling verdict is missing: the record
8998/// is not trustworthy, and *not trustworthy* is a different bucket from *try
8999/// again*. A name invented here would be served through the operator API as the
9000/// person who did it.
9001const UNATTRIBUTED: &str = "the chain records this ending and not who asked for it — an \
9002                            operator act with nobody on it cannot be answered by reading further";
9003
9004/// The recorded status of a run that must not be resumed.
9005///
9006/// The outcomes that close a run to recovery:
9007///
9008/// * **Succeeded** — there is nothing outstanding. Re-executing would repeat
9009///   work that is not an effect (a case-state write, say), which is the same
9010///   class of bug the effect protocol prevents, arriving through a side door.
9011/// * **Cancelled** — a stopped run stays stopped, or the next inbound event
9012///   resumes it and it carries on doing the thing somebody intervened to
9013///   prevent.
9014/// * **Abandoned** — a person already decided there will never be an answer.
9015/// * **Quarantined, until a person answers it.** Resuming an unanswered
9016///   quarantine would re-hit whatever could not be decided, and burying that in
9017///   a retry loop is exactly how an undecidable situation becomes an unnoticed
9018///   one. A recorded [`QuarantineDecided`](RecordKind::QuarantineDecided)
9019///   lifts it — the *precondition* is that a named person looked, and the
9020///   verdict is still the runtime's: a reopened run re-decides from a history
9021///   that now holds whatever they established, and reaches the same quarantine
9022///   again if they were wrong or incomplete.
9023///
9024/// A **failed** run is deliberately *not* terminal here: a process that died
9025/// mid-flight records a failure, and recovering it is the entire point.
9026///
9027/// # Why the seal, and not the last step
9028///
9029/// Scanning backwards for a `StepFinished` and reading its outcome is wrong
9030/// twice, and the second is severe:
9031///
9032/// * A step's outcome is not the run's. They coincide only in a one-step plan.
9033/// * `find_map` **skips** a record it does not recognise and keeps looking. A
9034///   run whose last step failed after earlier steps succeeded would match an
9035///   *earlier* `StepFinished { outcome: "succeeded" }` and be reported
9036///   closed-and-succeeded: every multi-step run suspended after a failure —
9037///   every saga waiting on an approval to finish unwinding — could never be
9038///   resumed, and would report success while stuck.
9039///
9040/// `RunSealed` is written by `conclude` for exactly the runs that reached a
9041/// conclusion, and never for a suspended one. That is the fact this needs, so
9042/// it is the fact it reads.
9043pub(crate) fn resume_is_closed(records: &[Record]) -> Option<RunStatus> {
9044    let outcome = records.iter().rev().find_map(|r| match r.kind() {
9045        RecordKind::RunConcluded { outcome, .. } => Some(outcome.as_str()),
9046        _ => None,
9047    })?;
9048
9049    match outcome {
9050        "succeeded" => Some(RunStatus::Succeeded),
9051        "quarantined" => quarantine_decision(records).is_none().then(|| {
9052            RunStatus::Quarantined(
9053                "recorded as quarantined; a named person has to answer it before it can run \
9054                 again — reopen it once the doubt is resolved, or abandon it"
9055                    .into(),
9056            )
9057        }),
9058        // A conclusion whose attribution record is missing is **quarantined**,
9059        // not attributed to a person called "unknown". The arm for an exhausted
9060        // run with no typed ceiling verdict already reads this way; these three
9061        // fabricated a name instead, and a fabricated one is worse than a
9062        // missing one because it is indistinguishable from a real operator with
9063        // that name and it is served through the operator API as fact.
9064        //
9065        // The reason is the one the person gave, from the record that holds
9066        // it, rather than a sentence written here: an operator refused a
9067        // resume is asking *why this run is closed*, and a canned restatement
9068        // of the outcome answers a question they did not ask.
9069        "abandoned" => Some(recorded_abandonment(records).map_or_else(
9070            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9071            |(actor, reason)| RunStatus::Abandoned { actor, reason },
9072        )),
9073        // A stopped run stays stopped. Otherwise the next inbound event resumes
9074        // it and it carries on doing the thing somebody intervened to prevent —
9075        // and the intervention would look, from the journal, like it worked.
9076        "cancelled" => Some(recorded_cancellation(records).map_or_else(
9077            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9078            |(actor, reason)| RunStatus::Cancelled { actor, reason },
9079        )),
9080        // The two conclusions that deliberately do not close a run: a failed
9081        // run resumes with its completed effects read back from history, and an
9082        // exhausted one continues once somebody raises the ceiling.
9083        //
9084        // Their relationship to [`RunStatus::seals`] is the load-bearing part
9085        // and is pinned by `a_sealing_conclusion_is_never_resumable`. The
9086        // direction that matters is *no sealing status may be resumable*: a
9087        // status that seals froze the journal and published a Merkle leaf, so
9088        // resuming it would grow the history past the leaf every later
9089        // checkpoint attests. The reverse direction is deliberately **not** an
9090        // equality — `Suspended` and `Replanning` also do not seal, and never
9091        // reach here at all, because neither is ever a recorded conclusion.
9092        // Stating that as "the two that do not seal" was wrong; there are four,
9093        // and only two of them can be a `RunSealed` outcome.
9094        // A withdrawal joins them, and is the clearest case of the three: the
9095        // run did nothing wrong, its mutations stand, and lifting the halt is
9096        // what continues it. Closing it would turn *stop acting under this
9097        // credential* into *destroy this work*, which is not what an operator
9098        // withdrawing a credential asked for — the operator who wants that
9099        // cancels, and cancelling unwinds.
9100        "failed" | "exhausted" | WITHHELD_OUTCOME => None,
9101        // Fail closed. An outcome this build does not recognise — a sweep's
9102        // `swept`, a future variant, a corrupted string — is not permission to
9103        // resume; it is a run whose recorded ending this code cannot interpret,
9104        // and continuing it would graft new behaviour onto a history that says
9105        // it ended.
9106        other => Some(RunStatus::Quarantined(format!(
9107            "recorded as '{other}', which this build does not recognise as resumable"
9108        ))),
9109    }
9110}
9111
9112/// Refuse a resume that would report success over work its own history reversed.
9113///
9114/// A run whose journal holds compensation-phase records is not resumable once it
9115/// stands concluded. The unwind *reversed* work — a hold released, a refund
9116/// issued — and a resume that replays the forward history and continues would
9117/// conclude success over a world where the work was undone: the journal would
9118/// say "done" about effects its own later records say were taken back.
9119///
9120/// Scoped to runs with a standing conclusion, deliberately: a run that is
9121/// mid-unwind (a compensation suspended for four eyes, a crash between
9122/// reversals) has concluded nothing, and resuming it is how the unwind finishes.
9123fn refuse_resume_over_undone_work(run: RunId, records: &[Record]) -> Result<(), RuntimeError> {
9124    let Some(outcome) = recorded_conclusion(records) else {
9125        return Ok(());
9126    };
9127    if !records
9128        .iter()
9129        .any(|r| matches!(r.kind(), RecordKind::StepCompensated { .. }))
9130    {
9131        return Ok(());
9132    }
9133    Err(RuntimeError::PlanContract(format!(
9134        "run {run} concluded '{outcome}' after compensating completed steps — the work was \
9135         reversed, and resuming over undone work would report success about a world where it no \
9136         longer stands. Start a fresh run"
9137    )))
9138}
9139
9140/// The person's answer standing against this run's quarantine, if there is one.
9141///
9142/// Scoped to what follows the **last** conclusion, so an answer is spent by the
9143/// pass it authorized. A reopened run that quarantines again is quarantined
9144/// again: the earlier decision was made about a history that has since moved,
9145/// and reading it as standing would turn one person's judgement into a
9146/// permanent licence to retry.
9147fn quarantine_decision(records: &[Record]) -> Option<(crate::core::QuarantineDecision, &Record)> {
9148    records
9149        .iter()
9150        .rev()
9151        .take_while(|r| !matches!(r.kind(), RecordKind::RunConcluded { .. }))
9152        .find_map(|r| match r.kind() {
9153            RecordKind::QuarantineDecided { decision, .. } => Some((*decision, r)),
9154            _ => None,
9155        })
9156}
9157
9158/// The abandonment this run's history asks for but has not yet recorded.
9159///
9160/// A decision is written before it is acted on, so a crash in between leaves
9161/// the instruction standing rather than lost. This is what the next resume
9162/// reads to finish the job — and it is why abandonment is not simply performed
9163/// inline by the verb that requests it.
9164fn pending_abandonment(records: &[Record]) -> Option<RunStatus> {
9165    let (decision, record) = quarantine_decision(records)?;
9166    if decision != crate::core::QuarantineDecision::Abandon {
9167        return None;
9168    }
9169    match record.kind() {
9170        RecordKind::QuarantineDecided {
9171            decider, reason, ..
9172        } => Some(RunStatus::Abandoned {
9173            actor: decider.clone(),
9174            reason: reason.clone(),
9175        }),
9176        _ => None,
9177    }
9178}
9179
9180/// The longest admission key this runtime accepts.
9181///
9182/// Bounded because the key is a **counterparty-controlled** string that becomes
9183/// part of a storage key: unbounded, it is an index entry an emitter chooses the
9184/// size of. 512 bytes is far above what an identity needs — a `CloudEvents`
9185/// `source` and `id` together are tens of bytes — and far below what a btree
9186/// index refuses, so the refusal is this crate's with a reason rather than the
9187/// backend's with an error code.
9188pub const MAX_ADMISSION_KEY_BYTES: usize = 512;
9189
9190/// Refuse a key that cannot mean what the caller thinks it means.
9191///
9192/// An **empty** key is the important one, and it is refused rather than
9193/// accepted as a value. A missing header or an unset variable arrives here as
9194/// `""`, and `""` is a perfectly good key: the first message claims it and every
9195/// later message — a *different* message — is answered with the first one's run.
9196/// That is silent, total message loss produced by a configuration mistake, and
9197/// it is exactly what an at-most-once control must not do quietly.
9198fn validate_admission_key(key: &str) -> Result<(), RuntimeError> {
9199    if key.trim().is_empty() {
9200        return Err(RuntimeError::PlanContract(
9201            "an admission key is empty — an unset header or variable arrives here as \
9202             an empty string, and accepting it would answer every later message with \
9203             the first one's run. Use the message's own identity, such as \
9204             `InboundEvent::dedup_key()`"
9205                .into(),
9206        ));
9207    }
9208    if key.len() > MAX_ADMISSION_KEY_BYTES {
9209        return Err(RuntimeError::PlanContract(format!(
9210            "an admission key of {} bytes exceeds the {MAX_ADMISSION_KEY_BYTES}-byte \
9211             limit — the key is chosen by the emitter and becomes part of a storage \
9212             key, so its size is not theirs to decide",
9213            key.len()
9214        )));
9215    }
9216    Ok(())
9217}
9218
9219/// The run id a store named as an admission key's holder.
9220///
9221/// An unparseable holder is corruption in a derived index. Refused rather than
9222/// admitted around: "the index is unreadable" is not evidence the key is free.
9223fn parse_holder(run: &str) -> Result<RunId, RuntimeError> {
9224    RunId::parse(run).map_err(|e| {
9225        RuntimeError::PlanContract(format!(
9226            "an admission key is held by run '{run}', which does not parse ({e}) — refusing rather than admitting a second run under a key whose holder this build cannot read"
9227        ))
9228    })
9229}
9230
9231/// What a run's history says its state is, if it says anything.
9232///
9233/// **One reader, for every surface that answers the question.** Idempotent
9234/// admission, the operator API and the A2A projection are three views of one
9235/// fact, and three copies of this match agree right up until somebody adds a
9236/// record kind or a conclusion arrives that this build cannot read — after
9237/// which a caller polling and the same caller streaming are told different
9238/// things about one run, which is worse than either answer being wrong. The
9239/// laxest copy is the one that decides what an operator sees, so there is one.
9240///
9241/// `None` means the run is still going: the last record is neither a suspension
9242/// nor a conclusion. It is not this function's business whether that means
9243/// *running* or *never existed* — an empty history answers `None` too, and the
9244/// caller knows which it asked for.
9245///
9246/// **The last record, not any record.** A run that waited, was resumed and
9247/// carried on has a suspension in its history and is not suspended now.
9248pub(crate) fn observed_status(records: &[Record]) -> Option<RunStatus> {
9249    Some(match records.last()?.kind() {
9250        RecordKind::RunSuspended { reason } => RunStatus::Suspended(reason.clone()),
9251        RecordKind::RunConcluded {
9252            outcome,
9253            reason,
9254            exhaustion,
9255            ..
9256        } => recorded_status(outcome, reason.as_deref(), exhaustion.as_ref(), records),
9257        _ => return None,
9258    })
9259}
9260
9261/// A sealed run's status, rebuilt from its recorded conclusion.
9262///
9263/// Distinct from `resume_is_closed`, which asks the narrower *may this run be
9264/// resumed* and returns `None` for the conclusions that stay resumable. Every
9265/// conclusion has an answer here: a failed run reported as "still working"
9266/// would send a redelivery into a second admission.
9267fn recorded_status(
9268    outcome: &str,
9269    reason: Option<&str>,
9270    exhaustion: Option<&crate::core::BudgetExceeded>,
9271    records: &[Record],
9272) -> RunStatus {
9273    // A conclusion that carries no reason gets a statement of that fact rather
9274    // than an empty string: the empty summary is what `RunStatus::reason`
9275    // exists to prevent, and this is the surface a retried delivery reads.
9276    let said = || {
9277        reason.map_or_else(
9278            || "concluded without a recorded reason".to_owned(),
9279            ToOwned::to_owned,
9280        )
9281    };
9282    match outcome {
9283        "succeeded" => RunStatus::Succeeded,
9284        "failed" => RunStatus::Failed(said()),
9285        "replanning" => RunStatus::Replanning(said()),
9286        "exhausted" => exhaustion.cloned().map_or_else(
9287            || {
9288                RunStatus::Quarantined(
9289                    "recorded as exhausted without the typed ceiling verdict".to_owned(),
9290                )
9291            },
9292            RunStatus::Exhausted,
9293        ),
9294        "quarantined" => RunStatus::Quarantined(said()),
9295        "cancelled" => recorded_cancellation(records).map_or_else(
9296            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9297            |(actor, reason)| RunStatus::Cancelled { actor, reason },
9298        ),
9299        "abandoned" => recorded_abandonment(records).map_or_else(
9300            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9301            |(actor, reason)| RunStatus::Abandoned { actor, reason },
9302        ),
9303        // A withdrawal, read from the record that holds both halves. Without
9304        // this arm the catch-all answered *quarantined, and this build does
9305        // not recognise 'withheld'* — the plane calling its own record
9306        // foreign, on the surface an operator uses to decide whether to lift a
9307        // halt or to cancel.
9308        WITHHELD_OUTCOME => recorded_withholding(records).map_or_else(
9309            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9310            |(subject, reason)| RunStatus::Withheld { subject, reason },
9311        ),
9312        super::sweeper::SWEEP_OUTCOME => RunStatus::Swept,
9313        // Read back as itself, like every other ending this build writes. The
9314        // catch-all below would answer `Quarantined` — this plane calling its
9315        // own record foreign, on a run whose whole point is that somebody else
9316        // ran it.
9317        OBSERVED_OUTCOME => RunStatus::Observed,
9318        BREAK_GLASS_OUTCOME => recorded_crossing(records).map_or_else(
9319            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9320            |(actor, reason)| RunStatus::BrokeGlass { actor, reason },
9321        ),
9322        HALT_LIFTED_OUTCOME => recorded_halt_lift(records).map_or_else(
9323            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9324            |actor| RunStatus::HaltLifted { actor },
9325        ),
9326        HOLD_RELEASED_OUTCOME => recorded_hold_release(records).map_or_else(
9327            || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9328            |actor| RunStatus::HoldReleased { actor },
9329        ),
9330        // Fail closed, as the resume path does: a conclusion this build cannot
9331        // interpret is not permission to treat the run as ordinary. Every
9332        // outcome this build *writes* has an arm above, and a guard holds the
9333        // two lists together — an ending answered here as unrecognised would be
9334        // this plane calling its own record foreign, and the surface it reaches
9335        // is the one an operator uses to read a break-glass crossing.
9336        other => RunStatus::Quarantined(format!(
9337            "recorded as '{other}', which this build does not recognise"
9338        )),
9339    }
9340}
9341
9342/// Whether this value is a sealed payload rather than a readable one.
9343///
9344/// Always false without `keyring`: nothing seals, so nothing can be sealed, and
9345/// a build with no key ring must not carry a dependency on the module that
9346/// defines the wrapper.
9347fn sealed_payload(value: &serde_json::Value) -> bool {
9348    #[cfg(feature = "keyring")]
9349    {
9350        crate::journal::payload::is_sealed(value)
9351    }
9352    #[cfg(not(feature = "keyring"))]
9353    {
9354        let _ = value;
9355        false
9356    }
9357}
9358
9359/// How the run's history says it last ended, or that it is waiting.
9360///
9361/// The last conclusion or suspension on the record, as the outcome word a
9362/// replay's own status reports, so the two compare directly. `None` for a run
9363/// that has neither: it is still in flight, and a strict replay of it would
9364/// stop at its frontier and call that divergence.
9365fn recorded_ending(records: &[Record]) -> Option<String> {
9366    records.iter().rev().find_map(|r| match r.kind() {
9367        RecordKind::RunConcluded { outcome, .. } => Some(outcome.clone()),
9368        RecordKind::RunSuspended { .. } => Some("suspended".to_owned()),
9369        _ => None,
9370    })
9371}
9372
9373/// Whether an ending was an act rather than something the run's steps reached.
9374///
9375/// A cancellation, an abandonment, a sweep, a break-glass crossing, a lift, a
9376/// withdrawal and an observation are each recorded by whoever took them; a
9377/// strict replay re-derives steps, and no step decided these.
9378fn ended_by_an_act(outcome: &str) -> bool {
9379    matches!(outcome, "cancelled" | "abandoned")
9380        || [
9381            super::sweeper::SWEEP_OUTCOME,
9382            BREAK_GLASS_OUTCOME,
9383            HALT_LIFTED_OUTCOME,
9384            HOLD_RELEASED_OUTCOME,
9385            WITHHELD_OUTCOME,
9386            OBSERVED_OUTCOME,
9387        ]
9388        .contains(&outcome)
9389}
9390
9391/// The plan the run was admitted under, and each successor a replan produced,
9392/// oldest first. Read back rather than recompiled, which could produce a graph
9393/// that never governed the run.
9394///
9395/// All or nothing. Successors are walked by position, so skipping one this
9396/// build cannot read would put every later replan on the wrong plan.
9397///
9398/// # Errors
9399///
9400/// [`RuntimeError::PayloadsErased`] when a plan is still sealed, named before
9401/// it is parsed: a sealed payload deserialises into a *missing field*, and that
9402/// sentence sends an operator looking for a corrupt journal rather than telling
9403/// them the data was erased on request.
9404fn frozen_plans(run: RunId, records: &[Record]) -> Result<(PlanIR, Vec<PlanIR>), RuntimeError> {
9405    let mut plans = records
9406        .iter()
9407        .filter_map(|r| match r.kind() {
9408            RecordKind::PlanFrozen { plan, .. } => Some(plan),
9409            _ => None,
9410        })
9411        .map(|plan| {
9412            if sealed_payload(plan) {
9413                return Err(RuntimeError::PayloadsErased {
9414                    run: run.to_string(),
9415                });
9416            }
9417            serde_json::from_value(plan.clone()).map_err(RuntimeError::Encoding)
9418        })
9419        .collect::<Result<Vec<PlanIR>, _>>()?
9420        .into_iter();
9421    let admitted = plans
9422        .next()
9423        .ok_or_else(|| RuntimeError::PlanContract("journal has no PlanFrozen record".into()))?;
9424    Ok((admitted, plans.collect()))
9425}
9426
9427/// Whether this run's history is sealed to a key that no longer exists.
9428///
9429/// Read from the frozen plan, which every admitted run has and which is sealed
9430/// under the run's own erasure scope — so it is gone exactly when the run's
9431/// data is gone. Checking a payload rather than asking the key ring keeps this
9432/// a property of the records in hand: the same answer offline, on a restored
9433/// copy, and in a build with no ring wired.
9434fn payloads_erased(records: &[Record]) -> bool {
9435    records.iter().any(|r| match r.kind() {
9436        RecordKind::PlanFrozen { plan, .. } => sealed_payload(plan),
9437        _ => false,
9438    })
9439}
9440
9441/// Who asked for the stop, read back from the chain.
9442///
9443/// Read rather than remembered, for the same reason every other fact about a run
9444/// is: the journal gives the same answer on every subsequent read, and an
9445/// operator asking "who stopped this?" six weeks later is asking history.
9446///
9447/// **Both halves, from the one record that holds them.** The conclusion does
9448/// not repeat the reason — see [`RecordKind::RunConcluded`]'s `reason` for why
9449/// a second copy of it would be the worse copy.
9450fn recorded_cancellation(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9451    records.iter().rev().find_map(|r| match r.kind() {
9452        RecordKind::RunCancelled { actor, reason } => Some((actor.clone(), reason.clone())),
9453        _ => None,
9454    })
9455}
9456
9457/// Who answered the quarantine and what they said, read back from the chain.
9458fn recorded_abandonment(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9459    records.iter().rev().find_map(|r| match r.kind() {
9460        RecordKind::QuarantineDecided {
9461            decider, reason, ..
9462        } => Some((decider.clone(), reason.clone())),
9463        _ => None,
9464    })
9465}
9466
9467/// Who crossed into this tenant and why, read back from the chain.
9468fn recorded_crossing(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9469    records.iter().rev().find_map(|r| match r.kind() {
9470        RecordKind::BreakGlass { actor, reason, .. } => Some((actor.clone(), reason.clone())),
9471        _ => None,
9472    })
9473}
9474
9475/// Who lifted the halt this run records, read back from its lift record.
9476fn recorded_halt_lift(records: &[Record]) -> Option<crate::core::Operator> {
9477    records.iter().rev().find_map(|r| match r.kind() {
9478        RecordKind::HaltLifted { by, .. } => Some(by.clone()),
9479        _ => None,
9480    })
9481}
9482
9483/// Who released the hold this run records, read back from its release record.
9484fn recorded_hold_release(records: &[Record]) -> Option<crate::core::Operator> {
9485    records.iter().rev().find_map(|r| match r.kind() {
9486        RecordKind::HoldReleased { by, .. } => Some(by.clone()),
9487        _ => None,
9488    })
9489}
9490
9491/// Whose authority was withdrawn, and what the operator said when withdrawing
9492/// it — read back from the withholding record.
9493///
9494/// The last one standing: a withdrawal may be lifted and thrown again within a
9495/// run's history, and the conclusion is about the one that stopped it.
9496fn recorded_withholding(records: &[Record]) -> Option<(String, String)> {
9497    records.iter().rev().find_map(|r| match r.kind() {
9498        RecordKind::AuthorityWithheld {
9499            subject, reason, ..
9500        } => Some((subject.clone(), reason.clone())),
9501        _ => None,
9502    })
9503}
9504
9505/// Wall-clock read for the case's `opened_at` stamp, and for a rate
9506/// reservation's instant.
9507///
9508/// Admission happens before any step exists, so there is no `StepCtx` to
9509/// journal through. Both values are store metadata and never participate in
9510/// replay: a rate refusal is journaled on the run and read back, never
9511/// re-counted — run-visible time still goes through `StepCtx::now`, which
9512/// journals it.
9513#[allow(clippy::disallowed_methods)]
9514pub(super) fn now_for_admission() -> crate::core::Timestamp {
9515    crate::core::Timestamp::now_utc()
9516}
9517
9518/// A lift was recorded in `run` and the register's row was not removed.
9519///
9520/// The control still stands; the operator is told the lift failed and which
9521/// run recorded the attempt.
9522fn removal_failed(run: RunId, e: &crate::core::StoreError) -> RuntimeError {
9523    RuntimeError::ControlStands {
9524        run: run.to_string(),
9525        detail: e.to_string(),
9526    }
9527}
9528
9529/// An operator's lift or release that found a control standing and recorded
9530/// ending it.
9531#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9532pub struct ControlLifted {
9533    /// The run holding the record.
9534    pub record: RunId,
9535    /// Whether this call removed the register row. `false` when another lift
9536    /// or release removed it between this call's read and its removal: the
9537    /// control is gone either way, and the record still stands.
9538    pub removed: bool,
9539}
9540
9541/// The epoch an operator act's own run is written under.
9542///
9543/// Such a run has no competing writer to fence against — it is created,
9544/// written and sealed in one call — so a constant is honest here for the same
9545/// reason it is in the sweeper.
9546const BREAK_GLASS_EPOCH: crate::core::Epoch = LEASE_FREE_EPOCH;
9547
9548/// How a break-glass run ends. Not a run status: it neither succeeded nor
9549/// failed at a goal, which is why a sweep seals as `swept` rather than
9550/// borrowing one.
9551const BREAK_GLASS_OUTCOME: &str = "broke-glass";
9552
9553/// How a run recording an operator lifting a halt ends.
9554pub const HALT_LIFTED_OUTCOME: &str = "halt-lifted";
9555
9556/// How a run recording an operator releasing a legal hold ends.
9557pub const HOLD_RELEASED_OUTCOME: &str = "hold-released";
9558
9559/// How a run ends when the authority it acts under is withdrawn under it.
9560///
9561/// Resumable, so it is deliberately absent from both
9562/// [`SEALED_OUTCOMES`] and [`OUTCOMES_OF_RECORD`]: the run has not ended, it is
9563/// waiting for somebody to lift a halt or to cancel it properly.
9564pub const WITHHELD_OUTCOME: &str = "withheld";
9565
9566/// The outcome a session this plane only **observed** is sealed under.
9567///
9568/// Its own word, and not `succeeded`: a session this runtime did not execute
9569/// has no success to report, and answering with one would put somebody else's
9570/// agent's work in the column an operator reads as *this plane did it*. The
9571/// distinction is the whole reason the status exists.
9572pub const OBSERVED_OUTCOME: &str = "observed";
9573
9574/// The epoch every run written without a lease is written under: a sweep's
9575/// record, an operator act's, an observed session's. Nothing takes such a run
9576/// over, so there is no ownership for the epoch to arbitrate, and one constant
9577/// lets the pass that finishes their seals use the epoch they were written at.
9578pub(crate) const LEASE_FREE_EPOCH: crate::core::Epoch = 1;
9579
9580/// The sealing outcomes whose runs are written without a lease.
9581///
9582/// Recovery selects a run by its lapsed lease, so it never finds one of these
9583/// whose writer died between conclusion and seal;
9584/// [`Runtime::seal_unsealed_conclusions`] is what finishes them.
9585pub(crate) const LEASE_FREE_OUTCOMES: &[&str] = &[
9586    super::sweeper::SWEEP_OUTCOME,
9587    BREAK_GLASS_OUTCOME,
9588    HALT_LIFTED_OUTCOME,
9589    HOLD_RELEASED_OUTCOME,
9590    OBSERVED_OUTCOME,
9591];
9592
9593/// One governed identity: a declaration and the skills that serve it.
9594///
9595/// A runtime **runs** agents; it is not one. It owns the journal, the stores,
9596/// the model drivers and the policy engine — infrastructure, shared. An agent
9597/// owns a manifest and its skills — governance, per-identity. Several agents on
9598/// one plane share a journal and are still separately declared, separately
9599/// bounded, and separately answerable.
9600///
9601/// Conflating the two forced a runtime per agent, which meant a lease owner per
9602/// agent for what is one process, a model driver registered once per agent, and
9603/// nowhere in the journal to record *which* agent governed a run.
9604#[cfg(feature = "manifest")]
9605#[derive(Debug, Default)]
9606pub struct Agent {
9607    manifest: Option<Arc<crate::manifest::Manifest>>,
9608    /// Who vouched for the declaration, when it came from a verified resolution.
9609    publisher: Option<crate::core::KeyId>,
9610    skills: Vec<Arc<dyn Skill>>,
9611}
9612
9613#[cfg(feature = "manifest")]
9614impl Agent {
9615    /// An agent governed by this declaration.
9616    ///
9617    /// Nobody has vouched for it. Prefer [`Agent::published_by`] where the
9618    /// manifest came from a verified registry resolution.
9619    #[must_use]
9620    pub fn new(manifest: &crate::manifest::Manifest) -> Self {
9621        Self {
9622            manifest: Some(Arc::new(manifest.clone())),
9623            publisher: None,
9624            skills: Vec::new(),
9625        }
9626    }
9627
9628    /// Record who vouched for this declaration.
9629    ///
9630    /// Takes the [`KeyId`](crate::core::KeyId) that
9631    /// [`Registry::resolve_verified`](crate::manifest::Registry::resolve_verified)
9632    /// returned beside the manifest — which is otherwise dropped on the floor,
9633    /// so a verified resolution and a parsed file become indistinguishable the
9634    /// moment they reach the runtime.
9635    ///
9636    /// # Why this is the grouping a policy wants
9637    ///
9638    /// A rule has to name *a set of agents*, and the obvious candidates do not
9639    /// survive contact with a deployment:
9640    ///
9641    /// * the **workload identity** is per-instance, so a rule naming one is a
9642    ///   rule rewritten on every deploy;
9643    /// * the agent **name**, its **role**, or any group label in the manifest is
9644    ///   self-asserted — a file claims it, so a rule granting authority to one
9645    ///   grants it to any file that types the same string;
9646    /// * the **digest** is unforgeable but names exactly one revision, so every
9647    ///   edit is a policy change.
9648    ///
9649    /// A publisher key is the only one that is both a group — many agents, many
9650    /// versions — and impossible to claim without holding the key. Bind the rule
9651    /// to the publisher, keep the digest for "this exact revision", and leave
9652    /// the name for humans reading logs.
9653    #[must_use]
9654    pub fn published_by(mut self, key_id: impl Into<crate::core::KeyId>) -> Self {
9655        self.publisher = Some(key_id.into());
9656        self
9657    }
9658
9659    /// Give it a skill.
9660    #[must_use]
9661    pub fn skill(mut self, skill: impl Skill + 'static) -> Self {
9662        self.skills.push(Arc::new(skill));
9663        self
9664    }
9665}
9666
9667/// Receives every model call's live output, with the run it belongs to.
9668///
9669/// What [`ModelStreamObserver`](crate::model::ModelStreamObserver) receives for
9670/// one call, for every call a plane makes — so a surface can show a run's model
9671/// writing while the journal holds only the finished answer.
9672pub trait RunStreamObserver: Send + Sync + std::fmt::Debug {
9673    /// Delivery is advisory and must not block the call.
9674    fn event(&self, run: RunId, event: Tainted<crate::model::ModelStreamEvent>);
9675}
9676
9677/// Assembles a [`Runtime`].
9678#[derive(Debug)]
9679pub struct RuntimeBuilder {
9680    store: Arc<dyn JournalStore>,
9681    /// The witnesses this plane submits checkpoints to, and the bar a round
9682    /// is held to. Both or neither: a witness list with no declared quorum is
9683    /// witnessing whose evidence is whatever happened to arrive.
9684    witnesses: Vec<Arc<dyn crate::journal::Witness>>,
9685    quorum: Option<crate::journal::WitnessQuorum>,
9686    witness_interval: Option<std::time::Duration>,
9687    signer: Option<Arc<dyn crate::core::Signer>>,
9688    skills: Vec<Arc<dyn Skill>>,
9689    #[cfg(feature = "manifest")]
9690    tools: Option<(
9691        Arc<crate::tools::ToolCatalog>,
9692        Arc<dyn crate::tools::ToolClient>,
9693    )>,
9694    #[cfg(feature = "manifest")]
9695    egress: Option<Arc<crate::core::Egress>>,
9696    #[cfg(feature = "manifest")]
9697    checkers: crate::content::Checkers,
9698    streams: Option<Arc<dyn RunStreamObserver>>,
9699    journal_ceiling: Option<crate::core::Sensitivity>,
9700    tenant: crate::core::TenantId,
9701    owner: Option<String>,
9702    lease_ttl: Duration,
9703    memories: Option<Arc<dyn crate::memory::MemoryStore>>,
9704    semantic: Option<Arc<super::SemanticMemory>>,
9705    authorities: Option<Arc<dyn crate::authority::AuthorityStore>>,
9706    peers: Option<Arc<PeerWiring>>,
9707    tenant_label: super::telemetry::TenantLabel,
9708    quotas: Option<Arc<dyn crate::quota::QuotaStore>>,
9709    quota: crate::quota::TenantQuota,
9710    budget: Budget,
9711    require_verifier: bool,
9712    /// Typed tools whose coherence with every agent is checked at `build`.
9713    #[cfg(feature = "manifest")]
9714    toolbox: Option<crate::tools::ToolBox>,
9715    /// Tool servers reached by some transport other than the box, by name.
9716    #[cfg(feature = "manifest")]
9717    tool_servers: Vec<(String, Arc<dyn crate::tools::ToolClient>)>,
9718    /// Agents registered on this plane, each with its own declaration.
9719    #[cfg(feature = "manifest")]
9720    agents: Vec<Agent>,
9721    /// Drivers by the name a manifest calls them.
9722    #[cfg(feature = "manifest")]
9723    providers: HashMap<String, Arc<dyn crate::model::ModelProvider>>,
9724    cases: Option<Arc<dyn CaseStore>>,
9725    events: Option<Arc<dyn EventStore>>,
9726    tasks: Option<Arc<dyn TaskStore>>,
9727    timers: Option<Arc<dyn TimerStore>>,
9728    blobs: Option<Arc<dyn crate::blob::BlobStore>>,
9729    disclosures: Option<Arc<dyn crate::disclosure::DisclosureRegister>>,
9730    #[cfg(feature = "push")]
9731    push: Option<Arc<dyn crate::push::PushStore>>,
9732    #[cfg(feature = "keyring")]
9733    keyring: Option<Arc<dyn crate::keyring::KeyRing>>,
9734    batches: Option<Arc<dyn crate::batch::BatchStore>>,
9735    policy: Option<Arc<dyn crate::core::PolicyEngine>>,
9736    identity: Option<crate::core::Delegation>,
9737    replanner: Option<Arc<dyn crate::plan::Replanner>>,
9738    calendar: Option<Arc<dyn Calendar>>,
9739    /// Destinations this deployment sends its own run events to.
9740    #[cfg(feature = "push")]
9741    outbox: Option<Arc<crate::push::Outbox>>,
9742}
9743
9744impl RuntimeBuilder {
9745    #[must_use]
9746    pub fn skill(mut self, s: impl Skill) -> Self {
9747        self.skills.push(Arc::new(s));
9748        self
9749    }
9750
9751    /// The workload identity this plane signs its outward claims with.
9752    ///
9753    /// What it buys is that a tool or peer can *check* who called it. Without a
9754    /// signer the provenance block still travels — a server can correlate on it
9755    /// — but it is an assertion any intermediary could have written, and a
9756    /// callee must not authorize on it.
9757    ///
9758    /// Give the store the same signer ([`signing_as`] there) so records and
9759    /// outward claims carry one identity. They are separate settings because a
9760    /// plane can legitimately have one without the other.
9761    ///
9762    /// [`signing_as`]: crate::store::RedbStore::signing_as
9763    #[must_use]
9764    pub fn signing_as(mut self, signer: Arc<dyn crate::core::Signer>) -> Self {
9765        self.signer = Some(signer);
9766        self
9767    }
9768
9769    /// How long this plane's run leases last.
9770    ///
9771    /// The trade is recovery speed against tolerance for a slow instance: a
9772    /// crashed owner's runs stay unclaimable for this long, and a live owner
9773    /// must renew within it. The runtime heartbeats while a run executes, so
9774    /// this bounds *crash* detection rather than how long a run may take.
9775    ///
9776    /// A TTL below [`MIN_LEASE_TTL`] is refused at
9777    /// [`build`](Self::build)/[`try_build`](Self::try_build) as
9778    /// [`BuildError::LeaseUnrenewable`](crate::runtime::BuildError::LeaseUnrenewable).
9779    /// Both stores keep lease expiry in **whole seconds** and treat
9780    /// `expires_at <= now` as lapsed, so a one-second lease expires the moment
9781    /// the clock ticks past the second it was written in — no matter how often
9782    /// it is renewed. Such a lease cannot be held by a live run, and a run that
9783    /// cannot hold its lease is one any instance may take away mid-flight.
9784    /// Refused where a manifest-serving plane can report it as a diagnostic
9785    /// rather than left as a panic in a setter, which `try_build`'s callers
9786    /// could never see as a value.
9787    #[must_use]
9788    pub fn lease_ttl(mut self, ttl: Duration) -> Self {
9789        self.lease_ttl = ttl;
9790        self
9791    }
9792
9793    /// Put this plane's tenant on what it reports about itself.
9794    ///
9795    /// Off by default, and one policy for both signals: the `tenant` field of
9796    /// every metric event, and `agentplane.tenant` on the run span. Read
9797    /// [`telemetry::TenantLabel`](super::telemetry::TenantLabel) before turning
9798    /// it on — a tenant name is often a customer name, and an observability
9799    /// backend is usually the least protected system in a deployment.
9800    ///
9801    /// Cardinality is bounded by construction — the label is *this plane's*
9802    /// tenant, so the number of streams is the number of planes configured, and
9803    /// no request can grow it.
9804    #[must_use]
9805    pub const fn tenant_label(mut self, label: super::telemetry::TenantLabel) -> Self {
9806        self.tenant_label = label;
9807        self
9808    }
9809
9810    /// Give this plane's agents a memory.
9811    ///
9812    /// Optional, and absent by default: an agent with no memory is a normal
9813    /// agent, and one that quietly gained persistent state because a store was
9814    /// wired for something else would be a surprise.
9815    ///
9816    /// Read [`crate::memory`] before wiring one. Writable memory is delayed
9817    /// code: what is written today is read into a context window tomorrow, where
9818    /// a model treats it as established fact.
9819    #[must_use]
9820    pub fn memory(mut self, memories: Arc<dyn crate::memory::MemoryStore>) -> Self {
9821        self.memories = Some(memories);
9822        self
9823    }
9824
9825    /// Give this plane a semantic index, and the embedder that speaks its
9826    /// language.
9827    ///
9828    /// Both together, never one at a time: `build` holds
9829    /// [`Embedder::revision`] to the [`IndexIdentity::query_revision`] the
9830    /// index declares it accepts, and refuses the pair otherwise
9831    /// ([`BuildError::EmbeddingSpaceMismatch`] says what a mismatch costs).
9832    ///
9833    /// Also needs [`memory`](Self::memory): the index holds only commitments,
9834    /// and every hit is materialised from the authoritative store before its
9835    /// content is exposed.
9836    ///
9837    /// [`Embedder::revision`]: crate::memory::Embedder::revision
9838    /// [`IndexIdentity::query_revision`]: crate::memory::IndexIdentity::query_revision
9839    /// [`BuildError::EmbeddingSpaceMismatch`]: crate::runtime::BuildError::EmbeddingSpaceMismatch
9840    #[must_use]
9841    pub fn semantic_memory(
9842        mut self,
9843        embedder: Arc<dyn crate::memory::Embedder>,
9844        retriever: Arc<dyn crate::memory::SemanticRetriever>,
9845    ) -> Self {
9846        self.semantic = Some(Arc::new(super::SemanticMemory {
9847            embedder,
9848            retriever,
9849        }));
9850        self
9851    }
9852
9853    /// Whether the plane's budget, or any registered agent's, caps money.
9854    fn states_a_money_ceiling(&self) -> bool {
9855        #[cfg(feature = "manifest")]
9856        if self
9857            .agents
9858            .iter()
9859            .filter_map(|agent| agent.manifest.as_ref())
9860            .any(|manifest| manifest.budget().max_minor_units.is_some())
9861        {
9862            return true;
9863        }
9864        self.budget.max_minor_units.is_some()
9865    }
9866
9867    /// Attach durable standing-authority accounting.
9868    ///
9869    /// The ceiling neither of the other two can express. A budget bounds one
9870    /// run; a quota bounds a tenant over a billing period. A standing authority
9871    /// bounds *an authorization* — what one customer approved, spanning as many
9872    /// runs as it takes, revocable when they change their mind.
9873    ///
9874    /// Without one, [`StepCtx::draw`](crate::runtime::StepCtx::draw) refuses
9875    /// rather than falling back to an in-process counter. That fallback would
9876    /// fail **open** the moment a second instance started, which is exactly when
9877    /// a shared ceiling was needed.
9878    #[must_use]
9879    pub fn authorities(mut self, authorities: Arc<dyn crate::authority::AuthorityStore>) -> Self {
9880        self.authorities = Some(authorities);
9881        self
9882    }
9883
9884    /// Bound what this tenant may consume, durably.
9885    ///
9886    /// Budgets bound one run; this bounds the tenant. Both are needed: a caller
9887    /// that can start runs can start a thousand, each within its own ceiling.
9888    ///
9889    /// The accounting lives in the store, so the ceiling survives a second
9890    /// instance — an in-process counter would silently double the moment
9891    /// somebody scales out, which is exactly when it was needed.
9892    ///
9893    /// Read [`crate::quota`] for what each ceiling does and does not bound; a
9894    /// limit believed to bound something it does not is worse than none.
9895    #[must_use]
9896    pub fn quota(
9897        mut self,
9898        quotas: Arc<dyn crate::quota::QuotaStore>,
9899        quota: crate::quota::TenantQuota,
9900    ) -> Self {
9901        self.quotas = Some(quotas);
9902        self.quota = quota;
9903        self
9904    }
9905
9906    /// This **process instance's** identity, as it appears in run leases.
9907    ///
9908    /// Not the agent's name. Fencing rests on the epoch — a claim never renews,
9909    /// and a renewal names the exact epoch it holds — so a shared string costs
9910    /// no safety; it costs the answer to *which process holds this run*, which
9911    /// is what an operator asks of a stuck one and what `LeaseHeld` reports.
9912    ///
9913    /// So keep it unique per running process, which is what the default is —
9914    /// override it only with a better instance identity than a random one,
9915    /// such as a pod name. An agent's *name* is
9916    /// [`Manifest::metadata`](crate::manifest::Metadata::name).
9917    ///
9918    /// The owner lives in the lease table and never in the chain, so it has no
9919    /// bearing on replay.
9920    #[must_use]
9921    pub fn owner(mut self, o: impl Into<String>) -> Self {
9922        self.owner = Some(o.into());
9923        self
9924    }
9925
9926    /// Cap what a run may consume.
9927    ///
9928    /// Defaults to [`Budget::unlimited`], which is right for a runtime whose
9929    /// effects are all free and local, and wrong the moment one of them calls a
9930    /// metered API.
9931    #[must_use]
9932    pub fn budget(mut self, budget: Budget) -> Self {
9933        self.budget = budget;
9934        self
9935    }
9936
9937    /// Refuse a plan in which nothing checks the work.
9938    ///
9939    /// A node that declares [`verifies`](crate::core::PlanNode::verifies) must
9940    /// already depend on what it checks; this makes carrying one a condition of
9941    /// admission. It binds every plan this runtime validates, and the one that
9942    /// matters is the one the embedder did not write: a
9943    /// [`Replanner`](crate::plan::Replanner)'s successor is a graph proposed
9944    /// mid-run, and a control that held for the first plan and not for its
9945    /// replacement is a control a replan removes.
9946    #[must_use]
9947    pub fn require_verifier(mut self) -> Self {
9948        self.require_verifier = true;
9949        self
9950    }
9951
9952    /// Attach long-lived case storage, enabling correlation and deadlines.
9953    #[must_use]
9954    pub fn cases(mut self, cases: Arc<dyn CaseStore>) -> Self {
9955        self.cases = Some(cases);
9956        self
9957    }
9958
9959    /// Send this plane's own events to destinations the deployment configured.
9960    ///
9961    /// Every run registers each destination **at admission**, so there is no
9962    /// window in which a run exists and nothing is watching it. Delivery is a
9963    /// separate, operator-scheduled sweep — see
9964    /// [`DeliveryWorker`](crate::push::DeliveryWorker) — reading each run's
9965    /// journal past a cursor that advances only on 2xx.
9966    ///
9967    /// This is the mirror image of A2A push, and the mirror is the point: there
9968    /// the *caller* names a URL and three controls exist because of it. Here the
9969    /// deployment names it, and the run's own history is the outbox rather than
9970    /// a queue that can fall out of sync with it.
9971    #[cfg(feature = "push")]
9972    #[must_use]
9973    pub fn outbox(mut self, outbox: Arc<crate::push::Outbox>) -> Self {
9974        self.outbox = Some(outbox);
9975        self
9976    }
9977
9978    /// Attach the webhook registration store.
9979    ///
9980    /// The same handle the A2A server and the [`Outbox`](crate::push::Outbox)
9981    /// are given: this is the *durable* half of push, and it belongs to the
9982    /// plane because its failures do. A receiver that answers permanently, or
9983    /// stays silent past the retry ceiling, has its registration
9984    /// [parked](crate::push::PushStore::park) rather than deleted — the cursor
9985    /// is the only record of how far that receiver got — and a parked row is a
9986    /// backlog an operator reads and re-arms on the operator API. Without this
9987    /// the plane serves the delivery and cannot answer for it.
9988    #[cfg(feature = "push")]
9989    #[must_use]
9990    pub fn push(mut self, push: Arc<dyn crate::push::PushStore>) -> Self {
9991        self.push = Some(push);
9992        self
9993    }
9994
9995    /// Attach inbound-event storage, enabling durable waits.
9996    #[must_use]
9997    pub fn events(mut self, events: Arc<dyn EventStore>) -> Self {
9998        self.events = Some(events);
9999        self
10000    }
10001
10002    /// Attach a worklist, enabling human tasks.
10003    #[must_use]
10004    pub fn tasks(mut self, tasks: Arc<dyn TaskStore>) -> Self {
10005        self.tasks = Some(tasks);
10006        self
10007    }
10008
10009    /// Supply the planner that produces successor plans.
10010    ///
10011    /// Without one, a step asking to replan fails with that as the reason —
10012    /// which is the honest outcome, not a silent no-op.
10013    #[must_use]
10014    pub fn replanner(mut self, r: Arc<dyn crate::plan::Replanner>) -> Self {
10015        self.replanner = Some(r);
10016        self
10017    }
10018
10019    /// Supply the durable-timer store.
10020    ///
10021    /// Needed by `StepCtx::sleep` and `sleep_until`, and by the sweep that wakes
10022    /// them. A runtime without one refuses to sleep rather than falling back to
10023    /// an in-process wait that a restart would forget.
10024    #[must_use]
10025    pub fn timers(mut self, timers: Arc<dyn TimerStore>) -> Self {
10026        self.timers = Some(timers);
10027        self
10028    }
10029
10030    /// Register an agent on this plane.
10031    ///
10032    /// A runtime runs agents; it is not one. This is where a declaration and
10033    /// its skills arrive together, so several agents can share one journal, one
10034    /// set of drivers and one process identity while each stays separately
10035    /// governed.
10036    ///
10037    /// The declaration **binds** for that agent's steps: an effect naming a
10038    /// model or tool its manifest never listed is refused before dispatch and
10039    /// journaled, and the egress and delegation ceilings combine with the sink's
10040    /// own — the stricter wins. Its budget bounds its runs. Architectural
10041    /// injection patterns are deliberately absent from the schema, because this
10042    /// runtime cannot prove that arbitrary skill code follows one.
10043    ///
10044    /// An agent declaring `spec.execution` needs no skill: the runtime supplies
10045    /// the behaviour. See [`provider`](Self::provider) for the driver mapping it
10046    /// needs.
10047    ///
10048    /// It does **not** set the lease owner. That identifies a *process*, and one
10049    /// plane running four agents is still one process — see
10050    /// [`owner`](Self::owner).
10051    ///
10052    /// # This call never fails
10053    ///
10054    /// It records the agent and returns. Every refusal an agent can cause — a
10055    /// capability none of its skills provide, a provider no driver is registered
10056    /// for, oversight with nowhere to put a decision — is raised at
10057    /// [`build`](Self::build), which panics, or returned by
10058    /// [`try_build`](Self::try_build), which does not.
10059    ///
10060    /// That split matters for a daemon assembling a plane from files it did not
10061    /// write: a bad manifest is an *input* there, and a panic takes down every
10062    /// other tenant in the process to report it. A `# Panics` section here,
10063    /// describing `build`'s refusals, would send readers looking for a
10064    /// fallible variant of the wrong call — the refusals are documented where
10065    /// they fire.
10066    #[cfg(feature = "manifest")]
10067    #[must_use]
10068    pub fn agent(mut self, agent: Agent) -> Self {
10069        self.agents.push(agent);
10070        self
10071    }
10072
10073    /// Register a model driver under the name a manifest uses for it.
10074    ///
10075    /// The seam a declarative agent needs. A manifest says `provider: anthropic`
10076    /// — a string a reviewer can read — and something has to map that to a
10077    /// driver holding a credential. That mapping is deployment wiring, not a
10078    /// property of the agent, which is exactly why it lives here and not in the
10079    /// file: an agent's declaration should not change when its API key does.
10080    ///
10081    /// Required only for [`ExecutionKind::Completion`] and the other declarative
10082    /// kinds. A hand-written skill constructs its own `ModelCall` and never
10083    /// consults this.
10084    ///
10085    /// [`ExecutionKind::Completion`]: crate::manifest::ExecutionKind::Completion
10086    #[cfg(feature = "manifest")]
10087    #[must_use]
10088    pub fn provider(
10089        mut self,
10090        name: impl Into<String>,
10091        provider: Arc<dyn crate::model::ModelProvider>,
10092    ) -> Self {
10093        self.providers.insert(name.into(), provider);
10094        self
10095    }
10096
10097    /// Supply content-addressed blob storage.
10098    ///
10099    /// Needed by `StepCtx::store_blob`, which is how bytes too large for a
10100    /// journal record get somewhere durable while the chain keeps only their
10101    /// digest. A runtime without one refuses rather than silently inlining
10102    /// megabytes into an append-only chain that can never take them back.
10103    #[must_use]
10104    pub fn blobs(mut self, blobs: Arc<dyn crate::blob::BlobStore>) -> Self {
10105        self.blobs = Some(blobs);
10106        self
10107    }
10108
10109    /// Where disclosures of a matter are recorded, so a retention pass names
10110    /// the copies of what it erased.
10111    #[must_use]
10112    pub fn disclosures(mut self, register: Arc<dyn crate::disclosure::DisclosureRegister>) -> Self {
10113        self.disclosures = Some(register);
10114        self
10115    }
10116
10117    /// The operator's tool catalogue, and the client that reaches those tools.
10118    ///
10119    /// Required by a `tool-calling` agent and by nothing else: a skill that
10120    /// calls tools builds its own [`ToolCall`](crate::tools::ToolCall), because
10121    /// it knows which client it means. A declarative agent has no code to make
10122    /// that choice, so the plane makes it once.
10123    ///
10124    /// The catalogue is the authority. A manifest grants a subset of it, the
10125    /// model is offered exactly that subset, and a name the model returns is
10126    /// matched against it byte for byte.
10127    #[cfg(feature = "manifest")]
10128    #[must_use]
10129    pub fn tools(
10130        mut self,
10131        catalog: Arc<crate::tools::ToolCatalog>,
10132        client: Arc<dyn crate::tools::ToolClient>,
10133    ) -> Self {
10134        self.tools = Some((catalog, client));
10135        self
10136    }
10137
10138    /// Typed tools, with their catalogue derived and their coherence enforced.
10139    ///
10140    /// The one-call form, and the reason it exists is not brevity. Deriving the
10141    /// catalogue and checking it against every agent's manifest were both
10142    /// possible before and both **optional**, and a control a caller may forget
10143    /// is not a control — it is advice that reads like one.
10144    ///
10145    /// So this does three things that were three things:
10146    ///
10147    /// * derives the catalogue from each agent's declaration, so a grant, its
10148    ///   ceiling and its protected fields are stated once;
10149    /// * refuses to build if the tools this binary implements and the manifests
10150    ///   a reviewer approved have drifted apart;
10151    /// * wires the box as the client.
10152    ///
10153    /// The work happens in [`build`](Self::build) rather than here, and that is
10154    /// the whole reason it is trustworthy: checking on this call would check
10155    /// against the agents registered *so far*, so `.toolbox(..).agent(..)` would
10156    /// pass by having nothing to disagree with. An enforcement that depends on
10157    /// the order a builder was written is not one.
10158    #[cfg(feature = "manifest")]
10159    #[must_use]
10160    pub fn toolbox(mut self, tools: crate::tools::ToolBox) -> Self {
10161        self.toolbox = Some(tools);
10162        self
10163    }
10164
10165    /// A tool server this plane reaches over some transport of its own.
10166    ///
10167    /// An MCP connection is the usual one. Registering it does three things:
10168    ///
10169    /// * a plane may reach **several** servers, because the router resolves the
10170    ///   `tool://server/name` a grant carries rather than handing every id to one
10171    ///   client;
10172    /// * typed in-process tools and remote servers can be used by the *same*
10173    ///   agent, which is the ordinary shape;
10174    /// * a grant naming a server nobody wired is refused at build, in the same
10175    ///   breath as a grant nothing implements — both mean the model would be
10176    ///   offered a tool that fails when chosen.
10177    ///
10178    /// Composes with [`toolbox`](Self::toolbox); the box answers for the servers
10179    /// its own tools name and these answer for theirs. A server claimed twice is
10180    /// a panic, because registration order deciding which transport carries a
10181    /// call is the defect [`ToolRouter`](crate::tools::ToolRouter) exists to
10182    /// remove.
10183    #[cfg(feature = "manifest")]
10184    #[must_use]
10185    pub fn tool_server(
10186        mut self,
10187        name: impl Into<String>,
10188        client: Arc<dyn crate::tools::ToolClient>,
10189    ) -> Self {
10190        self.tool_servers.push((name.into(), client));
10191        self
10192    }
10193
10194    /// Where this plane's **tool transports** may connect.
10195    ///
10196    /// A `ToolClient` is the embedder's code, and the split is the one every
10197    /// other outbound path here makes: **the client owns the connection, the
10198    /// plane owns the destination.** A transport says where it goes with
10199    /// [`ToolClient::destination`](crate::tools::ToolClient::destination), and
10200    /// this decides whether that is somewhere it may go — before the effect
10201    /// exists, so nothing leaves, nothing is journaled and nothing is metered.
10202    ///
10203    /// A [`Destination::Local`](crate::tools::Destination::Local) transport is
10204    /// not judged: tools compiled into the binary and a stdio child process
10205    /// contact no host of this plane's choosing, so there is nothing for an
10206    /// allowlist to decide. What that does not claim is that the far side
10207    /// reaches nothing — the same residual a compromised allowlisted endpoint
10208    /// carries.
10209    ///
10210    /// Unset means no egress control on the tool path, spelled the way an
10211    /// absent policy engine is: by configuring nothing, rather than by
10212    /// configuring something that looks like a control and is not.
10213    ///
10214    /// ```ignore
10215    /// let plane = Runtime::builder(store)
10216    ///     .tool_server("tickets", mcp_over_https)
10217    ///     .egress(Egress::new().allow("mcp.tickets.example"))
10218    ///     .build();
10219    /// ```
10220    #[cfg(feature = "manifest")]
10221    #[must_use]
10222    pub fn egress(mut self, egress: crate::core::Egress) -> Self {
10223        self.egress = Some(Arc::new(egress));
10224        self
10225    }
10226
10227    /// Forward every model call's live output, with the run it belongs to.
10228    ///
10229    /// Advisory and live-only: the journal's completion stays the canonical
10230    /// answer, a replay performs no call and so streams nothing, and an
10231    /// observer that drops events changes nothing a run does.
10232    #[must_use]
10233    pub fn observe_model_streams(mut self, observer: Arc<dyn RunStreamObserver>) -> Self {
10234        self.streams = Some(observer);
10235        self
10236    }
10237
10238    /// Register a content checker, by its [`name`](crate::content::ContentChecker::name).
10239    ///
10240    /// A manifest check naming a checker no builder registered, or mapping a
10241    /// category the checker does not declare, refuses the build: a declared
10242    /// check nothing could run is a control a reviewer approved that does not
10243    /// exist.
10244    #[cfg(feature = "manifest")]
10245    #[must_use]
10246    pub fn content_checker(mut self, checker: Arc<dyn crate::content::ContentChecker>) -> Self {
10247        Arc::make_mut(&mut self.checkers).insert(checker.name().to_owned(), checker);
10248        self
10249    }
10250
10251    /// The highest sensitivity this plane will write into an append-only record.
10252    ///
10253    /// # What this is, and what the key ring is
10254    ///
10255    /// Two different answers to *this data must be erasable*, and they compose.
10256    /// [`keyring`](Self::keyring) is the **seal it** half: payloads become
10257    /// ciphertext whose key an erasure destroys, which reaches every replica
10258    /// and every backup at once. This is the **refuse it** half: an argument
10259    /// more sensitive than the deployment is willing to make permanent is
10260    /// refused at dispatch, before it reaches the chain.
10261    ///
10262    /// The plane-level twin of `spec.security.max_sensitivity_journaled`, for
10263    /// a plane of hand-written skills that runs under no manifest. The default
10264    /// is the unsafe one — a journal keeps everything, indefinitely — so a
10265    /// plane must be able to state the ceiling whether or not it is
10266    /// declarative. It is an enforcement point at the same gate the manifest's
10267    /// field reaches, not a warning; where both are present **the stricter
10268    /// wins**, as a reviewed tool grant may only tighten what the operator's
10269    /// catalogue allows.
10270    ///
10271    /// Absent means no ceiling, spelled the way an absent policy engine is.
10272    ///
10273    /// ```ignore
10274    /// let plane = Runtime::builder(store)
10275    ///     // Personal data goes in a blob and the chain gets the digest.
10276    ///     .max_sensitivity_journaled(Sensitivity::Internal)
10277    ///     .skill(Triage)
10278    ///     .build();
10279    /// ```
10280    #[must_use]
10281    pub const fn max_sensitivity_journaled(mut self, ceiling: crate::core::Sensitivity) -> Self {
10282        self.journal_ceiling = Some(ceiling);
10283        self
10284    }
10285
10286    /// Which tenant this plane runs as.
10287    ///
10288    /// **One plane, one tenant — but one process, many planes.** A plane is the
10289    /// unit that is bound to a tenant; serving several is
10290    /// [`Planes`](crate::api::Planes)' job, and it resolves the plane from the
10291    /// authenticated caller's tenant rather than from the request, so a handler
10292    /// cannot reach a store it did not resolve. An unregistered tenant is
10293    /// refused rather than defaulted.
10294    ///
10295    /// The name scopes **data keys**, so one tenant's cryptographic erasure
10296    /// cannot reach another's bytes, and it reaches the **policy request**, so a
10297    /// rule can be written per tenant.
10298    ///
10299    /// It does **not** scope the store — that is a separate handle, scoped by
10300    /// `RedbStore::for_tenant` or `PostgresStore::for_tenant`. Two tenants may
10301    /// share one store, because the tenant is a key component of every row on
10302    /// both backends rather than a filter. Setting one and not the other is
10303    /// refused at [`build`](Self::build) rather than discovered later: a plane
10304    /// whose store is scoped elsewhere works perfectly and writes its runs into
10305    /// somebody else's keyspace.
10306    ///
10307    /// Defaults to `default`, which is a real tenant rather than an absence: the
10308    /// single-tenant path is then the same code as the multi-tenant one, and a
10309    /// special "no tenant" case is a second path that would not get tested.
10310    #[must_use]
10311    pub fn tenant(mut self, tenant: crate::core::TenantId) -> Self {
10312        self.tenant = tenant;
10313        self
10314    }
10315
10316    /// Seal payload bytes, and make erasure reach copies deletion cannot.
10317    ///
10318    /// With a key ring configured, everything written through
10319    /// [`StepCtx::blobs`](crate::runtime::StepCtx::blobs) — including
10320    /// [`store_blob`](crate::runtime::StepCtx::store_blob) and governed media —
10321    /// is encrypted under a data key belonging to the run's **case**. Erasing
10322    /// that case destroys the key, so every copy of those bytes becomes
10323    /// unreadable at once: the live store, the replicas, and every backup ever
10324    /// taken. Expiring blobs only reaches the first of those.
10325    ///
10326    /// The case is the erasure unit because it is already the retention unit —
10327    /// bytes are linked to their case at write time, and a second, differently
10328    /// shaped unit for keys would let the two disagree about what an erasure
10329    /// covered.
10330    ///
10331    /// It also seals the stores the plane holds — the journal's payloads, case
10332    /// state, task proposals and buffered event payloads — at
10333    /// [`build`](Self::build), so the order they were registered in cannot lose
10334    /// the guarantee — a store registered after this call is sealed just the
10335    /// same.
10336    ///
10337    /// **Governed memory is the one store this does not reach.**
10338    /// [`EncryptedMemoryStore`](crate::keyring::EncryptedMemoryStore)
10339    /// serialises subject erasure against writes and legal-hold changes with a
10340    /// process-local mutex, so it holds its contract on a single-writer
10341    /// deployment and nowhere else; wrapping it here would hand that adapter to
10342    /// an active-active `PostgreSQL` plane, where the mutex coordinates nothing
10343    /// and the hold race it exists to prevent is the result. Its erasure unit
10344    /// is one item version (`memory-item/<id>@<version>`) and outlives every
10345    /// case, so `erase_case` is not the act that reaches it either. Wrap it
10346    /// yourself, where the deployment's topology is visible:
10347    ///
10348    /// ```ignore
10349    /// let memories = EncryptedMemoryStore::new(inner, keys.clone(), tenant.clone());
10350    /// Runtime::builder(store).memory(Arc::new(memories)).keyring(keys).build()
10351    /// ```
10352    ///
10353    /// Without one, bytes are stored as given and erasure remains deletion.
10354    #[cfg(feature = "keyring")]
10355    #[must_use]
10356    pub fn keyring(mut self, keyring: Arc<dyn crate::keyring::KeyRing>) -> Self {
10357        self.keyring = Some(keyring);
10358        self
10359    }
10360
10361    /// Wrap every store a key ring can seal, at **build** time.
10362    ///
10363    /// One call, one guarantee. A deployer who configures a key ring reads it
10364    /// as *this plane is encrypted*, so the journal, the case store, the
10365    /// worklist, the event buffer and the blobs are sealed together. Five
10366    /// independent wrapping calls would be a control that can be forgotten
10367    /// four times, and forgetting looks exactly like remembering.
10368    ///
10369    /// Wrapped here rather than inside `keyring()` for the reason the tool
10370    /// catalogue is checked here: a wrap applied when the ring is supplied
10371    /// would cover only the stores registered so far, so `keyring(..)` before
10372    /// `cases(..)` would silently seal less than `keyring(..)` after it. An
10373    /// enforcement a reordering can lose is one a reformatter can delete.
10374    ///
10375    /// Every decorator is handed `self.tenant` rather than reading a name back
10376    /// out of the store it wraps, so all four derive the erasure scope from one
10377    /// value. `keyring()`'s own documentation names the one store deliberately
10378    /// left out, and why.
10379    #[cfg(feature = "keyring")]
10380    fn seal_stores(&mut self) {
10381        let Some(keys) = self.keyring.clone() else {
10382            return;
10383        };
10384        let tenant = self.tenant.clone();
10385        self.store = crate::keyring::SealedJournal::wrap(
10386            Arc::clone(&self.store),
10387            Arc::clone(&keys),
10388            tenant.clone(),
10389        );
10390        if let Some(cases) = self.cases.take() {
10391            self.cases = Some(crate::keyring::SealedCases::wrap(
10392                cases,
10393                Arc::clone(&keys),
10394                tenant.clone(),
10395            ));
10396        }
10397        if let Some(events) = self.events.take() {
10398            self.events = Some(crate::keyring::SealedEvents::wrap(
10399                events,
10400                Arc::clone(&keys),
10401                tenant.clone(),
10402            ));
10403        }
10404        // The outbox's store keeps the operator destinations' bearer tokens —
10405        // credentials like any caller's, and the one store here reached through
10406        // a handle the embedder built rather than registered. Sealing it in the
10407        // same sweep is what keeps `keyring()` one decision rather than five
10408        // and a sixth.
10409        #[cfg(feature = "push")]
10410        if let Some(outbox) = self.outbox.take() {
10411            self.outbox = Some(Arc::new(
10412                outbox
10413                    .as_ref()
10414                    .clone()
10415                    .sealed(Arc::clone(&keys), tenant.clone()),
10416            ));
10417        }
10418        if let Some(tasks) = self.tasks.take() {
10419            self.tasks = Some(crate::keyring::SealedTasks::wrap(tasks, keys, tenant));
10420        }
10421    }
10422
10423    /// Without the `keyring` feature there is nothing to seal.
10424    ///
10425    /// `&mut self` matches the sealing arm above, which does mutate. Taking
10426    /// `&self` here would make the signature depend on a feature flag, and
10427    /// every caller with it.
10428    #[cfg(not(feature = "keyring"))]
10429    #[allow(clippy::unused_self, clippy::needless_pass_by_ref_mut)]
10430    fn seal_stores(&mut self) {}
10431
10432    /// Supply the store that tracks batch items.
10433    ///
10434    /// Only needed for [`Runtime::run_batch`]; a plane that runs no batches does
10435    /// not need one, and asking for it unconditionally would make the common
10436    /// case carry the uncommon one's setup.
10437    #[must_use]
10438    pub fn batches(mut self, batches: Arc<dyn crate::batch::BatchStore>) -> Self {
10439        self.batches = Some(batches);
10440        self
10441    }
10442
10443    /// Supply the authorization engine.
10444    ///
10445    /// Without one there is no policy layer — the information-flow gates still
10446    /// apply, but nothing asks whether the principal was allowed. That is a
10447    /// deliberate absence rather than a permissive default: see `core::policy`
10448    /// on why there is no `AllowAll` to configure by mistake.
10449    ///
10450    /// The engine's complete immutable bundle identity is recorded at admission,
10451    /// so both whether policy was on and exactly which executable semantics
10452    /// governed the run are answerable from the journal. An open run may resume
10453    /// only under that same identity.
10454    #[must_use]
10455    pub fn policy(mut self, policy: Arc<dyn crate::core::PolicyEngine>) -> Self {
10456        self.policy = Some(policy);
10457        self
10458    }
10459
10460    /// The chain this plane's own runs act under.
10461    ///
10462    /// The identity of a run the embedder starts in-process — checked against
10463    /// the plan at admission (the plan is the authorization graph, so one that
10464    /// exceeds the chain's authority never starts), held to the chain's
10465    /// audience and validity, and journaled, so "on whose behalf" is
10466    /// answerable from history rather than reconstructed from timestamps.
10467    ///
10468    /// It is **not** the identity of a run a served surface admits for a
10469    /// caller: those act under the chain the caller presented, or under none
10470    /// when it presented none ([`RunTerms::served`]). A plane that bound its own
10471    /// chain to every peer's run would be an ambient credential.
10472    ///
10473    /// Verification of the *credential* belongs to whatever authenticates the
10474    /// caller; what arrives here is already a chain, and its attenuation is
10475    /// guaranteed by its own constructors however it was obtained.
10476    #[must_use]
10477    pub fn acting_as(mut self, chain: crate::core::Delegation) -> Self {
10478        self.identity = Some(chain);
10479        self
10480    }
10481
10482    /// The peers this plane may call, and the transport that reaches them.
10483    ///
10484    /// The registry is wiring — what each peer is granted, which credential
10485    /// reaches it, how a failed call recovers. What a run may *ask* a peer
10486    /// for is a manifest grant, `tool://<peer>/<capability>`, dispatched
10487    /// through [`StepCtx::call_peer`](crate::runtime::StepCtx::call_peer).
10488    /// Refused at build: a peer named `agent` or like a wired tool server,
10489    /// and a grant naming a capability outside the peer's registry scope.
10490    #[must_use]
10491    pub fn peers(
10492        mut self,
10493        registry: crate::peers::PeerRegistry,
10494        client: Arc<dyn crate::peers::PeerClient>,
10495    ) -> Self {
10496        self.peers = Some(Arc::new(PeerWiring { registry, client }));
10497        self
10498    }
10499
10500    /// Submit this plane's checkpoints to witnesses, and hold each round to a
10501    /// quorum.
10502    ///
10503    /// This is the only control here whose value comes from *not* being this
10504    /// plane. The hash chain proves no record was edited, the signatures say
10505    /// which workload wrote them, and the Merkle log proves no sealed run was
10506    /// removed — but every input to all three comes from the party an auditor
10507    /// is being asked to trust. None of them detects a whole run deleted
10508    /// before anyone took a checkpoint, or two internally perfect histories
10509    /// shown to two auditors. A witness keeps the last checkpoint it saw and
10510    /// cosigns only a checkpoint that provably extends it, so those two stop
10511    /// being invisible.
10512    ///
10513    /// Availability never waits on it: submission runs on
10514    /// [`sweep`](Runtime::sweep), after sealing and off the run path, and a
10515    /// round that falls short is a report rather than a refusal. A run whose
10516    /// witnesses are unreachable finished long ago, and making the plane's
10517    /// availability depend on a third party is the wrong trade for evidence
10518    /// that is read after the fact.
10519    ///
10520    /// Point it at witnesses somebody else runs. A witness hosted beside the
10521    /// history it vouches for is an anchor one compromise removes, and
10522    /// [`MemoryWitness`](crate::journal::MemoryWitness) says so at its own
10523    /// definition.
10524    ///
10525    /// # Errors
10526    ///
10527    /// [`build`](RuntimeBuilder::build) refuses a quorum larger than the
10528    /// number of witnesses configured, and a witness list with no quorum.
10529    #[must_use]
10530    pub fn witnesses(
10531        mut self,
10532        witnesses: Vec<Arc<dyn crate::journal::Witness>>,
10533        quorum: crate::journal::WitnessQuorum,
10534    ) -> Self {
10535        self.witnesses = witnesses;
10536        self.quorum = Some(quorum);
10537        self
10538    }
10539
10540    /// Declare the maximum time between checkpoint submissions to each
10541    /// witness: [`sweep`](Runtime::sweep) re-submits an unchanged checkpoint
10542    /// once it has passed since the last round that met quorum, so an idle
10543    /// plane still carries a fresh witness timestamp. Without it an unchanged
10544    /// log is never re-submitted.
10545    ///
10546    /// The interval is kept only as often as `sweep` runs; a deployment
10547    /// sweeping on its own scheduler owns that cadence.
10548    ///
10549    /// # Errors
10550    ///
10551    /// [`build`](RuntimeBuilder::build) refuses an interval with no witnesses.
10552    #[must_use]
10553    pub fn witness_interval(mut self, interval: std::time::Duration) -> Self {
10554        self.witness_interval = Some(interval);
10555        self
10556    }
10557
10558    /// Supply the calendar that resolves deadline descriptions to instants.
10559    ///
10560    /// Defaults to [`WallClock`], which understands plain offsets and refuses
10561    /// anything it does not know rather than approximating it. Domain calendars
10562    /// — working days, holidays, cut-off hours — are the adapter's job.
10563    #[must_use]
10564    pub fn calendar(mut self, calendar: Arc<dyn Calendar>) -> Self {
10565        self.calendar = Some(calendar);
10566        self
10567    }
10568
10569    /// Derive the tool catalogue from the agents and refuse a disagreement.
10570    ///
10571    /// Every agent, not the first: a plane may host several, and a tool granted
10572    /// to none of them is still a tool this binary can be asked for.
10573    ///
10574    /// # Errors
10575    ///
10576    /// If the box and any agent's manifest disagree. A build-time
10577    /// misconfiguration has a fix and no recovery, and it is refused the way the
10578    /// tenant mismatch beside it is: before anything runs.
10579    #[cfg(feature = "manifest")]
10580    fn settle_toolbox(&mut self) -> Result<(), BuildError> {
10581        let servers = std::mem::take(&mut self.tool_servers);
10582        let tools = self.toolbox.take();
10583        // A peer's name is a server name a grant can carry, so it must mean
10584        // exactly one thing on this plane: not `agent`, and not a name some
10585        // tool transport also answers for.
10586        let peer_names: std::collections::BTreeSet<String> = self
10587            .peers
10588            .as_ref()
10589            .map(|wiring| wiring.registry.peers().map(ToString::to_string).collect())
10590            .unwrap_or_default();
10591        if peer_names.contains(crate::tools::AGENT_SERVER) {
10592            return Err(BuildError::ReservedToolServer);
10593        }
10594        for name in &peer_names {
10595            if servers.iter().any(|(server, _)| server == name)
10596                || tools
10597                    .as_ref()
10598                    .is_some_and(|t| t.servers().any(|s| s == name))
10599            {
10600                return Err(BuildError::PeerIsAlsoAToolServer {
10601                    server: name.clone(),
10602                });
10603            }
10604        }
10605        if tools.is_none() && servers.is_empty() {
10606            return Ok(());
10607        }
10608        let tools = tools.unwrap_or_default();
10609        let remote_servers: std::collections::BTreeSet<String> =
10610            servers.iter().map(|(name, _)| name.clone()).collect();
10611        // What a grant may name without this box implementing it: a wired
10612        // transport, or a registered peer.
10613        let reachable_elsewhere: std::collections::BTreeSet<String> =
10614            remote_servers.union(&peer_names).cloned().collect();
10615        // `agent` names agents on this plane, and only them. A transport or a
10616        // typed tool under that name would let a deployment decide whether a
10617        // reviewed grant means "an agent here" or "somebody's server".
10618        if remote_servers.contains(crate::tools::AGENT_SERVER)
10619            || tools
10620                .servers()
10621                .any(|server| server == crate::tools::AGENT_SERVER)
10622        {
10623            return Err(BuildError::ReservedToolServer);
10624        }
10625        if remote_servers.len() != servers.len() {
10626            // The set lost an entry, so some name appears twice. Naming it beats
10627            // reporting a count a reader then has to go and diff by hand.
10628            let mut seen = std::collections::BTreeSet::new();
10629            let duplicate = servers
10630                .iter()
10631                .map(|(name, _)| name)
10632                .find(|name| !seen.insert((*name).clone()))
10633                .cloned()
10634                .unwrap_or_default();
10635            return Err(BuildError::DuplicateToolServer { server: duplicate });
10636        }
10637        // Both forms wired is not a merge and must not silently be one. The
10638        // hand-built catalogue is the operator saying something deliberate; the
10639        // derived one is the agent's declaration. Overwriting either with the
10640        // other would run a plane under grants nobody chose.
10641        if self.tools.is_some() {
10642            return Err(BuildError::ToolsWiredTwice);
10643        }
10644        let mut catalog = crate::tools::ToolCatalog::new();
10645        let mut declared = 0usize;
10646        // Which agent's declaration a tool's catalogue entry came from, so a
10647        // second agent declaring the same tool *differently* is a build-time
10648        // refusal rather than a silent overwrite.
10649        //
10650        // A plane has one catalogue and its agents have one manifest each, so
10651        // two agents granting `tool://ledger/read` with different protected
10652        // fields cannot both be satisfied. Merging by last-writer would resolve
10653        // it by **registration order** — the one thing this builder already
10654        // says an enforcement must never depend on — and it fails at a
10655        // distance: `declared` compares each agent's manifest against the
10656        // catalogue-derived descriptor exactly, so the agent that lost the race
10657        // is refused *every* call to that tool, in production, with a message
10658        // blaming a code-versus-manifest drift that neither file exhibits.
10659        let mut source: BTreeMap<crate::tools::ToolId, (String, crate::tools::ToolSafety)> =
10660            BTreeMap::new();
10661        for agent in &self.agents {
10662            let Some(manifest) = agent.manifest.as_ref() else {
10663                continue;
10664            };
10665            declared += 1;
10666            tools
10667                .check_against(manifest, &reachable_elsewhere)
10668                .map_err(|problems| BuildError::ToolDrift {
10669                    agent: manifest.metadata.name.clone(),
10670                    problems,
10671                })?;
10672            for (id, safety) in crate::tools::ToolCatalog::from_manifest(manifest).entries() {
10673                if let Some((first, existing)) = source.get(&id) {
10674                    if existing != &safety {
10675                        return Err(BuildError::ToolDeclaredTwoWays {
10676                            tool: id.reference(),
10677                            first: first.clone(),
10678                            second: manifest.metadata.name.clone(),
10679                        });
10680                    }
10681                    continue;
10682                }
10683                source.insert(id.clone(), (manifest.metadata.name.clone(), safety.clone()));
10684                catalog = catalog.allow(id, safety);
10685            }
10686        }
10687        // A box with nothing to be coherent *with* is the same defect one step
10688        // earlier: tools wired to a plane where no declaration admits them, so
10689        // nothing a reviewer reads describes what this binary can reach.
10690        if declared == 0 {
10691            return Err(BuildError::ToolsWithoutDeclaration);
10692        }
10693        // The typed argument type is the schema source. Overlay its
10694        // presentation only after every manifest has been checked, so the
10695        // model sees exactly what the body will deserialize rather than the
10696        // old permissive `{ type: object }` fallback.
10697        for id in tools.ids() {
10698            let (description, schema, _) = tools
10699                .declared(id)
10700                .expect("every registered typed tool has a declaration");
10701            let reviewed_description = catalog
10702                .declaration(id)
10703                .map_or_else(|| description.to_owned(), |(text, _)| text.to_owned());
10704            catalog = catalog.declare(id.clone(), reviewed_description, schema.clone());
10705        }
10706        // One client per server, resolved by the name a grant carries. A single
10707        // client handed every id could not tell `tool://ledger/read` from
10708        // `tool://tickets/read`, and a transport that never reads the server
10709        // component answers both — from whichever server it happens to hold.
10710        let router = servers.into_iter().fold(
10711            crate::tools::ToolRouter::new().toolbox(&Arc::new(tools)),
10712            |router, (name, client)| router.server(name, client),
10713        );
10714        self.tools = Some((
10715            Arc::new(catalog),
10716            Arc::new(router) as Arc<dyn crate::tools::ToolClient>,
10717        ));
10718        Ok(())
10719    }
10720
10721    /// Settle the tool catalogue: derive it if asked, then hold it to the
10722    /// declarations.
10723    ///
10724    /// One call because the two halves are one decision seen from either side.
10725    /// [`settle_toolbox`](Self::settle_toolbox) covers the derived catalogue,
10726    /// where code and manifest could disagree; the check after it covers the
10727    /// stated one, where operator and manifest could. Whichever way the
10728    /// catalogue arrived, it is checked before anything runs.
10729    #[cfg(feature = "manifest")]
10730    fn settle_tools(&mut self) -> Result<(), BuildError> {
10731        self.settle_toolbox()?;
10732        self.check_catalogue_not_laxer_than_grants()
10733    }
10734
10735    /// Refuse a stated catalogue that is **laxer** than a reviewed grant.
10736    ///
10737    /// `toolbox(..)` derives the catalogue from the manifests, so the two
10738    /// cannot drift. `tools(..)` states it by hand, and there the operator's
10739    /// entry and the agent's declaration are two copies of one decision — with
10740    /// nothing, until this, that noticed them disagreeing.
10741    ///
10742    /// Only one direction is a defect, the same one
10743    /// [`ToolBox::check_against`](crate::tools::ToolBox::check_against) refuses.
10744    /// An operator being **more** cautious than the declaration is fine and
10745    /// often right. An operator being **less** cautious is not, and it changes
10746    /// two things at once:
10747    ///
10748    /// * the whole-value taint gate stops firing, so model-chosen arguments
10749    ///   reach something that changes the world;
10750    /// * `ToolSafety::read_only` carries `Recovery::Retry`, so a timed-out call
10751    ///   to a money-moving tool is sent a second time.
10752    ///
10753    /// The dispatch gates already take the stricter `mutates` of the two, so
10754    /// the first is contained at runtime. The second is not — recovery is read
10755    /// from the catalogue alone — and neither should have to be, because this
10756    /// is a wiring mistake with a fix and no recovery. It is refused here,
10757    /// beside the rest of them.
10758    #[cfg(feature = "manifest")]
10759    fn check_catalogue_not_laxer_than_grants(&self) -> Result<(), BuildError> {
10760        let Some((catalog, _)) = self.tools.as_ref() else {
10761            return Ok(());
10762        };
10763        let mut problems = Vec::new();
10764        for agent in &self.agents {
10765            let Some(manifest) = agent.manifest.as_ref() else {
10766                continue;
10767            };
10768            for grant in &manifest.spec.tools {
10769                if !grant.mutates {
10770                    continue;
10771                }
10772                let Some(id) = crate::tools::ToolId::parse(&grant.reference) else {
10773                    continue;
10774                };
10775                if catalog.safety(&id).is_some_and(|s| !s.mutates) {
10776                    problems.push(format!(
10777                        "agent '{}' grants '{}' as mutating and the stated catalogue \
10778                         calls it read-only",
10779                        manifest.metadata.name, grant.reference
10780                    ));
10781                }
10782            }
10783        }
10784        if problems.is_empty() {
10785            Ok(())
10786        } else {
10787            Err(BuildError::CatalogueLaxerThanGrant { problems })
10788        }
10789    }
10790
10791    /// Assemble the runtime, or panic naming the wiring mistake.
10792    ///
10793    /// The ordinary entry point. Every refusal below is a bug in code the author
10794    /// is looking at, so propagating it through `?` to a `main` that prints it
10795    /// is ceremony around an abort — and each is caught here at startup rather
10796    /// than at dispatch, in production, where the cost is a run that has already
10797    /// begun.
10798    ///
10799    /// Use [`try_build`](Self::try_build) where a manifest arrives at *runtime*
10800    /// — read from disk, pinned by a registry, or supplied per tenant. There a
10801    /// bad declaration is an input rather than a bug, and a panic would take
10802    /// every other tenant in the process down to report it.
10803    ///
10804    /// # Panics
10805    ///
10806    /// On any [`BuildError`]:
10807    ///
10808    /// * A manifest declares a capability in `spec.capabilities.provides` that
10809    ///   no registered skill provides. **An agent has skills**, so a declaration
10810    ///   advertising one it cannot perform is a card that lies.
10811    /// * Two agents claim the same capability. Dispatch resolves a capability to
10812    ///   one skill *and to the manifest governing it*, so a second claim would
10813    ///   silently take the first's work out from under the first's budget,
10814    ///   model grants and egress ceiling.
10815    /// * Two skills share a name. A name is what a capability resolves to and
10816    ///   what governance is keyed on; two of them make both lookups arbitrary.
10817    /// * A stated catalogue calls a tool read-only that a reviewed manifest
10818    ///   grants as mutating. That exemption drops the whole-value taint gate
10819    ///   and makes a timed-out money-moving call retryable — the one direction
10820    ///   an operator cannot be right about.
10821    /// * A declarative agent names a provider no driver is registered for, or
10822    ///   declares `spec.execution` without a privileged model to call.
10823    /// * A plane and its store — or its blob store — are scoped to different
10824    ///   tenants. The two are set
10825    ///   separately — this builder's tenant scopes data keys and the policy
10826    ///   request, `for_tenant` scopes the store's keys — and the mismatch does
10827    ///   not show up at runtime. It *works*, and writes this tenant's runs into
10828    ///   another's keyspace while every erasure and every policy request names
10829    ///   the right one.
10830    ///
10831    /// # Long-running services want [`try_build`](Self::try_build)
10832    ///
10833    /// This panics, which is the honest answer for a binary wiring its own
10834    /// skills: every variant above is a bug in code the author is looking at,
10835    /// and aborting reports it at the moment it can be fixed. A daemon is a
10836    /// different case — it wants to exit with a diagnostic, and a plane
10837    /// assembled from a manifest that arrived at runtime is handling an *input*,
10838    /// where a panic reports one tenant's typo by killing every other tenant's
10839    /// in-flight run. `try_build` returns the same `BuildError` instead. One
10840    /// implementation underneath both, so they cannot disagree about what is
10841    /// refused.
10842    #[must_use]
10843    pub fn build(self) -> Arc<Runtime> {
10844        // Not `expect`. That formats the error with `Debug`, which would print
10845        // `AdvertisesWhatItCannotProvide { agent: "…", missing: [...] }` — the
10846        // variant's *shape* — while the sentence explaining what to do about it
10847        // lives in `Display`. Panicking with `{error}` keeps the two entry
10848        // points telling one story, which is the whole point of `build` being
10849        // `try_build` underneath.
10850        match self.try_build() {
10851            Ok(runtime) => runtime,
10852            Err(error) => panic!("{error}"),
10853        }
10854    }
10855
10856    /// The driver for a declarative agent's privileged model — named rather
10857    /// than defaulted, because falling back to some other registered driver
10858    /// would run the agent on a model its own declaration does not name.
10859    /// Every declared content check names a registered checker, and maps only
10860    /// categories that checker declares.
10861    #[cfg(feature = "manifest")]
10862    fn check_content_checkers(
10863        checkers: &crate::content::Checkers,
10864        m: &crate::manifest::Manifest,
10865    ) -> Result<(), BuildError> {
10866        let Some(content) = m.spec.security.content.as_ref() else {
10867            return Ok(());
10868        };
10869        for check in &content.checks {
10870            let refused = |detail: String| BuildError::ContentCheck {
10871                agent: m.metadata.name.clone(),
10872                check: check.id.clone(),
10873                detail,
10874            };
10875            let Some(checker) = checkers.get(&check.checker) else {
10876                return Err(refused(format!(
10877                    "no checker named '{}' is registered on this plane",
10878                    check.checker
10879                )));
10880            };
10881            if let Some(unknown) = check
10882                .on
10883                .keys()
10884                .find(|category| !checker.categories().contains(*category))
10885            {
10886                return Err(refused(format!(
10887                    "'{unknown}' is not a category checker '{}' declares",
10888                    check.checker
10889                )));
10890            }
10891        }
10892        Ok(())
10893    }
10894
10895    #[cfg(feature = "manifest")]
10896    fn declared_provider(
10897        providers: &HashMap<String, Arc<dyn crate::model::ModelProvider>>,
10898        m: &crate::manifest::Manifest,
10899    ) -> Result<Arc<dyn crate::model::ModelProvider>, BuildError> {
10900        let model = m
10901            .spec
10902            .models
10903            .as_ref()
10904            .and_then(|x| x.privileged.as_ref())
10905            .ok_or_else(|| BuildError::DeclarativeWithoutModel {
10906                agent: m.metadata.name.clone(),
10907            })?;
10908        providers
10909            .get(&model.provider)
10910            .map(Arc::clone)
10911            .ok_or_else(|| BuildError::UnknownProvider {
10912                agent: m.metadata.name.clone(),
10913                provider: model.provider.clone(),
10914            })
10915    }
10916
10917    /// Assemble the runtime, or say why it cannot be.
10918    ///
10919    /// The same checks as [`build`](Self::build), returned rather than raised.
10920    /// One implementation behind both, so they cannot come to disagree about
10921    /// what is refused.
10922    ///
10923    /// # Errors
10924    ///
10925    /// Any [`BuildError`] — see [`build`](Self::build) for what each means.
10926    // `mut` is for `settle_toolbox`, which only exists when manifests do.
10927    #[cfg_attr(not(feature = "manifest"), allow(unused_mut))]
10928    #[allow(clippy::too_many_lines)]
10929    pub fn try_build(mut self) -> Result<Arc<Runtime>, BuildError> {
10930        if self.lease_ttl < MIN_LEASE_TTL {
10931            return Err(BuildError::LeaseUnrenewable {
10932                ttl: self.lease_ttl,
10933                minimum: MIN_LEASE_TTL,
10934            });
10935        }
10936        // A ceiling of zero is a budget already spent, and it refuses the
10937        // first effect of every run this plane will ever make. The manifest
10938        // refuses one at parse; a plane wired in Rust reaches the same `Budget`
10939        // without passing a parser, so the rule is applied at both doors from
10940        // the one definition in `Budget::bricked_ceiling`.
10941        if let Some(field) = self.budget.bricked_ceiling() {
10942            return Err(BuildError::BudgetPermitsNothing { field });
10943        }
10944        if let Some(engine) = self.policy.as_ref() {
10945            preflight_policy(engine.as_ref(), self.identity.as_ref())?;
10946        }
10947        // Before `seal_stores`, necessarily: sealing wraps each handle in one
10948        // scoped to the *plane's* tenant, after which every store agrees with
10949        // the plane and the question cannot be asked.
10950        // Both halves of push, because either can be wired to another
10951        // tenant's rows and the consequence is the same: registrations,
10952        // cursors and credentials filed under a scope this plane's erasure
10953        // never names.
10954        #[cfg(feature = "push")]
10955        let push = self
10956            .push
10957            .as_ref()
10958            .map(|s| s.tenant())
10959            .or_else(|| self.outbox.as_ref().map(|o| o.store_tenant()));
10960        #[cfg(not(feature = "push"))]
10961        let push: Option<&str> = None;
10962        check_same_tenant(
10963            self.store.as_ref(),
10964            self.blobs.as_ref(),
10965            self.memories.as_ref(),
10966            &[
10967                ("case", self.cases.as_ref().map(|s| s.tenant())),
10968                ("event", self.events.as_ref().map(|s| s.tenant())),
10969                ("task", self.tasks.as_ref().map(|s| s.tenant())),
10970                ("memory", self.memories.as_ref().map(|s| s.tenant())),
10971                ("quota", self.quotas.as_ref().map(|s| s.tenant())),
10972                ("timer", self.timers.as_ref().map(|s| s.tenant())),
10973                ("batch", self.batches.as_ref().map(|s| s.tenant())),
10974                ("authority", self.authorities.as_ref().map(|s| s.tenant())),
10975                ("push", push),
10976            ],
10977            &self.tenant,
10978        )?;
10979        // The one wiring mistake here that would otherwise never fail: a
10980        // cross-space query ranks rather than refusing.
10981        if let Some(semantic) = &self.semantic {
10982            let embedder = semantic.embedder.revision();
10983            let index = semantic.retriever.index().query_revision;
10984            if embedder != index {
10985                return Err(BuildError::EmbeddingSpaceMismatch { embedder, index });
10986            }
10987            if semantic.embedder.pricing().is_none() && self.states_a_money_ceiling() {
10988                return Err(BuildError::UnpricedEmbedder { embedder });
10989            }
10990            let Some(memories) = self.memories.take() else {
10991                return Err(BuildError::SemanticMemoryWithoutStore);
10992            };
10993            // Every erasure through the plane's store — and through a sealed
10994            // store's own subject erasure — reaches the index built from it.
10995            let reached = memories.erasure_index();
10996            let indexed: Arc<dyn crate::memory::MemoryStore> = match reached {
10997                Some(index) if Arc::ptr_eq(&index, &semantic.retriever) => memories,
10998                Some(_) => return Err(BuildError::MemoryIndexedElsewhere),
10999                None if memories.seals() => {
11000                    return Err(BuildError::SealedMemoryMissesIndex);
11001                }
11002                None => Arc::new(crate::memory::IndexedMemoryStore::new(
11003                    Arc::clone(&memories),
11004                    Arc::clone(&semantic.retriever),
11005                )),
11006            };
11007            self.memories = Some(indexed);
11008        }
11009        self.seal_stores();
11010        #[cfg(feature = "manifest")]
11011        self.settle_tools()?;
11012
11013        let mut skills = HashMap::new();
11014        let mut by_capability = HashMap::new();
11015        #[cfg(feature = "manifest")]
11016        let mut governed_by: HashMap<String, Arc<crate::manifest::Manifest>> = HashMap::new();
11017        #[cfg(feature = "manifest")]
11018        let mut published_by: HashMap<String, crate::core::KeyId> = HashMap::new();
11019
11020        // Skills registered directly belong to the plane's anonymous agent: no
11021        // declaration, so nothing to enforce against them beyond the runtime's
11022        // own budget. That is a legitimate shape — not every agent needs a
11023        // manifest — and it is why `skill()` still exists beside `agent()`.
11024        for s in self.skills {
11025            register_skill(s, &mut by_capability, &mut skills)?;
11026        }
11027
11028        #[cfg(feature = "manifest")]
11029        for agent in self.agents {
11030            if let (Some(m), Some(key)) = (agent.manifest.as_ref(), agent.publisher.clone()) {
11031                published_by.insert(m.metadata.name.clone(), key);
11032            }
11033            let Some(m) = agent.manifest.clone() else {
11034                for s in agent.skills {
11035                    register_skill(s, &mut by_capability, &mut skills)?;
11036                }
11037                continue;
11038            };
11039
11040            // The capabilities this agent's *own* skills provide, which is what
11041            // its declaration is checked against below.
11042            let mut mine: HashSet<Capability> = HashSet::new();
11043            for s in agent.skills {
11044                mine.extend(s.descriptor().capabilities());
11045                let name = register_skill(s, &mut by_capability, &mut skills)?;
11046                governed_by.insert(name, Arc::clone(&m));
11047            }
11048
11049            // A declarative agent needs no skill: the runtime supplies the
11050            // behaviour its manifest asked for.
11051            if let Some(execution) = &m.spec.execution {
11052                // A `call` calls no model, so it needs no provider.
11053                let provider = if execution.kind == crate::manifest::ExecutionKind::Call {
11054                    None
11055                } else {
11056                    Some(Self::declared_provider(&self.providers, &m)?)
11057                };
11058                if m.spec.capabilities.provides.is_empty() {
11059                    return Err(BuildError::DeclarativeProvidesNothing {
11060                        agent: m.metadata.name.clone(),
11061                    });
11062                }
11063                // A plane whose only tools are agents needs no toolbox and no
11064                // transport: the catalogue is derived from the declaration and
11065                // dispatch is `commission`. The empty router is deliberate —
11066                // an agent-server call never reaches it, and anything else
11067                // arriving there is refused as unreachable rather than
11068                // silently absorbed.
11069                let tools = self.tools.clone().or_else(|| {
11070                    // Agents on this plane and registered peers: both
11071                    // dispatch through the runtime's own effects rather than
11072                    // a tool transport, so a declaration made only of them
11073                    // needs no box and no router.
11074                    let all_agent = !m.spec.tools.is_empty()
11075                        && m.spec.tools.iter().all(|g| {
11076                            crate::tools::ToolId::parse(&g.reference).is_some_and(|id| {
11077                                id.server == crate::tools::AGENT_SERVER
11078                                    || self.peers.as_ref().is_some_and(|wiring| {
11079                                        wiring
11080                                            .registry
11081                                            .grant(&crate::peers::PeerId::new(&id.server))
11082                                            .is_some()
11083                                    })
11084                            })
11085                        });
11086                    all_agent.then(|| {
11087                        (
11088                            Arc::new(crate::tools::ToolCatalog::from_manifest(&m)),
11089                            Arc::new(crate::tools::ToolRouter::new())
11090                                as Arc<dyn crate::tools::ToolClient>,
11091                        )
11092                    })
11093                });
11094                // A tool loop with nothing to reach fails on every run with the
11095                // same sentence, and the manifest that says so was read at
11096                // build. `planned` only needs a catalogue when it has grants;
11097                // `tool-calling` needs one to exist at all, since offering a
11098                // model no tools is not a tool loop.
11099                if tools.is_none() {
11100                    let needs = match execution.kind {
11101                        crate::manifest::ExecutionKind::ToolCalling => {
11102                            Some(if m.spec.tools.is_empty() {
11103                                "no tool grants".to_owned()
11104                            } else {
11105                                format!("{} tool grant(s)", m.spec.tools.len())
11106                            })
11107                        }
11108                        crate::manifest::ExecutionKind::Planned
11109                        | crate::manifest::ExecutionKind::Call
11110                            if !m.spec.tools.is_empty() =>
11111                        {
11112                            Some(format!("{} tool grant(s)", m.spec.tools.len()))
11113                        }
11114                        _ => None,
11115                    };
11116                    if let Some(grants) = needs {
11117                        return Err(BuildError::DeclarativeToolsUnreachable {
11118                            agent: m.metadata.name.clone(),
11119                            kind: execution.kind.as_str(),
11120                            grants,
11121                        });
11122                    }
11123                }
11124
11125                // Oversight needs somewhere to put the decision, and both
11126                // halves are known here. Left to run time it surfaces at the
11127                // first real approval — a person already waiting, on the code
11128                // path a test suite is least likely to reach.
11129                let declared = if m.spec.oversight.is_some() {
11130                    Some("`spec.oversight`".to_owned())
11131                } else if m.spec.tools.iter().any(|g| g.requires_approval) {
11132                    Some("a grant with `requires_approval: true`".to_owned())
11133                } else {
11134                    None
11135                };
11136                if let Some(declared) = declared {
11137                    // Ordered by what a reader fixes first: without a case
11138                    // there is nothing for a task to hang off.
11139                    //
11140                    // Timers are deliberately **not** required, and the test
11141                    // suite is why: four approval tests wire a case store and a
11142                    // worklist, no timers, and run an approval end to end. A
11143                    // task is opened, the run suspends, a person decides. What
11144                    // a timer store adds is the sweeper firing `on_expiry`, and
11145                    // refusing a configuration that works would be this check
11146                    // asserting more than it knows — the failure mode it exists
11147                    // to prevent, arriving from the other side.
11148                    let missing = if self.cases.is_none() {
11149                        Some(("case store", "cases"))
11150                    } else if self.tasks.is_none() {
11151                        Some(("worklist", "tasks"))
11152                    } else {
11153                        None
11154                    };
11155                    if let Some((missing, remedy)) = missing {
11156                        return Err(BuildError::OversightUnreachable {
11157                            agent: m.metadata.name.clone(),
11158                            declared,
11159                            missing,
11160                            remedy,
11161                        });
11162                    }
11163                }
11164
11165                // A recall reads before the first model call and formation
11166                // writes after the answer, so a missing store costs a run its
11167                // first effect or its whole budget depending on which half was
11168                // declared. Both facts are here: the file says memories are
11169                // reached, the plane says there is nowhere to reach.
11170                if let Some(memory) = &m.spec.memory {
11171                    let declared: [(&'static str, Option<&crate::manifest::MemorySubject>); 2] = [
11172                        (
11173                            "spec.memory.recall",
11174                            memory.recall.as_ref().map(|r| &r.subject),
11175                        ),
11176                        (
11177                            "spec.memory.formation",
11178                            memory.formation.as_ref().map(|f| &f.subject),
11179                        ),
11180                    ];
11181                    for (field, subject) in declared {
11182                        let Some(subject) = subject else { continue };
11183                        if self.memories.is_none() {
11184                            return Err(BuildError::MemoryWithoutStore {
11185                                agent: m.metadata.name.clone(),
11186                                declared: field,
11187                            });
11188                        }
11189                        if subject.needs_case() && self.cases.is_none() {
11190                            return Err(BuildError::MemorySubjectUnbindable {
11191                                agent: m.metadata.name.clone(),
11192                                subject: subject.as_written(),
11193                            });
11194                        }
11195                    }
11196                }
11197
11198                for cap in &m.spec.capabilities.provides {
11199                    let skill: Arc<dyn Skill> = Arc::new(super::declarative::Declarative::new(
11200                        execution.kind,
11201                        cap.clone(),
11202                        m.metadata.name.clone(),
11203                        provider.clone(),
11204                        tools.clone(),
11205                        execution.max_turns,
11206                    ));
11207                    mine.insert(Capability::new(cap.as_str()));
11208                    let name = register_skill(skill, &mut by_capability, &mut skills)?;
11209                    governed_by.insert(name, Arc::clone(&m));
11210                }
11211            }
11212
11213            check_declaration_matches_skills(&m, &mine)?;
11214        }
11215
11216        // Agent grants are validated against the finished plane, because the
11217        // capability they name may belong to an agent registered *later* — a
11218        // check inside the loop would pass or fail on registration order.
11219        #[cfg(feature = "manifest")]
11220        {
11221            let mut checked = std::collections::BTreeSet::new();
11222            for m in governed_by.values() {
11223                if !checked.insert(m.metadata.name.clone()) {
11224                    continue;
11225                }
11226                Self::check_content_checkers(&self.checkers, m)?;
11227                for grant in &m.spec.tools {
11228                    let Some(id) = crate::tools::ToolId::parse(&grant.reference) else {
11229                        continue;
11230                    };
11231                    // A grant naming a registered peer is a delegating hop,
11232                    // and both things that would refuse it on every run are
11233                    // knowable here: a capability the registry never gave the
11234                    // peer, and a granting agent whose own ceiling is zero.
11235                    if let Some(wiring) = self.peers.as_ref()
11236                        && let Some(peer_grant) = wiring
11237                            .registry
11238                            .grant(&crate::peers::PeerId::new(&id.server))
11239                    {
11240                        if !peer_grant.scope.permits(&Capability::new(id.tool.as_str())) {
11241                            return Err(BuildError::PeerGrantOutsideScope {
11242                                agent: m.metadata.name.clone(),
11243                                peer: id.server,
11244                                capability: id.tool,
11245                            });
11246                        }
11247                        let cannot_delegate = m
11248                            .spec
11249                            .topology
11250                            .as_ref()
11251                            .is_some_and(|t| t.role == crate::manifest::Role::Specialist)
11252                            || m.spec.security.max_delegation_depth == Some(0);
11253                        if cannot_delegate {
11254                            return Err(BuildError::PeerGrantOnASpecialist {
11255                                agent: m.metadata.name.clone(),
11256                                peer: id.server,
11257                            });
11258                        }
11259                        continue;
11260                    }
11261                    if id.server != crate::tools::AGENT_SERVER {
11262                        continue;
11263                    }
11264                    if m.spec.capabilities.provides.contains(&id.tool) {
11265                        return Err(BuildError::AgentToolSelfReference {
11266                            agent: m.metadata.name.clone(),
11267                            capability: id.tool,
11268                        });
11269                    }
11270                    if !by_capability.contains_key(&Capability::new(id.tool.as_str())) {
11271                        return Err(BuildError::AgentToolUnknownCapability {
11272                            agent: m.metadata.name.clone(),
11273                            capability: id.tool,
11274                        });
11275                    }
11276                }
11277            }
11278        }
11279
11280        // Every ceiling any declaration on this plane states, by tool: the
11281        // count is per tool, so the tightest binds every agent calling it.
11282        #[cfg(feature = "manifest")]
11283        let rates = {
11284            let mut declarations: Vec<&Arc<crate::manifest::Manifest>> =
11285                governed_by.values().collect();
11286            declarations.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
11287            let mut ceilings: BTreeMap<String, Vec<crate::quota::RateCeiling>> = BTreeMap::new();
11288            let mut first = None;
11289            for m in declarations {
11290                for grant in &m.spec.tools {
11291                    let Some(rate) = grant.rate_limit else {
11292                        continue;
11293                    };
11294                    first.get_or_insert_with(|| (m.metadata.name.clone(), grant.reference.clone()));
11295                    let stated = ceilings.entry(grant.reference.clone()).or_default();
11296                    let ceiling = crate::quota::RateCeiling::from(rate);
11297                    if !stated.contains(&ceiling) {
11298                        stated.push(ceiling);
11299                        stated.sort();
11300                    }
11301                }
11302            }
11303            if ceilings.is_empty() {
11304                None
11305            } else if let Some(store) = self.quotas.clone() {
11306                Some(Arc::new(super::ctx::Rates { store, ceilings }))
11307            } else {
11308                let (agent, grant) = first.unwrap_or_default();
11309                return Err(BuildError::RateLimitWithoutQuotaStore { agent, grant });
11310            }
11311        };
11312
11313        // Both or neither, checked here because a builder cannot see the other
11314        // half at the moment either is set.
11315        if self.witness_interval.is_some() && self.witnesses.is_empty() {
11316            return Err(BuildError::IntervalWithoutWitnesses);
11317        }
11318        let witnesses = match (self.witnesses.is_empty(), self.quorum) {
11319            (true, None) => None,
11320            (false, Some(quorum)) if quorum.required() <= self.witnesses.len() => {
11321                Some(Arc::new(Witnessing {
11322                    witnesses: self.witnesses,
11323                    quorum,
11324                    submitted: std::sync::atomic::AtomicU64::new(0),
11325                    interval: self.witness_interval,
11326                    last_met: std::sync::Mutex::new(None),
11327                }))
11328            }
11329            (empty, quorum) => {
11330                return Err(BuildError::WitnessQuorumUnreachable {
11331                    declared: quorum.map_or(0, |q| q.required()),
11332                    configured: if empty { 0 } else { self.witnesses.len() },
11333                });
11334            }
11335        };
11336
11337        Ok(Arc::new_cyclic(|self_ref| Runtime {
11338            self_ref: self_ref.clone(),
11339            witnesses,
11340            inflight: Arc::default(),
11341            signer: self.signer,
11342            store: self.store,
11343            skills,
11344            by_capability,
11345            #[cfg(feature = "manifest")]
11346            published_by,
11347            #[cfg(feature = "manifest")]
11348            providers: self.providers,
11349            meter: super::metrics::Meter::new(self.tenant_label, &self.tenant),
11350            tenant: self.tenant,
11351            owner: self.owner.unwrap_or_else(default_owner),
11352            lease_ttl: self.lease_ttl,
11353            memories: self.memories,
11354            semantic: self.semantic,
11355            authorities: self.authorities,
11356            peers: self.peers,
11357            #[cfg(feature = "manifest")]
11358            tools: self.tools,
11359            journal_ceiling: self.journal_ceiling,
11360            #[cfg(feature = "manifest")]
11361            egress: self.egress,
11362            #[cfg(feature = "manifest")]
11363            checkers: self.checkers,
11364            streams: self.streams,
11365            quotas: self.quotas,
11366            #[cfg(feature = "manifest")]
11367            rates,
11368            quota: self.quota,
11369            budget: self.budget,
11370            require_verifier: self.require_verifier,
11371            cases: self.cases,
11372            events: self.events,
11373            tasks: self.tasks,
11374            timers: self.timers,
11375            blobs: self.blobs,
11376            disclosures: self.disclosures,
11377            #[cfg(feature = "keyring")]
11378            keyring: self.keyring,
11379            batches: self.batches,
11380            policy: self.policy,
11381            identity: self.identity,
11382            replanner: self.replanner,
11383            calendar: self.calendar.unwrap_or_else(|| Arc::new(WallClock)),
11384            #[cfg(feature = "push")]
11385            push: self.push,
11386            #[cfg(feature = "push")]
11387            outbox: self.outbox,
11388            #[cfg(feature = "manifest")]
11389            governed_by,
11390        }))
11391    }
11392}
11393
11394/// Ask the policy set the questions this plane will ask, before it asks them
11395/// for real.
11396///
11397/// Cedar evaluates **every** rule against **every** request, so a rule reading
11398/// an attribute a request does not carry does not quietly fail to match — it
11399/// errors, and an unevaluable rule may be the `forbid` that would have stopped
11400/// the call, so the gate refuses. One unguarded rule therefore denies every
11401/// effect of every run, from a policy set that compiled cleanly and validated
11402/// against its schema.
11403///
11404/// The probes cover every shape a runtime request takes: an effect that reads
11405/// and one that mutates, each with and without a label (only `sink` carries
11406/// one), a release, and an admission. `mutates` matters on its own: `&&`
11407/// short-circuits, so `context.mutates && context.label.trust == ..` evaluates
11408/// cleanly on a read and errors on every mutating call that carries no label.
11409///
11410/// The delegation attributes follow the plane. A plane with a chain carries
11411/// `owner`, `scope` and `delegation_depth` on every request, and a rule reading
11412/// them unguarded is correct there. A plane without one never carries them on
11413/// its own runs — and still admits runs under a caller's chain from a served
11414/// surface, so both shapes are asked.
11415///
11416/// What this does **not** establish: that the rules are *right*. A set that
11417/// permits everything passes here, as does one whose rules error only on a
11418/// shape carrying an attribute of an unexpected type. It answers one question
11419/// — can this plane's own requests be evaluated at all — which is the question
11420/// whose wrong answer looks like a working plane that refuses everything. The
11421/// served surfaces ask their own actions with their own contexts, and check
11422/// those where they are built.
11423#[cfg_attr(not(feature = "manifest"), allow(dead_code))]
11424fn preflight_policy(
11425    engine: &dyn crate::core::PolicyEngine,
11426    identity: Option<&crate::core::Delegation>,
11427) -> Result<(), BuildError> {
11428    let problems = runtime_policy_problems(engine, identity);
11429    if problems.is_empty() {
11430        return Ok(());
11431    }
11432    Err(BuildError::PolicyUnevaluable {
11433        problems: problems.join("; "),
11434    })
11435}
11436
11437/// The runtime tier's probes, answered by `engine`.
11438fn runtime_policy_problems(
11439    engine: &dyn crate::core::PolicyEngine,
11440    identity: Option<&crate::core::Delegation>,
11441) -> Vec<String> {
11442    use crate::policy::requests;
11443
11444    // Values are placeholders shaped like the real ones: a rule reading
11445    // `context.label.trust` on a release must find a label there, or the probe
11446    // reports a defect the real request would never hit. Nulls are absent for
11447    // the same reason — no gate sends one, and the adapter strips them.
11448    let run = RunId(ulid::Ulid::nil());
11449    let step = StepId(0);
11450    let label = crate::core::Label::untrusted(crate::core::SourceId::new("preflight"));
11451    let release = crate::core::Release::whole(
11452        crate::core::ReleaseScope::trust(),
11453        "preflight",
11454        "tool://preflight/probe",
11455        ["preflight"],
11456    );
11457    // A tool call's arguments are the one shape the runtime fixes for a kind,
11458    // so a rule scoped to `tool.call` may read `context.args.server` as-is.
11459    let tool_args = serde_json::json!({
11460        "server": "preflight",
11461        "tool": "preflight",
11462        "arguments": {},
11463        "protected_fields": [],
11464    });
11465
11466    // The chain shapes this plane's requests take. A chain of the plane's own
11467    // is on every request; without one, a run may still carry its caller's.
11468    let caller = crate::core::Delegation::root(crate::core::Principal::new(
11469        "preflight",
11470        crate::core::Scope::root(),
11471    ));
11472    let chains: Vec<Option<&crate::core::Delegation>> = match identity {
11473        Some(chain) => vec![Some(chain)],
11474        None => vec![None, Some(&caller)],
11475    };
11476
11477    // Built by the gates' own builders, so a probe cannot drift from the
11478    // shape it stands in for: an effect that reads and one that mutates, each
11479    // with and without a label (only `sink` carries one), a release, and an
11480    // admission.
11481    let mut built = Vec::new();
11482    for chain in chains {
11483        let acting = requests::Acting {
11484            tenant: "preflight",
11485            capability: "preflight.capability",
11486            agent: None,
11487            chain,
11488        };
11489        for (kind, args) in [
11490            ("preflight.effect", serde_json::json!({})),
11491            ("tool.call", tool_args.clone()),
11492        ] {
11493            for (mutates, labelled) in [(false, false), (true, false), (false, true), (true, true)]
11494            {
11495                built.push(requests::effect(
11496                    &acting,
11497                    run,
11498                    step,
11499                    kind,
11500                    &args,
11501                    mutates,
11502                    labelled.then_some(&label),
11503                ));
11504            }
11505        }
11506        built.push(requests::release(&acting, run, step, &release, &label));
11507        built.push(requests::admission(&acting, &serde_json::json!({})));
11508    }
11509    let requests: Vec<crate::core::PolicyRequest<'_>> = built
11510        .iter()
11511        .map(requests::GatedRequest::as_request)
11512        .collect();
11513
11514    // Asked through the engine's own preflight rather than by calling
11515    // `authorize` here. An engine written as Rust code has no missing-attribute
11516    // trap and nothing to report; the engine that has the trap implements the
11517    // method, and knows which resources its own rules name.
11518    let mut problems = engine.preflight(&requests);
11519    problems.sort();
11520    problems.dedup();
11521    problems
11522}
11523
11524/// Refuse a plane whose stores answer for somebody else.
11525///
11526/// The plane's tenant scopes its data keys and names its policy requests; each
11527/// store handle is scoped separately. Nothing at runtime looks wrong when the
11528/// two disagree — the work is written into another tenant's keyspace while
11529/// every erasure and every policy request names the right one — so the mismatch
11530/// is easy to make and invisible once made. Asking each store who it serves is
11531/// what turns it into a startup refusal.
11532///
11533/// **A store that does not override its accessor answers `default`.** That is
11534/// the honest answer for a single-tenant deployment and the reason the check
11535/// can be unconditional, but it is also why this catches a wiring mistake
11536/// rather than proving isolation: an implementation that is scoped to `acme`
11537/// and never overrode the accessor reports `default` and is refused, which is
11538/// the safe direction — the unsafe one would be an implementation that claims a
11539/// tenant it does not enforce, and no build-time question can settle that.
11540fn check_same_tenant(
11541    store: &dyn JournalStore,
11542    blobs: Option<&Arc<dyn crate::blob::BlobStore>>,
11543    memories: Option<&Arc<dyn crate::memory::MemoryStore>>,
11544    state: &[(&'static str, Option<&str>)],
11545    tenant: &crate::core::TenantId,
11546) -> Result<(), BuildError> {
11547    if let Some(blobs) = blobs
11548        && blobs.tenant() != tenant.as_str()
11549    {
11550        return Err(BuildError::BlobStoreTenant {
11551            plane: tenant.to_string(),
11552            store: blobs.tenant().to_owned(),
11553        });
11554    }
11555    if store.tenant() != tenant.as_str() {
11556        return Err(BuildError::JournalStoreTenant {
11557            plane: tenant.to_string(),
11558            store: store.tenant().to_owned(),
11559        });
11560    }
11561
11562    // The stores whose state a key ring seals, in the caller's fixed order so a
11563    // plane with two mismatches reports the same one every build — an error
11564    // that varies between runs of the same wiring sends an operator looking for
11565    // a race that is not there.
11566    for &(store, serves) in state {
11567        if let Some(serves) = serves
11568            && serves != tenant.as_str()
11569        {
11570            return Err(BuildError::StateStoreTenant {
11571                store,
11572                plane: tenant.to_string(),
11573                tenant: serves.to_owned(),
11574            });
11575        }
11576    }
11577
11578    // After the tenant checks, deliberately: a store scoped to the wrong tenant
11579    // is the more basic fault, and reporting the erasure lock first would send
11580    // an operator to fix the second-most-wrong thing.
11581    //
11582    // A process-local erasure lock beside a store two instances can write is a
11583    // control that reads as present and is not. The window it fails to close is
11584    // between an erasure's legal-hold check and its key destruction: the other
11585    // instance writes an item, that item is sealed under a scope about to stop
11586    // existing, and the erasure reports success. Both facts are here, so the
11587    // refusal is here.
11588    if store.is_shared() && memories.is_some_and(|m| m.erasure_is_distributed() == Some(false)) {
11589        return Err(BuildError::ErasureCoordinatorNotShared);
11590    }
11591    Ok(())
11592}
11593
11594/// Refuse an event from outside whose kind this plane mints for itself.
11595fn refuse_reserved_kind(event: &InboundEvent) -> Result<(), RuntimeError> {
11596    if crate::core::is_reserved_kind(&event.kind) {
11597        return Err(RuntimeError::ReservedEventKind {
11598            kind: event.kind.clone(),
11599        });
11600    }
11601    Ok(())
11602}
11603
11604/// Inbound event delivery.
11605impl Runtime {
11606    /// Deliver an inbound event, resuming whichever run was waiting for it.
11607    ///
11608    /// # Ordering
11609    ///
11610    /// The event is **stored before** anyone looks for a waiter. That ordering
11611    /// is the whole reason this works: a message can arrive before its run
11612    /// reaches the wait, and one that is matched-then-discarded leaves that run
11613    /// waiting forever for something that already happened.
11614    ///
11615    /// A [`Delivery::Buffered`] result is therefore normal and not an error —
11616    /// it means "held until someone asks". Only the sweep
11617    /// ([`EventStore::sweep_unclaimed`](crate::case::EventStore::sweep_unclaimed))
11618    /// decides an event is genuinely unroutable, because that is a claim about
11619    /// the future rather than about this instant.
11620    ///
11621    /// # Errors
11622    ///
11623    /// [`RuntimeError::ReservedEventKind`] for a kind in the `agentplane.`
11624    /// namespace, which only this plane mints; [`RuntimeError`] for a store
11625    /// failure or a refused resume.
11626    pub async fn deliver(&self, event: &InboundEvent) -> Result<Delivery, RuntimeError> {
11627        refuse_reserved_kind(event)?;
11628        self.deliver_buffered(event, true).await
11629    }
11630
11631    /// Deliver an event this plane minted itself — a worklist decision.
11632    ///
11633    /// The one path into the reserved namespace [`deliver`](Self::deliver)
11634    /// refuses from outside.
11635    pub(crate) async fn deliver_minted(
11636        &self,
11637        event: &InboundEvent,
11638    ) -> Result<Delivery, RuntimeError> {
11639        self.deliver_buffered(event, false).await
11640    }
11641
11642    /// Store `event`, then offer it to the oldest eligible waiter.
11643    ///
11644    /// `rematch` offers a duplicate again: the buffer and the match are two
11645    /// writes, and a counterparty's retry is the one chance to repair a crash
11646    /// between them. A minted decision is never re-offered, because its
11647    /// duplicate is the answer *already on record*, and offering that stored
11648    /// answer under a second decider's call would report it as theirs.
11649    async fn deliver_buffered(
11650        &self,
11651        event: &InboundEvent,
11652        rematch: bool,
11653    ) -> Result<Delivery, RuntimeError> {
11654        let events = self.events.as_ref().ok_or_else(|| {
11655            RuntimeError::PlanContract(
11656                "this runtime has no event store — build it with `.events(store)`".into(),
11657            )
11658        })?;
11659
11660        let now = now_for_admission();
11661
11662        // Durable first. Deduplication by event id makes a counterparty's retry
11663        // — and they all retry — harmless.
11664        let fresh = events
11665            .buffer(event, now)
11666            .await
11667            .map_err(RuntimeError::from_store)?;
11668
11669        // The match claims only an unclaimed, live row, so a retry of a
11670        // message already consumed matches nothing.
11671        if !fresh && !rematch {
11672            return Ok(Delivery::Duplicate);
11673        }
11674        let Some(sub) = events
11675            .match_waiter(event, now)
11676            .await
11677            .map_err(RuntimeError::from_store)?
11678        else {
11679            return Ok(if fresh {
11680                Delivery::Buffered
11681            } else {
11682                Delivery::Duplicate
11683            });
11684        };
11685
11686        self.resume_subscription(events, sub, event).await
11687    }
11688
11689    /// Deliver an inbound event to exactly `run`.
11690    ///
11691    /// This is the task-addressed counterpart to [`Runtime::deliver`]. It is
11692    /// used by protocols such as A2A where a follow-up carries a concrete task
11693    /// id. Correlation alone is insufficient there: two tasks may wait on the
11694    /// same business key, and resuming the oldest would violate the request.
11695    ///
11696    /// The event store atomically inserts and claims the event for this run. A
11697    /// run that is not waiting leaves no buffered event behind for another run.
11698    ///
11699    /// # Errors
11700    ///
11701    /// As [`deliver`](Self::deliver), and [`RuntimeError::PlanContract`] when
11702    /// `run` is not waiting for this input. Addressing a run by id is no way
11703    /// around the reserved namespace: a run waiting on a human task is waiting
11704    /// for exactly the kind this refuses.
11705    pub async fn deliver_to(
11706        &self,
11707        run: RunId,
11708        event: &InboundEvent,
11709    ) -> Result<Delivery, RuntimeError> {
11710        refuse_reserved_kind(event)?;
11711        let events = self.events.as_ref().ok_or_else(|| {
11712            RuntimeError::PlanContract(
11713                "this runtime has no event store — build it with `.events(store)`".into(),
11714            )
11715        })?;
11716        match events
11717            .deliver_to(run, event, now_for_admission())
11718            .await
11719            .map_err(RuntimeError::from_store)?
11720        {
11721            crate::case::TargetedDelivery::Duplicate => Ok(Delivery::Duplicate),
11722            crate::case::TargetedDelivery::NotWaiting => Err(RuntimeError::PlanContract(format!(
11723                "run {run} is not waiting for this input"
11724            ))),
11725            crate::case::TargetedDelivery::Matched(sub) => {
11726                self.resume_subscription(events, sub, event).await
11727            }
11728        }
11729    }
11730
11731    async fn resume_subscription(
11732        &self,
11733        events: &Arc<dyn crate::case::EventStore>,
11734        sub: crate::core::Subscription,
11735        event: &InboundEvent,
11736    ) -> Result<Delivery, RuntimeError> {
11737        // Record the event as the awaited effect's result, then let replay do
11738        // the rest: the resumed run reads it back like any other completed
11739        // effect, and none of the suspension machinery exists twice.
11740        //
11741        // The lease may be briefly held: an event can arrive in the window
11742        // between the run registering its subscription and its owner finishing
11743        // the suspension bookkeeping — the owner is *concluding*, and will
11744        // release within milliseconds. Failing that race would be fatal in the
11745        // quietest possible way: the delivery fails `LeaseHeld` **after** the
11746        // event is durably claimed for this run, the counterparty's retry
11747        // deduplicates against the claimed event, and nothing ever resumes the
11748        // run — a message that arrived in time, parked forever. So the claim
11749        // is retried under a bounded backoff; if the owner is genuinely
11750        // stalled past the bound, the event is *left claimed* for this run
11751        // and reported as buffered — the sweep's redelivery pass finds a
11752        // waiting subscription with a claimed event and finishes the job on a
11753        // later tick, so the bound bounds latency, never delivery.
11754        let mut backoff = Duration::from_millis(25);
11755        let lease = loop {
11756            match self
11757                .store
11758                .acquire(sub.run, &self.owner, self.lease_ttl)
11759                .await
11760            {
11761                Ok(lease) => break lease,
11762                Err(crate::core::StoreError::LeaseHeld { .. })
11763                    if backoff < Duration::from_secs(1) =>
11764                {
11765                    tokio::time::sleep(backoff).await;
11766                    backoff *= 2;
11767                }
11768                Err(crate::core::StoreError::LeaseHeld { .. }) => {
11769                    // The claim retired the subscription, so nothing lists
11770                    // this run as waiting any more — and the claimed event
11771                    // blocks every dedup'd retry of itself. Parking the pair
11772                    // is what keeps it findable: the sweep's redelivery pass
11773                    // walks parked waits, re-claims (a claim by the same run
11774                    // is returned, not filtered), and finishes the delivery
11775                    // once the owner concludes.
11776                    events
11777                        .park_wait(&sub, now_for_admission())
11778                        .await
11779                        .map_err(RuntimeError::from_store)?;
11780                    tracing::warn!(
11781                        run = %sub.run,
11782                        event = %event.id,
11783                        "an event was claimed for a run whose owner did not conclude \
11784                         within the retry window; the sweep will deliver it"
11785                    );
11786                    return Ok(Delivery::Buffered);
11787                }
11788                Err(e) => return Err(RuntimeError::from_store(e)),
11789            }
11790        };
11791
11792        let history = self
11793            .store
11794            .read(sub.run, 1)
11795            .await
11796            .map_err(RuntimeError::from_store)?;
11797        let already_recorded = history.iter().any(|record| {
11798            record.effect_key() == Some(sub.effect)
11799                && matches!(record.kind(), RecordKind::EffectDone { .. })
11800        });
11801        // A run whose conclusion is durable consumes nothing. An answer
11802        // appended after `RunConcluded` is one the run never acted on, and it
11803        // leaves a run its seal refuses forever. The resume below finishes it
11804        // instead — seals it and retires its waits, which hands this message,
11805        // still unconsumed, to the next waiter on its key.
11806        let closed = resume_is_closed(&history).is_some();
11807        if !already_recorded && !closed {
11808            // Judged here, where the output is recorded, under the run's own
11809            // declaration: the step that resumes reads the verdict back.
11810            let content = self.delivered_verdict(&history, &event.payload);
11811            self.store
11812                .append(
11813                    lease.epoch,
11814                    vec![{
11815                        let mut a = Append::new(
11816                            sub.run,
11817                            RecordKind::EffectDone {
11818                                output: event.payload.clone(),
11819                                // The sender, so a replayed run rebuilds the same
11820                                // provenance this delivery gave the value.
11821                                source: Some(event.source.clone()),
11822                                // And the operator who minted it, where one
11823                                // did: the same record on the same path, so a
11824                                // decision that resumes a suspended run is
11825                                // attributed exactly as one consumed inline.
11826                                by: event.by.clone(),
11827                                spend: crate::core::Spend::default(),
11828                                // An inbound payload is another party's data,
11829                                // which is exactly the point this lattice value
11830                                // names — the same one the run's own delivery
11831                                // path records.
11832                                declared: crate::core::DeclaredOutput::untrusted(),
11833                                content,
11834                                elapsed_ms: None,
11835                            },
11836                        )
11837                        .effect(sub.effect)
11838                        .step(sub.step)
11839                        .phase(sub.phase);
11840                        // Every record of a case-bound run carries its case, and a
11841                        // record written from outside the run is no exception.
11842                        if let Some(c) = sub.case {
11843                            a = a.case(c);
11844                        }
11845                        a
11846                    }],
11847                )
11848                .await
11849                .map_err(RuntimeError::from_store)?;
11850        }
11851
11852        if !closed {
11853            events
11854                .unsubscribe(sub.run, sub.effect)
11855                .await
11856                .map_err(RuntimeError::from_store)?;
11857        }
11858
11859        // The event is recorded as the awaited effect's result, and the resume
11860        // continues under the *same* lease. Releasing here and letting the
11861        // resume re-acquire opened the window this function's own doc names —
11862        // a crash between the release and the acquire left a released lease
11863        // over a run whose subscription was just retired: no driver, and
11864        // invisible to the abandonment queue, which lists only leases that
11865        // expired while still naming an owner. The handover keeps the run
11866        // owned continuously from delivery to conclusion.
11867        match self.resume_holding(sub.run, lease).await {
11868            // A `Fenced` means this lease lapsed mid-resume and somebody
11869            // claimed the run — a concurrent delivery, a recovery sweep. The
11870            // event is durably recorded, so whoever holds the lease replays
11871            // it; this delivery still delivered, and a sender told otherwise
11872            // retries into a duplicate.
11873            Ok(_) | Err(RuntimeError::Fenced { .. }) => {}
11874            // This plane holds no agent for the run — an operator's terminal
11875            // recording a decision. The answer is durable, and the lease is
11876            // kept so it lapses owned: that puts the run on the abandonment
11877            // listing, where the plane that runs the agent recovers it from
11878            // the recorded answer. Failing here would leave a recorded
11879            // decision reported as lost and the task it answers still claimed.
11880            // A plane holding no key ring for a sealed run, or no case store
11881            // for a case-bound one, is the same terminal: the plane that holds
11882            // it continues the run.
11883            Err(
11884                RuntimeError::NoProvider { .. }
11885                | RuntimeError::NoCaseStore { .. }
11886                | RuntimeError::PayloadsSealed { .. },
11887            ) => {
11888                return Ok(Delivery::Buffered);
11889            }
11890            Err(e) => return Err(e),
11891        }
11892        if closed {
11893            return Ok(Delivery::Buffered);
11894        }
11895        Ok(Delivery::Resumed { run: sub.run })
11896    }
11897
11898    /// Retire everything a run that concluded closed was still waiting on:
11899    /// its timers, its event subscriptions and its unanswered tasks.
11900    ///
11901    /// A sealed run can record nothing, so each of these left behind is work
11902    /// the plane keeps doing for nobody — a timer claimed and failed every
11903    /// lease period, the oldest waiter on a key taking the next event from a
11904    /// live run behind it, a task offered for a decision no answer reaches.
11905    ///
11906    /// Called after every seal of a concluded run — the conclusion's own and
11907    /// the one resume writes to repair a crash between conclusion and seal —
11908    /// and idempotent, so retiring twice is harmless.
11909    ///
11910    /// After the seal, and best-effort: the conclusion is already durable, and
11911    /// the stores pass over a sealed run's timers and subscriptions on their
11912    /// own, so a retirement that fails here costs clutter rather than a
11913    /// delivery. A task left pending by that failure is withdrawn by nothing
11914    /// else, which is why the failure is logged loudly.
11915    async fn retire_waits(&self, run: RunId) {
11916        if let Some(timers) = &self.timers
11917            && let Err(error) = timers.disarm_run(run).await
11918        {
11919            tracing::error!(%run, %error, "a closed run's timers could not be retired");
11920        }
11921        // The waits the run announced and never had answered: a message
11922        // claimed for one of them reached nobody and goes to the next waiter,
11923        // and a task opened for one is withdrawn. A task opened beside an
11924        // answer completes its own effect and is not among them: it is a
11925        // finding for a desk, and outlives the run by design.
11926        let unanswered = match self.store.read(run, 1).await {
11927            Ok(records) => unanswered_waits(&records),
11928            Err(error) => {
11929                tracing::error!(%run, %error, "a closed run's waits could not be read to retire them");
11930                return;
11931            }
11932        };
11933        if let Some(events) = &self.events {
11934            match events.unsubscribe_run(run, &unanswered).await {
11935                Ok(retired) => {
11936                    for event in retired.released {
11937                        self.offer_released(events, &event).await;
11938                    }
11939                }
11940                Err(error) => {
11941                    tracing::error!(%run, %error, "a closed run's subscriptions could not be retired");
11942                }
11943            }
11944        }
11945        if let Some(tasks) = &self.tasks {
11946            let awaited: Vec<_> = unanswered
11947                .iter()
11948                .map(|effect| crate::core::TaskId::derive(run, *effect))
11949                .collect();
11950            if let Err(error) = tasks.withdraw_run(run, &awaited).await {
11951                tracing::error!(%run, %error, "a closed run's open tasks could not be withdrawn");
11952            }
11953        }
11954    }
11955
11956    /// Offer a message a closed run held, and never consumed, to the next
11957    /// waiter on its key — the delivery it would have had if that run had not
11958    /// concluded first. Best effort, like the retirement that released it: a
11959    /// message no waiter takes now stays buffered for the next one.
11960    async fn offer_released(
11961        &self,
11962        events: &Arc<dyn crate::case::EventStore>,
11963        event: &InboundEvent,
11964    ) {
11965        let sub = match events.match_waiter(event, now_for_admission()).await {
11966            Ok(Some(sub)) => sub,
11967            Ok(None) => return,
11968            Err(error) => {
11969                tracing::error!(event = %event.id, %error, "a released message could not be offered");
11970                return;
11971            }
11972        };
11973        if let Err(error) = Box::pin(self.resume_subscription(events, sub, event)).await {
11974            tracing::error!(event = %event.id, %error, "a released message could not be delivered");
11975        }
11976    }
11977
11978    /// Finish deliveries that died between the claim and the resume.
11979    ///
11980    /// A delivery that raced a live owner leaves an event durably claimed for
11981    /// a run nothing resumed, and the counterparty's retries deduplicate
11982    /// against that claim — so no future delivery drives the run, and a
11983    /// message that arrived in time is stranded. The giving-up delivery parks
11984    /// the pair precisely so it stays findable; this pass walks the parked
11985    /// waits — never every registered wait, whose long legitimate members
11986    /// would fill the page ahead of a pair parked after them — re-claims
11987    /// (idempotently: a claim by the same run is returned, not filtered), and
11988    /// finishes the delivery. Runs whose owner is *still* live are skipped and
11989    /// found again next tick, so the delivery's retry bound bounds latency,
11990    /// never delivery.
11991    ///
11992    /// What this does **not** cover, honestly: a crash inside the store
11993    /// commit between an event's claim and the resume retires the
11994    /// subscription with nothing left to re-register it, and that window
11995    /// stays what it always was — recorded rather than closed, with later
11996    /// events for the same correlation dead-lettering as its symptom.
11997    pub(crate) async fn redeliver_claimed(
11998        &self,
11999        limit: usize,
12000    ) -> Result<crate::runtime::Redelivered, RuntimeError> {
12001        let Some(events) = self.events.as_ref() else {
12002            return Ok(crate::runtime::Redelivered::default());
12003        };
12004        let parked = events
12005            .parked_waits(limit)
12006            .await
12007            .map_err(RuntimeError::from_store)?;
12008        let examined = parked.len();
12009        let mut delivered = 0usize;
12010        for sub in parked {
12011            let Some(buffered) = events
12012                .claim_for(&sub, now_for_admission())
12013                .await
12014                .map_err(RuntimeError::from_store)?
12015            else {
12016                continue;
12017            };
12018            match self
12019                .resume_subscription(events, sub.clone(), &buffered.event)
12020                .await
12021            {
12022                Ok(Delivery::Resumed { .. }) => delivered += 1,
12023                // Still held live: the owner is working, and will either
12024                // consume the claim itself or conclude and be found next tick.
12025                Ok(_) => {}
12026                Err(error) => {
12027                    tracing::error!(
12028                        run = %sub.run,
12029                        %error,
12030                        "a claimed event's redelivery failed; retried next tick",
12031                    );
12032                }
12033            }
12034        }
12035        Ok(crate::runtime::Redelivered {
12036            finished: delivered,
12037            examined,
12038        })
12039    }
12040
12041    /// Retire events that nobody claimed within `grace`.
12042    ///
12043    /// A non-empty dead-letter list means a correlation key is wrong somewhere:
12044    /// the message arrived, was held, and no run ever asked for it. That is the
12045    /// failure which otherwise presents as a process silently never completing,
12046    /// so it is worth alerting on rather than logging.
12047    /// `grace` is a `std::time::Duration` for the reason
12048    /// [`StepCtx::deadline`](crate::runtime::StepCtx::deadline)'s `warn_before`
12049    /// is: a negative grace window is meaningless, and the signed type could
12050    /// express it — a cutoff moved *forward* of now, retiring events that had
12051    /// not yet had their chance. It is also the `Duration` the caller has.
12052    pub async fn sweep_events(&self, grace: std::time::Duration) -> Result<usize, RuntimeError> {
12053        let events = self
12054            .events
12055            .as_ref()
12056            .ok_or_else(|| RuntimeError::PlanContract("this runtime has no event store".into()))?;
12057        // Saturating rather than fallible, at both steps: a grace window
12058        // beyond what the calendar can hold means "retire nothing", and
12059        // refusing the call would be a worse answer to a caller asking for a
12060        // longer hold. The subtraction has to saturate too — `time` panics on
12061        // underflow, so converting the window to `Duration::MAX` and then
12062        // subtracting it aborts the tick that also breaches obligations and
12063        // recovers abandoned runs. The first instant the calendar can name is
12064        // a cutoff nothing is older than, which is exactly "retire nothing".
12065        let grace = time::Duration::try_from(grace).unwrap_or(time::Duration::MAX);
12066        let cutoff = now_for_admission()
12067            .checked_sub(grace)
12068            .unwrap_or_else(crate::core::first_instant);
12069        let retired = events
12070            .sweep_unclaimed(cutoff, "no run claimed this event within the grace window")
12071            .await
12072            .map_err(RuntimeError::from_store)?;
12073
12074        if retired > 0 {
12075            // A non-empty dead-letter list means a correlation key is wrong
12076            // somewhere: the message arrived, was held, and no run ever asked
12077            // for it. That is the failure which otherwise presents as a process
12078            // silently never completing.
12079            tracing::error!(target: telemetry::DEAD_LETTERED, count = retired, %cutoff);
12080            self.meter
12081                .count_by(metrics::DEAD_LETTERS, "", retired as u64);
12082        }
12083        Ok(retired)
12084    }
12085}
12086
12087/// Every `RunStatus`, so adding one forces a decision everywhere one is owed.
12088///
12089/// Written out rather than derived, because Rust cannot enumerate a
12090/// data-carrying enum. What keeps it honest is **not** the count assertions in
12091/// its consumers: those fire when somebody edits this list, which is the case
12092/// that needs no help. A variant added to the enum and forgotten here changes
12093/// nothing they can see, and every agreement test then walks a list that no
12094/// longer describes the type — passing over the one variant nobody decided
12095/// about.
12096///
12097/// `every_run_status_variant_is_listed` in `tests/guards` is what closes that:
12098/// it reads the enum's variants and this function's arms out of the source and
12099/// holds them to each other, which is the only check available for a list a
12100/// language cannot derive.
12101///
12102/// Crate-visible so those consumers share **one** list. Two copies would be two
12103/// places to remember, and the second would be the one that went stale — which
12104/// is the same failure the tests using it exist to catch.
12105#[cfg(test)]
12106pub(crate) fn every_status() -> Vec<RunStatus> {
12107    use crate::core::{BudgetExceeded, CorrelationKey, SuspendReason, Timestamp};
12108    vec![
12109        RunStatus::Succeeded,
12110        RunStatus::Failed("because".into()),
12111        RunStatus::Suspended(SuspendReason::AwaitingEvent {
12112            kind: "reply".into(),
12113            correlation: vec![CorrelationKey::new("claim", "CLM-1")],
12114            until: Timestamp::from_unix_timestamp(1_760_000_000).expect("time"),
12115        }),
12116        RunStatus::Exhausted(BudgetExceeded::Steps { allowed: 1 }),
12117        RunStatus::Quarantined("unknown outcome".into()),
12118        RunStatus::Replanning("try again".into()),
12119        RunStatus::Cancelled {
12120            actor: crate::core::Operator::asserted("ops").expect("a name"),
12121            reason: "stop".into(),
12122        },
12123        RunStatus::Abandoned {
12124            actor: crate::core::Operator::asserted("ops").expect("a name"),
12125            reason: "nobody could establish what happened".into(),
12126        },
12127        RunStatus::Swept,
12128        RunStatus::BrokeGlass {
12129            actor: crate::core::Operator::asserted("ops").expect("a name"),
12130            reason: "INC-42".into(),
12131        },
12132        RunStatus::HaltLifted {
12133            actor: crate::core::Operator::asserted("ops").expect("a name"),
12134        },
12135        RunStatus::HoldReleased {
12136            actor: crate::core::Operator::asserted("ops").expect("a name"),
12137        },
12138        RunStatus::Withheld {
12139            subject: "alice".into(),
12140            reason: "credential withdrawn".into(),
12141        },
12142        RunStatus::Observed,
12143    ]
12144}
12145
12146#[cfg(test)]
12147mod resume_agreement_tests {
12148    use super::{QuotaPass, RunStatus, resume_is_closed};
12149    use crate::journal::RecordKind;
12150
12151    // Through the crate re-export rather than `super::`, so the one path both
12152    // consumers use is exercised in every build. Reaching it directly here
12153    // would leave the re-export unused whenever `a2a-server` is off — which is
12154    // a lint failure in exactly the feature configuration nobody runs locally.
12155    use crate::runtime::every_status;
12156
12157    /// **A conclusion that sealed the journal may never be resumed.**
12158    ///
12159    /// `RunStatus::seals` decides whether a conclusion freezes the run and
12160    /// publishes a Merkle leaf; `resume_is_closed` decides whether a recorded
12161    /// conclusion may be continued. A status that did the first and permits the
12162    /// second would grow the history past the leaf every later checkpoint
12163    /// attests — the failure the "a conclusion is not a closure" work removed
12164    /// for `failed`, reachable again the moment the two disagree.
12165    ///
12166    /// The executor's own comment claimed "a test pins the agreement" while no
12167    /// such test existed, which is the unfalsifiable-guarantee shape this
12168    /// project treats as a defect in itself: the rule was real, checked by
12169    /// nobody, and deletable in silence.
12170    #[test]
12171    fn a_sealing_conclusion_is_never_resumable() {
12172        let statuses = every_status();
12173        assert_eq!(
12174            statuses.len(),
12175            14,
12176            "a RunStatus variant was added or removed — decide whether it seals \
12177             and whether a resume may continue from it, then update this list"
12178        );
12179
12180        for status in &statuses {
12181            let records = sealed_as(status.as_str());
12182            let verdict = resume_is_closed(&records);
12183            if status.seals() {
12184                assert!(
12185                    verdict.is_some(),
12186                    "'{}' seals the journal and enters the Merkle log, yet a resume \
12187                     is permitted from it — the resume would grow the history past \
12188                     the leaf every later checkpoint attests",
12189                    status.as_str()
12190                );
12191            }
12192        }
12193    }
12194
12195    /// The availability half: the conclusions that stay open really do.
12196    ///
12197    /// Asserted separately and by name rather than as the converse of the rule
12198    /// above, because the converse is false — `Suspended` and `Replanning` do
12199    /// not seal either, and neither is ever a recorded conclusion.
12200    ///
12201    /// `withheld` is here because it is a **pause**: the authority a run acts
12202    /// under was withdrawn, its mutations stand, and lifting the halt is what
12203    /// lets it continue. A withheld run that could not be resumed would make the
12204    /// withdrawal a destruction rather than a stop.
12205    #[test]
12206    fn a_failed_or_exhausted_run_may_still_be_resumed() {
12207        for outcome in ["failed", "exhausted", super::WITHHELD_OUTCOME] {
12208            assert!(
12209                resume_is_closed(&sealed_as(outcome)).is_none(),
12210                "'{outcome}' is a conclusion a resume must be able to continue \
12211                 from — its completed effects are read back from history, which \
12212                 is the point of having a journal"
12213            );
12214        }
12215    }
12216
12217    /// `SEALED_OUTCOMES` and `RunStatus::seals` are one rule in two spellings,
12218    /// and this holds them together variant by variant.
12219    ///
12220    /// The list exists so a caller wanting every sealed run — the export CLI —
12221    /// does not restate the rule as string literals in a binary, which is where
12222    /// a new sealing outcome would have been silently dropped from exactly the
12223    /// artifact an auditor asks for. Both directions matter: a sealing status
12224    /// missing from the list loses runs, and a non-sealing status present in it
12225    /// makes the export ask the outcome index about a backlog that drains.
12226    #[test]
12227    fn the_sealed_outcome_list_agrees_with_the_sealing_rule() {
12228        for status in super::every_status() {
12229            assert_eq!(
12230                super::SEALED_OUTCOMES.contains(&status.as_str()),
12231                status.seals(),
12232                "'{}' disagrees between RunStatus::seals and SEALED_OUTCOMES — \
12233                 one rule, two spellings, and the export reads the list",
12234                status.as_str()
12235            );
12236        }
12237    }
12238
12239    /// **Every outcome this build writes is one its own reader recognises.**
12240    ///
12241    /// The reader is a `match` over `&str` with a catch-all, so it goes on
12242    /// compiling forever while the writer's list grows — and the catch-all is
12243    /// not benign: it answers `Quarantined`, so a run this plane sealed itself
12244    /// reads back as *the runtime could not decide what happened*. Two endings
12245    /// sat there, `swept` and `broke-glass`, and a break-glass crossing — the
12246    /// one control whose entire content is who crossed and why — answered the
12247    /// operator surface with the words "this build does not recognise it" in
12248    /// place of the reason.
12249    ///
12250    /// Held from the writer's side, and the writer's side is
12251    /// [`every_status`](super::every_status) — every variant the type can
12252    /// take. `OUTCOMES_OF_RECORD` is the *export's* list and deliberately
12253    /// omits the endings that stay resumable, so a guard reading it asks about
12254    /// a subset of what the writer writes and passes while the rest fall
12255    /// through the catch-all. `withheld` sat there: the one ending whose whole
12256    /// content is *whose credential was withdrawn*, answering the operator API
12257    /// with "this build does not recognise it".
12258    #[test]
12259    fn every_outcome_this_build_writes_is_one_it_can_read_back() {
12260        for written in super::every_status() {
12261            // A suspension is not a conclusion; it has its own record and its
12262            // own arm in `observed_status`.
12263            if written.is_suspended() {
12264                continue;
12265            }
12266            let outcome = written.as_str();
12267            let status = super::recorded_status(
12268                outcome,
12269                written.own_account().as_deref(),
12270                match &written {
12271                    RunStatus::Exhausted(limit) => Some(limit),
12272                    _ => None,
12273                },
12274                &sealed_as(outcome),
12275            );
12276            assert_eq!(
12277                status.as_str(),
12278                outcome,
12279                "this build concludes runs as '{outcome}' and reads that back as \
12280                 '{}' — a catch-all answering `Quarantined` about a record this \
12281                 plane wrote itself",
12282                status.as_str()
12283            );
12284        }
12285    }
12286
12287    /// **An ending a person caused reports the words that person gave.**
12288    ///
12289    /// The conclusion does not carry them: one sentence, one record. So the
12290    /// reader has to reach the attribution record for the reason exactly as it
12291    /// already does for the actor, and a reader that only learned the second
12292    /// half would answer an incident review with a blank where the operator
12293    /// wrote why.
12294    #[test]
12295    fn an_ending_a_person_caused_reports_their_own_words() {
12296        for (outcome, expected) in [
12297            ("cancelled", "stop"),
12298            ("abandoned", "nobody could establish what happened"),
12299            (super::BREAK_GLASS_OUTCOME, "INC-42"),
12300            (super::WITHHELD_OUTCOME, "credential withdrawn"),
12301        ] {
12302            // `None`, as the conclusion this build writes now carries.
12303            let status = super::recorded_status(outcome, None, None, &sealed_as(outcome));
12304            assert_eq!(
12305                status.reason().as_deref(),
12306                Some(expected),
12307                "'{outcome}' lost the reason the person gave: {status:?}"
12308            );
12309        }
12310    }
12311
12312    /// **An ending nobody is named on is quarantined, not given a name.**
12313    ///
12314    /// The chain this build writes always carries the attribution record before
12315    /// the conclusion; a chain that carries only the conclusion is a crash
12316    /// between the two appends, or a chain somebody edited. Either way the
12317    /// record is not trustworthy, and *not trustworthy* is a different bucket
12318    /// from *an operator called unknown did this*.
12319    ///
12320    /// A fabricated actor would be worse than a missing one: indistinguishable
12321    /// from a real operator with that name, and served through the operator API
12322    /// as fact.
12323    #[test]
12324    fn an_ending_with_no_attribution_record_is_not_given_a_name() {
12325        for outcome in ["cancelled", "abandoned", super::BREAK_GLASS_OUTCOME] {
12326            let status =
12327                super::recorded_status(outcome, Some("why"), None, &concluded_only(outcome));
12328            assert!(
12329                matches!(status, RunStatus::Quarantined(_)),
12330                "'{outcome}' with no record of who asked read back as {status:?} — \
12331                 an operator act with nobody on it was given a name"
12332            );
12333        }
12334    }
12335
12336    /// The offline sweep covers every ending, plus the backlog that is not one.
12337    ///
12338    /// Held to `SEALED_OUTCOMES` rather than restated, because the failure this
12339    /// prevents is silent in exactly the artifact nobody re-reads: an ending
12340    /// added to the runtime and not to the export drops those runs from the
12341    /// file an auditor is handed, and the file looks complete.
12342    #[test]
12343    fn the_offline_sweep_covers_every_ending_and_the_quarantine_backlog() {
12344        for outcome in super::SEALED_OUTCOMES {
12345            assert!(
12346                super::OUTCOMES_OF_RECORD.contains(outcome),
12347                "'{outcome}' is an ending this plane can reach and the export's \
12348                 default sweep would silently omit it"
12349            );
12350        }
12351        assert!(
12352            super::OUTCOMES_OF_RECORD.contains(&"quarantined"),
12353            "a quarantine is the plane's outstanding unresolved risk and the one \
12354             thing an auditor is most likely to have come for"
12355        );
12356        for open in ["failed", "exhausted", "suspended"] {
12357            assert!(
12358                !super::OUTCOMES_OF_RECORD.contains(&open),
12359                "'{open}' is work in progress that a resume moves on its own — \
12360                 exporting it is a snapshot of a moving target, not a record"
12361            );
12362        }
12363    }
12364
12365    /// An outcome this build cannot interpret is never permission to resume.
12366    #[test]
12367    fn an_unrecognised_conclusion_fails_closed() {
12368        let verdict = resume_is_closed(&sealed_as("swept"));
12369        assert!(
12370            matches!(verdict, Some(RunStatus::Quarantined(_))),
12371            "an unrecognised conclusion must quarantine rather than resume: {verdict:?}"
12372        );
12373    }
12374
12375    /// A chain carrying the conclusion and **not** the attribution record.
12376    ///
12377    /// The damaged shape, which no healthy write produces: a crash between the
12378    /// two appends, or an edit. Built by taking the real chain and dropping
12379    /// everything before the conclusion, so it cannot drift from what
12380    /// [`sealed_as`] produces.
12381    fn concluded_only(outcome: &str) -> Vec<crate::journal::Record> {
12382        let full = sealed_as(outcome);
12383        full.into_iter()
12384            .filter(|r| matches!(r.kind(), RecordKind::RunConcluded { .. }))
12385            .collect()
12386    }
12387
12388    /// The chain this build actually writes for an ending.
12389    ///
12390    /// **Including the attribution record**, where the ending has one. A
12391    /// conclusion is never the only record of a cancellation, an abandonment or
12392    /// a crossing — the runtime appends who asked first — so a fixture holding
12393    /// the conclusion alone asserts a round trip over a chain this plane does
12394    /// not produce, and would keep passing if the reader began inventing a
12395    /// name for the missing half.
12396    fn sealed_as(outcome: &str) -> Vec<crate::journal::Record> {
12397        use crate::core::{Digest, Epoch};
12398        use crate::journal::{Record, RecordBody};
12399
12400        let run = crate::core::RunId::generate();
12401        let who = || crate::core::Operator::asserted("ops").expect("a name");
12402        let mut kinds: Vec<RecordKind> = match outcome {
12403            "cancelled" => vec![RecordKind::RunCancelled {
12404                actor: who(),
12405                reason: "stop".into(),
12406            }],
12407            "abandoned" => vec![RecordKind::QuarantineDecided {
12408                decider: who(),
12409                reason: "nobody could establish what happened".into(),
12410                decision: crate::core::QuarantineDecision::Abandon,
12411            }],
12412            super::BREAK_GLASS_OUTCOME => vec![RecordKind::BreakGlass {
12413                actor: who(),
12414                roles: vec!["oncall".into()],
12415                reason: "INC-42".into(),
12416            }],
12417            super::HALT_LIFTED_OUTCOME => vec![RecordKind::HaltLifted {
12418                scope: "tenant".into(),
12419                by: who(),
12420                at: crate::core::Timestamp::UNIX_EPOCH,
12421                reason: "INC-42".into(),
12422                thrown_by: who(),
12423                thrown_at: crate::core::Timestamp::UNIX_EPOCH,
12424            }],
12425            super::HOLD_RELEASED_OUTCOME => vec![RecordKind::HoldReleased {
12426                by: who(),
12427                at: crate::core::Timestamp::UNIX_EPOCH,
12428                placed_by: who(),
12429                placed_at: crate::core::Timestamp::UNIX_EPOCH,
12430            }],
12431            super::WITHHELD_OUTCOME => vec![RecordKind::AuthorityWithheld {
12432                subject: "alice".into(),
12433                reason: "credential withdrawn".into(),
12434                by: who(),
12435            }],
12436            _ => Vec::new(),
12437        };
12438        kinds.push(RecordKind::RunConcluded {
12439            outcome: outcome.to_owned(),
12440            chain_head: Digest::of(b""),
12441            reason: None,
12442            exhaustion: None,
12443            live_spend: crate::core::Spend::default(),
12444        });
12445
12446        let mut prev = Digest::of(b"");
12447        let mut out = Vec::new();
12448        for (i, kind) in kinds.into_iter().enumerate() {
12449            let body = RecordBody {
12450                seq: u64::try_from(i + 1).expect("a small fixture"),
12451                run,
12452                case: None,
12453                step: None,
12454                phase: super::Phase::Forward,
12455                epoch: Epoch::default(),
12456                v: 1,
12457                effect_key: None,
12458                kind,
12459            };
12460            let record = Record::seal(body, prev).expect("a sealed record");
12461            prev = record.hash;
12462            out.push(record);
12463        }
12464        out
12465    }
12466
12467    /// One execution pass is authorized and accrued against one period even
12468    /// when it finishes after a billing boundary.
12469    #[test]
12470    fn a_quota_pass_keeps_the_period_it_started_in() {
12471        let quota = crate::quota::TenantQuota {
12472            max_tokens_per_period: Some(1_000),
12473            period: crate::quota::Period::Daily,
12474            ..crate::quota::TenantQuota::default()
12475        };
12476        let before_midnight = crate::core::Timestamp::from_unix_timestamp(86_399).unwrap();
12477        let after_midnight = crate::core::Timestamp::from_unix_timestamp(86_400).unwrap();
12478
12479        let pass = QuotaPass::at(&quota, before_midnight, true);
12480        assert_eq!(
12481            pass.period(),
12482            Some(quota.period.key_for(before_midnight).as_str())
12483        );
12484        assert_ne!(
12485            pass.period(),
12486            Some(quota.period.key_for(after_midnight).as_str()),
12487            "settlement recomputed the period at completion, so this pass was \
12488             authorized against one ledger and charged to another"
12489        );
12490    }
12491}