agentplane/runtime/executor.rs
1//! The executor: admission, step dispatch, sealing, and replay.
2//!
3//! M0 executes a single step. The DAG scheduler, plan contract, and topology
4//! checks slot in above this without changing the effect protocol below it —
5//! which is the point of putting the determinism boundary at the effect rather
6//! than at the plan.
7
8#[cfg(feature = "manifest")]
9use std::collections::HashSet;
10use std::collections::{BTreeMap, BTreeSet, HashMap};
11use std::sync::Arc;
12use std::time::Duration;
13
14use serde_json::Value;
15
16use crate::case::{CaseStore, EventStore, TaskStore, TimerStore};
17use crate::core::{
18 ArgSource, Budget, Calendar, Capability, Consumed, CorrelationKey, Delivery, Digest,
19 InboundEvent, Ledger, Outcome, Phase, PlanIR, PlanNode, PolicyBundleIdentity, RunId,
20 RuntimeError, Skill, SkillDescriptor, Spend, StepId, Tainted, WallClock,
21};
22use crate::journal::{Append, JournalStore, Record, RecordKind, ReplayCursor, StepCursor};
23use crate::runtime::BuildError;
24
25use super::ctx::{CaseContext, Mode, Pass, StepCtx};
26use super::metrics;
27use super::telemetry;
28use tracing::Instrument;
29
30#[derive(Debug, Clone)]
31pub(crate) enum CaseBinding {
32 Correlate {
33 kind: String,
34 keys: Vec<CorrelationKey>,
35 },
36 Existing(crate::core::CaseId),
37}
38
39/// What a run is bound to, beyond its target and input: who it acts for,
40/// which matter it belongs to, who asked, which declaration it was reviewed
41/// against. Handed to [`RunRequest::terms`].
42///
43/// The value a front door computes per caller — above all a chain **per run**:
44/// a door that authenticates many callers admits each one's run under the
45/// chain that caller presented. Whether the run happens at most once is not a
46/// term; it is [`RunRequest::once`], because it changes what the answer is.
47///
48/// **Authority is chosen once, by constructor**: [`default`](Self::default)
49/// acts as the plane, [`acting_as`](Self::acting_as) under a caller's chain,
50/// [`served`](Self::served) for a served caller that may have presented none.
51///
52/// ```no_run
53/// # use agentplane::runtime::RunTerms;
54/// # fn f(chain: agentplane::core::Delegation, keys: &[agentplane::core::CorrelationKey]) {
55/// let terms = RunTerms::acting_as(chain).correlated("order", keys);
56/// # }
57/// ```
58///
59/// ```compile_fail
60/// # use agentplane::runtime::RunTerms;
61/// # fn f(chain: agentplane::core::Delegation) {
62/// let terms = RunTerms::served(None).acting_as(chain);
63/// # }
64/// ```
65#[derive(Debug, Clone, Default)]
66pub struct RunTerms {
67 /// Which matter this run belongs to, if any.
68 case: Option<CaseBinding>,
69 /// The admission key to claim. Travels to the `RunAdmitted` record, because
70 /// that is where the store claims it — see
71 /// [`JournalStore::admitted_as`](crate::journal::JournalStore::admitted_as).
72 idempotency_key: Option<String>,
73 /// On whose authority this run acts.
74 acting_as: Acting,
75 /// Who asked for this run, when a person or peer did. Journaled on
76 /// `RunAdmitted`, and barred from deciding the run's own tasks.
77 admitted_by: Option<String>,
78 /// The declaration revision the caller reviewed, when it pinned one.
79 expected_declaration: Option<Digest>,
80 /// Data subjects the embedder names for this run, beside any its
81 /// declaration binds.
82 subjects: Vec<String>,
83}
84
85/// On whose authority a run acts.
86///
87/// Three answers, not two, because "no chain was presented" means different
88/// things at different doors. An in-process embedder that names no chain is
89/// the plane itself, and acts under the plane's
90/// ([`RuntimeBuilder::acting_as`]). A served caller whose credential carried
91/// none is **somebody else** — and falling back to the plane's chain for them
92/// would make the operator's authority an ambient credential every
93/// authenticated peer holds.
94///
95/// Acting under a chain and being bounded by one are separate questions. A
96/// served caller with no chain acts under none, and is still bounded by the
97/// plane's: that chain is the most this plane may do for anybody, so its
98/// scope and its admissibility gate the plan
99/// ([`ceiling`](Self::ceiling)) while nothing of it reaches the policy
100/// context, the journal or a withdrawal lookup.
101#[derive(Debug, Clone, Default)]
102enum Acting {
103 /// The plane's own chain, if it has one: a run the embedder started.
104 #[default]
105 Plane,
106 /// The chain an authenticated caller presented.
107 Caller(crate::core::Delegation),
108 /// A served caller that presented no chain: this run acts under none.
109 Nobody,
110 /// A chain delegated from the plane's own, for a run the plane's run
111 /// commissioned: it acts under that chain, and as the plane, so a peer
112 /// it calls is presented the plane's credential rather than one naming
113 /// the plane's owner as if a person had asked.
114 PlaneDelegate(crate::core::Delegation),
115}
116
117impl Acting {
118 /// The chain this run acts under, given the plane's own.
119 const fn resolve<'a>(
120 &'a self,
121 plane: Option<&'a crate::core::Delegation>,
122 ) -> Option<&'a crate::core::Delegation> {
123 match self {
124 Self::Plane => plane,
125 Self::Caller(chain) | Self::PlaneDelegate(chain) => Some(chain),
126 Self::Nobody => None,
127 }
128 }
129
130 /// The chain that bounds what this run may be admitted for: the one it
131 /// acts under, or — for a served caller that presented none — the
132 /// plane's, which it does not act under.
133 const fn ceiling<'a>(
134 &'a self,
135 plane: Option<&'a crate::core::Delegation>,
136 ) -> Option<&'a crate::core::Delegation> {
137 match self {
138 Self::Plane | Self::Nobody => plane,
139 Self::Caller(chain) | Self::PlaneDelegate(chain) => Some(chain),
140 }
141 }
142}
143
144impl RunTerms {
145 /// Run inside a case that already exists.
146 #[must_use]
147 pub fn in_case(mut self, case: crate::core::CaseId) -> Self {
148 self.case = Some(CaseBinding::Existing(case));
149 self
150 }
151
152 /// Join or open a case by business key before running.
153 #[must_use]
154 pub fn correlated(mut self, kind: &str, keys: &[CorrelationKey]) -> Self {
155 self.case = Some(CaseBinding::Correlate {
156 kind: kind.to_owned(),
157 keys: keys.to_vec(),
158 });
159 self
160 }
161
162 /// Act under this chain rather than the plane's own.
163 ///
164 /// Checked at admission exactly as the plane's chain is: the plan must be
165 /// inside its scope, and [`Delegation::admissible`] must accept this
166 /// plane at this instant. Journaled as the run's `IdentityBound`, so
167 /// "on whose behalf" names the caller and not the plane.
168 ///
169 /// [`Delegation::admissible`]: crate::core::Delegation::admissible
170 #[must_use]
171 pub fn acting_as(chain: crate::core::Delegation) -> Self {
172 Self {
173 acting_as: Acting::Caller(chain),
174 ..Self::default()
175 }
176 }
177
178 /// Act under `chain` as the plane: a commission from a run the plane
179 /// itself admitted, whose sub-run is the plane's work one link down.
180 pub(crate) fn acting_as_plane(mut self, chain: crate::core::Delegation) -> Self {
181 self.acting_as = Acting::PlaneDelegate(chain);
182 self
183 }
184
185 /// Act for a served caller: under the chain its credential carried, or
186 /// under **none** — never the plane's.
187 ///
188 /// The door every surface that admits work for somebody else goes through.
189 /// [`acting_as`](Self::acting_as) alone cannot say "this caller presented
190 /// nothing", and leaving it unset means the plane's own chain.
191 #[must_use]
192 pub fn served(chain: Option<crate::core::Delegation>) -> Self {
193 Self {
194 acting_as: chain.map_or(Acting::Nobody, Acting::Caller),
195 ..Self::default()
196 }
197 }
198
199 /// Record who asked for this run: the authenticated caller a served
200 /// surface admitted it for, or the operator an embedder started it on
201 /// behalf of.
202 ///
203 /// Journaled as `RunAdmitted.admitted_by`, and excluded from deciding
204 /// every task the run opens — the four-eyes control, applied to the one
205 /// actor a declarative agent's approvals could not otherwise name. A
206 /// caller who could approve the call their own request produced would be
207 /// one person on both sides of the review.
208 #[must_use]
209 pub fn admitted_by(mut self, actor: impl Into<String>) -> Self {
210 self.admitted_by = Some(actor.into());
211 self
212 }
213
214 /// Attribute what this run takes in to a data subject.
215 ///
216 /// Recorded in the run's `DataSubjectBound` after any subjects its
217 /// declaration binds, so `agentplane subject` follows the run's input,
218 /// events and case state to the effects they reached. Attribution only:
219 /// no gate reads it.
220 #[must_use]
221 pub fn subject(mut self, subject: impl Into<String>) -> Self {
222 self.subjects.push(subject.into());
223 self
224 }
225
226 /// Admit only if this declaration revision governs the capability.
227 ///
228 /// For a caller that reviewed one revision and schedules runs against a
229 /// plane it does not control: another revision — or no declaration at
230 /// all, including on a build without the `manifest` feature — is refused
231 /// as [`RuntimeError::DeclarationPinMismatch`] before authorization, and
232 /// nothing is recorded.
233 #[must_use]
234 pub const fn expect_declaration(mut self, digest: Digest) -> Self {
235 self.expected_declaration = Some(digest);
236 self
237 }
238
239 fn keyed(mut self, key: &str) -> Self {
240 self.idempotency_key = Some(key.to_owned());
241 self
242 }
243}
244
245/// A commissioned sub-run this process is executing: the run that started it,
246/// and the task running it.
247type Working = (
248 RunId,
249 tokio::task::JoinHandle<Result<RunOutcome, RuntimeError>>,
250);
251
252/// Default lease duration. A crashed owner's runs become claimable this long
253/// after its last heartbeat.
254///
255/// This bounds how long a *dead* owner's runs are stranded, not how long a run
256/// may take: a live run renews while it executes. Set per plane with
257/// [`RuntimeBuilder::lease_ttl`].
258pub const LEASE_TTL: Duration = Duration::from_secs(30);
259
260/// The shortest lease a live run can actually hold.
261///
262/// Both stores keep expiry in whole seconds and lapse on `expires_at <= now`, so
263/// a one-second lease is expired for part of every second it exists. Two is the
264/// smallest value a renewal can stay ahead of.
265pub const MIN_LEASE_TTL: Duration = Duration::from_secs(2);
266
267/// What a run produced.
268#[derive(Debug, Clone)]
269pub struct RunOutcome {
270 pub run_id: RunId,
271 pub status: RunStatus,
272 /// What this run consumed — steps, effects, spend, elapsed, refusals.
273 ///
274 /// Reported rather than left in the ledger because "what did the settlement
275 /// run cost" has to be answerable per item, and a batch sums its items. A
276 /// figure that only exists inside a dropped `Ledger` is a figure nobody can
277 /// bill against.
278 ///
279 /// The whole tally rather than only the money: the counts are what a
280 /// ceiling is sized against, and an operator raising `max_effects` after a
281 /// run stopped short is asking exactly the question this answers.
282 pub consumed: Consumed,
283 /// Terminal hash of the run's chain — what a signature would cover.
284 pub chain_head: Digest,
285 /// What the run produced, **with its label**.
286 ///
287 /// Labelled rather than bare, and the difference is not cosmetic: a
288 /// caller acting on a run's answer needs to know whether a model, a peer
289 /// or a person wrote it. The label was stripped here until an A2A reply
290 /// projection read a marker key out of an untrusted answer and let a
291 /// remote peer choose the envelope its own reply arrived in — a
292 /// confused-deputy reachable because the one fact that would have refused
293 /// it had been dropped at the boundary.
294 pub output: Option<Tainted<Value>>,
295 /// The effect a failed run ended on — its kind, disposition, permanence
296 /// and the peer's code — read from the run's records by
297 /// [`failed_effect`](crate::journal::failed_effect), so a replay answers
298 /// as the live run did. `None` unless the run failed on an effect.
299 pub failed_effect: Option<crate::journal::FailedEffect>,
300}
301
302impl RunOutcome {
303 /// What this run spent, in metered units — the money half of
304 /// [`consumed`](Self::consumed).
305 #[must_use]
306 pub const fn spend(&self) -> Spend {
307 self.consumed.spend
308 }
309
310 /// Why this run ended — [`RunStatus::reason`] without the match.
311 ///
312 /// `None` for a success. The accessor is here as well as on the status
313 /// because this is the type an embedder holds after `run(..)`, and a
314 /// summary field that silently stays empty is the shape this exists to
315 /// prevent.
316 #[must_use]
317 pub fn reason(&self) -> Option<std::borrow::Cow<'_, str>> {
318 self.status.reason()
319 }
320
321 /// The output if the run succeeded, or a [`RunFailure`] saying why not.
322 ///
323 /// Every caller that acts on a run's answer asks the same two questions —
324 /// did it work, and what did it say — and pattern-matching
325 /// [`status`](Self::status) to learn the first is how the second gets read
326 /// off a run that quarantined. This folds the pair into one `?`:
327 ///
328 /// ```ignore
329 /// let answer = runtime.start("greet", input).run().await?.success()?;
330 /// ```
331 ///
332 /// `status` stays public for the callers whose branches genuinely differ —
333 /// a suspended run is not a failed one, and an operator surface treats
334 /// them differently. This is for the caller to whom anything short of an
335 /// answer is an error.
336 ///
337 /// # Errors
338 ///
339 /// [`RunFailure`] for every non-succeeded status, carrying the run id and
340 /// the status itself.
341 pub fn success(self) -> Result<Tainted<Value>, RunFailure> {
342 match self.status {
343 // A succeeded run's answer; `Null` for the degenerate run that
344 // finished without producing one, which is still a success.
345 RunStatus::Succeeded => {
346 Ok(self.output.unwrap_or_else(|| Tainted::trusted(Value::Null)))
347 }
348 status => Err(RunFailure {
349 run_id: self.run_id,
350 status,
351 }),
352 }
353 }
354}
355
356/// What an idempotent admission did.
357///
358/// A duplicate comes back as an answer rather than an error: a caller that
359/// retried wants the original run, not a conflict it has to interpret.
360///
361/// Exhaustive, unlike the error types: a key is unused, or held by a run that
362/// has rested, or held by one that has not. There is no fourth state to leave
363/// room for, and `#[non_exhaustive]` would cost every caller a `_` arm that
364/// means nothing.
365#[derive(Debug, Clone)]
366pub enum Admission {
367 /// The key was unused. This run is the one it admitted.
368 Fresh(RunOutcome),
369 /// The key already admitted a run that has reached a resting point —
370 /// concluded, or suspended waiting for something. Its answer, again.
371 ///
372 /// A suspension counts, and that is the case this exists for: a run parked
373 /// on a four-eyes decision has already opened the task, so a redelivery
374 /// that admitted would put a second identical approval in front of a
375 /// reviewer.
376 ///
377 /// Carries no `output`. A step's result is reconstructed by replay rather
378 /// than stored, so what the journal answers is *what happened* — status,
379 /// reason, chain head. Re-derive the value with [`replay`](Runtime::replay)
380 /// under [`Mode::Strict`], which performs no external effect.
381 Replayed(RunOutcome),
382 /// The key already admitted a run that has not rested yet — still
383 /// executing, or abandoned and awaiting the recovery sweep.
384 ///
385 /// The honest answer to an emitter is *accepted, already in progress* — a
386 /// 200 or 202 naming this run, never a retry-provoking failure.
387 InFlight(RunId),
388}
389
390impl Admission {
391 /// The run this key admitted, whichever of the three this is.
392 #[must_use]
393 pub const fn run_id(&self) -> RunId {
394 match self {
395 Self::Fresh(outcome) | Self::Replayed(outcome) => outcome.run_id,
396 Self::InFlight(run) => *run,
397 }
398 }
399
400 /// Whether this call is the one that admitted the run.
401 #[must_use]
402 pub const fn is_fresh(&self) -> bool {
403 matches!(self, Self::Fresh(_))
404 }
405
406 /// What the run produced, if it has produced anything yet.
407 #[must_use]
408 pub const fn outcome(&self) -> Option<&RunOutcome> {
409 match self {
410 Self::Fresh(outcome) | Self::Replayed(outcome) => Some(outcome),
411 Self::InFlight(_) => None,
412 }
413 }
414}
415
416/// What a keyed [`spawn`](RunRequest::spawn) did.
417///
418/// Separate from [`Admission`] because a spawn returns before the work happens,
419/// so it has no outcome to report.
420#[derive(Debug, Clone, Copy, PartialEq, Eq)]
421pub struct Spawned {
422 /// The run holding the key: the one just started, or the one that already
423 /// held it.
424 pub run: RunId,
425 /// False when the key was already held, so nothing new was started.
426 pub fresh: bool,
427}
428
429/// What a run starts from.
430#[derive(Debug, Clone)]
431pub enum Target {
432 /// A capability a registered skill answers — the degenerate one-node plan.
433 Capability(String),
434 /// A plan, validated and frozen before its first step: one that would
435 /// fail at step seven must not begin at step one.
436 Plan(PlanIR),
437}
438
439impl From<&str> for Target {
440 fn from(capability: &str) -> Self {
441 Self::Capability(capability.to_owned())
442 }
443}
444
445impl From<String> for Target {
446 fn from(capability: String) -> Self {
447 Self::Capability(capability)
448 }
449}
450
451impl From<&String> for Target {
452 fn from(capability: &String) -> Self {
453 Self::Capability(capability.clone())
454 }
455}
456
457impl From<PlanIR> for Target {
458 fn from(plan: PlanIR) -> Self {
459 Self::Plan(plan)
460 }
461}
462
463/// A request with no admission key: answered by what the run did.
464#[derive(Debug, Clone, Copy)]
465pub struct Unkeyed;
466
467/// A request under an admission key: answered by what the key admitted.
468#[derive(Debug, Clone)]
469pub struct Keyed(String);
470
471/// A run about to start — see [`Runtime::start`].
472///
473/// The answer's type follows the key. Unkeyed, a run answers with its
474/// [`RunOutcome`] and a spawn with its [`RunId`]; under
475/// [`once`](Self::once), with an [`Admission`] or a [`Spawned`], because a
476/// key already held is answered by the run that holds it, and a caller must
477/// say what it does then.
478///
479/// ```compile_fail
480/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
481/// let _: agentplane::runtime::RunOutcome = rt.start("c", input).once("k").run().await.unwrap();
482/// # }
483/// ```
484///
485/// ```compile_fail
486/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
487/// rt.start("c", input).once("a").once("b");
488/// # }
489/// ```
490///
491/// ```no_run
492/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
493/// let _: agentplane::runtime::RunOutcome = rt.start("c", input.clone()).run().await.unwrap();
494/// let _: agentplane::runtime::Admission = rt.start("c", input).once("k").run().await.unwrap();
495/// # }
496/// ```
497#[must_use = "a run request does nothing until it is run, spawned or settled"]
498#[derive(Debug)]
499pub struct RunRequest<'a, K> {
500 runtime: &'a Runtime,
501 target: Target,
502 input: Tainted<Value>,
503 terms: RunTerms,
504 key: K,
505}
506
507impl<K> RunRequest<'_, K> {
508 /// Bind the run: a case, a caller's chain, who asked, a pinned
509 /// declaration. A second call replaces the first, as one value.
510 pub fn terms(mut self, terms: RunTerms) -> Self {
511 self.terms = terms;
512 self
513 }
514}
515
516impl<'a> RunRequest<'a, Unkeyed> {
517 /// Admit at most once per tenant under this key.
518 ///
519 /// The door for an at-least-once emitter: a bus that retries until it sees
520 /// a 2xx, a webhook fan-out, a queue with redelivery. The store claims the
521 /// key in the same transaction that writes the run's first record, so a
522 /// second request under it cannot admit a second run — not on a retry, not
523 /// across a restart, not from another instance sharing the store.
524 ///
525 /// **The key must carry its producer.** An id is unique only within one
526 /// emitter, so a bare message id lets two counterparties swallow each
527 /// other's messages as apparent retries.
528 /// [`InboundEvent::dedup_key`](crate::core::InboundEvent::dedup_key) is the
529 /// ready-made spelling.
530 ///
531 /// Not the event buffer: an event nobody subscribes to is dead-lettered by
532 /// [`sweep`](Runtime::sweep), so using the buffer as a ledger would spend
533 /// that signal to buy deduplication.
534 pub fn once(self, key: impl Into<String>) -> RunRequest<'a, Keyed> {
535 RunRequest {
536 runtime: self.runtime,
537 target: self.target,
538 input: self.input,
539 terms: self.terms,
540 key: Keyed(key.into()),
541 }
542 }
543
544 /// Run, and wait until it rests.
545 ///
546 /// # Errors
547 ///
548 /// [`RuntimeError::NoProvider`] when no skill provides the capability, and
549 /// whatever admission refuses — policy, quota, a halted tenant, an
550 /// instance draining, a chain this plane may not act under.
551 pub async fn run(self) -> Result<RunOutcome, RuntimeError> {
552 self.runtime
553 .admit_target(self.target, self.input, self.terms)
554 .await
555 }
556
557 /// Admit, and let the run proceed in the background, returning its id.
558 ///
559 /// **Admission happens before this returns.** The policy gate, the lease
560 /// and the admission records are written first, so a refusal is an error
561 /// here and not a task that never appears, and the id can be read at once.
562 /// The run is durable: a process that dies mid-flight leaves a journal
563 /// another instance resumes.
564 ///
565 /// # Panics
566 ///
567 /// Outside a Tokio runtime, as the rest of this crate's timing does.
568 ///
569 /// # Errors
570 ///
571 /// As [`run`](Self::run).
572 pub async fn spawn(self) -> Result<RunId, RuntimeError> {
573 self.runtime
574 .owned()
575 .spawn_target(self.target, self.input, self.terms)
576 .await
577 }
578}
579
580impl RunRequest<'_, Keyed> {
581 /// Run under the key. A key already held comes back as
582 /// [`Admission::Replayed`] or [`Admission::InFlight`], not an error.
583 ///
584 /// # Errors
585 ///
586 /// As [`RunRequest::run`], and [`RuntimeError::PlanContract`] for an empty
587 /// key or one over [`MAX_ADMISSION_KEY_BYTES`].
588 pub async fn run(self) -> Result<Admission, RuntimeError> {
589 let runtime = self.runtime;
590 let terms = self.terms.keyed(&self.key.0);
591 // Boxed: admission is the largest future in the crate, and the claim
592 // would otherwise carry it inline into every keyed caller's future.
593 let admit = Box::pin(runtime.admit_target(self.target, self.input, terms));
594 match runtime.claimed(&self.key.0, admit).await? {
595 Claimed::Fresh(outcome) => Ok(Admission::Fresh(outcome)),
596 Claimed::Held(run) => runtime.answer_with(run).await,
597 }
598 }
599
600 /// Spawn under the key. A key already held names its run, with `fresh`
601 /// false.
602 ///
603 /// # Panics
604 ///
605 /// Outside a Tokio runtime.
606 ///
607 /// # Errors
608 ///
609 /// As [`run`](Self::run).
610 pub async fn spawn(self) -> Result<Spawned, RuntimeError> {
611 let runtime = self.runtime.owned();
612 let terms = self.terms.keyed(&self.key.0);
613 let admit = Box::pin(runtime.spawn_target(self.target, self.input, terms));
614 Ok(match runtime.claimed(&self.key.0, admit).await? {
615 Claimed::Fresh(run) => Spawned { run, fresh: true },
616 Claimed::Held(run) => Spawned { run, fresh: false },
617 })
618 }
619
620 /// Run under the key until the run it admitted rests, whoever admitted it,
621 /// and answer with its outcome.
622 ///
623 /// The shape a process that may have been killed mid-run wants on restart:
624 /// the same request again. An unused key runs. A key whose run has rested
625 /// answers with that run, its value re-derived by strict replay, which
626 /// performs nothing. A key whose run is still working is resumed when
627 /// nobody holds its lease — its holder has gone — and otherwise followed
628 /// on the journal until it rests or its lease lapses.
629 ///
630 /// # Errors
631 ///
632 /// As [`run`](Self::run), and whatever resuming or replaying the held run
633 /// refuses.
634 pub async fn settle(self) -> Result<RunOutcome, RuntimeError> {
635 let runtime = self.runtime;
636 match self.run().await? {
637 Admission::Fresh(outcome) => Ok(outcome),
638 Admission::Replayed(rested) => runtime.with_output(rested).await,
639 Admission::InFlight(run) => runtime.settled(run).await,
640 }
641 }
642}
643
644/// What a keyed claim found.
645enum Claimed<T> {
646 /// The key was unused, and this admitted.
647 Fresh(T),
648 /// The key already admitted this run.
649 Held(RunId),
650}
651
652/// The stores a full plane runs on, as handles rather than as one type.
653///
654/// [`Runtime::builder_on`] is the shorthand for one backend implementing all
655/// of them, and is the wrong shape for a caller that picks its backend at **run
656/// time** — an operator naming either a file or a connection string holds
657/// `Arc<dyn JournalStore>`, and unsizing a generic parameter needs it `Sized`,
658/// so the wiring is a value that `builder_on` builds.
659///
660/// Here rather than in [`store`](crate::store) because that module is the
661/// backends, gated on `redb` or `postgres`, while this exists whether or not
662/// either is compiled in.
663#[derive(Clone)]
664pub struct Stores {
665 pub journal: std::sync::Arc<dyn crate::journal::JournalStore>,
666 pub cases: std::sync::Arc<dyn crate::case::CaseStore>,
667 pub tasks: std::sync::Arc<dyn crate::case::TaskStore>,
668 pub events: std::sync::Arc<dyn crate::case::EventStore>,
669 pub timers: std::sync::Arc<dyn crate::case::TimerStore>,
670 pub memory: std::sync::Arc<dyn crate::memory::MemoryStore>,
671 /// Where halts, subject withdrawals, rate windows and tenant ceilings live.
672 pub quotas: std::sync::Arc<dyn crate::quota::QuotaStore>,
673 /// Where disclosures of a matter are recorded.
674 pub disclosures: std::sync::Arc<dyn crate::disclosure::DisclosureRegister>,
675}
676
677impl std::fmt::Debug for Stores {
678 /// The handles are trait objects with no identity to print, so what a reader
679 /// wants here is *that the set is complete*, which it is by construction.
680 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
681 f.write_str("Stores { journal, cases, tasks, events, timers, memory, quotas, disclosures }")
682 }
683}
684
685impl Stores {
686 /// Every store from one backend that implements all of them.
687 #[must_use]
688 pub fn on<B: FullBackend + 'static>(store: std::sync::Arc<B>) -> Self {
689 Self {
690 journal: std::sync::Arc::clone(&store) as _,
691 cases: std::sync::Arc::clone(&store) as _,
692 tasks: std::sync::Arc::clone(&store) as _,
693 events: std::sync::Arc::clone(&store) as _,
694 timers: std::sync::Arc::clone(&store) as _,
695 memory: std::sync::Arc::clone(&store) as _,
696 quotas: std::sync::Arc::clone(&store) as _,
697 disclosures: store as _,
698 }
699 }
700}
701
702/// How far into a run's history [`Runtime::running_runs`] reads to attribute it.
703///
704/// `RunAdmitted` is seq 1 and `IdentityBound` follows it when the caller
705/// presented a chain, with the quota pass and the frozen plan between them — so
706/// a handful of records carries both and a long-running agent's history is not
707/// scanned to answer *who is this*.
708const ATTRIBUTION_RECORDS: usize = 8;
709
710/// One run this tenant currently holds an admission slot for.
711///
712/// What [`Runtime::running_runs`] answers with. The ids alone would be a listing
713/// an operator reads out; the attribution is what makes it one they can act on.
714#[derive(Debug, Clone)]
715pub struct LiveRun {
716 pub run: RunId,
717 /// The declared agent governing it, when a declared one does.
718 ///
719 /// `None` for a skill registered directly on the plane — a legitimate shape,
720 /// and one an incident about *a bad revision* can therefore rule out.
721 pub governed_by: Option<crate::journal::AgentIdentity>,
722 /// Who this run acts for: the root of the delegation chain bound at
723 /// admission, when the caller presented one.
724 pub subject: Option<String>,
725 /// The slot is held by a run whose lease has lapsed, so no process is
726 /// executing it.
727 ///
728 /// Not a cancellation candidate: the recovery sweep resumes these, and
729 /// cancelling one unwinds work that was about to continue.
730 pub stranded: bool,
731}
732
733/// A run that stopped short of an answer, as an error a caller can `?`.
734///
735/// Carries the [`RunStatus`] whole rather than a flattened string, so a caller
736/// that started with [`RunOutcome::success`] and later needs to tell a
737/// suspension from a quarantine can match on [`status`](Self::status) without
738/// re-plumbing the call site.
739#[derive(Debug, Clone)]
740pub struct RunFailure {
741 pub run_id: RunId,
742 pub status: RunStatus,
743}
744
745/// **Total, including over the status this type is not supposed to hold.**
746///
747/// `status` is public and `RunStatus` carries `Succeeded`, so a
748/// `RunFailure { status: Succeeded }` is a value any caller can build. Asserting
749/// that away would make *formatting an error* the operation that panics, and a
750/// `Display` impl runs wherever an error is logged — disproportionately during
751/// an incident. So it prints the contradiction instead.
752impl std::fmt::Display for RunFailure {
753 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
754 if matches!(self.status, RunStatus::Succeeded) {
755 return write!(
756 f,
757 "run {} is recorded as a failure carrying a success — \
758 whatever built this value did so wrongly",
759 self.run_id
760 );
761 }
762 write!(f, "run {} did not succeed: ", self.run_id)?;
763 match &self.status {
764 RunStatus::Succeeded => write!(f, "it succeeded"),
765 RunStatus::Failed(reason) => write!(f, "it failed — {reason}"),
766 RunStatus::Suspended(reason) => write!(f, "it is suspended — {reason:?}"),
767 RunStatus::Exhausted(limit) => write!(f, "a budget stopped it — {limit}"),
768 RunStatus::Quarantined(reason) => write!(f, "it is quarantined — {reason}"),
769 RunStatus::Replanning(reason) => write!(f, "it asked to replan — {reason}"),
770 RunStatus::Observed => write!(
771 f,
772 "it is a session this plane observed rather than executed, so it \
773 neither succeeded nor failed"
774 ),
775 RunStatus::Cancelled { actor, reason } => {
776 write!(f, "{actor} cancelled it — {reason}")
777 }
778 RunStatus::Swept => write!(f, "the plane recorded a pass over its own state"),
779 RunStatus::BrokeGlass { actor, reason } => {
780 write!(f, "{actor} crossed into this tenant — {reason}")
781 }
782 RunStatus::HaltLifted { actor } => write!(f, "{actor} lifted an emergency stop"),
783 RunStatus::HoldReleased { actor } => write!(f, "{actor} released a legal hold"),
784 RunStatus::Abandoned { actor, reason } => {
785 write!(
786 f,
787 "{actor} abandoned it, unresolved and not unwound — {reason}"
788 )
789 }
790 RunStatus::Withheld { subject, reason } => {
791 write!(
792 f,
793 "the authority it acts for — '{subject}' — was withdrawn: {reason}"
794 )
795 }
796 }
797 }
798}
799
800impl std::error::Error for RunFailure {}
801
802#[derive(Debug, Clone, PartialEq)]
803pub enum RunStatus {
804 /// All terminal work completed.
805 ///
806 /// Structural, never self-reported: a skill saying "done" is not what makes
807 /// a run succeed. Agents confidently announce success on unmet objectives,
808 /// so completion is determined by the runtime, not claimed by the workload.
809 Succeeded,
810 Failed(String),
811 /// Waiting for something that has not happened. The frame is persisted and
812 /// the task is gone: a suspended run costs disk, not a thread.
813 Suspended(crate::core::SuspendReason),
814 /// A limit stopped it. Not a fault — the run did what it was told, and what
815 /// it was told included a ceiling.
816 Exhausted(crate::core::BudgetExceeded),
817 /// Needs human resolution before anything else may happen to it.
818 Quarantined(String),
819 /// A step asked for a different plan.
820 ///
821 /// Never observed by a caller: the executor either produces a successor and
822 /// keeps going, or turns this into a failure with the reason it refused.
823 /// It exists as a status so a step's request travels the same path every
824 /// other outcome does.
825 Replanning(String),
826 /// An operator stopped it.
827 ///
828 /// Distinct from `Failed` on purpose. A failure is the run discovering it
829 /// cannot proceed; a cancellation is a human deciding it should not. They
830 /// call for different responses — one is investigated, the other was
831 /// intended — and an operator scanning for failures should not have to
832 /// mentally subtract their own interventions.
833 ///
834 /// Completed steps are unwound exactly as they are for a failure: stopping a
835 /// run that has moved money and leaving the movement in place is not
836 /// stopping it.
837 Cancelled {
838 actor: crate::core::Operator,
839 reason: String,
840 },
841 /// An operator closed a run whose outcome could never be established.
842 ///
843 /// The end of the road for a quarantine nobody can answer. Distinct from
844 /// `Cancelled` in the one way that matters to whoever reads it next:
845 /// **nothing was unwound**. A cancellation puts the world back; this leaves
846 /// it exactly as the run left it, because compensating around an unknown
847 /// outcome is the single thing quarantine exists to forbid, and an
848 /// operator's patience running out is not new evidence.
849 ///
850 /// So the doubt does not go away when the run does. It stops being a
851 /// *status* — which is a mutable thing a later action can overwrite — and
852 /// becomes an `agentplane audit` finding, which is derived from the journal
853 /// and outlives every status the run will ever have.
854 Abandoned {
855 actor: crate::core::Operator,
856 reason: String,
857 },
858 /// A pass this plane made over its own state, journaled as a run of its own.
859 ///
860 /// The sweep records what it decided — which obligations breached, which
861 /// tasks it escalated, which runs it took over — rather than logging it, so
862 /// *who decided what* is answerable from the chain six weeks later. It is
863 /// not `Succeeded` because nothing asked for it and there is no answer to
864 /// hand back: a sweep is the plane reporting on itself.
865 Swept,
866 /// An operator crossed into this tenant deliberately, and the reason they
867 /// gave for it.
868 ///
869 /// The one exception to tenant isolation, and the exception's entire value
870 /// is this record. Both halves are carried because an incident review asks
871 /// two questions and neither is worth less than the other: `actor` is read
872 /// back from the crossing record, `reason` from the conclusion, and a
873 /// crossing cannot be recorded without one.
874 BrokeGlass {
875 actor: crate::core::Operator,
876 reason: String,
877 },
878 /// An operator lifted an emergency stop, recorded as a run of its own.
879 ///
880 /// `actor` is read back from the run's
881 /// [`HaltLifted`](crate::journal::RecordKind::HaltLifted) record, which
882 /// also holds the stop it ended.
883 HaltLifted {
884 actor: crate::core::Operator,
885 },
886 /// An operator released a legal hold, recorded as a run of its own.
887 ///
888 /// `actor` is read back from the run's
889 /// [`HoldReleased`](crate::journal::RecordKind::HoldReleased) record.
890 HoldReleased {
891 actor: crate::core::Operator,
892 },
893 /// The authority this run acts under was withdrawn while it was running.
894 ///
895 /// **A pause, not a fault**, for the reason exhaustion is one: the run did
896 /// nothing wrong. Its mutations stand and it stays resumable — lift the halt
897 /// and it continues. Unwinding would reverse correct work for a reason
898 /// unrelated to it; cancelling is the verb for that.
899 ///
900 /// Carries the subject, because *whose* authority is what an operator
901 /// matches against the withdrawal they made.
902 Withheld {
903 subject: String,
904 reason: String,
905 },
906 /// A session this plane **observed** rather than executed, closed.
907 ///
908 /// The one status that is not about work this runtime did. A seat beside
909 /// somebody else's agent produces records at the *asserted* rung
910 /// ([`RecordKind::Observed`](crate::journal::RecordKind::Observed)), in a
911 /// run of their own with no admission — the shape a sweep's run already
912 /// takes — so that they share the chain, the Merkle log and the witness
913 /// while sharing no record kind with a dispatched effect.
914 ///
915 /// **It carries nothing, and the absence is the design.** There is no
916 /// reason to give: the session ended because the party running it ended
917 /// it, and this plane is in no position to say why. The session's own
918 /// identifier is on every observation record rather than here, so a plane
919 /// that stopped mid-session still leaves evidence that says which session
920 /// it describes.
921 ///
922 /// Sealed like any other ending, so an external verifier checks it without
923 /// being taught what an observation is. Never resumable: there is nothing
924 /// to resume onto, because nothing here was executing.
925 Observed,
926}
927
928impl RunStatus {
929 /// Why the run ended, whatever kind of ending it was.
930 ///
931 /// `None` only for [`Succeeded`](Self::Succeeded), which has no reason to
932 /// give. Every other conclusion carries one, and the point of gathering
933 /// them here is that an embedder mapping outcomes onto its own wire type
934 /// should not have to match every variant to find the sentence, and an
935 /// embedder that reads only the status ships an empty summary on failure.
936 ///
937 /// Borrowed where the variant already holds a string and formatted where
938 /// it holds a typed value: a suspension names what it waits for and an
939 /// exhaustion names the ceiling it hit, and both are worth more than the
940 /// word "suspended". `Cancelled` yields the operator's reason; the actor
941 /// who gave it stays on the variant, because *why* and *who* are two
942 /// questions and only one of them is the reason.
943 #[must_use]
944 pub fn reason(&self) -> Option<std::borrow::Cow<'_, str>> {
945 use std::borrow::Cow;
946 match self {
947 // Nothing to say, for three different reasons and one answer: a
948 // success has no why, a sweep's records carry theirs, and an
949 // observed session ended because somebody else ended it.
950 // A lift's record carries what it ended; the lifter gave no reason.
951 Self::Succeeded
952 | Self::Swept
953 | Self::Observed
954 | Self::HaltLifted { .. }
955 | Self::HoldReleased { .. } => None,
956 Self::Failed(reason)
957 | Self::Quarantined(reason)
958 | Self::Replanning(reason)
959 | Self::Cancelled { reason, .. }
960 | Self::Abandoned { reason, .. }
961 | Self::BrokeGlass { reason, .. }
962 | Self::Withheld { reason, .. } => Some(Cow::Borrowed(reason.as_str())),
963 Self::Suspended(reason) => Some(Cow::Owned(reason.to_string())),
964 Self::Exhausted(exceeded) => Some(Cow::Owned(exceeded.to_string())),
965 }
966 }
967
968 /// The run's **own** account of why it ended, for the journal.
969 ///
970 /// [`reason`](Self::reason) answers *what should a person be shown*, which
971 /// includes the words an operator typed when they stopped the run. This
972 /// answers the narrower question the conclusion record asks: *is there a
973 /// sentence here that nothing else on the chain already records?*
974 ///
975 /// `None` for every ending a person caused. Those write their own record —
976 /// [`RunCancelled`](crate::journal::RecordKind::RunCancelled),
977 /// [`QuarantineDecided`](crate::journal::RecordKind::QuarantineDecided),
978 /// [`BreakGlass`](crate::journal::RecordKind::BreakGlass),
979 /// [`AuthorityWithheld`](crate::journal::RecordKind::AuthorityWithheld) —
980 /// carrying the actor and the reason together, in the clear. Repeating the
981 /// reason on the conclusion would make one sentence two facts, and seal the
982 /// copy that an erasure then destroys.
983 ///
984 /// `None` for an exhaustion too: the typed ceiling verdict is the record,
985 /// and the prose was a rendering of it.
986 #[must_use]
987 pub fn own_account(&self) -> Option<std::borrow::Cow<'_, str>> {
988 use std::borrow::Cow;
989 match self {
990 // The runtime's own endings: a skill's words, a provider's words,
991 // a divergence this build found. Nothing else on the chain says it.
992 Self::Failed(reason) | Self::Quarantined(reason) | Self::Replanning(reason) => {
993 Some(Cow::Borrowed(reason.as_str()))
994 }
995 // Nothing to add, for two different reasons and one answer. The
996 // first four are somebody's doing and their own record already
997 // carries both halves. The rest either write no conclusion with a
998 // why — a success, a sweep, a suspension — or carry it typed
999 // instead, which is what `exhaustion` is.
1000 //
1001 // Enumerated rather than left to a wildcard: a status added later
1002 // must be made to answer this question, and `_ => None` would
1003 // answer it silently and wrongly for anything a person causes.
1004 Self::Cancelled { .. }
1005 | Self::Abandoned { .. }
1006 | Self::BrokeGlass { .. }
1007 | Self::HaltLifted { .. }
1008 | Self::HoldReleased { .. }
1009 | Self::Withheld { .. }
1010 | Self::Succeeded
1011 | Self::Swept
1012 // The account is the observation records themselves, and this plane
1013 // has none of its own to add: it did not run the session.
1014 | Self::Observed
1015 | Self::Suspended(_)
1016 | Self::Exhausted(_) => None,
1017 }
1018 }
1019
1020 /// Whether the run stopped without reaching a conclusion.
1021 #[must_use]
1022 pub fn is_suspended(&self) -> bool {
1023 matches!(self, Self::Suspended(_))
1024 }
1025
1026 /// Whether the recorded history can no longer be trusted to describe this
1027 /// code, so a human must look before anything else happens.
1028 #[must_use]
1029 pub fn is_quarantined(&self) -> bool {
1030 matches!(self, Self::Quarantined(_))
1031 }
1032
1033 #[must_use]
1034 pub fn as_str(&self) -> &'static str {
1035 match self {
1036 Self::Succeeded => "succeeded",
1037 Self::Failed(_) => "failed",
1038 Self::Suspended(_) => "suspended",
1039 Self::Exhausted(_) => "exhausted",
1040 Self::Quarantined(_) => "quarantined",
1041 Self::Replanning(_) => "replanning",
1042 Self::Cancelled { .. } => "cancelled",
1043 Self::Abandoned { .. } => "abandoned",
1044 Self::Swept => super::sweeper::SWEEP_OUTCOME,
1045 Self::BrokeGlass { .. } => BREAK_GLASS_OUTCOME,
1046 Self::HaltLifted { .. } => HALT_LIFTED_OUTCOME,
1047 Self::HoldReleased { .. } => HOLD_RELEASED_OUTCOME,
1048 Self::Withheld { .. } => WITHHELD_OUTCOME,
1049 Self::Observed => OBSERVED_OUTCOME,
1050 }
1051 }
1052
1053 /// Who brought this run to this state, where a person did.
1054 ///
1055 /// `None` for every ending the runtime reached on its own, which is most of
1056 /// them. One accessor rather than three `matches!` at each surface, because
1057 /// *who* is the question an operator asks immediately after *what* — and a
1058 /// surface that reads the reason and drops the actor answers half of it.
1059 #[must_use]
1060 pub fn actor(&self) -> Option<&crate::core::Operator> {
1061 match self {
1062 Self::Cancelled { actor, .. }
1063 | Self::Abandoned { actor, .. }
1064 | Self::BrokeGlass { actor, .. }
1065 | Self::HaltLifted { actor }
1066 | Self::HoldReleased { actor } => Some(actor),
1067 // A withdrawal names a *subject*, not an actor: the operator who
1068 // threw the halt and the credential it covers are different
1069 // parties, and answering "who" with the subject would report the
1070 // person whose authority was taken as the one who took it.
1071 Self::Succeeded
1072 | Self::Failed(_)
1073 | Self::Suspended(_)
1074 | Self::Exhausted(_)
1075 | Self::Quarantined(_)
1076 | Self::Replanning(_)
1077 | Self::Withheld { .. }
1078 | Self::Swept
1079 // Somebody was at the keyboard, and it was not this plane's
1080 // operator. Naming the observed party's user here would report a
1081 // person this runtime never authenticated as one of its own.
1082 | Self::Observed => None,
1083 }
1084 }
1085
1086 /// Whether an operator stopped this run.
1087 #[must_use]
1088 pub fn is_cancelled(&self) -> bool {
1089 matches!(self, Self::Cancelled { .. })
1090 }
1091
1092 /// Whether this conclusion freezes the journal and enters the Merkle log.
1093 ///
1094 /// Only conclusions that nothing may resume seal. A failed run may be
1095 /// resumed — its completed effects are read back from history rather than
1096 /// performed again, which is the point of having a journal — and an
1097 /// exhausted run may continue once somebody raises the ceiling. Sealing
1098 /// either would commit the Merkle log to a leaf its own resume is then
1099 /// permitted to grow past: the checkpoint would attest a prefix of a
1100 /// history that kept moving, and the store's refusal of appends after a
1101 /// seal would turn every legitimate resume into an error.
1102 ///
1103 /// **A quarantine does not seal**, and that follows from the same rule
1104 /// rather than being an exception to it. A quarantine is the runtime saying
1105 /// it does not know, which is a story that is not over; a Merkle leaf is a
1106 /// claim that a history is complete. Sealing one published a proof of
1107 /// completeness over an admittedly incomplete record, and — the practical
1108 /// half — froze the journal against the one record that answers it, so
1109 /// "a human must resolve it before it can run again" named a resolution the
1110 /// format made impossible. It enters the log when it reaches a real ending,
1111 /// including [`Abandoned`](Self::Abandoned), which is a person deciding
1112 /// there will never be one.
1113 ///
1114 /// This is the **one** implementation of that rule. `resume_is_closed`
1115 /// refuses exactly the outcomes this method seals — a test pins the
1116 /// agreement — because two copies of one rule agree everywhere except the
1117 /// boundary nobody probed.
1118 #[must_use]
1119 pub fn seals(&self) -> bool {
1120 matches!(
1121 self,
1122 Self::Succeeded
1123 | Self::Cancelled { .. }
1124 | Self::Abandoned { .. }
1125 | Self::Swept
1126 | Self::BrokeGlass { .. }
1127 | Self::HaltLifted { .. }
1128 | Self::HoldReleased { .. }
1129 | Self::Observed
1130 )
1131 }
1132}
1133
1134/// Every outcome a sealed run can carry, as the outcome index spells them.
1135///
1136/// The store indexes runs *by* outcome and deliberately has no "all runs"
1137/// query — a backlog listing is a range read rather than a table scan — so any
1138/// caller that wants every sealed run has to name the outcomes one by one. This
1139/// is the list to name them from, and it lives beside [`RunStatus::seals`]
1140/// because it is the same rule in its other spelling: exactly the statuses that
1141/// seal, with no exception clause. A sweep (`swept`) and a break-glass crossing
1142/// (`broke-glass`) are sealed at birth with no goal to have succeeded or failed
1143/// at, and they are statuses all the same — an ending this plane writes and its
1144/// own reader cannot name is an ending it reports as unrecognised.
1145///
1146/// It lived in the CLI first, as string literals — two implementations of one
1147/// rule, where a new sealing outcome would have been exported by nobody and
1148/// the export that silently omitted it would have dropped exactly the runs an
1149/// auditor came for. A test holds this list to `RunStatus::seals` variant by
1150/// variant, which the literals in a binary could never be held to.
1151pub const SEALED_OUTCOMES: &[&str] = &[
1152 "succeeded",
1153 "cancelled",
1154 "abandoned",
1155 super::sweeper::SWEEP_OUTCOME,
1156 BREAK_GLASS_OUTCOME,
1157 HALT_LIFTED_OUTCOME,
1158 HOLD_RELEASED_OUTCOME,
1159 OBSERVED_OUTCOME,
1160];
1161
1162/// What an offline verb sweeps when the operator names no outcome.
1163///
1164/// Every ending — [`SEALED_OUTCOMES`] — **plus `quarantined`**, which is not an
1165/// ending and is exactly why it belongs: a run set aside for a person is the
1166/// plane's outstanding unresolved risk, and an artifact holding everything that
1167/// finished and nothing that did not is not the one an auditor came for. It is
1168/// also the outcome most likely to be the reason they came.
1169///
1170/// Failed and exhausted runs are deliberately absent, and the distinction is
1171/// not sealing but *whether anything is waiting on a person*. Those two are
1172/// work in progress that a resume moves on its own; a quarantine moves only
1173/// when somebody decides, so it can sit for weeks and still be current.
1174///
1175/// A test holds this to [`SEALED_OUTCOMES`] rather than restating it, so a new
1176/// ending is swept here the day it exists.
1177pub const OUTCOMES_OF_RECORD: &[&str] = &[
1178 "succeeded",
1179 "cancelled",
1180 "abandoned",
1181 "quarantined",
1182 super::sweeper::SWEEP_OUTCOME,
1183 BREAK_GLASS_OUTCOME,
1184 HALT_LIFTED_OUTCOME,
1185 HOLD_RELEASED_OUTCOME,
1186 OBSERVED_OUTCOME,
1187];
1188
1189/// How a resume came to hold its lease, which decides what a failure does
1190/// with it.
1191///
1192/// A lease that lapses while still naming an owner is what the recovery sweep
1193/// lists, so keeping one on failure is how a run stays findable; releasing it
1194/// is right only where the run is findable without it.
1195#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1196enum Claim {
1197 /// Claimed by the resume itself, over a run whose journal already lists
1198 /// it — failed, exhausted, withheld, quarantined, waiting.
1199 Standing,
1200 /// Handed over by a wake or a delivery that recorded the awaited result
1201 /// under it, which took the run off the waiting listing.
1202 Unlisted,
1203 /// Claimed by the recovery sweep over a lease its owner died holding.
1204 Recovering,
1205}
1206
1207/// A run that exists but has not run.
1208///
1209/// Produced by admission and consumed by execution, so the two can happen in
1210/// different places — the same request for a blocking call, and a background
1211/// task for a non-blocking one.
1212struct Admitted {
1213 run: RunId,
1214 epoch: crate::core::Epoch,
1215 quota: QuotaPass,
1216 budget: Budget,
1217 agent: String,
1218 plan: PlanIR,
1219 input: Tainted<Value>,
1220 case: Option<CaseContext>,
1221 /// The chain admission bound — the caller's where one was presented,
1222 /// the plane's otherwise — and what every step of this run acts under.
1223 identity: Option<crate::core::Delegation>,
1224 /// The references its `DataSubjectBound` record makes available.
1225 subjects: BTreeSet<crate::core::SubjectRef>,
1226}
1227
1228/// The quota identity of one live execution pass.
1229///
1230/// Captured once at admission or resume and carried to settlement. Computing
1231/// the period again after the work finishes can cross a day or month boundary,
1232/// authorizing against one ledger and charging another.
1233#[derive(Debug, Clone, Default)]
1234struct QuotaPass {
1235 period: Option<String>,
1236 enabled: bool,
1237 release_slot: bool,
1238 /// How many steps this pass may dispatch at once, when the tenant bounds
1239 /// spend: the width the run's reservation was sized for. `None` leaves
1240 /// the run's own budget as the only bound.
1241 width: Option<usize>,
1242}
1243
1244impl QuotaPass {
1245 fn at(
1246 quota: &crate::quota::TenantQuota,
1247 at: crate::core::Timestamp,
1248 release_slot: bool,
1249 ) -> Self {
1250 Self {
1251 period: quota.bounds_spend().then(|| quota.period.key_for(at)),
1252 enabled: true,
1253 release_slot,
1254 width: None,
1255 }
1256 }
1257
1258 const fn disabled() -> Self {
1259 Self {
1260 period: None,
1261 enabled: false,
1262 release_slot: false,
1263 width: None,
1264 }
1265 }
1266
1267 /// Held to the width a spend reservation was sized for, whenever the
1268 /// tenant bounds spend.
1269 fn sized_for(mut self, width: usize) -> Self {
1270 if self.period.is_some() {
1271 self.width = Some(width);
1272 }
1273 self
1274 }
1275
1276 /// How wide this pass may dispatch: the budget's own width, narrowed to
1277 /// the one its reservation was sized for. A replan may widen the plan; it
1278 /// does not widen what the period holds for it.
1279 fn parallelism(&self, budget: &Budget) -> usize {
1280 self.width
1281 .map_or_else(|| budget.parallelism(), |w| budget.parallelism().min(w))
1282 }
1283
1284 fn period(&self) -> Option<&str> {
1285 self.period.as_deref()
1286 }
1287
1288 fn started(&self) -> Option<RecordKind> {
1289 self.enabled.then(|| RecordKind::QuotaPassStarted {
1290 period: self.period.clone(),
1291 release_slot: self.release_slot,
1292 })
1293 }
1294
1295 fn settlement(
1296 &self,
1297 run: RunId,
1298 epoch: crate::core::Epoch,
1299 spend: Spend,
1300 concludes: bool,
1301 ) -> Option<crate::quota::QuotaSettlement> {
1302 self.enabled.then(|| crate::quota::QuotaSettlement {
1303 run,
1304 epoch,
1305 period: self.period.clone(),
1306 spend,
1307 release_slot: self.release_slot,
1308 concludes,
1309 })
1310 }
1311}
1312
1313/// How many steps a run may have in flight: its budget's width, or its
1314/// admitted plan's node count when that is narrower or nothing narrows it.
1315///
1316/// What a spend reservation multiplies the per-call bound by, since each step
1317/// in flight may hold one operation the ledger admitted and has not billed.
1318fn reserved_width(budget: &Budget, plan: &PlanIR) -> usize {
1319 budget.parallelism().min(plan.nodes.len()).max(1)
1320}
1321
1322/// The peers a plane may call, and how it reaches them.
1323///
1324/// One value because the two are one decision: a registry says what each
1325/// peer is granted and which credential reaches it, and the client is the
1326/// transport those grants are spent over. Held by the plane and handed to
1327/// every step, so a skill calling a peer reaches it through the same checked
1328/// registry a declarative agent does — never one it carries itself.
1329#[derive(Debug)]
1330pub(crate) struct PeerWiring {
1331 pub(crate) registry: crate::peers::PeerRegistry,
1332 pub(crate) client: Arc<dyn crate::peers::PeerClient>,
1333}
1334
1335/// Keeps a run's lease alive for as long as it is executing.
1336///
1337/// A lease answers one question — *is this owner dead?* — and it answers by
1338/// expiry. Without renewal it also answers a question it was never asked: a
1339/// healthy run that outlives its TTL looks exactly like a crashed one, and agent
1340/// runs routinely outlive a lease because a single model call can. Another
1341/// instance then takes the run over, bumps the epoch, and the original is fenced
1342/// on its next append: killed mid-flight, having already done real work.
1343///
1344/// Renewing while executing separates the two. The TTL then bounds how long a
1345/// *crashed* owner strands its runs, which is what it is for, and stops bounding
1346/// how long a run may take, which it should never have bounded.
1347///
1348/// Aborted on drop, so the renewal stops the moment execution returns — by any
1349/// path, including a panic unwinding through it. A heartbeat that outlived its
1350/// run would hold a lease nobody is using and strand it for a full TTL after a
1351/// crash, which is the failure this exists to prevent, arriving late.
1352struct Heartbeat(tokio::task::JoinHandle<()>);
1353
1354impl Drop for Heartbeat {
1355 fn drop(&mut self) {
1356 self.0.abort();
1357 }
1358}
1359
1360/// The runtime.
1361#[derive(Debug, Clone)]
1362pub struct Runtime {
1363 store: Arc<dyn JournalStore>,
1364 skills: HashMap<String, Arc<dyn Skill>>,
1365 by_capability: HashMap<Capability, String>,
1366 /// A handle to this plane, for steps that commission other agents on it.
1367 ///
1368 /// Weak, and that is not an optimisation: a strong self-reference would
1369 /// leak every runtime ever built. It exists because commissioning belongs to
1370 /// the *runtime* — a skill holding an `Arc<Runtime>` cannot work, since the
1371 /// runtime needs the skill before the skill can have the runtime.
1372 self_ref: std::sync::Weak<Self>,
1373 /// The declaration governing each skill, by skill name.
1374 ///
1375 /// Per skill rather than per runtime, because a plane runs several agents
1376 /// and a step must be judged against *its own* agent's manifest.
1377 #[cfg(feature = "manifest")]
1378 governed_by: HashMap<String, Arc<crate::manifest::Manifest>>,
1379 /// Which tenant this plane runs as.
1380 ///
1381 /// One plane serves one tenant; a **process** may run several and serve
1382 /// them all, because `api::Planes` resolves a plane from the authenticated
1383 /// caller's tenant. That is what the composite keys on both backends bought:
1384 /// the tenant leads every key, lease, correlation and blob path, so a query
1385 /// that forgets it misses rather than returning somebody else's rows.
1386 tenant: crate::core::TenantId,
1387 /// Who vouched for each declaration, by agent name.
1388 ///
1389 /// Separate from the manifest because it is *not* in it: a document cannot
1390 /// state who signed it. It arrives beside the manifest from a verified
1391 /// registry resolution, and an absent entry means nobody vouched.
1392 #[cfg(feature = "manifest")]
1393 published_by: HashMap<String, crate::core::KeyId>,
1394 /// Model drivers by the name a manifest calls them.
1395 ///
1396 /// Kept after build — not only consumed by it — so a *hand-written* skill
1397 /// can reach the driver its manifest's model role names through
1398 /// [`StepCtx::complete`], instead of smuggling an `Arc<dyn ModelProvider>`
1399 /// field past the declaration.
1400 #[cfg(feature = "manifest")]
1401 providers: HashMap<String, Arc<dyn crate::model::ModelProvider>>,
1402 owner: String,
1403 /// How long a run's lease lasts, and how long a crashed owner's runs stay
1404 /// unclaimable.
1405 lease_ttl: Duration,
1406 /// Where this plane's agents remember things, when a deployment wires one.
1407 memories: Option<Arc<dyn crate::memory::MemoryStore>>,
1408 semantic: Option<Arc<super::SemanticMemory>>,
1409 authorities: Option<Arc<dyn crate::authority::AuthorityStore>>,
1410 /// The peers this plane may call, and the transport that reaches them.
1411 peers: Option<Arc<PeerWiring>>,
1412 /// The catalogue and transport a **skill** reaches tools through.
1413 ///
1414 /// Held by the plane rather than handed to each skill, and that is the
1415 /// point. A skill that built its own [`ToolCatalog`] could declare a tool
1416 /// read-only that its manifest calls mutating — and `ToolSafety::read_only`
1417 /// carries `Recovery::Retry`, so a timed-out money-moving call gets sent
1418 /// again. `check_catalogue_not_laxer_than_grants` refuses exactly that for
1419 /// the plane's catalogue at build; a hand-built one never passed under it.
1420 /// `StepCtx::call_tool` is the path that cannot.
1421 #[cfg(feature = "manifest")]
1422 tools: Option<(
1423 Arc<crate::tools::ToolCatalog>,
1424 Arc<dyn crate::tools::ToolClient>,
1425 )>,
1426 /// The highest sensitivity this plane will write into an append-only
1427 /// record, when it says.
1428 journal_ceiling: Option<crate::core::Sensitivity>,
1429 /// Where this plane's tool transports may connect.
1430 ///
1431 /// Behind `manifest` because the tool path is.
1432 ///
1433 /// Here rather than on each `ToolClient` because it is a **plane**
1434 /// decision: the client owns the connection, the plane owns the
1435 /// destination, exactly as it does for a model driver's base URL. Absent
1436 /// means no egress control on the tool path, spelled the same way an
1437 /// absent policy engine is.
1438 #[cfg(feature = "manifest")]
1439 egress: Option<Arc<crate::core::Egress>>,
1440 #[cfg(feature = "manifest")]
1441 checkers: crate::content::Checkers,
1442 /// Where live model output is forwarded, when anything listens.
1443 streams: Option<Arc<dyn RunStreamObserver>>,
1444 /// How this plane attributes its metrics.
1445 meter: super::metrics::Meter,
1446 /// Durable per-tenant ceilings, when a deployment wires them.
1447 quotas: Option<Arc<dyn crate::quota::QuotaStore>>,
1448 /// The rate ceilings this plane's declarations state, and the quota store
1449 /// that counts them. `None` when no grant declares one.
1450 #[cfg(feature = "manifest")]
1451 rates: Option<Arc<super::ctx::Rates>>,
1452 quota: crate::quota::TenantQuota,
1453 budget: Budget,
1454 /// Whether every plan this runtime admits must carry a verifier.
1455 require_verifier: bool,
1456
1457 cases: Option<Arc<dyn CaseStore>>,
1458 events: Option<Arc<dyn EventStore>>,
1459 tasks: Option<Arc<dyn TaskStore>>,
1460 timers: Option<Arc<dyn TimerStore>>,
1461 blobs: Option<Arc<dyn crate::blob::BlobStore>>,
1462 /// Where disclosures are recorded, read by a retention pass so it names
1463 /// the copies of what it erased.
1464 disclosures: Option<Arc<dyn crate::disclosure::DisclosureRegister>>,
1465 /// Where webhook registrations and their cursors live.
1466 ///
1467 /// Held by the plane rather than only by whoever delivers, because the
1468 /// registrations that stop being delivered are a **backlog**, and a backlog
1469 /// is answered by an operator on the plane's own surface — not by the
1470 /// worker that gave up on it.
1471 #[cfg(feature = "push")]
1472 push: Option<Arc<dyn crate::push::PushStore>>,
1473 /// Where data keys live, when payload bytes are sealed.
1474 #[cfg(feature = "keyring")]
1475 keyring: Option<Arc<dyn crate::keyring::KeyRing>>,
1476 batches: Option<Arc<dyn crate::batch::BatchStore>>,
1477 policy: Option<Arc<dyn crate::core::PolicyEngine>>,
1478 identity: Option<crate::core::Delegation>,
1479 replanner: Option<Arc<dyn crate::plan::Replanner>>,
1480 calendar: Arc<dyn Calendar>,
1481 signer: Option<Arc<dyn crate::core::Signer>>,
1482 /// Where this plane sends its own events, when a deployment configures it.
1483 ///
1484 /// Registered per run **at admission**, and again by every resume that
1485 /// executes: a destination attached after the first record would silently
1486 /// miss it, and "the journal is the outbox" only holds if the cursor starts
1487 /// at sequence one.
1488 #[cfg(feature = "push")]
1489 outbox: Option<Arc<crate::push::Outbox>>,
1490 /// The parties this plane submits its checkpoints to, and how many must
1491 /// answer.
1492 ///
1493 /// The one control here whose whole value is that it is **not** this
1494 /// plane: the chain, the signatures and the Merkle log all verify against
1495 /// inputs the operator supplies, so none of them detects a run removed or
1496 /// two histories shown to two auditors. Absent means the plane's history
1497 /// is anchored nowhere outside itself, which is a deployment decision and
1498 /// is reported as one by `audit`.
1499 witnesses: Option<Arc<Witnessing>>,
1500 /// The runs this instance put on a background task, and the gate that
1501 /// closes admission when it is asked to stop.
1502 ///
1503 /// Process-local on purpose: it answers *what is this instance executing
1504 /// right now*, which no other instance can be told and no store can hold.
1505 /// The durable cousin is [`Runtime::set_halt`], which stops a tenant
1506 /// everywhere and survives a restart — a different question with a
1507 /// different audience.
1508 inflight: Arc<super::drain::InFlight>,
1509 /// Sub-runs that started commissions set working in this process, by run: the
1510 /// awaiting run takes the outcome from the task that executed it, rather
1511 /// than replaying a run that already did its work here.
1512 commissions: Arc<std::sync::Mutex<HashMap<RunId, Working>>>,
1513}
1514
1515/// The witness set and the bar a submission round is held to.
1516#[derive(Debug)]
1517pub(crate) struct Witnessing {
1518 pub(crate) witnesses: Vec<Arc<dyn crate::journal::Witness>>,
1519 pub(crate) quorum: crate::journal::WitnessQuorum,
1520 /// The size last submitted under a **met** quorum.
1521 ///
1522 /// In memory, deliberately. It exists only to keep a periodic pass from
1523 /// re-submitting a log that has not grown, and the authority on where a
1524 /// witness actually is has always been the witness — which answers `409`
1525 /// with its own size and gets a proof built from there. So a restart
1526 /// re-submits once and the protocol absorbs it; a durable copy would be a
1527 /// second answer to a question the counterparty already answers.
1528 pub(crate) submitted: std::sync::atomic::AtomicU64,
1529 /// The declared maximum time between submissions of an unchanged log;
1530 /// `None` never re-submits one.
1531 pub(crate) interval: Option<std::time::Duration>,
1532 /// The sweep instant of the last round that met quorum, in memory for the
1533 /// reason `submitted` is.
1534 pub(crate) last_met: std::sync::Mutex<Option<crate::core::Timestamp>>,
1535}
1536
1537/// Where a run is entering this plane from.
1538///
1539/// The one thing the drain gate needs to know, and it cannot be derived at the
1540/// gate: by the time admission runs, an agent commissioning another one and a
1541/// peer sending a message look identical. The difference is whose work it is —
1542/// a commission is a step of a run this process is *already* executing and has
1543/// promised to finish, so refusing it would make the drain manufacture the
1544/// undecided effects it exists to prevent.
1545#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1546pub(crate) enum Entry {
1547 /// A caller outside this process: an embedder, an operator, a peer, a batch
1548 /// pass. Refused while draining.
1549 Outside,
1550 /// A run already executing here, ordering work it needs in order to
1551 /// conclude. Never refused for draining — the parent is what the drain is
1552 /// waiting for.
1553 Nested,
1554}
1555
1556/// A backend that implements every store a full plane runs on.
1557///
1558/// Blanket-implemented, so it is a *description* rather than an obligation:
1559/// any type providing the store traits — journal, cases, tasks, events,
1560/// timers, memory, quotas, disclosures — is a `FullBackend` without saying so, and both
1561/// shipped backends are. The quota store is in the set because halts live
1562/// there: a plane without it would admit past every emergency stop. What it buys is [`Runtime::builder_on`], where the
1563/// alternative was a cast of one `Arc` per store that every deployment spelled out
1564/// and none could get differently.
1565pub trait FullBackend:
1566 JournalStore
1567 + CaseStore
1568 + TaskStore
1569 + EventStore
1570 + TimerStore
1571 + crate::memory::MemoryStore
1572 + crate::quota::QuotaStore
1573 + crate::disclosure::DisclosureRegister
1574{
1575}
1576
1577impl<B> FullBackend for B where
1578 B: JournalStore
1579 + CaseStore
1580 + TaskStore
1581 + EventStore
1582 + TimerStore
1583 + crate::memory::MemoryStore
1584 + crate::quota::QuotaStore
1585 + crate::disclosure::DisclosureRegister
1586{
1587}
1588
1589impl Runtime {
1590 /// A builder with the whole case layer wired to one backend.
1591 ///
1592 /// The one-call form of the seven-cast litany: journal, cases, tasks,
1593 /// events, timers, memory and quotas all backed by `store`, which is what a
1594 /// deployment on a single `RedbStore` or `PostgresStore` means anyway. The
1595 /// à-la-carte methods remain — [`cases`](RuntimeBuilder::cases) and
1596 /// friends override an individual store afterwards, and
1597 /// [`builder`](Self::builder) still starts from the journal alone for a
1598 /// plane that wants nothing else.
1599 ///
1600 /// Blob storage is deliberately not included: bytes routinely live in a
1601 /// different system than rows, so [`blobs`](RuntimeBuilder::blobs) stays
1602 /// an explicit decision.
1603 #[must_use]
1604 pub fn builder_on<B: FullBackend + 'static>(store: Arc<B>) -> RuntimeBuilder {
1605 Self::builder_with(Stores::on(store))
1606 }
1607
1608 /// A builder with the whole case layer wired from handles.
1609 ///
1610 /// What [`builder_on`](Self::builder_on) is in terms of: the seven casts live
1611 /// in one place, so a deployment that chose its backend at run time — an
1612 /// operator naming either a file or a connection string — wires exactly what
1613 /// a deployment on one concrete type wires, rather than a hand-written
1614 /// approximation of it that drifts the day an eighth store is added.
1615 #[must_use]
1616 pub fn builder_with(stores: Stores) -> RuntimeBuilder {
1617 Self::builder(stores.journal)
1618 .cases(stores.cases)
1619 .tasks(stores.tasks)
1620 .events(stores.events)
1621 .timers(stores.timers)
1622 .memory(stores.memory)
1623 .quota(stores.quotas, crate::quota::TenantQuota::default())
1624 .disclosures(stores.disclosures)
1625 }
1626
1627 #[must_use]
1628 pub fn builder(store: Arc<dyn JournalStore>) -> RuntimeBuilder {
1629 RuntimeBuilder {
1630 store,
1631 signer: None,
1632 skills: Vec::new(),
1633 owner: None,
1634 lease_ttl: LEASE_TTL,
1635 memories: None,
1636 semantic: None,
1637 authorities: None,
1638 peers: None,
1639 tenant_label: super::telemetry::TenantLabel::default(),
1640 quotas: None,
1641 quota: crate::quota::TenantQuota::default(),
1642 budget: Budget::unlimited(),
1643 require_verifier: false,
1644 cases: None,
1645 events: None,
1646 tasks: None,
1647 timers: None,
1648 blobs: None,
1649 disclosures: None,
1650 #[cfg(feature = "push")]
1651 push: None,
1652 #[cfg(feature = "keyring")]
1653 keyring: None,
1654 #[cfg(feature = "manifest")]
1655 tools: None,
1656 journal_ceiling: None,
1657 #[cfg(feature = "manifest")]
1658 egress: None,
1659 #[cfg(feature = "manifest")]
1660 checkers: crate::content::Checkers::default(),
1661 streams: None,
1662 tenant: crate::core::TenantId::default(),
1663 batches: None,
1664 policy: None,
1665 identity: None,
1666 replanner: None,
1667 calendar: None,
1668 #[cfg(feature = "manifest")]
1669 toolbox: None,
1670 #[cfg(feature = "manifest")]
1671 tool_servers: Vec::new(),
1672 #[cfg(feature = "manifest")]
1673 agents: Vec::new(),
1674 #[cfg(feature = "manifest")]
1675 providers: HashMap::new(),
1676 #[cfg(feature = "push")]
1677 outbox: None,
1678 witnesses: Vec::new(),
1679 quorum: None,
1680 witness_interval: None,
1681 }
1682 }
1683
1684 /// The driver registered under a manifest's name for it, if any.
1685 #[cfg(feature = "manifest")]
1686 pub(crate) fn model_provider(
1687 &self,
1688 name: &str,
1689 ) -> Option<Arc<dyn crate::model::ModelProvider>> {
1690 self.providers.get(name).map(Arc::clone)
1691 }
1692
1693 /// The case store, if this runtime has one.
1694 #[must_use]
1695 pub fn cases(&self) -> Option<&Arc<dyn CaseStore>> {
1696 self.cases.as_ref()
1697 }
1698
1699 /// The worklist, if this runtime has one.
1700 #[must_use]
1701 pub fn tasks(&self) -> Option<&Arc<dyn TaskStore>> {
1702 self.tasks.as_ref()
1703 }
1704
1705 /// The inbound-event store, if this runtime has one.
1706 #[must_use]
1707 pub fn events(&self) -> Option<&Arc<dyn EventStore>> {
1708 self.events.as_ref()
1709 }
1710
1711 /// The ceilings every run under this runtime starts with.
1712 ///
1713 /// Readable because "what is this plane allowed to spend" is a question an
1714 /// operator asks of a running system, and answering it by re-reading the
1715 /// config that *should* have been applied is how a misapplied budget stays
1716 /// invisible.
1717 #[must_use]
1718 pub const fn budget(&self) -> &Budget {
1719 &self.budget
1720 }
1721
1722 /// The blob store, if this runtime has one.
1723 #[must_use]
1724 pub fn blobs(&self) -> Option<&Arc<dyn crate::blob::BlobStore>> {
1725 self.blobs.as_ref()
1726 }
1727
1728 /// Run the live half of the case-layer drill with this plane's own stores.
1729 ///
1730 /// The wiring is the point of this method existing: the drill's value is
1731 /// holding each case's references against the stores the plane *actually
1732 /// runs with*, and an embedder assembling `drill::Stores` by hand could
1733 /// hand it a different blob store than the one their runs write to —
1734 /// a drill that passes over the wrong bucket. Stores this runtime does
1735 /// not have are reported as unchecked by the drill itself, which is a
1736 /// different and better answer than an error: a plane with no blob store
1737 /// has no bytes to lose.
1738 ///
1739 /// **The clock is the caller's**, the same decision
1740 /// [`attention`](Self::attention) and the push sweep make. A rehearsal's
1741 /// instant is a fact about the outside world rather than a journaled
1742 /// observation, and taking it here would be a fourth ambient-clock escape
1743 /// inside the deterministic zone for a value the caller already has. A
1744 /// fixed instant also makes the recorded verdict testable.
1745 ///
1746 /// # Errors
1747 ///
1748 /// If this runtime has no case store — the drill walks cases, so there is
1749 /// nothing to drill — or if the case layer cannot be enumerated.
1750 pub async fn drill(
1751 &self,
1752 at: crate::core::Timestamp,
1753 ) -> Result<crate::drill::DrillReport, RuntimeError> {
1754 let cases = self.cases().ok_or_else(|| {
1755 RuntimeError::PlanContract(
1756 "this runtime has no case store — the drill walks cases, so there is \
1757 nothing to drill; build it with `.cases(store)`"
1758 .into(),
1759 )
1760 })?;
1761 let stores = crate::drill::Stores {
1762 cases,
1763 blobs: self.blobs(),
1764 #[cfg(feature = "keyring")]
1765 keys: self.keyring.as_ref(),
1766 tenant: &self.tenant,
1767 };
1768 let report = crate::drill::drill(&stores)
1769 .await
1770 .map_err(RuntimeError::from_store)?;
1771
1772 // **The verdict is recorded, not only returned.** What an audit asks
1773 // is *when did you last rehearse, and did it pass* — and left to the
1774 // caller that answer lives in whatever ran the verb, which is a CI log
1775 // or a wiki page. The checkpoint pins which store it ran against, so a
1776 // drill over a restored copy cannot be read as one over production.
1777 let checkpoint = self
1778 .store
1779 .checkpoint()
1780 .await
1781 .map_err(RuntimeError::from_store)?;
1782 cases
1783 .record_drill(&crate::case::DrillRecord {
1784 at,
1785 sound: report.is_sound(),
1786 cases: report.cases as u64,
1787 findings: report.findings.len() as u64,
1788 not_checked: report.not_checked.len() as u64,
1789 origin: checkpoint.origin.clone(),
1790 size: checkpoint.size,
1791 })
1792 .await
1793 .map_err(RuntimeError::from_store)?;
1794 Ok(report)
1795 }
1796
1797 /// The last recovery rehearsal this plane ran, or `None` if it never has.
1798 ///
1799 /// The half that makes `drill` evidence rather than a command: an audit
1800 /// asks *when did you last rehearse and did it pass*, and that answer has
1801 /// to come from the plane rather than from the thing that scheduled it.
1802 ///
1803 /// # Errors
1804 ///
1805 /// [`RuntimeError::PlanContract`] when this plane has no case store — the
1806 /// same condition that makes a drill impossible in the first place — and
1807 /// [`RuntimeError`] for a store failure.
1808 pub async fn last_drill(&self) -> Result<Option<crate::case::DrillRecord>, RuntimeError> {
1809 self.cases()
1810 .ok_or_else(|| {
1811 RuntimeError::PlanContract(
1812 "this runtime has no case store, so it has never drilled one".into(),
1813 )
1814 })?
1815 .last_drill()
1816 .await
1817 .map_err(RuntimeError::from_store)
1818 }
1819
1820 /// Erase every closed case opened before `older_than`. The retention verb.
1821 ///
1822 /// The wiring is the point, exactly as it is for [`drill`](Self::drill):
1823 /// erasure derives blob addresses and key scopes from the tenant and the
1824 /// case, so a caller assembling the stores by hand could tombstone
1825 /// addresses no run ever wrote and report a clean pass over nothing.
1826 ///
1827 /// `reason` lands on every tombstone and on each key destruction, so a
1828 /// later read says *expired, on this date, for this reason* rather than
1829 /// *missing* — the distinction the drill's three-way verdict turns on.
1830 ///
1831 /// `at` is a parameter rather than a clock read, for the reason the
1832 /// sweeper's is: a pass that read the clock could not be tested against a
1833 /// year of ageing cases.
1834 ///
1835 /// Read [`RetentionReport::not_erasable`](crate::retention::RetentionReport::not_erasable)
1836 /// before concluding anything. Without a key ring this expires blob
1837 /// tombstones in the live store and **journal payloads stay verbatim**,
1838 /// which is the residual the erasure page names and the one an erasure
1839 /// request actually turns on.
1840 ///
1841 /// # Errors
1842 ///
1843 /// If this runtime has no case store, or the case layer cannot be walked.
1844 /// A store that fails on one case is a report entry, not an error.
1845 pub async fn retain(
1846 &self,
1847 older_than: crate::core::Timestamp,
1848 at: crate::core::Timestamp,
1849 reason: &str,
1850 ) -> Result<crate::retention::RetentionReport, RuntimeError> {
1851 let cases = self.cases().ok_or_else(|| {
1852 RuntimeError::PlanContract(
1853 "this runtime has no case store — retention walks cases, so there is \
1854 nothing to retain; build it with `.cases(store)`"
1855 .into(),
1856 )
1857 })?;
1858 let stores = crate::retention::Stores {
1859 cases,
1860 blobs: self.blobs(),
1861 #[cfg(feature = "keyring")]
1862 keys: self.keyring.as_ref(),
1863 tenant: &self.tenant,
1864 disclosures: self.disclosures.as_ref(),
1865 };
1866 crate::retention::retain(&stores, self.store.as_ref(), older_than, at, reason)
1867 .await
1868 .map_err(RuntimeError::from_store)
1869 }
1870
1871 /// The timer store, if this runtime has one.
1872 #[must_use]
1873 pub fn timers(&self) -> Option<&Arc<dyn TimerStore>> {
1874 self.timers.as_ref()
1875 }
1876
1877 #[must_use]
1878 pub fn batches(&self) -> Option<&Arc<dyn crate::batch::BatchStore>> {
1879 self.batches.as_ref()
1880 }
1881
1882 #[cfg(feature = "push")]
1883 #[must_use]
1884 pub fn push(&self) -> Option<&Arc<dyn crate::push::PushStore>> {
1885 self.push.as_ref()
1886 }
1887
1888 /// Re-arm a webhook registration a delivery worker gave up on.
1889 ///
1890 /// A parked registration is not delivered and not deleted: it keeps its
1891 /// cursor, so re-arming resumes at the first record the receiver never
1892 /// acknowledged rather than at the head of the run. That is the whole
1893 /// reason parking exists, and the reason this verb is on the plane — a
1894 /// backlog nobody can empty is a warning line in yesterday's logs.
1895 ///
1896 /// The clock is read here rather than taken from the caller, unlike the
1897 /// delivery worker's: a worker's schedule has to be assertable in a test,
1898 /// and an operator asking for *now* means now.
1899 ///
1900 /// Returns whether a parked registration was found. `false` is an answer —
1901 /// already live, or never existed — and is told rather than swallowed,
1902 /// because an operator left waiting for a sweep with nothing to do has no
1903 /// way to tell the difference.
1904 ///
1905 /// # Errors
1906 ///
1907 /// [`RuntimeError::PlanContract`] if this plane has no push store, and
1908 /// [`RuntimeError::Store`] if it is unreachable.
1909 #[cfg(feature = "push")]
1910 pub async fn rearm_push(&self, task: RunId, id: &str) -> Result<bool, RuntimeError> {
1911 let push = self
1912 .push
1913 .as_ref()
1914 .ok_or_else(|| RuntimeError::PlanContract("this plane has no push store".into()))?;
1915 let at = now_for_admission().unix_timestamp().max(0).unsigned_abs();
1916 push.unpark(task, id, at)
1917 .await
1918 .map_err(RuntimeError::from_store)
1919 }
1920
1921 /// Ask a run to stop, and drive the stop if nothing else will.
1922 ///
1923 /// The request is durable before this returns, so an operator who gets an
1924 /// acknowledgement has one whether or not the run was reachable. What
1925 /// happens next depends on where the run is:
1926 ///
1927 /// * **Suspended** — nothing is executing, so this resumes the run itself.
1928 /// It observes the request at its first step boundary and unwinds.
1929 /// * **Running here or elsewhere** — the owner observes the request at its
1930 /// next step boundary. Nothing is interrupted mid-effect, deliberately:
1931 /// stopping between "announced" and "recorded" manufactures the in-doubt
1932 /// case the effect protocol exists to avoid.
1933 /// * **Quarantined** — refused with [`RuntimeError::CannotUnwind`]: the
1934 /// unwind a stop promises is what a quarantine forbids, and reopening or
1935 /// abandoning it are the verbs that apply.
1936 /// * **Concluded and closed** (succeeded, abandoned, cancelled) — refused
1937 /// with [`RuntimeError::AlreadyConcluded`] and nothing is recorded; a run
1938 /// somebody already stopped answers `false`. A failed or exhausted run is
1939 /// resumable, and stopping it unwinds its work.
1940 /// * **Held by no provider on this plane** — an operator's terminal: the
1941 /// request is recorded and not driven, and the plane that holds the run's
1942 /// agent observes it on its next resume.
1943 ///
1944 /// Returns whether *this* call recorded the request. A second caller gets
1945 /// `false`: the first asker stays on the record, because "who intervened"
1946 /// must not be rewritten by a retry.
1947 ///
1948 /// # Errors
1949 ///
1950 /// [`RuntimeError::PlanContract`] for a blank reason,
1951 /// [`RuntimeError::CannotUnwind`] for a quarantined run,
1952 /// [`RuntimeError::AlreadyConcluded`] for a closed one, and
1953 /// [`RuntimeError`] if the store is unreachable or resuming a suspended run
1954 /// fails.
1955 pub async fn request_cancel(
1956 &self,
1957 run: RunId,
1958 actor: &crate::core::Operator,
1959 reason: &str,
1960 ) -> Result<bool, RuntimeError> {
1961 // An intervention with no stated reason documents nobody's judgement,
1962 // and it is the one field of the record the requester writes.
1963 if reason.trim().is_empty() {
1964 return Err(RuntimeError::PlanContract(
1965 "stopping a run needs a reason — what the person stopping it saw".to_owned(),
1966 ));
1967 }
1968 // Checked before recording. Writing first and failing afterwards leaves a
1969 // request standing against an id that does not exist, and the operator's
1970 // retry then comes back "somebody else already asked" — which is a
1971 // confusing way to say "you mistyped".
1972 let records = self
1973 .store
1974 .read(run, 1)
1975 .await
1976 .map_err(RuntimeError::from_store)?;
1977 if records.is_empty() {
1978 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
1979 run.to_string(),
1980 )));
1981 }
1982
1983 // A quarantine is refused here rather than recorded and ignored. The
1984 // stop this verb promises is *unwind and put the world back*, and that
1985 // is the one thing a quarantined run may not do — so the request landed
1986 // durably, answered the operator `recorded: true`, and the resume then
1987 // walked straight past it. An operator who believes they stopped a run
1988 // holding an unresolved payment is worse off than one who was told no.
1989 //
1990 // Read from the recorded conclusion, not from a live status: a run that
1991 // quarantines *after* this call is a genuine race the standing request
1992 // loses, and that is correct — the request was made about a run nobody
1993 // had given up on.
1994 if recorded_conclusion(&records).as_deref() == Some("quarantined") {
1995 return Err(RuntimeError::CannotUnwind {
1996 run: run.to_string(),
1997 });
1998 }
1999 // Refused only where the conclusion closes the run. A failed or an
2000 // exhausted run is resumable and its work stands, so stopping it is the
2001 // operator's remedy — the unwind it asks for is exactly what those
2002 // conclusions left undone. Read from the latest conclusion, not the
2003 // last record: an answer to a doubt may follow a conclusion, and the
2004 // run is no less closed for it. A run already stopped answers `false`
2005 // rather than an error, so a second asker is told the intervention of
2006 // record is somebody else's.
2007 if let Some(status) = resume_is_closed(&records) {
2008 if self.cancellation(run).await?.is_some() {
2009 return Ok(false);
2010 }
2011 return Err(RuntimeError::AlreadyConcluded {
2012 run: run.to_string(),
2013 outcome: recorded_conclusion(&records)
2014 .unwrap_or_else(|| status.as_str().to_owned()),
2015 });
2016 }
2017
2018 let fresh = self
2019 .store
2020 .request_cancel(run, actor, reason)
2021 .await
2022 .map_err(RuntimeError::from_store)?;
2023
2024 // Drive it. A suspended run has no thread to notice anything, so an
2025 // operator's stop would sit unobserved until the deadline swept it —
2026 // which for a run waiting on a six-week obligation is not a stop.
2027 //
2028 // Resuming a run that has already concluded is a no-op inside `replay`,
2029 // which reads the recorded status back rather than re-executing — so
2030 // there is no "is it finished?" check to race with here.
2031 //
2032 // Driven **only when the run is genuinely idle**: `replay` claims the
2033 // lease before it reads, and a `LeaseHeld` refusal means somebody is
2034 // executing the run right now — here or on another instance. That is
2035 // not a failure of this call. The request is already durable, and the
2036 // live owner checks for it at every step boundary (the cancellation
2037 // read at the top of the executor's ready-set loop), so the stop
2038 // arrives without a second executor ever touching the run. Resuming
2039 // anyway would, on the owner's own instance, hand the resume the *same
2040 // epoch* the live execution is writing under — two executors on one
2041 // chain that fencing, by construction, cannot tell apart.
2042 //
2043 // A plane holding no provider, case store or key ring for the run — an
2044 // operator's terminal, a library plane — records the request and
2045 // leaves the driving to the plane that does: that plane's next resume
2046 // finds the request standing.
2047 if fresh {
2048 match self.replay(run, Mode::Resume).await {
2049 Ok(_)
2050 | Err(
2051 RuntimeError::LeaseHeld { .. }
2052 | RuntimeError::NoProvider { .. }
2053 | RuntimeError::NoCaseStore { .. }
2054 | RuntimeError::PayloadsSealed { .. },
2055 ) => {}
2056 Err(e) => return Err(e),
2057 }
2058 }
2059 Ok(fresh)
2060 }
2061
2062 /// Answer a quarantine, and let the runtime decide the run again.
2063 ///
2064 /// A quarantine is this runtime saying it does not know, and the one
2065 /// conclusion it cannot reach past on its own. Everything else that stops a
2066 /// run has a party who can move it — a resume for a crash, a raised ceiling
2067 /// for an exhaustion, a reviewer for a suspension. This is the party for
2068 /// this one.
2069 ///
2070 /// # What this does and does not decide
2071 ///
2072 /// The person decides *whether the run may be judged again*, or that it
2073 /// never will be. They do not decide the outcome. A
2074 /// [`Reopen`](crate::core::QuarantineDecision::Reopen) hands the run back
2075 /// to the executor, which re-derives its verdict from a history that now
2076 /// holds whatever the person established with
2077 /// [`reconcile_effect`](Self::reconcile_effect) — and quarantines it again,
2078 /// on the record, if the answer was wrong or incomplete. An
2079 /// [`Abandon`](crate::core::QuarantineDecision::Abandon) closes it where it
2080 /// stands, unwinding nothing.
2081 ///
2082 /// The decision is journaled **before** it is acted on, so a crash between
2083 /// the two leaves the instruction standing rather than lost: the next
2084 /// resume finds it and finishes the job.
2085 ///
2086 /// Returns the outcome of the pass this drove — so an operator who reopens
2087 /// a run learns immediately whether it got any further, rather than having
2088 /// to go and look.
2089 ///
2090 /// # Errors
2091 ///
2092 /// [`RuntimeError::NotQuarantined`] if the run's last conclusion is
2093 /// anything else, [`RuntimeError::LeaseHeld`] if somebody is executing it,
2094 /// and [`RuntimeError`] for a store failure or a refused resume.
2095 ///
2096 /// # Panics
2097 ///
2098 /// Never: the `expect` covers a lease this function has just acquired.
2099 pub async fn decide_quarantine(
2100 &self,
2101 run: RunId,
2102 decider: &crate::core::Operator,
2103 reason: &str,
2104 decision: crate::core::QuarantineDecision,
2105 ) -> Result<RunOutcome, RuntimeError> {
2106 // The whole weight of this record is that a named person took
2107 // responsibility for a fact the runtime could not establish, and
2108 // "unknown decided this for no stated reason" documents nothing while
2109 // looking like process. Only the reason is checked here: `Operator` has
2110 // no empty value, so the decider is guaranteed by the type on every
2111 // path that builds one rather than by a check on this one.
2112 if reason.trim().is_empty() {
2113 return Err(RuntimeError::PlanContract(
2114 "answering a quarantine needs a decider and a reason — who looked, and what they \
2115 found"
2116 .to_owned(),
2117 ));
2118 }
2119
2120 let lease = self
2121 .store
2122 .acquire(run, &self.owner, self.lease_ttl)
2123 .await
2124 .map_err(RuntimeError::from_store)?;
2125 let appended = self
2126 .record_decision(run, &lease, decider, reason, decision)
2127 .await;
2128 if let Err(e) = appended {
2129 // The lease was claimed by this call and never handed to a resume,
2130 // so it is this call's to give back. Leaving it would strand the
2131 // run for a TTL and hand it to the recovery sweep, which would
2132 // "recover" a run nobody was running.
2133 let _ = self.store.release_lease(run, lease.epoch).await;
2134 return Err(e);
2135 }
2136 // Handed over rather than released and re-taken: the same window a wake
2137 // path closes. Both decisions travel this path — an abandonment is
2138 // concluded inside the resume, from the record just written. The run
2139 // still stands quarantined on the record, so it keeps its listing
2140 // whatever this pass does.
2141 self.replay_releasing(run, Pass::Resume, Some(lease), Claim::Standing)
2142 .await
2143 }
2144
2145 /// Answer a quarantine **without driving the run**.
2146 ///
2147 /// [`decide_quarantine`](Self::decide_quarantine) records the decision and
2148 /// then hands the run to the executor, which is what an operator on a
2149 /// running plane wants: they learn immediately whether a reopened run got
2150 /// any further.
2151 ///
2152 /// A terminal holding a store cannot do the second half. It has the
2153 /// journal and not the agent — a declarative run's program is a manifest
2154 /// this process was not given — so driving the run would mean refusing the
2155 /// verb for want of an argument the operator may not have during an
2156 /// incident.
2157 ///
2158 /// **That is a supported state rather than a lesser one.** The decision is
2159 /// journaled before it is acted on precisely so a crash between the two
2160 /// leaves the instruction standing, and this is the same position reached
2161 /// deliberately: the next resume finds the record and finishes the job —
2162 /// so what the caller must not do is report the run as moved.
2163 ///
2164 /// # Errors
2165 ///
2166 /// As [`decide_quarantine`](Self::decide_quarantine), minus everything the
2167 /// resume could have raised.
2168 pub async fn record_quarantine_decision(
2169 &self,
2170 run: RunId,
2171 decider: &crate::core::Operator,
2172 reason: &str,
2173 decision: crate::core::QuarantineDecision,
2174 ) -> Result<(), RuntimeError> {
2175 if reason.trim().is_empty() {
2176 return Err(RuntimeError::PlanContract(
2177 "answering a quarantine needs a decider and a reason — who looked, and what they \
2178 found"
2179 .to_owned(),
2180 ));
2181 }
2182 let lease = self
2183 .store
2184 .acquire(run, &self.owner, self.lease_ttl)
2185 .await
2186 .map_err(RuntimeError::from_store)?;
2187 let appended = self
2188 .record_decision(run, &lease, decider, reason, decision)
2189 .await;
2190 // Always given back: this verb never executes the run, so the lease is
2191 // held only for the width of one append — the rule
2192 // `reconcile_effect` follows.
2193 let _ = self.store.release_lease(run, lease.epoch).await;
2194 appended
2195 }
2196
2197 /// Append the decision, once the run is known to be asking for one.
2198 async fn record_decision(
2199 &self,
2200 run: RunId,
2201 lease: &crate::journal::Lease,
2202 decider: &crate::core::Operator,
2203 reason: &str,
2204 decision: crate::core::QuarantineDecision,
2205 ) -> Result<(), RuntimeError> {
2206 let records = self
2207 .store
2208 .read(run, 1)
2209 .await
2210 .map_err(RuntimeError::from_store)?;
2211 if records.is_empty() {
2212 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2213 run.to_string(),
2214 )));
2215 }
2216 let status = recorded_conclusion(&records);
2217 // A run whose payloads were erased is asking the same question a
2218 // quarantine asks — *the runtime cannot go on, and a person must
2219 // decide* — so it is answered by the same verb whatever outcome it was
2220 // left under. Without this it has no verb at all: it cannot be
2221 // resumed, and a cancellation promises an unwind that erased arguments
2222 // make impossible. Abandonment is the honest ending and this is the
2223 // door to it.
2224 if status.as_deref() != Some("quarantined") && !payloads_erased(&records) {
2225 return Err(RuntimeError::NotQuarantined {
2226 run: run.to_string(),
2227 status: status.unwrap_or_else(|| "still running".to_owned()),
2228 });
2229 }
2230 let mut append = Append::new(
2231 run,
2232 RecordKind::QuarantineDecided {
2233 decider: decider.clone(),
2234 reason: reason.to_owned(),
2235 decision,
2236 },
2237 );
2238 if let Some(case) = records.iter().find_map(|r| r.body.case) {
2239 append = append.case(case);
2240 }
2241 self.store
2242 .append(lease.epoch, vec![append])
2243 .await
2244 .map_err(RuntimeError::from_store)?;
2245 Ok(())
2246 }
2247
2248 /// Supply, out of band, the outcome of an effect the journal cannot decide.
2249 ///
2250 /// The manual twin of [`Effect::reconcile`](crate::core::Effect::reconcile),
2251 /// and it produces the same record for the same reason: an operator who
2252 /// looks the charge up in the provider's console is answering exactly the
2253 /// question a probe asks, so the answer belongs in the place a probe's
2254 /// answer goes, where replay reads it back and nothing re-derives it.
2255 ///
2256 /// What separates the two is on the record too. A probe's verdict is
2257 /// attributed to the run; this one names the person, because "the provider
2258 /// told us" and "somebody asserted it" are different evidence.
2259 ///
2260 /// **It supplies a missing fact and never replaces a recorded one.** Only
2261 /// an effect [`undecided_effects`](crate::journal::undecided_effects) names
2262 /// may be answered. Without that rule an operator could talk a run out of
2263 /// compensating work that is standing in the world, and the journal would
2264 /// show an orderly reconciliation while it happened.
2265 ///
2266 /// This changes no status by itself: the run stays quarantined until
2267 /// [`decide_quarantine`](Self::decide_quarantine) hands it back. Several
2268 /// doubts can be answered before the run is judged again, and answering one
2269 /// of three is not a decision.
2270 ///
2271 /// # Errors
2272 ///
2273 /// [`RuntimeError::NotUndecided`] if that effect's outcome is already on
2274 /// the record, [`RuntimeError::LeaseHeld`] if somebody is executing the
2275 /// run, and [`RuntimeError`] for a store failure.
2276 pub async fn reconcile_effect(
2277 &self,
2278 run: RunId,
2279 effect: crate::core::EffectKey,
2280 assertion: crate::core::Assertion,
2281 asserted_by: &crate::core::Operator,
2282 note: &str,
2283 ) -> Result<(), RuntimeError> {
2284 // `asserted_by` needs no emptiness check: `Operator` has none to fail.
2285 if note.trim().is_empty() {
2286 return Err(RuntimeError::PlanContract(
2287 "an assertion about an effect needs a name and a note — who established this, and \
2288 how"
2289 .to_owned(),
2290 ));
2291 }
2292 let lease = self
2293 .store
2294 .acquire(run, &self.owner, self.lease_ttl)
2295 .await
2296 .map_err(RuntimeError::from_store)?;
2297 let result = self
2298 .record_assertion(run, &lease, effect, assertion, asserted_by, note)
2299 .await;
2300 // Always given back: this verb never executes the run, so the lease is
2301 // held only for the width of one append.
2302 let _ = self.store.release_lease(run, lease.epoch).await;
2303 result
2304 }
2305
2306 async fn record_assertion(
2307 &self,
2308 run: RunId,
2309 lease: &crate::journal::Lease,
2310 effect: crate::core::EffectKey,
2311 assertion: crate::core::Assertion,
2312 asserted_by: &crate::core::Operator,
2313 note: &str,
2314 ) -> Result<(), RuntimeError> {
2315 let records = self
2316 .store
2317 .read(run, 1)
2318 .await
2319 .map_err(RuntimeError::from_store)?;
2320 if records.is_empty() {
2321 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2322 run.to_string(),
2323 )));
2324 }
2325 let Some(undecided) = crate::journal::undecided_effects(&records)
2326 .into_iter()
2327 .find(|u| u.effect == effect)
2328 else {
2329 return Err(RuntimeError::NotUndecided {
2330 run: run.to_string(),
2331 effect: effect.to_string(),
2332 });
2333 };
2334
2335 let output = match &assertion {
2336 crate::core::Assertion::Landed(value) => Some(value.clone()),
2337 crate::core::Assertion::DidNotHappen => None,
2338 };
2339 let mut append = Append::new(
2340 run,
2341 RecordKind::EffectReconciled {
2342 disposition: assertion.disposition(),
2343 // Stated on the record rather than left to a reader's default.
2344 // A durable format that does not say what it means is one a
2345 // later build guesses about, and the guess here is the label on
2346 // a value the run is about to act on.
2347 declared: output
2348 .is_some()
2349 .then(crate::core::DeclaredOutput::untrusted),
2350 output,
2351 // A person's assertion costs nothing. What the call itself
2352 // consumed, if it consumed anything, is already on the record
2353 // of the attempt this answers.
2354 spend: Spend::ZERO,
2355 // No probe ran: this is a person's assertion, so the
2356 // provider-facing field stays empty and their account goes in
2357 // the field that is read without a key.
2358 detail: None,
2359 note: Some(note.to_owned()),
2360 asserted_by: Some(asserted_by.clone()),
2361 },
2362 )
2363 .step(undecided.step)
2364 .phase(undecided.phase)
2365 .effect(effect);
2366 if let Some(case) = records.iter().find_map(|r| r.body.case) {
2367 append = append.case(case);
2368 }
2369 self.store
2370 .append(lease.epoch, vec![append])
2371 .await
2372 .map_err(RuntimeError::from_store)?;
2373 tracing::info!(
2374 target: telemetry::RECONCILED,
2375 %run,
2376 step = %undecided.step,
2377 verdict = ?assertion.disposition(),
2378 actor = %asserted_by,
2379 );
2380 self.meter
2381 .count(metrics::RECONCILIATIONS, assertion.disposition().as_str());
2382 Ok(())
2383 }
2384
2385 /// Every mutating effect this run's journal cannot decide the outcome of.
2386 ///
2387 /// What an operator holding a quarantine has to go and look up, and the
2388 /// keys they quote back to [`reconcile_effect`](Self::reconcile_effect).
2389 ///
2390 /// # Errors
2391 ///
2392 /// If the store is unreachable.
2393 pub async fn undecided(&self, run: RunId) -> Result<Vec<crate::core::Undecided>, RuntimeError> {
2394 let records = self
2395 .store
2396 .read(run, 1)
2397 .await
2398 .map_err(RuntimeError::from_store)?;
2399 Ok(crate::journal::undecided_effects(&records))
2400 }
2401
2402 /// Stop this tenant from starting new work, or let it start again.
2403 ///
2404 /// The emergency stop. `Some(reason)` halts, `None` lifts, and the reason is
2405 /// required because the next person to look will be somebody else, possibly
2406 /// at three in the morning, and *why* is the whole question.
2407 ///
2408 /// **What it stops, precisely.** New admissions, across every instance,
2409 /// because the flag is in the store rather than in this process — a switch
2410 /// that stops only the instance it was thrown on is the in-process-counter
2411 /// failure arriving during an incident. Refusals are their own error, not a
2412 /// ceiling: a ceiling means *not right now* and invites a retry, which is
2413 /// exactly what somebody pulling this switch is trying to stop.
2414 ///
2415 /// **What a workload-scoped halt does not stop, deliberately.** Runs
2416 /// already executing, and suspended runs resuming. Those are existing work,
2417 /// and refusing to let them continue would strand them mid-saga with
2418 /// reversals unrun — turning an incident into a second one. To stop work in
2419 /// flight, cancel it: that unwinds what it did and records who asked. This
2420 /// is the front door, not a power cut, and saying so is the difference
2421 /// between a control an operator can reason about and one they discover the
2422 /// shape of during an outage.
2423 ///
2424 /// **How wide it reaches** is the [`HaltScope`](crate::quota::HaltScope),
2425 /// and the scopes divide on *which axis they name* rather than on how
2426 /// broad they are. Three name a workload — the tenant, every revision of
2427 /// one declared agent, or one exact reviewed digest — and stop admission.
2428 /// The fourth names an **authority**, any principal on the delegation
2429 /// chain a run was admitted under, and is the one that reaches work
2430 /// already running:
2431 /// there the incident *is* the credential, so a run carrying on under it is
2432 /// the harm rather than a side effect, and it pauses at its next step
2433 /// boundary with its completed work standing
2434 /// ([`withdraws_from`](crate::quota::HaltScope::withdraws_from) is the
2435 /// predicate, so the distinction is checked rather than remembered).
2436 /// Scopes are independent rows — lifting a narrow halt leaves a broad one
2437 /// standing. An ungoverned run, with no manifest, is stopped only by a
2438 /// tenant halt; there is nothing narrower to key it on.
2439 ///
2440 /// `at` is the caller's, never a clock read here: every lifecycle instant in
2441 /// this crate is supplied for the reason a hold's is — a pass that reads its
2442 /// own clock cannot be tested against an ageing plane, and [`I1`] forbids the
2443 /// runtime an ambient clock outside the effect protocol.
2444 ///
2445 /// Requires a quota store; without one there is nowhere durable to keep the
2446 /// flag, and an emergency stop that a restart forgets is not one.
2447 ///
2448 /// [`I1`]: crate::journal
2449 ///
2450 /// # Errors
2451 ///
2452 /// If no quota store is wired, or the store is unreachable.
2453 pub async fn set_halt(
2454 &self,
2455 scope: &crate::quota::HaltScope,
2456 by: &crate::core::Operator,
2457 at: crate::core::Timestamp,
2458 reason: &str,
2459 ) -> Result<(), RuntimeError> {
2460 let quotas = self.quota_store()?;
2461 quotas
2462 .set_halt(scope, by, at, reason)
2463 .await
2464 .map_err(RuntimeError::Store)?;
2465 self.forget_credentials(scope);
2466 Ok(())
2467 }
2468
2469 /// Drop the peer credentials held for a subject a halt names, when it is
2470 /// thrown and when it is lifted — not only when a hop next meets it, which
2471 /// may never happen while the halt stands.
2472 fn forget_credentials(&self, scope: &crate::quota::HaltScope) {
2473 if let (crate::quota::HaltScope::Subject { id }, Some(wiring)) =
2474 (scope, self.peers.as_ref())
2475 {
2476 wiring.registry.forget(id);
2477 }
2478 }
2479
2480 /// Lift one, and let work at that scope start again — recording who did.
2481 ///
2482 /// Answers the run holding the lift record and whether this call removed
2483 /// the row, or `None` when nothing was standing at `scope`: an operator who
2484 /// lifts a scope nobody halted is told so rather than told *done*, and
2485 /// nothing is written.
2486 ///
2487 /// **The record comes first, then the row goes.** The lift is written as
2488 /// the sole [`HaltLifted`](crate::journal::RecordKind::HaltLifted) record
2489 /// of a sealed run of its own, carrying `by`, `at` and the whole stop it
2490 /// ended; only then is the register's row removed. The journal and the
2491 /// register share no transaction, so a lift that cannot be recorded does
2492 /// not happen, and a removal that fails after the record is written is an
2493 /// error naming the record's run — the halt still stands, the register is
2494 /// the answer to what is in force, and lifting again writes a second
2495 /// record. The record attests the instruction and the row as read, not
2496 /// that the removal succeeded.
2497 ///
2498 /// **Only the row the record names goes.** The removal is
2499 /// [`QuotaStore::lift_halt_if`](crate::quota::QuotaStore::lift_halt_if)
2500 /// with that row, so a halt re-thrown between the read and the removal is
2501 /// left standing and the call fails naming the record's run. A row another
2502 /// lift already removed answers `removed: false`.
2503 ///
2504 /// [`QuotaStore::lift_halt`](crate::quota::QuotaStore::lift_halt) is the
2505 /// raw register verb; a direct call to it is unrecorded.
2506 ///
2507 /// # Errors
2508 ///
2509 /// If no quota store is wired, the store is unreachable, or the record
2510 /// cannot be written; [`RuntimeError::ControlStands`] if the row cannot be
2511 /// removed after the record was written, or was re-thrown in between.
2512 pub async fn lift_halt(
2513 &self,
2514 scope: &crate::quota::HaltScope,
2515 by: &crate::core::Operator,
2516 at: crate::core::Timestamp,
2517 ) -> Result<Option<ControlLifted>, RuntimeError> {
2518 let quotas = self.quota_store()?;
2519 let standing = quotas.halts().await.map_err(RuntimeError::Store)?;
2520 let Some(halt) = standing.into_iter().find(|h| &h.scope == scope) else {
2521 return Ok(None);
2522 };
2523 let run = self
2524 .seal_operator_record(
2525 RecordKind::HaltLifted {
2526 scope: scope.key(),
2527 by: by.clone(),
2528 at,
2529 reason: halt.reason.clone(),
2530 thrown_by: halt.by.clone(),
2531 thrown_at: halt.at,
2532 },
2533 None,
2534 HALT_LIFTED_OUTCOME,
2535 None,
2536 )
2537 .await?;
2538 let removed = quotas
2539 .lift_halt_if(&halt)
2540 .await
2541 .map_err(|e| removal_failed(run, &e))?;
2542 self.forget_credentials(scope);
2543 if !removed {
2544 let now = quotas.halts().await.map_err(|e| removal_failed(run, &e))?;
2545 if now.iter().any(|h| &h.scope == scope) {
2546 return Err(RuntimeError::ControlStands {
2547 run: run.to_string(),
2548 detail: format!(
2549 "the halt on '{}' was re-thrown after this lift read it, and the new \
2550 halt still stands",
2551 scope.key()
2552 ),
2553 });
2554 }
2555 }
2556 Ok(Some(ControlLifted {
2557 record: run,
2558 removed,
2559 }))
2560 }
2561
2562 /// Release the legal hold on `case` — recording who did.
2563 ///
2564 /// The hold's counterpart of [`lift_halt`](Self::lift_halt), in the same
2565 /// order for the same reason: the
2566 /// [`HoldReleased`](crate::journal::RecordKind::HoldReleased) record is
2567 /// written, stamped with `case`, before the row goes. It names the releaser
2568 /// and the hold's placing operator and instant, and not the hold's reason,
2569 /// so the erasure the release permits leaves its authorization readable.
2570 ///
2571 /// `None` when the matter is not held, with nothing written; `removed:
2572 /// false` when another release removed the row first.
2573 ///
2574 /// **Only the hold the record names goes.** The removal is
2575 /// [`CaseStore::release_hold_if`](crate::case::CaseStore::release_hold_if)
2576 /// with that hold, so a hold placed again between the read and the removal
2577 /// is left standing and the call fails naming the record's run.
2578 /// [`CaseStore::release_hold`](crate::case::CaseStore::release_hold) is the
2579 /// raw register verb; a direct call to it is unrecorded.
2580 ///
2581 /// # Errors
2582 ///
2583 /// [`StoreError::NotFound`](crate::core::StoreError::NotFound) if the case
2584 /// does not exist; [`RuntimeError::ControlStands`] if the row cannot be
2585 /// removed after the record was written, or was placed again in between;
2586 /// otherwise if no case store is
2587 /// wired, the store is unreachable, or the record cannot be written.
2588 pub async fn release_hold(
2589 &self,
2590 case: crate::core::CaseId,
2591 by: &crate::core::Operator,
2592 at: crate::core::Timestamp,
2593 ) -> Result<Option<ControlLifted>, RuntimeError> {
2594 let cases = self.cases.as_ref().ok_or_else(|| {
2595 RuntimeError::PlanContract("releasing a hold needs a case store".to_owned())
2596 })?;
2597 let Some(standing) = cases.hold(case).await.map_err(RuntimeError::Store)? else {
2598 return match cases.case(case).await.map_err(RuntimeError::Store)? {
2599 Some(_) => Ok(None),
2600 None => Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2601 format!("case {case}"),
2602 ))),
2603 };
2604 };
2605 let run = self
2606 .seal_operator_record(
2607 RecordKind::HoldReleased {
2608 by: by.clone(),
2609 at,
2610 placed_by: standing.by.clone(),
2611 placed_at: standing.placed_at,
2612 },
2613 Some(case),
2614 HOLD_RELEASED_OUTCOME,
2615 None,
2616 )
2617 .await?;
2618 let removed = cases
2619 .release_hold_if(case, &standing)
2620 .await
2621 .map_err(|e| removal_failed(run, &e))?;
2622 if !removed
2623 && cases
2624 .hold(case)
2625 .await
2626 .map_err(|e| removal_failed(run, &e))?
2627 .is_some()
2628 {
2629 return Err(RuntimeError::ControlStands {
2630 run: run.to_string(),
2631 detail: format!(
2632 "the hold on case {case} was placed again after this release read it, \
2633 and the new hold still stands"
2634 ),
2635 });
2636 }
2637 Ok(Some(ControlLifted {
2638 record: run,
2639 removed,
2640 }))
2641 }
2642
2643 /// Every recorded halt lift, newest first: the one record of each run.
2644 ///
2645 /// # Errors
2646 ///
2647 /// If the journal is unreachable, or a lift run does not hold a lift record.
2648 pub async fn lifted_halts(&self, limit: usize) -> Result<Vec<Record>, RuntimeError> {
2649 self.operator_records(HALT_LIFTED_OUTCOME, limit).await
2650 }
2651
2652 /// Every recorded hold release, newest first: the one record of each run.
2653 ///
2654 /// # Errors
2655 ///
2656 /// If the journal is unreachable, or a release run does not hold a release
2657 /// record.
2658 pub async fn released_holds(&self, limit: usize) -> Result<Vec<Record>, RuntimeError> {
2659 self.operator_records(HOLD_RELEASED_OUTCOME, limit).await
2660 }
2661
2662 /// The first record of each run sealed under `outcome`, in the index's
2663 /// newest-first order. A run whose first record is not the act its outcome
2664 /// names is a store error, never skipped: a history that silently thins is
2665 /// one that reads as complete.
2666 async fn operator_records(
2667 &self,
2668 outcome: &str,
2669 limit: usize,
2670 ) -> Result<Vec<Record>, RuntimeError> {
2671 let runs = self
2672 .store
2673 .runs_by_outcome(outcome, limit)
2674 .await
2675 .map_err(RuntimeError::from_store)?;
2676 let mut found = Vec::with_capacity(runs.len());
2677 for run in runs {
2678 let first = self
2679 .store
2680 .read_page(run, 1, 1)
2681 .await
2682 .map_err(RuntimeError::from_store)?
2683 .into_iter()
2684 .next();
2685 match first {
2686 Some(record)
2687 if matches!(
2688 (outcome, record.kind()),
2689 (HALT_LIFTED_OUTCOME, RecordKind::HaltLifted { .. })
2690 | (HOLD_RELEASED_OUTCOME, RecordKind::HoldReleased { .. })
2691 ) =>
2692 {
2693 found.push(record);
2694 }
2695 _ => {
2696 return Err(RuntimeError::Store(crate::core::StoreError::Corrupt {
2697 seq: 1,
2698 detail: format!(
2699 "run {run} is sealed as '{outcome}' but holds no such record"
2700 ),
2701 }));
2702 }
2703 }
2704 }
2705 Ok(found)
2706 }
2707
2708 /// The quota store, when one is wired.
2709 pub(crate) fn quota_store_if_wired(&self) -> Option<&Arc<dyn crate::quota::QuotaStore>> {
2710 self.quotas.as_ref()
2711 }
2712
2713 /// Every run holding spend against this tenant's periods, and how much.
2714 ///
2715 /// The runs a [`QuotaError::SpentOut`](crate::quota::QuotaError::SpentOut)
2716 /// refusal counts as *reserved*. A run holds its reservation until it
2717 /// concludes, so the ones that stopped waiting on a person — quarantined,
2718 /// exhausted, withheld, failed — hold it until that person acts;
2719 /// [`attention`](Self::attention) names those.
2720 ///
2721 /// # Errors
2722 ///
2723 /// If no quota store is wired, or it is unreachable.
2724 pub async fn reservations(
2725 &self,
2726 limit: usize,
2727 ) -> Result<Vec<crate::quota::Held>, RuntimeError> {
2728 let quotas = self.quotas.as_ref().ok_or_else(|| {
2729 RuntimeError::Store(crate::core::StoreError::Backend(
2730 "no quota store is wired, so no run holds spend against a period".to_owned(),
2731 ))
2732 })?;
2733 quotas
2734 .reservations(limit)
2735 .await
2736 .map_err(RuntimeError::Store)
2737 }
2738
2739 /// The quota store, or the refusal that says why an emergency stop needs one.
2740 fn quota_store(&self) -> Result<&std::sync::Arc<dyn crate::quota::QuotaStore>, RuntimeError> {
2741 self.quotas.as_ref().ok_or_else(|| {
2742 RuntimeError::Store(crate::core::StoreError::Backend(
2743 "an emergency stop needs a quota store to keep the flag in — an \
2744 in-process one is forgotten by a restart and never seen by a \
2745 second instance"
2746 .to_owned(),
2747 ))
2748 })
2749 }
2750
2751 /// Every standing emergency stop on this tenant, and why.
2752 ///
2753 /// The operator's question — *what is stopped right now?* — which a
2754 /// per-scope lookup cannot answer without already knowing what to ask
2755 /// about.
2756 ///
2757 /// # Errors
2758 ///
2759 /// If no quota store is wired, or the store is unreachable.
2760 pub async fn halts(&self) -> Result<Vec<crate::quota::Halt>, RuntimeError> {
2761 let quotas = self.quotas.as_ref().ok_or_else(|| {
2762 RuntimeError::Store(crate::core::StoreError::Backend(
2763 "no quota store is wired, so no emergency stop can be set or read".to_owned(),
2764 ))
2765 })?;
2766 quotas.halts().await.map_err(RuntimeError::Store)
2767 }
2768
2769 /// Which runs are executing **right now**, and under whose authority.
2770 ///
2771 /// [`runs_by_outcome`](crate::journal::JournalStore::runs_by_outcome)
2772 /// answers from the journal's index, which only *concluded* runs are in; the
2773 /// live set is the tenant's admission slots.
2774 ///
2775 /// **The join is the point, not the ids.** An incident is rarely *cancel
2776 /// this run* but a revoked credential or a bad revision, so each entry
2777 /// carries the declared agent and the delegation subject it was admitted
2778 /// under.
2779 ///
2780 /// A slot whose lease has lapsed is marked `stranded` rather than omitted:
2781 /// the recovery sweep resumes those, and cancelling one undoes work that was
2782 /// about to continue.
2783 ///
2784 /// Bounded by `limit`, and reads a page of each run's header records to
2785 /// attribute it — an operator question rather than a hot path.
2786 ///
2787 /// # Errors
2788 ///
2789 /// If no quota store is wired, or either store is unreachable.
2790 pub async fn running_runs(&self, limit: usize) -> Result<Vec<LiveRun>, RuntimeError> {
2791 let quotas = self.quotas.as_ref().ok_or_else(|| {
2792 RuntimeError::Store(crate::core::StoreError::Backend(
2793 "no quota store is wired, so nothing tracks which runs hold this \
2794 tenant's admission slots"
2795 .to_owned(),
2796 ))
2797 })?;
2798 let live = quotas
2799 .running_runs(limit)
2800 .await
2801 .map_err(RuntimeError::Store)?;
2802 // One read, not one per run: the stranded set is small and the whole of
2803 // it is needed to answer any entry.
2804 let stranded: std::collections::BTreeSet<RunId> = self
2805 .store
2806 .abandoned_runs(limit)
2807 .await
2808 .map_err(RuntimeError::Store)?
2809 .into_iter()
2810 .collect();
2811
2812 let mut out = Vec::with_capacity(live.len());
2813 for run in live {
2814 // The header records only. `RunAdmitted` is the first and
2815 // `IdentityBound` follows it when the caller presented a chain, so a
2816 // short page carries both without reading a run's whole history —
2817 // which on a long-running agent is the difference between an
2818 // operator listing and a scan.
2819 let head = self
2820 .store
2821 .read_page(run, 1, ATTRIBUTION_RECORDS)
2822 .await
2823 .map_err(RuntimeError::Store)?;
2824 let governed_by = head.iter().find_map(|r| match r.kind() {
2825 crate::journal::RecordKind::RunAdmitted { governed_by, .. } => {
2826 governed_by.as_ref().map(|id| (**id).clone())
2827 }
2828 _ => None,
2829 });
2830 let subject = head.iter().find_map(|r| match r.kind() {
2831 crate::journal::RecordKind::IdentityBound { chain } => {
2832 chain.first().map(|p| p.id.clone())
2833 }
2834 _ => None,
2835 });
2836 out.push(LiveRun {
2837 stranded: stranded.contains(&run),
2838 run,
2839 governed_by,
2840 subject,
2841 });
2842 }
2843 Ok(out)
2844 }
2845
2846 /// The runs that are waiting, and what each one waits for.
2847 ///
2848 /// The third listing, beside concluded runs and running ones: a suspended
2849 /// run has no outcome and holds no admission slot, so neither of the others
2850 /// names it. Re-arming these is the recovery runbook's last step, and until
2851 /// this existed that step named a verb with no argument an operator could
2852 /// obtain.
2853 ///
2854 /// Soonest due first, bounded by `limit`. Straight through to the store's
2855 /// maintained index — no page of records is read, because what a run waits
2856 /// for is on the suspension itself.
2857 ///
2858 /// # Errors
2859 ///
2860 /// If the store is unreachable.
2861 pub async fn waiting_runs(
2862 &self,
2863 limit: usize,
2864 ) -> Result<Vec<crate::journal::WaitingRun>, RuntimeError> {
2865 self.store
2866 .waiting_runs(limit)
2867 .await
2868 .map_err(RuntimeError::from_store)
2869 }
2870
2871 /// Stop this instance taking on work, then wait for what it is already
2872 /// doing.
2873 ///
2874 /// A process killed mid-step leaves an announced effect with no terminal
2875 /// record, so its declared [`Recovery`](crate::core::Recovery) has to
2876 /// decide whether it happened — the right answer to an accident, and an
2877 /// expensive one to schedule on every deploy.
2878 ///
2879 /// Admission closes **first**: every later [`start`](Self::start) is
2880 /// refused
2881 /// [`RuntimeError::Draining`](crate::core::RuntimeError::Draining), leaving
2882 /// no lease, no slot and no journal. Then this waits up to `grace` for the
2883 /// runs it put on a background task to reach a journaled resting point.
2884 /// Closing first is what makes the answer worth having — a wait alone
2885 /// reports about an instant — and it is also the only signal a long-lived
2886 /// read gets, so an open A2A subscription ends rather than holding a
2887 /// graceful shutdown for as long as the run it watches.
2888 ///
2889 /// **Run it beside a server's own graceful shutdown, not after it** —
2890 /// `tokio::join!` of the two. Serialising deadlocks one on the other in
2891 /// whichever order you pick: a run awaited inside a request handler is
2892 /// finished by the server, and an open subscription is ended by this.
2893 ///
2894 /// Three things it deliberately does not do. It does not wait for runs on
2895 /// somebody else's task; those belong to that caller's own shutdown. It
2896 /// does not gate resumes or an agent commissioning another one — both
2897 /// continue work this plane already owns, and a commission is a step of a
2898 /// run this is waiting for. And it does not hand back the leases of runs it
2899 /// did not finish: a release says takeover is safe, and for a run still
2900 /// inside `perform` the next owner would re-perform the call beside the one
2901 /// in flight. Left to expire, the lease is the signal the recovery sweep
2902 /// reads as *an instance died holding this run* — which is what a
2903 /// drained-out run is.
2904 ///
2905 /// Idempotent: draining an instance that is already draining waits again.
2906 pub async fn drain(&self, grace: std::time::Duration) -> super::DrainReport {
2907 self.inflight.drain(grace).await
2908 }
2909
2910 /// Whether this instance has stopped admitting runs.
2911 ///
2912 /// For an embedder whose server keeps its listener open while it drains and
2913 /// therefore needs something to answer a readiness probe with. The shipped
2914 /// server needs none: it closes its listeners first, so a probe that
2915 /// connects already fails — which is why this crate's HTTP surface has no
2916 /// `/health` route to flip. See [`crate::api`] on why it has none at all.
2917 #[must_use]
2918 pub fn is_draining(&self) -> bool {
2919 self.inflight.is_draining()
2920 }
2921
2922 /// The cancellation recorded against a run, if one was.
2923 ///
2924 /// Reads the journal rather than a flag: a cancellation is a record, so
2925 /// this answers the same thing after a restart and on another instance.
2926 ///
2927 /// # Errors
2928 ///
2929 /// If the store is unreachable.
2930 pub async fn cancellation(
2931 &self,
2932 run: RunId,
2933 ) -> Result<Option<crate::journal::Cancellation>, RuntimeError> {
2934 self.store
2935 .cancellation(run)
2936 .await
2937 .map_err(RuntimeError::from_store)
2938 }
2939
2940 /// The policy engine, if this runtime has one.
2941 ///
2942 /// Exposed because a surface that faces strangers has to be able to ask
2943 /// whether one exists at all. Inside the process the caller is the
2944 /// embedder's own code and an absent engine is a choice; on a socket it is
2945 /// a hole, and the HTTP surface refuses to start without one.
2946 #[must_use]
2947 pub fn policy(&self) -> Option<&Arc<dyn crate::core::PolicyEngine>> {
2948 self.policy.as_ref()
2949 }
2950
2951 /// Which tenant this plane runs as.
2952 #[must_use]
2953 pub fn tenant(&self) -> &crate::core::TenantId {
2954 &self.tenant
2955 }
2956
2957 /// What this plane's policy engine cannot evaluate for a served caller
2958 /// that presented no chain.
2959 ///
2960 /// The build probes the chain shapes this plane's own runs take: its own
2961 /// chain where it has one, and both shapes where it has none. A served
2962 /// surface adds the one the build cannot know about — a caller acting
2963 /// under no chain on a plane that has one — so every surface that admits
2964 /// work for a stranger asks this before it starts.
2965 #[cfg(any(feature = "a2a-server", feature = "mcp-server"))]
2966 pub(crate) fn served_policy_problems(&self) -> Vec<String> {
2967 match (&self.policy, &self.identity) {
2968 (Some(engine), Some(_)) => runtime_policy_problems(engine.as_ref(), None),
2969 _ => Vec::new(),
2970 }
2971 }
2972
2973 /// How far from its owner the plane's own chain is, or zero without one.
2974 ///
2975 /// The depth a run bound by the plane's chain but acting under none starts
2976 /// from: acting under no chain does not put a served caller closer to the
2977 /// owner than the plane that serves it.
2978 pub(crate) fn own_depth(&self) -> usize {
2979 self.identity
2980 .as_ref()
2981 .map_or(0, crate::core::Delegation::depth)
2982 }
2983
2984 /// The journal, under the name the batch driver reads it by.
2985 #[must_use]
2986 pub(crate) fn meter(&self) -> &super::metrics::Meter {
2987 &self.meter
2988 }
2989
2990 /// Write an operator act as the sole record of a sealed run of its own.
2991 ///
2992 /// The act, then its conclusion, then the seal — so it carries the chain,
2993 /// the signature, Merkle inclusion and the witness, and an offline reader
2994 /// checks it without being taught what the act was. `case` stamps the
2995 /// record onto a matter's history; `reason` is the conclusion's, for the
2996 /// one act whose reason lives there.
2997 async fn seal_operator_record(
2998 &self,
2999 kind: RecordKind,
3000 case: Option<crate::core::CaseId>,
3001 outcome: &str,
3002 reason: Option<&str>,
3003 ) -> Result<RunId, RuntimeError> {
3004 let run = RunId::generate();
3005 let epoch = BREAK_GLASS_EPOCH;
3006 let mut act = Append::new(run, kind);
3007 if let Some(case) = case {
3008 act = act.case(case);
3009 }
3010 self.store
3011 .append(epoch, vec![act])
3012 .await
3013 .map_err(RuntimeError::from_store)?;
3014 let head = self
3015 .store
3016 .head(run)
3017 .await
3018 .map_err(RuntimeError::from_store)?;
3019 self.store
3020 .append(
3021 epoch,
3022 vec![Append::new(
3023 run,
3024 RecordKind::RunConcluded {
3025 outcome: outcome.to_owned(),
3026 reason: reason.map(str::to_owned),
3027 exhaustion: None,
3028 live_spend: Spend::default(),
3029 chain_head: head.hash,
3030 },
3031 )],
3032 )
3033 .await
3034 .map_err(RuntimeError::from_store)?;
3035 self.store
3036 .seal(run, epoch, outcome)
3037 .await
3038 .map_err(RuntimeError::from_store)?;
3039 Ok(run)
3040 }
3041
3042 /// Record an operator deliberately crossing into this tenant, and return
3043 /// the run that holds the record.
3044 ///
3045 /// Every other tenancy control keeps a cross-tenant read from being reached
3046 /// by accident. This is the designed exception, and the rule for it is that an
3047 /// exception without a record is indistinguishable from the breach it is
3048 /// meant to be. So the access is written into **this** tenant's journal —
3049 /// the one whose data is about to be reached — in a sealed run of its own,
3050 /// exactly as a sweep writes its decisions. It therefore inherits the hash
3051 /// chain, the per-record signature and the Merkle inclusion, and the
3052 /// offline audit tool reports it without being taught that break-glass
3053 /// exists.
3054 ///
3055 /// **The record is written before any data is served, and a failure to
3056 /// write it is a failure to access.** That direction is the whole control:
3057 /// the alternative — serve first, record best-effort — is a break-glass
3058 /// that works exactly as well when its own evidence is lost.
3059 ///
3060 /// # Errors
3061 ///
3062 /// If `reason` is blank, or if the record cannot be written. An
3063 /// unexplained exception is the thing this record exists to prevent, so it
3064 /// is refused rather than stored empty.
3065 pub async fn record_break_glass(
3066 &self,
3067 actor: &crate::core::Operator,
3068 roles: &[String],
3069 reason: &str,
3070 ) -> Result<RunId, RuntimeError> {
3071 if reason.trim().is_empty() {
3072 return Err(RuntimeError::PlanContract(
3073 "break-glass needs a reason: an unexplained crossing of the tenant \
3074 boundary is what this record exists to prevent"
3075 .to_owned(),
3076 ));
3077 }
3078 let run = self
3079 .seal_operator_record(
3080 RecordKind::BreakGlass {
3081 actor: actor.clone(),
3082 roles: roles.to_vec(),
3083 reason: reason.to_owned(),
3084 },
3085 None,
3086 BREAK_GLASS_OUTCOME,
3087 Some(reason),
3088 )
3089 .await?;
3090 tracing::warn!(
3091 tenant = %self.tenant(),
3092 %actor,
3093 %run,
3094 reason,
3095 "break-glass: an operator crossed the tenant boundary"
3096 );
3097 Ok(run)
3098 }
3099
3100 /// The journal, for reads the runtime does not mediate.
3101 ///
3102 /// This is also the embedder's **resumable output stream**:
3103 /// [`JournalStore::read`] is a seq-cursored read over a run's records, so
3104 /// polling from the last sequence seen is a durable, reconnect-safe
3105 /// progress feed that any instance can serve — the exact mechanism the
3106 /// A2A streaming surface is built on. There is deliberately no curated
3107 /// event type between an embedder and the records: a third vocabulary
3108 /// beside the journal's and the wire's would be a second truth wearing
3109 /// ergonomics, and the records are already the history the stream must
3110 /// agree with.
3111 pub fn journal(&self) -> &Arc<dyn JournalStore> {
3112 &self.store
3113 }
3114
3115 /// Which case a run belongs to, or `None` if it belongs to none.
3116 ///
3117 /// Read from the **journal**, not from a column beside it. The binding is
3118 /// stamped on the run's own records at admission, so answering from there
3119 /// is answering from the plan of record — a case-store column saying the
3120 /// same thing would be a second copy of one fact, and the two could
3121 /// disagree about a run the case layer never saw.
3122 ///
3123 /// It is the first question an operator surface asks, which is why it is a
3124 /// method rather than a documented one-liner: every caller was otherwise
3125 /// going to reach for the first record and read `body.case` off it, and a
3126 /// caller who reached for the *last* one instead would still be right today
3127 /// and wrong the moment a run is admitted before its case is known.
3128 ///
3129 /// # Errors
3130 ///
3131 /// If the journal cannot be read. An unknown run is `Ok(None)` rather than
3132 /// an error: *no such run* and *a run in no case* are both honest answers
3133 /// to this question, and neither is a fault.
3134 pub async fn case_of(&self, run: RunId) -> Result<Option<crate::core::CaseId>, RuntimeError> {
3135 Ok(self
3136 .store
3137 .read(run, 1)
3138 .await
3139 .map_err(RuntimeError::from_store)?
3140 .first()
3141 .and_then(|record| record.body.case))
3142 }
3143
3144 #[must_use]
3145 /// The witness set and quorum this plane was built with, if any.
3146 pub(crate) fn witnessing(&self) -> Option<&Arc<Witnessing>> {
3147 self.witnesses.as_ref()
3148 }
3149
3150 pub fn store(&self) -> &Arc<dyn JournalStore> {
3151 &self.store
3152 }
3153
3154 /// This **process instance's** identity, as it appears in run leases.
3155 ///
3156 /// Not the agent's name: several instances of one agent are normal. See
3157 /// [`RuntimeBuilder::owner`].
3158 ///
3159 /// Public because "which instance holds this run" is a question an operator
3160 /// asks of a stuck system, and the answer is otherwise only in a store row.
3161 #[must_use]
3162 pub fn owner_id(&self) -> &str {
3163 &self.owner
3164 }
3165
3166 /// The input as the declared admission rules leave it: refused with no
3167 /// record, as an admission policy denial is, or with its sensitivity
3168 /// raised to every matched classification — which `RunAdmitted` then
3169 /// records as the input's label.
3170 #[cfg(feature = "manifest")]
3171 fn admit_content(
3172 &self,
3173 agent: &str,
3174 input: Tainted<Value>,
3175 ) -> Result<Tainted<Value>, RuntimeError> {
3176 let Some(manifest) = self
3177 .resolve(agent)
3178 .ok()
3179 .and_then(|skill| self.governing(skill.as_ref()))
3180 else {
3181 return Ok(input);
3182 };
3183 let outcome = super::ctx::judged(
3184 manifest.spec.security.content.as_ref(),
3185 crate::content::At::Admission,
3186 input.peek(),
3187 )
3188 .map_err(RuntimeError::PolicyDenied)?;
3189 if let Some(hit) = outcome.refused.first() {
3190 return Err(RuntimeError::PolicyDenied(
3191 crate::core::PolicyError::Content {
3192 rule: hit.rule.clone(),
3193 pointer: hit.pointer.clone(),
3194 },
3195 ));
3196 }
3197 let raised = outcome.raise(input.label().sensitivity);
3198 if raised == input.label().sensitivity {
3199 return Ok(input);
3200 }
3201 Ok(input.raised_to(raised))
3202 }
3203
3204 #[cfg(not(feature = "manifest"))]
3205 #[allow(clippy::unused_self, clippy::unnecessary_wraps)]
3206 fn admit_content(
3207 &self,
3208 _agent: &str,
3209 input: Tainted<Value>,
3210 ) -> Result<Tainted<Value>, RuntimeError> {
3211 Ok(input)
3212 }
3213
3214 /// The content rules' verdict on an event delivered to a suspended run,
3215 /// under the declaration governing the run's skill; `None` where no rule
3216 /// matched or none is declared.
3217 #[cfg(feature = "manifest")]
3218 fn delivered_verdict(
3219 &self,
3220 history: &[crate::journal::Record],
3221 payload: &serde_json::Value,
3222 ) -> Option<crate::core::ContentVerdict> {
3223 let capability = history.iter().find_map(|r| match r.kind() {
3224 RecordKind::RunAdmitted { capability, .. } => Some(capability.clone()),
3225 _ => None,
3226 })?;
3227 let manifest = self.governing(self.resolve(&capability).ok()?.as_ref())?;
3228 super::ctx::verdict(super::ctx::judged(
3229 manifest.spec.security.content.as_ref(),
3230 crate::content::At::Source(super::ctx::AWAIT_KIND),
3231 payload,
3232 ))
3233 }
3234
3235 #[cfg(not(feature = "manifest"))]
3236 #[allow(clippy::unused_self)]
3237 fn delivered_verdict(
3238 &self,
3239 _history: &[crate::journal::Record],
3240 _payload: &serde_json::Value,
3241 ) -> Option<crate::core::ContentVerdict> {
3242 None
3243 }
3244
3245 /// The declaration governing a skill, if its agent has one.
3246 ///
3247 /// Per skill, not per plane: a runtime runs several agents, and a step must
3248 /// be judged against the manifest of the agent whose skill it is. Looking up
3249 /// one plane-wide manifest would apply another agent's ceilings.
3250 #[cfg(feature = "manifest")]
3251 fn governing(&self, skill: &dyn Skill) -> Option<Arc<crate::manifest::Manifest>> {
3252 self.governed_by.get(&skill.descriptor().name).cloned()
3253 }
3254
3255 /// Renew this run's lease until the returned guard is dropped.
3256 ///
3257 /// Requires a Tokio runtime, as the rest of this crate's timing does.
3258 fn heartbeat(&self, run: RunId, epoch: crate::core::Epoch) -> Heartbeat {
3259 let store = Arc::clone(&self.store);
3260 let owner = self.owner.clone();
3261 let ttl = self.lease_ttl;
3262 // A third of the TTL, so two renewals can be lost to a slow store before
3263 // the lease lapses. Renewing *at* the TTL would mean any hesitation
3264 // leaves it expired, and an expired lease is one anybody may take —
3265 // including, per `acquire`, this caller, which would fence the run with
3266 // its own heartbeat.
3267 let period = ttl / 3;
3268 Heartbeat(tokio::spawn(async move {
3269 loop {
3270 tokio::time::sleep(period).await;
3271 // `renew`, never `acquire`. A renewal extends only a lease
3272 // still held by exactly this `(owner, epoch)`; it cannot
3273 // claim. That distinction is load-bearing here: this task
3274 // races the run's own conclusion, and an acquire arriving
3275 // just after `conclude` released the lease would re-take it —
3276 // a live, never-released lease over a concluded run, which
3277 // the recovery sweep then "recovers" forever.
3278 //
3279 // Stop only when the store says the lease is gone — released
3280 // by the conclusion, lapsed and reclaimed, or taken over: the
3281 // run's next append will be fenced, which is the correct
3282 // outcome, and claiming now would only resurrect ownership
3283 // this instance no longer has. Any other failure is the store
3284 // not answering, and giving up then would let a run that is
3285 // still executing lapse into somebody else's recovery.
3286 match store.renew(run, &owner, epoch, ttl).await {
3287 Ok(_) => {}
3288 Err(crate::core::StoreError::LeaseNotHeld { .. }) => return,
3289 Err(error) => tracing::warn!(
3290 target: "agentplane::lease",
3291 run = %run,
3292 error = %error,
3293 "a lease renewal failed; the next one retries"
3294 ),
3295 }
3296 }
3297 }))
3298 }
3299
3300 /// The standing withdrawal that reaches `identity`, if one does.
3301 ///
3302 /// The one predicate the step boundary and a subject-bound hop both ask,
3303 /// so the two cannot disagree about whose authority is withdrawn. Read on
3304 /// a live pass only — a halt thrown today must not decide what a run did
3305 /// last year — and an unreachable store propagates, because carrying on
3306 /// would leave a withdrawal unenforced exactly while accounting is down.
3307 pub(crate) async fn withdrawal_against(
3308 &self,
3309 identity: Option<&crate::core::Delegation>,
3310 ) -> Result<Option<Withdrawal>, crate::core::StoreError> {
3311 let (Some(quotas), Some(chain)) = (self.quotas.as_ref(), identity) else {
3312 return Ok(None);
3313 };
3314 let halts = quotas.halts().await?;
3315 Ok(halts.into_iter().find_map(|halt| {
3316 let subject = halt.scope.withdraws_from(chain)?.to_owned();
3317 Some(Withdrawal {
3318 subject,
3319 reason: halt.reason,
3320 by: halt.by,
3321 })
3322 }))
3323 }
3324
3325 /// Stop a run because the authority it acts under was withdrawn.
3326 ///
3327 /// A **pause**: `stop` is reached with a status `maybe_unwind` passes
3328 /// straight through, so the run's mutations stand and it stays resumable.
3329 /// That is the treatment exhaustion gets and for the same reason — the run
3330 /// did nothing wrong, and reversing correct work because a credential lapsed
3331 /// would be a second incident.
3332 ///
3333 /// `record` is false on a resume that finds the withdrawal still standing:
3334 /// the withholding already on the chain says so, and a second one per resume
3335 /// attempt would turn a queue of retries into a queue of records.
3336 #[allow(clippy::too_many_arguments)]
3337 async fn withhold(
3338 &self,
3339 cx: Unwind<'_>,
3340 withdrawal: Withdrawal,
3341 record: bool,
3342 completed: &[(StepId, Capability)],
3343 outputs: &BTreeMap<StepId, Tainted<Value>>,
3344 cursor: &mut ReplayCursor,
3345 case_id: Option<crate::core::CaseId>,
3346 quota: &QuotaPass,
3347 ) -> Result<RunOutcome, RuntimeError> {
3348 let Withdrawal {
3349 subject,
3350 reason,
3351 by,
3352 } = withdrawal;
3353 if record {
3354 let stamped = (cx.stamp)(Append::new(
3355 cx.run,
3356 RecordKind::AuthorityWithheld {
3357 subject: subject.clone(),
3358 reason: reason.clone(),
3359 by: by.clone(),
3360 },
3361 ));
3362 self.store
3363 .append(cx.epoch, vec![stamped])
3364 .await
3365 .map_err(RuntimeError::from_store)?;
3366 }
3367 self.stop(
3368 cx,
3369 RunStatus::Withheld { subject, reason },
3370 completed,
3371 outputs,
3372 cursor,
3373 case_id,
3374 quota,
3375 )
3376 .await
3377 }
3378
3379 /// Refuse a run whose tenant is at a ceiling.
3380 ///
3381 /// Fails **closed**: an unreachable quota store refuses rather than admits,
3382 /// because a ceiling that yields when its accounting is down is a ceiling an
3383 /// attacker removes by taking the accounting down.
3384 ///
3385 /// Live admission only, and replay and resume are skipped for **different**
3386 /// reasons. Re-checking during replay would let a run that happened produce
3387 /// a different history when it is re-read: a ceiling crossed since admission
3388 /// would rewrite the past into a refusal. Resume is not that — it decides
3389 /// only what happens next — and it is ungated because refusing it strands a
3390 /// run mid-saga with its reversals unrun. What that costs the ceiling is
3391 /// stated where the ceiling is defined ([`crate::quota`]); it is a real cost
3392 /// rather than an absent one.
3393 async fn check_quota(
3394 &self,
3395 run: RunId,
3396 governed_by: Option<&crate::journal::AgentIdentity>,
3397 chain: Option<&crate::core::Delegation>,
3398 at: crate::core::Timestamp,
3399 (budget, width): (&Budget, usize),
3400 ) -> Result<QuotaPass, RuntimeError> {
3401 let Some(quotas) = self.quotas.as_ref() else {
3402 return Ok(QuotaPass::disabled());
3403 };
3404 let pass = QuotaPass::at(&self.quota, at, true).sized_for(width);
3405
3406 // The halt is checked **before** the unlimited shortcut, because an
3407 // emergency stop is not a ceiling and a tenant with no ceilings is
3408 // exactly the one an operator is most likely to need to stop. Reading it
3409 // fails closed for the same reason the ceilings do: a switch that yields
3410 // when its store is unreachable is a switch an attacker throws by taking
3411 // the store down.
3412 //
3413 // Every standing halt in one read, and the **narrowest match wins the
3414 // message**: an operator who stopped one agent and whoever is refused
3415 // are looking for different sentences, and "the tenant is halted" told
3416 // to the caller of agent 12 sends them to the wrong incident. Ordering
3417 // the scopes puts `Subject` before `Revision` before `Agent` before
3418 // `Tenant`, so the reason reported is the most specific one that
3419 // actually covers this run — and a withdrawn authority outranks every
3420 // workload scope, because it is the answer that tells the caller their
3421 // credential is the problem rather than the agent.
3422 match quotas.halts().await {
3423 Ok(halts) => {
3424 if let Some(halt) = halts
3425 .into_iter()
3426 .filter(|halt| halt.scope.covers(governed_by, chain))
3427 .max_by(|a, b| a.scope.cmp(&b.scope))
3428 {
3429 return Err(RuntimeError::QuotaExceeded(
3430 crate::quota::QuotaError::Halted {
3431 tenant: self.tenant.as_str().to_owned(),
3432 scope: halt.scope,
3433 reason: halt.reason,
3434 },
3435 ));
3436 }
3437 }
3438 Err(e) => {
3439 return Err(RuntimeError::QuotaExceeded(
3440 crate::quota::QuotaError::Unavailable(e.to_string()),
3441 ));
3442 }
3443 }
3444
3445 // What the run can cost at most, held against the period in the same
3446 // store transaction that checks the ceiling and takes the slot. A run
3447 // that leaves a bounded unit unbounded is refused here: admitted, it
3448 // would hold nothing against a period it can spend without limit.
3449 let hold = match crate::quota::reservation(&self.quota, budget, width as u64) {
3450 Ok(amount) => {
3451 amount
3452 .zip(pass.period())
3453 .map(|(amount, period)| crate::quota::SpendHold {
3454 period: period.to_owned(),
3455 amount,
3456 })
3457 }
3458 Err(field) => {
3459 return Err(RuntimeError::QuotaExceeded(
3460 crate::quota::QuotaError::Unbounded {
3461 tenant: self.tenant.as_str().to_owned(),
3462 unit: if field.contains("minor") {
3463 "money"
3464 } else {
3465 "tokens"
3466 },
3467 field,
3468 },
3469 ));
3470 }
3471 };
3472
3473 quotas
3474 .reserve(run, &self.quota, hold.as_ref(), at)
3475 .await
3476 .map_err(RuntimeError::QuotaExceeded)?;
3477 Ok(pass)
3478 }
3479
3480 /// Atomically settle this pass's spend and admission slot.
3481 ///
3482 /// Required before physical sealing and lease release. A failure keeps the
3483 /// lease owned so the abandonment sweep retries from journal evidence; a
3484 /// best-effort write here would turn a transient store outage into permanent
3485 /// under-accounting. The period is the one captured when this pass started,
3486 /// never recomputed here.
3487 async fn settle_quota(
3488 &self,
3489 run: RunId,
3490 epoch: crate::core::Epoch,
3491 spend: Spend,
3492 pass: &QuotaPass,
3493 concludes: bool,
3494 ) -> Result<(), RuntimeError> {
3495 let Some(quotas) = self.quotas.as_ref() else {
3496 return Ok(());
3497 };
3498 let pending = |error: crate::core::StoreError| RuntimeError::QuotaSettlementPending {
3499 run: run.to_string(),
3500 epoch,
3501 detail: error.to_string(),
3502 };
3503 if !pass.enabled {
3504 // A conclusion no pass opened — an abandonment decided while the
3505 // run was stopped — still ends the run, and a run that has ended
3506 // holds nothing against the period.
3507 if concludes {
3508 quotas.release(run).await.map_err(pending)?;
3509 }
3510 return Ok(());
3511 }
3512 let settlement = pass
3513 .settlement(run, epoch, spend, concludes)
3514 .expect("an enabled quota pass has a settlement");
3515 quotas.settle(&settlement).await.map_err(pending)
3516 }
3517
3518 /// Release a reservation for an admission whose journal never existed.
3519 pub(crate) async fn release_empty_quota_reservation(
3520 &self,
3521 run: RunId,
3522 ) -> Result<(), crate::core::StoreError> {
3523 match self.quotas.as_ref() {
3524 Some(quotas) => quotas.release(run).await,
3525 None => Ok(()),
3526 }
3527 }
3528
3529 /// Settle and release the admission slots sealed runs still hold.
3530 ///
3531 /// A plane with no quota store may seal a run whose admission holds a
3532 /// slot in this ledger, and no resume or recovery selects a sealed run
3533 /// again. Settlement is idempotent, so a slot another path already settled
3534 /// is not billed twice. Returns how many slots were given back.
3535 pub(crate) async fn release_sealed_slots(&self, limit: usize) -> Result<usize, RuntimeError> {
3536 let Some(quotas) = self.quotas.as_ref() else {
3537 return Ok(0);
3538 };
3539 let held = quotas
3540 .running_runs(limit)
3541 .await
3542 .map_err(RuntimeError::Store)?;
3543 let positions = self
3544 .store
3545 .log_positions(&held)
3546 .await
3547 .map_err(RuntimeError::from_store)?;
3548 let mut released = 0;
3549 for (run, sealed) in held.into_iter().zip(positions) {
3550 if sealed.is_none() {
3551 continue;
3552 }
3553 let records = self
3554 .store
3555 .read(run, 1)
3556 .await
3557 .map_err(RuntimeError::from_store)?;
3558 match self.settle_recorded_quota_passes(run, &records).await {
3559 Ok(()) => released += 1,
3560 Err(error) => tracing::warn!(
3561 %run,
3562 %error,
3563 "a sealed run's admission slot could not be released; the next sweep retries"
3564 ),
3565 }
3566 }
3567 Ok(released)
3568 }
3569
3570 /// Seal the lease-free runs whose sealing conclusion is written and whose
3571 /// seal is not.
3572 ///
3573 /// Each [`LEASE_FREE_OUTCOMES`] writer appends its conclusion and then
3574 /// seals in a second call, and holds no lease, so recovery never selects
3575 /// a run whose writer died between the two. The conclusion alone already
3576 /// lists the run under its outcome, so a history reads it as done while
3577 /// the audit finds it in no log. The conclusion is what was decided; the
3578 /// seal adds nothing to it but the log position, so finishing it here
3579 /// writes no decision the writer did not make.
3580 ///
3581 /// Looks at the newest `limit` runs of each outcome. Returns how many it
3582 /// sealed.
3583 pub(crate) async fn seal_unsealed_conclusions(
3584 &self,
3585 limit: usize,
3586 ) -> Result<usize, RuntimeError> {
3587 let mut sealed = 0;
3588 for &outcome in LEASE_FREE_OUTCOMES {
3589 let runs = self
3590 .store
3591 .runs_by_outcome(outcome, limit)
3592 .await
3593 .map_err(RuntimeError::from_store)?;
3594 let positions = self
3595 .store
3596 .log_positions(&runs)
3597 .await
3598 .map_err(RuntimeError::from_store)?;
3599 for (run, position) in runs.into_iter().zip(positions) {
3600 if position.is_some() {
3601 continue;
3602 }
3603 match self.store.seal(run, LEASE_FREE_EPOCH, outcome).await {
3604 Ok(_) => sealed += 1,
3605 Err(error) => tracing::warn!(
3606 %run,
3607 %error,
3608 "a concluded run's seal could not be finished; the next sweep retries"
3609 ),
3610 }
3611 }
3612 }
3613 Ok(sealed)
3614 }
3615
3616 /// Settle every quota pass present in history, idempotently.
3617 ///
3618 /// Reached before a resume can dispatch. A normally settled pass is a cheap
3619 /// receipt lookup; a pass whose process died is accrued here. Conclusions
3620 /// carry the exact in-memory live total, including a known outcome whose
3621 /// terminal effect record failed before a later append recovered. An
3622 /// unconcluded crash has only durable terminal effect records to sum.
3623 async fn settle_recorded_quota_passes(
3624 &self,
3625 run: RunId,
3626 records: &[Record],
3627 ) -> Result<(), RuntimeError> {
3628 // A plane with no ledger resumes a run only when no pass of it accrued
3629 // spend to a period, which would otherwise go unbilled. It cannot give
3630 // back the concurrency slot of an admission whose process died
3631 // mid-pass; the ledger's own plane does, on its next sweep once the
3632 // run is sealed.
3633 let Some(quotas) = self.quotas.as_ref() else {
3634 if records.iter().any(|record| {
3635 matches!(
3636 record.kind(),
3637 RecordKind::QuotaPassStarted {
3638 period: Some(_),
3639 ..
3640 }
3641 )
3642 }) {
3643 return Err(RuntimeError::PlanContract(
3644 "this run has passes that accrued spend to a quota period, and the resuming \
3645 plane has no quota store to settle them"
3646 .to_owned(),
3647 ));
3648 }
3649 return Ok(());
3650 };
3651
3652 for record in records {
3653 let RecordKind::QuotaPassStarted {
3654 period,
3655 release_slot,
3656 } = record.kind()
3657 else {
3658 continue;
3659 };
3660 let epoch = record.body.epoch;
3661 let spend = records
3662 .iter()
3663 .rev()
3664 .find_map(|candidate| (candidate.body.epoch == epoch).then_some(candidate.kind()))
3665 .and_then(|kind| match kind {
3666 RecordKind::RunConcluded { live_spend, .. } => Some(*live_spend),
3667 _ => None,
3668 })
3669 .unwrap_or_else(|| spend_recorded_in_epoch(records, epoch));
3670 quotas
3671 .settle(&crate::quota::QuotaSettlement {
3672 run,
3673 epoch,
3674 period: period.clone(),
3675 spend,
3676 release_slot: *release_slot,
3677 concludes: concluded_in(records, epoch),
3678 })
3679 .await
3680 .map_err(|error| RuntimeError::QuotaSettlementPending {
3681 run: run.to_string(),
3682 epoch,
3683 detail: error.to_string(),
3684 })?;
3685 }
3686 // A run whose history has ended holds nothing against the period,
3687 // whichever path ended it — including one no pass marker opened.
3688 if records.iter().any(is_sealing_conclusion) {
3689 quotas
3690 .release(run)
3691 .await
3692 .map_err(|error| RuntimeError::QuotaSettlementPending {
3693 run: run.to_string(),
3694 epoch: records.last().map_or(0, |r| r.body.epoch),
3695 detail: error.to_string(),
3696 })?;
3697 }
3698 Ok(())
3699 }
3700
3701 /// Repair quota receipts, and finish a conclusion an owner died after.
3702 ///
3703 /// Under recovery, a run whose last record is a conclusion lost its owner
3704 /// between concluding and handing the lease back. What remains is the seal and the release, never the business
3705 /// work: recovery is not permission to retry a failure, with or without a
3706 /// quota store. A run that crashed before concluding continues through
3707 /// replay.
3708 async fn recover_quota_before_resume(
3709 &self,
3710 run: RunId,
3711 records: &[Record],
3712 lease: &crate::journal::Lease,
3713 recovering: bool,
3714 ) -> Result<Option<RunOutcome>, RuntimeError> {
3715 self.settle_recorded_quota_passes(run, records).await?;
3716 if !recovering {
3717 return Ok(None);
3718 }
3719 let Some(last) = records.last() else {
3720 return Ok(None);
3721 };
3722 let RecordKind::RunConcluded {
3723 outcome,
3724 reason,
3725 exhaustion,
3726 ..
3727 } = last.kind()
3728 else {
3729 return Ok(None);
3730 };
3731
3732 let status = recorded_status(outcome, reason.as_deref(), exhaustion.as_ref(), records);
3733 let chain_head = if SEALED_OUTCOMES.contains(&outcome.as_str()) {
3734 let head = self
3735 .store
3736 .seal(run, lease.epoch, outcome)
3737 .await
3738 .map_err(RuntimeError::from_store)?;
3739 // A repaired seal is a conclusion like any other.
3740 self.retire_waits(run).await;
3741 head
3742 } else {
3743 self.store
3744 .head(run)
3745 .await
3746 .map_err(RuntimeError::from_store)?
3747 .hash
3748 };
3749 Ok(Some(RunOutcome {
3750 failed_effect: failure_of(&status, records),
3751 run_id: run,
3752 status,
3753 chain_head,
3754 consumed: Consumed::default(),
3755 output: None,
3756 }))
3757 }
3758
3759 /// The effect this run ended on, read from its records, when it failed.
3760 ///
3761 /// Read only for a failure: every other ending answers `None` without a
3762 /// read.
3763 async fn failed_effect(
3764 &self,
3765 run: RunId,
3766 status: &RunStatus,
3767 ) -> Result<Option<crate::journal::FailedEffect>, RuntimeError> {
3768 if !matches!(status, RunStatus::Failed(_)) {
3769 return Ok(None);
3770 }
3771 let records = self
3772 .store
3773 .read(run, 1)
3774 .await
3775 .map_err(RuntimeError::from_store)?;
3776 Ok(failure_of(status, &records))
3777 }
3778
3779 /// Return a closed run without continuing it, repairing a missing physical
3780 /// seal when a crash landed after the conclusion append.
3781 async fn closed_resume_outcome(
3782 &self,
3783 run: RunId,
3784 records: &[Record],
3785 lease: &crate::journal::Lease,
3786 ) -> Result<Option<RunOutcome>, RuntimeError> {
3787 let Some(status) = resume_is_closed(records) else {
3788 return Ok(None);
3789 };
3790 let outcome = recorded_conclusion(records).unwrap_or_else(|| status.as_str().to_owned());
3791 let chain_head = if SEALED_OUTCOMES.contains(&outcome.as_str()) {
3792 let head = self
3793 .store
3794 .seal(run, lease.epoch, &outcome)
3795 .await
3796 .map_err(RuntimeError::from_store)?;
3797 // The crash this repairs may have landed before the conclusion
3798 // retired the run's waits; retiring twice is harmless.
3799 self.retire_waits(run).await;
3800 head
3801 } else {
3802 self.store
3803 .head(run)
3804 .await
3805 .map_err(RuntimeError::from_store)?
3806 .hash
3807 };
3808 Ok(Some(RunOutcome {
3809 failed_effect: failure_of(&status, records),
3810 run_id: run,
3811 status,
3812 chain_head,
3813 output: None,
3814 consumed: Consumed::default(),
3815 }))
3816 }
3817
3818 /// Finish an abandonment a decision asked for and a crash interrupted.
3819 ///
3820 /// The instruction is journaled before it is acted on, so this is what
3821 /// makes the ending arrive whatever happened in between — and it is the
3822 /// same path the verb itself takes, rather than a repair beside it.
3823 ///
3824 /// Nothing is unwound and no step runs: the run seals holding whatever it
3825 /// left in the world, which is the whole content of the decision.
3826 async fn abandon_if_decided(
3827 &self,
3828 run: RunId,
3829 records: &[Record],
3830 lease: &crate::journal::Lease,
3831 ) -> Result<Option<RunOutcome>, RuntimeError> {
3832 let Some(status) = pending_abandonment(records) else {
3833 return Ok(None);
3834 };
3835 self.conclude(
3836 run,
3837 lease.epoch,
3838 status,
3839 None,
3840 true,
3841 recorded_case_id(records),
3842 Consumed::default(),
3843 Spend::ZERO,
3844 &QuotaPass::disabled(),
3845 )
3846 .await
3847 .map(Some)
3848 }
3849
3850 /// This plane, writing through a journal that puts `quota`'s marker on
3851 /// the pass's first append — so a pass that writes nothing leaves no
3852 /// marker. `None` for a pass with no marker.
3853 fn marking_pass(&self, run: RunId, quota: &QuotaPass) -> Option<Self> {
3854 let marker = quota.started()?;
3855 Some(Self {
3856 store: Arc::new(super::pass_journal::PassJournal::new(
3857 Arc::clone(&self.store),
3858 run,
3859 marker,
3860 )),
3861 ..self.clone()
3862 })
3863 }
3864
3865 /// Open the quota identity of the live tail a resume is about to execute.
3866 ///
3867 /// Its marker is not written here: it rides the pass's first append (see
3868 /// [`super::pass_journal`]), so a resume that writes nothing leaves none.
3869 async fn start_replay_quota_pass(
3870 &self,
3871 run: RunId,
3872 mode: Pass,
3873 width: usize,
3874 ) -> Result<QuotaPass, RuntimeError> {
3875 let (Pass::Resume, Some(quotas)) = (mode, self.quotas.as_ref()) else {
3876 return Ok(QuotaPass::disabled());
3877 };
3878 let pass = QuotaPass::at(&self.quota, now_for_admission(), false).sized_for(width);
3879 // The run spends in the period it resumes in, so what it still holds
3880 // moves there — never refused, because a resume is not.
3881 if let Some(period) = pass.period() {
3882 quotas
3883 .carry(run, period)
3884 .await
3885 .map_err(RuntimeError::Store)?;
3886 }
3887 Ok(pass)
3888 }
3889
3890 /// The ceilings a run gets: its agent's, or the plane's if it has no agent.
3891 ///
3892 /// Per agent, because that is who declared them. A plane-wide budget would
3893 /// let one agent's generosity bound another's runs, which is the whole
3894 /// reason a declaration belongs to an identity rather than to a process.
3895 fn budget_for(&self, target: &str) -> Budget {
3896 #[cfg(feature = "manifest")]
3897 if let Ok(skill) = self.resolve(target)
3898 && let Some(m) = self.governing(skill.as_ref())
3899 {
3900 return m.budget();
3901 }
3902 let _ = target;
3903 self.budget
3904 }
3905
3906 /// Which declaration governs runs of this capability, if a declared agent
3907 /// does.
3908 ///
3909 /// Resolved the same way the budget is, and for the same reason: a run is
3910 /// governed by the agent that answers its entry capability. A commissioned
3911 /// sub-run opens its own run and records its own governor, so every run in a
3912 /// room has exactly one — there is no case where this has to pick.
3913 ///
3914 /// A manifest that cannot produce a digest is one that could not have been
3915 /// published; recording `None` would claim the run was ungoverned, so the
3916 /// failure is surfaced as an absent identity rather than a false one.
3917 #[cfg(feature = "manifest")]
3918 fn identity_for(&self, target: &str) -> Option<crate::journal::AgentIdentity> {
3919 let skill = self.resolve(target).ok()?;
3920 self.identity_of(self.governing(skill.as_ref())?.as_ref())
3921 }
3922
3923 /// What the agent answering `capability` may do, as its declaration says.
3924 ///
3925 /// `None` when nothing on this plane answers it; a reach with no
3926 /// declaration when something does and no declaration governs it.
3927 #[cfg(feature = "manifest")]
3928 pub(crate) fn reach_of(&self, capability: &str) -> Option<crate::core::Reach> {
3929 let skill = self.resolve(capability).ok()?;
3930 let declaration = self.governing(skill.as_ref()).and_then(|manifest| {
3931 let digest = manifest.digest().ok()?;
3932 Some(super::reach::declared(&manifest, digest, |server| {
3933 self.peers.as_ref().is_some_and(|wiring| {
3934 wiring
3935 .registry
3936 .grant(&crate::peers::PeerId::new(server))
3937 .is_some()
3938 })
3939 }))
3940 });
3941 Some(crate::core::Reach {
3942 capability: capability.to_owned(),
3943 declaration,
3944 })
3945 }
3946
3947 /// The declaration's identity, as every gate reports it.
3948 ///
3949 /// One construction, because admission and each effect gate must name the
3950 /// same revision the same way: two spellings of *which agent is acting*
3951 /// would let a rule permitted at the door mean something else at the sink.
3952 #[cfg(feature = "manifest")]
3953 fn identity_of(&self, m: &crate::manifest::Manifest) -> Option<crate::journal::AgentIdentity> {
3954 Some(crate::journal::AgentIdentity {
3955 name: m.metadata.name.clone(),
3956 version: m.metadata.version.clone(),
3957 digest: m.digest().ok()?,
3958 publisher: self.published_by.get(&m.metadata.name).cloned(),
3959 })
3960 }
3961
3962 /// Whether anything on this plane answers `target`.
3963 ///
3964 /// For a catalogue built before the first call: a surface that advertises a
3965 /// capability nothing provides offers a caller — often a model — a verb
3966 /// that fails at admission every time, and the failure names the plane
3967 /// rather than the catalogue that lied about it.
3968 ///
3969 /// Gated on the feature that builds a catalogue, so a build without one
3970 /// answers *this question is not asked here* rather than carrying a method
3971 /// nothing reaches.
3972 #[cfg(feature = "mcp-server")]
3973 pub(crate) fn provides(&self, target: &str) -> bool {
3974 self.resolve(target).is_ok()
3975 }
3976
3977 /// Whether a grant's server component names a registered A2A peer — a
3978 /// call the run may wait on.
3979 #[cfg(feature = "mcp-server")]
3980 pub(crate) fn wires_peer(&self, server: &str) -> bool {
3981 self.peers.as_ref().is_some_and(|wiring| {
3982 wiring
3983 .registry
3984 .grant(&crate::peers::PeerId::new(server))
3985 .is_some()
3986 })
3987 }
3988
3989 fn resolve(&self, target: &str) -> Result<Arc<dyn Skill>, RuntimeError> {
3990 if let Some(s) = self.skills.get(target) {
3991 return Ok(Arc::clone(s));
3992 }
3993 let cap = Capability::new(target);
3994 if let Some(name) = self.by_capability.get(&cap)
3995 && let Some(s) = self.skills.get(name)
3996 {
3997 return Ok(Arc::clone(s));
3998 }
3999 // Capabilities rather than skill names: `run` is documented to take the
4000 // capability, and listing the names would answer a question nobody
4001 // asked with vocabulary that does not work in the call that failed.
4002 let mut available: Vec<String> = self
4003 .by_capability
4004 .keys()
4005 .map(std::string::ToString::to_string)
4006 .collect();
4007 available.sort();
4008 Err(RuntimeError::NoProvider {
4009 target: target.to_owned(),
4010 available,
4011 })
4012 }
4013
4014 /// Start a run: a capability a skill answers, or a plan.
4015 ///
4016 /// The one door. What the run is bound to — a case, a caller's chain, a
4017 /// pinned declaration — goes in [`RunRequest::terms`]; whether it runs at
4018 /// most once goes in [`RunRequest::once`], which changes what the answer
4019 /// is; and the terminal says whether to wait for it.
4020 ///
4021 /// ```no_run
4022 /// # use std::sync::Arc;
4023 /// # use agentplane::core::Tainted;
4024 /// # use agentplane::runtime::{Runtime, RunTerms};
4025 /// # async fn f(rt: Arc<Runtime>, input: Tainted<serde_json::Value>) -> Result<(), agentplane::RuntimeError> {
4026 /// let outcome = rt.start("order.settle", input.clone()).run().await?;
4027 /// let admission = rt
4028 /// .start("order.settle", input)
4029 /// .terms(RunTerms::default().correlated("order", &[]))
4030 /// .once("billing:msg-7")
4031 /// .run()
4032 /// .await?;
4033 /// # Ok(()) }
4034 /// ```
4035 ///
4036 /// The input arrives **labelled**, and that is the whole of the decision.
4037 /// [`Tainted::trusted`](crate::core::Tainted::trusted) is what an operator
4038 /// writes for a literal, a constant or a configuration value they vouch for;
4039 /// anything that came from outside — an inbound event, a queue message, a
4040 /// counterparty's payload — keeps the label it arrived with. A bare value
4041 /// admitted as trusted would satisfy `require_trusted` fields with
4042 /// attacker-chosen data and give the egress ceiling nothing to join with.
4043 pub fn start(
4044 &self,
4045 target: impl Into<Target>,
4046 input: Tainted<Value>,
4047 ) -> RunRequest<'_, Unkeyed> {
4048 RunRequest {
4049 runtime: self,
4050 target: target.into(),
4051 input,
4052 terms: RunTerms::default(),
4053 key: Unkeyed,
4054 }
4055 }
4056
4057 /// This plane as the handle a background run keeps.
4058 ///
4059 /// Every `Runtime` is built inside an `Arc` that holds `self_ref`, and a
4060 /// borrow of it is a borrow of that `Arc`, so the upgrade cannot fail
4061 /// while `&self` exists.
4062 fn owned(&self) -> Arc<Self> {
4063 self.self_ref
4064 .upgrade()
4065 .expect("a runtime is only reachable through the Arc its builder made")
4066 }
4067
4068 /// Admit and execute, blocking until the run rests.
4069 async fn admit_target(
4070 &self,
4071 target: Target,
4072 input: Tainted<Value>,
4073 terms: RunTerms,
4074 ) -> Result<RunOutcome, RuntimeError> {
4075 match target {
4076 Target::Capability(capability) => self.admit(&capability, input, terms).await,
4077 Target::Plan(plan) => self.admit_plan(plan, input, terms).await,
4078 }
4079 }
4080
4081 /// Admit, and let the run proceed in the background.
4082 async fn spawn_target(
4083 self: &Arc<Self>,
4084 target: Target,
4085 input: Tainted<Value>,
4086 terms: RunTerms,
4087 ) -> Result<RunId, RuntimeError> {
4088 // Resolved before the id is minted: an unknown capability is the
4089 // caller's mistake and must be an error, not a run that exists and
4090 // immediately fails. A plan is validated inside admission, which also
4091 // precedes the id becoming visible.
4092 let plan = match target {
4093 Target::Capability(capability) => {
4094 let skill = self.resolve(&capability)?;
4095 PlanIR::single(first_capability(&skill.descriptor()))
4096 }
4097 Target::Plan(plan) => plan,
4098 };
4099
4100 let run = RunId::generate();
4101 // Registered before admission, not after. Admission is several store
4102 // writes, and a drain beginning inside them would take its snapshot
4103 // without this run, report a complete stop, and let the process exit
4104 // while the run was still being admitted. Registered first, both orders
4105 // work out: a drain that starts after this waits for the run, and one
4106 // that starts before it is seen by the gate inside `admit_only`, which
4107 // refuses and drops the ticket on the way out.
4108 let ticket = self.inflight.enter(run);
4109 let admitted = self
4110 .admit_only(run, plan, input, terms, Entry::Outside)
4111 .await?;
4112
4113 let plane = Arc::clone(self);
4114 // In the caller's span, as a blocking run is: a background run's
4115 // trace is still the trace of whatever started it.
4116 tokio::spawn(
4117 async move {
4118 let _ticket = ticket;
4119 plane.execute_admitted(admitted).await
4120 }
4121 .in_current_span(),
4122 );
4123 Ok(run)
4124 }
4125
4126 /// Admission under a key: the one copy, for blocking and background,
4127 /// capability and plan.
4128 ///
4129 /// Two gates, and only the second arbitrates. The read below is an
4130 /// optimisation — it saves a redelivery the cost of a policy evaluation, a
4131 /// quota reservation, a lease and a correlation lookup — and settles
4132 /// nothing, since two instances racing both see nothing here. The
4133 /// constraint inside `append` decides, on the same division
4134 /// `CaseStore::correlate_or_open` runs on.
4135 async fn claimed<T, F>(&self, key: &str, admit: F) -> Result<Claimed<T>, RuntimeError>
4136 where
4137 F: Future<Output = Result<T, RuntimeError>>,
4138 {
4139 validate_admission_key(key)?;
4140 if let Some(held) = self.holder_of(key).await? {
4141 return Ok(Claimed::Held(held));
4142 }
4143 match admit.await {
4144 Ok(fresh) => Ok(Claimed::Fresh(fresh)),
4145 Err(RuntimeError::Store(crate::core::StoreError::DuplicateAdmission {
4146 run, ..
4147 })) => Ok(Claimed::Held(parse_holder(&run)?)),
4148 Err(e) => Err(e),
4149 }
4150 }
4151
4152 /// Wait until `run` rests and answer with its outcome — the value
4153 /// re-derived by strict replay when it concluded.
4154 ///
4155 /// A run whose lease is free is resumed here: its holder has gone. One
4156 /// whose lease is held is followed on the journal until it rests or the
4157 /// lease lapses, so a holder that dies mid-wait is taken over rather than
4158 /// waited for forever.
4159 async fn settled(&self, run: RunId) -> Result<RunOutcome, RuntimeError> {
4160 loop {
4161 if let Some(rested) = self.recorded_outcome(run).await? {
4162 return self.with_output(rested).await;
4163 }
4164 // Subscribed before the attempt, so an append between the two
4165 // still wakes the wait below.
4166 let mut follower = self.store.watch(run).await;
4167 match self.replay(run, Mode::Resume).await {
4168 Err(RuntimeError::LeaseHeld { remaining_secs, .. }) => {
4169 let lapse = Duration::from_secs(remaining_secs.max(1));
4170 match follower.as_mut() {
4171 Some(follower) => {
4172 let _ = tokio::time::timeout(lapse, follower.changed()).await;
4173 }
4174 None => tokio::time::sleep(lapse).await,
4175 }
4176 }
4177 resumed => return resumed,
4178 }
4179 }
4180 }
4181
4182 /// A rested run's outcome with its value: strict replay re-derives what a
4183 /// concluded run answered, performing nothing. A suspension has no answer
4184 /// yet, and is returned as recorded.
4185 async fn with_output(&self, rested: RunOutcome) -> Result<RunOutcome, RuntimeError> {
4186 if rested.status == RunStatus::Succeeded {
4187 self.replay(rested.run_id, Mode::Strict).await
4188 } else {
4189 Ok(rested)
4190 }
4191 }
4192
4193 /// Which run holds this admission key, if any.
4194 async fn holder_of(&self, key: &str) -> Result<Option<RunId>, RuntimeError> {
4195 self.store
4196 .admitted_as(key)
4197 .await
4198 .map_err(RuntimeError::from_store)
4199 }
4200
4201 /// Answer a duplicate with what the original run has on the record.
4202 async fn answer_with(&self, held: RunId) -> Result<Admission, RuntimeError> {
4203 Ok(match self.recorded_outcome(held).await? {
4204 Some(outcome) => Admission::Replayed(outcome),
4205 None => Admission::InFlight(held),
4206 })
4207 }
4208
4209 /// What a run has recorded, **without resuming it**.
4210 ///
4211 /// `None` while the run is still working — it has records but has reached
4212 /// neither a conclusion nor a suspension.
4213 ///
4214 /// No lease is taken and nothing is appended. The original is frequently
4215 /// live when its redelivery lands, and a read that acquired a lease would
4216 /// fence a healthy run mid-step in order to report on it.
4217 ///
4218 /// The **last** record decides, never any record: a run that suspended, was
4219 /// resumed and carried on has a suspension in its history and is not
4220 /// suspended now. Same rule as the A2A status projection, so the two
4221 /// surfaces cannot disagree about one run.
4222 ///
4223 /// # Errors
4224 ///
4225 /// If the journal cannot be read.
4226 pub async fn recorded_outcome(&self, run: RunId) -> Result<Option<RunOutcome>, RuntimeError> {
4227 let records = self
4228 .store
4229 .read(run, 1)
4230 .await
4231 .map_err(RuntimeError::from_store)?;
4232 let Some(last) = records.last() else {
4233 return Ok(None);
4234 };
4235 let Some(status) = observed_status(&records) else {
4236 return Ok(None);
4237 };
4238 Ok(Some(RunOutcome {
4239 failed_effect: failure_of(&status, &records),
4240 run_id: run,
4241 status,
4242 chain_head: last.hash,
4243 // Reading a record performs nothing, so it consumes nothing — the
4244 // same answer the closed-run path gives, and for the same reason:
4245 // the spend belongs to the pass that did the work.
4246 consumed: Consumed::default(),
4247 // A step's result is reconstructed by replay rather than stored, so
4248 // there is none to hand back here. See `Admission::Replayed`.
4249 output: None,
4250 }))
4251 }
4252
4253 /// Rebuild a run's case binding **from its history**, never by re-deriving.
4254 ///
4255 /// Two facts are read back rather than recomputed, and each has its own
4256 /// failure if it is not.
4257 ///
4258 /// The **case id**, because re-correlating could land on a different case if
4259 /// the keys were since released — silently rewriting which business fact the
4260 /// run belongs to.
4261 ///
4262 /// The **business keys**, because a case accumulates them over months. A run
4263 /// that resolved `memory.formation.subject: $correlation/meter` would
4264 /// otherwise resolve it against a set the live run never saw, write a second
4265 /// memory under a second subject, and produce a history that disagrees with
4266 /// itself with nothing on the record explaining why.
4267 fn recorded_case(&self, records: &[Record]) -> Option<CaseContext> {
4268 let (case_id, correlation) = records.iter().find_map(|r| match r.kind() {
4269 RecordKind::CaseBound { correlation, .. } => {
4270 r.body.case.map(|case_id| (case_id, correlation.clone()))
4271 }
4272 _ => None,
4273 })?;
4274 self.cases.as_ref().map(|cases| CaseContext {
4275 cases: Arc::clone(cases),
4276 tasks: self.tasks.clone(),
4277 events: self.events.clone(),
4278 calendar: Arc::clone(&self.calendar),
4279 case_id,
4280 correlation,
4281 })
4282 }
4283
4284 /// The contract this runtime enforces on every plan.
4285 pub(crate) fn contract(&self) -> crate::plan::Contract {
4286 let contract = crate::plan::Contract::new(self.by_capability.keys().cloned());
4287 if self.require_verifier {
4288 contract.require_verifier()
4289 } else {
4290 contract
4291 }
4292 }
4293
4294 async fn admit(
4295 &self,
4296 target: &str,
4297 input: Tainted<Value>,
4298 terms: RunTerms,
4299 ) -> Result<RunOutcome, RuntimeError> {
4300 // A bare target is the degenerate plan: one node, terminal.
4301 let skill = self.resolve(target)?;
4302 let capability = first_capability(&skill.descriptor());
4303 self.admit_plan(PlanIR::single(capability), input, terms)
4304 .await
4305 }
4306
4307 async fn admit_plan(
4308 &self,
4309 plan: PlanIR,
4310 input: Tainted<Value>,
4311 terms: RunTerms,
4312 ) -> Result<RunOutcome, RuntimeError> {
4313 self.admit_plan_as(RunId::generate(), plan, input, terms, Entry::Outside)
4314 .await
4315 }
4316
4317 /// Admit and set working the sub-run a started commission orders, under
4318 /// `key`, answering with its id.
4319 ///
4320 /// A key already held answers with the run that holds it: the start was
4321 /// re-performed after a crash and its sub-run exists. A commission, not an
4322 /// outside admission, so the drain gate does not see it — but the sub-run
4323 /// is tracked in flight, so a drain waits for it.
4324 pub(crate) async fn start_commission_run(
4325 &self,
4326 parent: RunId,
4327 target: &str,
4328 input: Tainted<Value>,
4329 terms: RunTerms,
4330 key: &str,
4331 ) -> Result<RunId, RuntimeError> {
4332 let skill = self.resolve(target)?;
4333 let plan = PlanIR::single(first_capability(&skill.descriptor()));
4334 let run = RunId::generate();
4335 let ticket = self.inflight.enter(run);
4336 let admit = Box::pin(self.admit_only(run, plan, input, terms.keyed(key), Entry::Nested));
4337 match self.claimed(key, admit).await? {
4338 Claimed::Fresh(admitted) => {
4339 let plane = self.owned();
4340 let working = tokio::spawn(
4341 async move {
4342 let _ticket = ticket;
4343 plane.execute_admitted(admitted).await
4344 }
4345 .in_current_span(),
4346 );
4347 self.commissions
4348 .lock()
4349 .unwrap_or_else(std::sync::PoisonError::into_inner)
4350 .insert(run, (parent, working));
4351 Ok(run)
4352 }
4353 Claimed::Held(held) => Ok(held),
4354 }
4355 }
4356
4357 /// A commissioned sub-run once it rests, with its answer and everything
4358 /// it spent.
4359 ///
4360 /// From the task that executed it, when this process set it working: that
4361 /// pass did all its work, so its outcome is the whole. Otherwise — the
4362 /// commissioning run resumed after a crash, or another instance started
4363 /// it — the run is settled and read back by strict replay, which reads the
4364 /// answer and every pass's spend from the journal; a resumed pass reports
4365 /// only its own.
4366 pub(crate) async fn rested_commission(&self, run: RunId) -> Result<RunOutcome, RuntimeError> {
4367 let working = self
4368 .commissions
4369 .lock()
4370 .unwrap_or_else(std::sync::PoisonError::into_inner)
4371 .remove(&run);
4372 if let Some((_, working)) = working {
4373 return working.await.map_err(|e| {
4374 RuntimeError::Store(crate::core::StoreError::Backend(format!(
4375 "the commissioned run's task ended without an outcome: {e}"
4376 )))
4377 })?;
4378 }
4379 self.settled(run).await?;
4380 self.replay(run, Mode::Strict).await
4381 }
4382
4383 /// Let go of the tasks running `parent`'s unawaited commissions, once its
4384 /// pass ends: the sub-runs carry on, and a later await reads them back
4385 /// from the journal.
4386 fn forget_commissions_of(&self, parent: RunId) {
4387 self.commissions
4388 .lock()
4389 .unwrap_or_else(std::sync::PoisonError::into_inner)
4390 .retain(|_, (started_by, _)| *started_by != parent);
4391 }
4392
4393 /// Record the chain this run acts under, beside the plan it authorizes.
4394 ///
4395 /// The two are read back together: a chain without its plan says nothing
4396 /// about what it was allowed to do, and a plan without its chain says
4397 /// nothing about who was allowed to run it.
4398 fn bind_identity(
4399 run: RunId,
4400 chain: Option<&crate::core::Delegation>,
4401 records: &mut Vec<Append>,
4402 ) {
4403 if let Some(chain) = chain {
4404 records.push(Append::new(
4405 run,
4406 RecordKind::IdentityBound {
4407 chain: chain.links().cloned().collect(),
4408 },
4409 ));
4410 }
4411 }
4412
4413 /// Check the plan against the delegation chain's authority.
4414 ///
4415 /// The plan is the authorization graph, so this is where authority belongs:
4416 /// a plan that names a capability outside the chain's scope must never
4417 /// start, rather than failing at whichever step happens to reach it first.
4418 /// Checking here also makes the refusal deterministic — it depends only on
4419 /// the frozen plan and the chain, both of which are recorded.
4420 ///
4421 /// The chain's other two bounds are checked beside it, and only here: a
4422 /// chain issued for another plane or past its validity is refused before
4423 /// the run exists, and a run admitted under a live chain is never re-judged
4424 /// — replay reads the recorded chain back.
4425 ///
4426 /// `acts_under` is whether the run acts under `ceiling` or is only bounded
4427 /// by it. A refusal names the chain's subject only in the first case: a
4428 /// caller bounded by the plane's chain is not its subject, and a denial
4429 /// naming the operator would say the operator was refused.
4430 fn authorize_scope(
4431 &self,
4432 plan: &PlanIR,
4433 ceiling: Option<&crate::core::Delegation>,
4434 acts_under: bool,
4435 ) -> Result<(), RuntimeError> {
4436 let Some(chain) = ceiling else {
4437 return Ok(());
4438 };
4439 chain.admissible(self.tenant.as_str(), now_for_admission())?;
4440 let scope = chain.effective_scope();
4441 for node in &plan.nodes {
4442 if !scope.permits(&node.capability) {
4443 return Err(RuntimeError::PolicyDenied(
4444 crate::core::PolicyError::Denied {
4445 principal: if acts_under {
4446 chain.subject().id.clone()
4447 } else {
4448 node.capability.to_string()
4449 },
4450 action: crate::core::ACTION_ADMIT.to_owned(),
4451 resource: node.capability.to_string(),
4452 },
4453 ));
4454 }
4455 }
4456 Ok(())
4457 }
4458
4459 /// Authorize starting a run, before the run exists.
4460 ///
4461 /// A denial here leaves no journal at all, which is correct: nothing
4462 /// happened, and a run record for something that was never allowed to start
4463 /// would be a run nobody can explain.
4464 fn authorize_admission(
4465 &self,
4466 capability: &str,
4467 governed_by: Option<&crate::journal::AgentIdentity>,
4468 chain: Option<&crate::core::Delegation>,
4469 input: &Value,
4470 ) -> Result<(), RuntimeError> {
4471 let Some(engine) = self.policy.as_ref() else {
4472 return Ok(());
4473 };
4474 // Who is acting, and what is being asked for. Passing one string as both
4475 // made every admission rule a tautology — `principal == resource` cannot
4476 // express "this agent may not run that capability", which is the whole
4477 // question at admission on a plane hosting several agents.
4478 //
4479 // The principal is an **authenticated** identity or it is nothing. The
4480 // scope check a few lines above already denies under the delegation
4481 // subject, so anything else here would give one refused run two answers
4482 // to "who was refused" in the same error type.
4483 //
4484 // The agent's `metadata.name` is deliberately *not* used, tempting as it
4485 // is. A name is self-asserted — a manifest is a file, and its name is
4486 // whatever the author typed — so a rule granting authority to a name
4487 // grants it to any file claiming that name. Worse, a fallback would be
4488 // *silent*: `principal == X` would mean an authenticated identity in a
4489 // deployment with a delegation chain and a self-asserted label in one
4490 // without, so the same rule would change meaning with the wiring.
4491 //
4492 // Rules that need to bind to the agent bind to `context.agent.digest`,
4493 // which is content-addressed and pins what the declaration actually
4494 // said — so "this exact declaration may admit, and an edited one may
4495 // not" is expressible. The capability is the fallback principal because
4496 // it claims nothing: it is what was asked for, not who asked.
4497 //
4498 // Built by the function `policy check` also calls, from the values
4499 // `RunAdmitted` and `IdentityBound` record.
4500 let request = crate::policy::requests::admission(
4501 &crate::policy::requests::Acting {
4502 tenant: self.tenant.as_str(),
4503 capability,
4504 agent: governed_by,
4505 chain,
4506 },
4507 input,
4508 );
4509 // Every refusal, not only a rule's. `let … else` over `Deny` alone
4510 // admits the run for every other decision, so a policy set that cannot
4511 // be evaluated would open the door it exists to hold — the fail-open a
4512 // widened vocabulary creates for free unless the permit is the arm that
4513 // has to be named.
4514 let decision = engine.authorize(&request.as_request());
4515 let malformed = decision.is_malformed();
4516 let Some(reason) = decision.reason().map(ToOwned::to_owned) else {
4517 return Ok(());
4518 };
4519
4520 tracing::error!(
4521 target: telemetry::POLICY_DENIED,
4522 action = crate::core::ACTION_ADMIT,
4523 resource = %capability,
4524 policy_error = malformed,
4525 %reason,
4526 );
4527 self.meter
4528 .count(metrics::POLICY_DENIALS, crate::core::ACTION_ADMIT);
4529 Err(RuntimeError::PolicyDenied(
4530 crate::core::PolicyError::Denied {
4531 principal: request.principal,
4532 action: crate::core::ACTION_ADMIT.to_owned(),
4533 resource: capability.to_owned(),
4534 },
4535 ))
4536 }
4537
4538 /// The record that opens a run.
4539 ///
4540 /// Its own function because the label matters: journaled rather than
4541 /// recomputed, so a replay reaches the same verdict at every taint gate as
4542 /// the run it reproduces.
4543 fn admission(
4544 &self,
4545 capability: &str,
4546 governed_by: Option<crate::journal::AgentIdentity>,
4547 input: &Tainted<Value>,
4548 idempotency_key: Option<String>,
4549 admitted_by: Option<String>,
4550 acting_as: &Acting,
4551 ) -> RecordKind {
4552 RecordKind::RunAdmitted {
4553 capability: capability.to_owned(),
4554 governed_by: governed_by.map(Box::new),
4555 input: input.peek().clone(),
4556 input_label: input.label().clone(),
4557 input_field_labels: input
4558 .field_labels()
4559 .map(|(path, label)| (path.to_owned(), label.clone()))
4560 .collect(),
4561 policy_bundle: self.policy.as_ref().map(|p| Box::new(p.bundle())),
4562 canon: crate::core::canon::VERSION,
4563 idempotency_key,
4564 admitted_by,
4565 served_unchained: matches!(acting_as, Acting::Nobody),
4566 plane_chain: match acting_as {
4567 Acting::Plane => self.identity.is_some(),
4568 Acting::PlaneDelegate(_) => true,
4569 Acting::Caller(_) | Acting::Nobody => false,
4570 },
4571 }
4572 }
4573
4574 /// Everything up to and including the admission records, and no work.
4575 ///
4576 /// The seam a non-blocking submit needs. Admission is what makes a run
4577 /// *exist*: the policy gate, the lease, and the records that say what is
4578 /// about to happen. Splitting there means a caller can be told the run was
4579 /// accepted — or refused — before any of it runs, and a refusal stays an
4580 /// immediate answer rather than becoming a task that silently never appears.
4581 #[allow(clippy::too_many_lines)]
4582 async fn admit_only(
4583 &self,
4584 run: RunId,
4585 plan: PlanIR,
4586 input: Tainted<Value>,
4587 terms: RunTerms,
4588 entry: Entry,
4589 ) -> Result<Admitted, RuntimeError> {
4590 // First, and before the plan is even read: a draining instance is going
4591 // away, and everything below this line takes on work it may not be able
4592 // to finish. Refusing here leaves no lease, no quota slot and no
4593 // journal — the same nothing a policy denial leaves.
4594 if entry == Entry::Outside && self.inflight.is_draining() {
4595 return Err(RuntimeError::Draining);
4596 }
4597 crate::plan::validate(&plan, &self.contract())
4598 .map_err(|e| RuntimeError::PlanContract(e.to_string()))?;
4599
4600 let agent = plan
4601 .nodes
4602 .first()
4603 .map_or_else(|| "plan".to_owned(), |n| n.capability.to_string());
4604
4605 // Resolved once: the gate and the record must agree about which
4606 // declaration governs this run, and computing it twice is how they
4607 // start disagreeing.
4608 #[cfg(feature = "manifest")]
4609 let governed_by = self.identity_for(&agent);
4610 #[cfg(not(feature = "manifest"))]
4611 let governed_by: Option<crate::journal::AgentIdentity> = None;
4612
4613 if let Some(expected) = terms.expected_declaration {
4614 let found = governed_by.as_ref().map(|id| id.digest);
4615 if found != Some(expected) {
4616 return Err(RuntimeError::DeclarationPinMismatch {
4617 agent,
4618 expected,
4619 found,
4620 });
4621 }
4622 }
4623
4624 // The caller's chain where one was presented, the plane's otherwise —
4625 // one choice, made once, so the gate, the policy context and the
4626 // record cannot disagree about whom this run acts for.
4627 let chain = terms.acting_as.resolve(self.identity.as_ref());
4628 self.authorize_scope(
4629 &plan,
4630 terms.acting_as.ceiling(self.identity.as_ref()),
4631 chain.is_some(),
4632 )?;
4633 self.authorize_admission(&agent, governed_by.as_ref(), chain, input.peek())?;
4634 let input = self.admit_content(&agent, input)?;
4635
4636 self.admit_reserved(run, plan, input, terms, agent, governed_by)
4637 .await
4638 }
4639
4640 /// Admission past the quota reservation: the lease, the records, the case
4641 /// binding, the outbox.
4642 ///
4643 /// Every error path hands the lease back. A lease left standing over a
4644 /// failed admission is not idle bookkeeping: it expires unreleased, which
4645 /// is the exact signature the recovery sweep reads as "an instance died
4646 /// holding this run" — so a half-admitted run (records appended, outbox
4647 /// registration failed) would be *executed* by the sweeper a TTL later,
4648 /// having never been admitted. The one error that concludes rather than
4649 /// merely unwinding is the post-append one, where records already exist:
4650 /// the run is concluded `failed` in its own chain first, so nothing later
4651 /// reads an open journal as work in progress.
4652 async fn admit_reserved(
4653 &self,
4654 run: RunId,
4655 plan: PlanIR,
4656 input: Tainted<Value>,
4657 terms: RunTerms,
4658 agent: String,
4659 governed_by: Option<crate::journal::AgentIdentity>,
4660 ) -> Result<Admitted, RuntimeError> {
4661 // Ownership precedes reservation. A process can die between any two
4662 // store calls; with this order, a slot reserved before the admission
4663 // batch always has an owned lease beside it, and the abandonment sweep
4664 // can release both. Reserving first could strand a slot over no run and
4665 // no lease — invisible to every recovery query.
4666 let lease = self
4667 .store
4668 .acquire(run, &self.owner, self.lease_ttl)
4669 .await
4670 .map_err(RuntimeError::from_store)?;
4671
4672 let budget = self.budget_for(&agent);
4673 // Boxed: the quota decision holds its store round trips across awaits,
4674 // and inlined it pushes every admission future past the size clippy
4675 // allows on the served surfaces that await one.
4676 let quota = match Box::pin(self.check_quota(
4677 run,
4678 governed_by.as_ref(),
4679 // **The chain this run acts under, not the plane's.** A served
4680 // surface admits each run under its caller's chain, so reading
4681 // the plane's here would check a withdrawal against the
4682 // operator's own subject and never match the caller's — a
4683 // refusal that does not happen, which leaves no trace anywhere.
4684 terms.acting_as.resolve(self.identity.as_ref()),
4685 now_for_admission(),
4686 (&budget, reserved_width(&budget, &plan)),
4687 ))
4688 .await
4689 {
4690 Ok(pass) => pass,
4691 Err(error) => {
4692 let _ = self.store.release_lease(run, lease.epoch).await;
4693 return Err(error);
4694 }
4695 };
4696
4697 let out = self
4698 .admit_under_lease(run, &lease, plan, input, terms, &agent, governed_by, quota)
4699 .await;
4700 if out.is_err() {
4701 if let Some(quotas) = self.quotas.as_ref()
4702 && let Err(error) = quotas.release(run).await
4703 {
4704 tracing::debug!(%run, %error, "could not release the quota slot of a failed admission");
4705 }
4706 if let Err(error) = self.store.release_lease(run, lease.epoch).await {
4707 tracing::debug!(
4708 %run,
4709 %error,
4710 "could not release the lease of a failed admission; it will expire on its own"
4711 );
4712 }
4713 }
4714 out
4715 }
4716
4717 #[allow(clippy::too_many_arguments, clippy::too_many_lines)]
4718 async fn admit_under_lease(
4719 &self,
4720 run: RunId,
4721 lease: &crate::journal::Lease,
4722 plan: PlanIR,
4723 input: Tainted<Value>,
4724 terms: RunTerms,
4725 agent: &str,
4726 governed_by: Option<crate::journal::AgentIdentity>,
4727 quota: QuotaPass,
4728 ) -> Result<Admitted, RuntimeError> {
4729 let RunTerms {
4730 case,
4731 idempotency_key,
4732 acting_as,
4733 admitted_by,
4734 expected_declaration: _,
4735 subjects,
4736 } = terms;
4737 let chain = acting_as.resolve(self.identity.as_ref());
4738 let mut records = vec![
4739 Append::new(
4740 run,
4741 self.admission(
4742 agent,
4743 governed_by,
4744 &input,
4745 idempotency_key,
4746 admitted_by,
4747 &acting_as,
4748 ),
4749 ),
4750 // From here the plan is an authorization graph: compiled from
4751 // trusted input, frozen before anything untrusted was read, and
4752 // recorded so the journal that follows can be checked against it.
4753 Append::new(
4754 run,
4755 RecordKind::PlanFrozen {
4756 steps: plan
4757 .nodes
4758 .iter()
4759 .map(|n| n.capability.to_string())
4760 .collect(),
4761 plan: serde_json::to_value(&plan)?,
4762 },
4763 ),
4764 ];
4765
4766 Self::bind_identity(run, chain, &mut records);
4767 if let Some(started) = quota.started() {
4768 records.push(Append::new(run, started));
4769 }
4770
4771 #[cfg(feature = "manifest")]
4772 let declared: Vec<crate::journal::SubjectBinding> = self
4773 .resolve(agent)
4774 .ok()
4775 .and_then(|skill| self.governing(skill.as_ref()))
4776 .map(|m| {
4777 m.spec
4778 .data_subjects
4779 .iter()
4780 .map(crate::manifest::DataSubject::binding)
4781 .collect()
4782 })
4783 .unwrap_or_default();
4784 #[cfg(not(feature = "manifest"))]
4785 let declared: Vec<crate::journal::SubjectBinding> = Vec::new();
4786 // Before correlation, because `correlate_or_open` commits a case: a
4787 // binding that refuses the run must refuse it before anything outside
4788 // the journal names it. `$case` alone needs the case, and the case is
4789 // bound whenever the run asked for one, so it cannot refuse later.
4790 let bound = bind_subjects(&declared, subjects, &input, case.is_some())?;
4791
4792 // Correlation is deterministic and runs before planning: which case a
4793 // message belongs to is a matter of fact, settled by a lookup.
4794 let case_ctx = match (case, self.cases.as_ref()) {
4795 (Some(CaseBinding::Correlate { kind, keys }), Some(cases)) => {
4796 let correlation = cases
4797 .correlate_or_open(&kind, &keys, now_for_admission())
4798 .await
4799 .map_err(RuntimeError::from_store)?;
4800 let case_id = correlation.case_id();
4801 // The case's own keys, not the ones this message carried. A run
4802 // that *joined* an existing case may have been admitted with one
4803 // key while the case holds three, and a binding naming any of
4804 // them must resolve — which is the difference between "the keys
4805 // in this request" and "the business facts this matter is
4806 // identified by".
4807 let bound_keys = case_correlation(cases.as_ref(), case_id, &keys).await?;
4808 // Stamp the case on the records already queued as well: every
4809 // record of a case-bound run carries its case, which is what
4810 // makes "show me everything about this matter" one range scan.
4811 for r in &mut records {
4812 r.case = Some(case_id);
4813 }
4814 records.push(
4815 Append::new(
4816 run,
4817 RecordKind::CaseBound {
4818 case_kind: kind,
4819 opened: correlation.is_new(),
4820 correlation: bound_keys.clone(),
4821 },
4822 )
4823 .case(case_id),
4824 );
4825 Some(CaseContext {
4826 cases: Arc::clone(cases),
4827 tasks: self.tasks.clone(),
4828 events: self.events.clone(),
4829 calendar: Arc::clone(&self.calendar),
4830 case_id,
4831 correlation: bound_keys,
4832 })
4833 }
4834 (Some(CaseBinding::Existing(case_id)), Some(cases)) => {
4835 let existing = cases
4836 .case(case_id)
4837 .await
4838 .map_err(RuntimeError::from_store)?
4839 .ok_or_else(|| {
4840 RuntimeError::PlanContract(format!("no such case: {case_id}"))
4841 })?;
4842 if existing.status.is_closed() {
4843 return Err(RuntimeError::PlanContract(format!(
4844 "case '{case_id}' is closed and cannot accept another run"
4845 )));
4846 }
4847 for record in &mut records {
4848 record.case = Some(case_id);
4849 }
4850 records.push(
4851 Append::new(
4852 run,
4853 RecordKind::CaseBound {
4854 case_kind: existing.kind,
4855 opened: false,
4856 correlation: existing.correlation.clone(),
4857 },
4858 )
4859 .case(case_id),
4860 );
4861 Some(CaseContext {
4862 cases: Arc::clone(cases),
4863 tasks: self.tasks.clone(),
4864 events: self.events.clone(),
4865 calendar: Arc::clone(&self.calendar),
4866 case_id,
4867 correlation: existing.correlation,
4868 })
4869 }
4870 (Some(_), None) => {
4871 return Err(RuntimeError::PlanContract(
4872 "this run was admitted with correlation keys but the runtime has no case \
4873 store — build it with `.cases(store)`"
4874 .into(),
4875 ));
4876 }
4877 (None, _) => None,
4878 };
4879
4880 let bound = fill_case_subjects(bound, case_ctx.as_ref());
4881 let subject_refs = subject_refs(run, &bound);
4882 if !bound.is_empty() {
4883 let mut append = Append::new(run, RecordKind::DataSubjectBound { bindings: bound });
4884 append.case = case_ctx.as_ref().map(CaseContext::id);
4885 records.push(append);
4886 }
4887
4888 // The append claims the admission key, so a key already held is refused
4889 // here, before anything outside the journal names the run.
4890 self.store
4891 .append(lease.epoch, records)
4892 .await
4893 .map_err(RuntimeError::from_store)?;
4894
4895 // Registered before the run is handed back, and its failure fails
4896 // admission: a case that does not list the run, or destinations that
4897 // never see its history.
4898 //
4899 // After the append rather than before: a registration written for a
4900 // run whose admission then failed — or whose process died — would name
4901 // a run no journal holds, and nothing could tell which case to take it
4902 // off. The price of that order is paid here instead: the admission
4903 // records above are already durable, so a registration failure must
4904 // **conclude** the run rather than merely return — an open journal
4905 // under a lease that then lapses is exactly what the recovery sweep
4906 // reads as a crashed run, and it would execute a run whose admission
4907 // failed. Concluded `failed`, in the chain, with the reason beside it;
4908 // the caller's cleanup then releases the lease, and a resume registers
4909 // again before it executes.
4910 if let Err(open_failed) = self
4911 .register(run, case_ctx.as_ref().map(CaseContext::id))
4912 .await
4913 {
4914 let head = self
4915 .store
4916 .head(run)
4917 .await
4918 .map_err(RuntimeError::from_store)?;
4919 self.store
4920 .append(
4921 lease.epoch,
4922 vec![Append::new(
4923 run,
4924 RecordKind::RunConcluded {
4925 outcome: RunStatus::Failed(String::new()).as_str().to_owned(),
4926 reason: Some(format!(
4927 "the run's registration was refused ({open_failed}), so this \
4928 run was concluded without ever executing"
4929 )),
4930 exhaustion: None,
4931 live_spend: Spend::default(),
4932 chain_head: head.hash,
4933 },
4934 )],
4935 )
4936 .await
4937 .map_err(RuntimeError::from_store)?;
4938 return Err(RuntimeError::from_store(open_failed));
4939 }
4940
4941 Ok(Admitted {
4942 run,
4943 epoch: lease.epoch,
4944 quota,
4945 budget: self.budget_for(agent),
4946 agent: agent.to_owned(),
4947 plan,
4948 input,
4949 case: case_ctx,
4950 identity: chain.cloned(),
4951 subjects: subject_refs,
4952 })
4953 }
4954
4955 /// Execute a run that has already been admitted.
4956 async fn execute_admitted(&self, a: Admitted) -> Result<RunOutcome, RuntimeError> {
4957 let mut cursor = ReplayCursor::default();
4958 // Named, not `_`: `let _ = ` drops immediately, which would renew
4959 // nothing at all while looking exactly like this.
4960 let _heartbeat = self.heartbeat(a.run, a.epoch);
4961 self.execute(
4962 Execution {
4963 run: a.run,
4964 epoch: a.epoch,
4965 quota: a.quota,
4966 plan: &a.plan,
4967 input: a.input,
4968 mode: Pass::Live,
4969 case: a.case,
4970 budget: a.budget,
4971 agent: a.agent,
4972 identity: a.identity,
4973 subjects: a.subjects,
4974 refusal: None,
4975 withheld_subject: None,
4976 successors: Vec::new(),
4977 started: BTreeSet::new(),
4978 finished: BTreeSet::new(),
4979 succeeded: BTreeSet::new(),
4980 recorded_groups: BTreeMap::new(),
4981 },
4982 &mut cursor,
4983 )
4984 .await
4985 }
4986
4987 /// Admit and execute, which is what every blocking entry point does.
4988 ///
4989 /// The run id is the caller's rather than minted here, for batches: an
4990 /// item's run id is written to the batch store *before* the run starts, so
4991 /// that a crash leaves a reservation pointing at a journal that can be
4992 /// replayed rather than an item that must be guessed about.
4993 pub(crate) async fn admit_plan_as(
4994 &self,
4995 run: RunId,
4996 plan: PlanIR,
4997 input: Tainted<Value>,
4998 terms: RunTerms,
4999 entry: Entry,
5000 ) -> Result<RunOutcome, RuntimeError> {
5001 let admitted = self.admit_only(run, plan, input, terms, entry).await?;
5002 self.execute_admitted(admitted).await
5003 }
5004
5005 /// Why an open run may not cross its history frontier on this plane, if it
5006 /// may not.
5007 ///
5008 /// Two facts recorded at admission are compared with what this plane holds:
5009 /// the policy bundle (who may authorize an effect) and, for a declarative
5010 /// agent, the declaration (what the agent *is*). A resume under either one
5011 /// changed would continue one run's journal under another's rules. The
5012 /// answer is a refusal to journal rather than an error to raise; the caller
5013 /// decides where it goes.
5014 fn resume_refusal(&self, records: &[Record]) -> Result<Option<RuntimeError>, RuntimeError> {
5015 if let Some(refusal) = self.policy_bundle_refusal(records)? {
5016 return Ok(Some(refusal));
5017 }
5018 #[cfg(feature = "manifest")]
5019 if let Some(refusal) = self.declaration_refusal(records)? {
5020 return Ok(Some(refusal));
5021 }
5022 Ok(None)
5023 }
5024
5025 /// [`RuntimeError::NoProvider`] when a capability any of the run's plans
5026 /// names resolves to nothing on this plane.
5027 ///
5028 /// Every plan the run froze is checked, not only the one in force: an
5029 /// unwind resolves the steps that ran, and those may belong to a plan a
5030 /// replan replaced.
5031 fn refuse_undrivable<'p>(
5032 &self,
5033 plans: impl IntoIterator<Item = &'p PlanIR>,
5034 ) -> Result<(), RuntimeError> {
5035 for node in plans.into_iter().flat_map(|p| &p.nodes) {
5036 self.resolve(&node.capability.0)?;
5037 }
5038 Ok(())
5039 }
5040
5041 /// Name the run where it is watched from outside its journal: on its case,
5042 /// and to every outbox destination.
5043 ///
5044 /// Only after the admission's append, and idempotent — an existing case
5045 /// row and an existing delivery cursor are kept — so every writing resume
5046 /// repeats it.
5047 async fn register(
5048 &self,
5049 run: RunId,
5050 case: Option<crate::core::CaseId>,
5051 ) -> Result<(), crate::core::StoreError> {
5052 if let (Some(case), Some(cases)) = (case, self.cases.as_ref()) {
5053 cases.attach_run(case, run).await?;
5054 }
5055 #[cfg(feature = "push")]
5056 if let Some(outbox) = &self.outbox {
5057 outbox.open(run).await?;
5058 }
5059 Ok(())
5060 }
5061
5062 /// [`RuntimeError::NoCaseStore`] when the run is bound to a case and this
5063 /// plane has no case store.
5064 ///
5065 /// Everything a resume writes belongs to the case, and without the store
5066 /// it would land under the run alone, where erasing the case misses it.
5067 /// About this plane rather than the run, so another plane may resume it.
5068 fn refuse_caseless(&self, run: RunId, records: &[Record]) -> Result<(), RuntimeError> {
5069 match recorded_case_id(records) {
5070 Some(case) if self.cases.is_none() => Err(RuntimeError::NoCaseStore {
5071 run: run.to_string(),
5072 case: case.to_string(),
5073 }),
5074 _ => Ok(()),
5075 }
5076 }
5077
5078 /// Journal a resume refusal as the run's quarantine, naming what moved.
5079 async fn quarantine_refused(
5080 &self,
5081 run: RunId,
5082 records: &[Record],
5083 lease: &crate::journal::Lease,
5084 refusal: &RuntimeError,
5085 ) -> Result<RunOutcome, RuntimeError> {
5086 self.conclude(
5087 run,
5088 lease.epoch,
5089 RunStatus::Quarantined(refusal.to_string()),
5090 None,
5091 true,
5092 recorded_case_id(records),
5093 Consumed::default(),
5094 Spend::ZERO,
5095 &QuotaPass::disabled(),
5096 )
5097 .await
5098 }
5099
5100 /// The bundle half of [`resume_refusal`](Self::resume_refusal).
5101 ///
5102 /// Resume can dispatch new effects after it reaches the end of history.
5103 /// They must be judged by the same complete bundle recorded at admission,
5104 /// or one run would claim one policy while later effects were authorized
5105 /// by another.
5106 fn policy_bundle_refusal(
5107 &self,
5108 records: &[Record],
5109 ) -> Result<Option<RuntimeError>, RuntimeError> {
5110 let recorded = records
5111 .iter()
5112 .find_map(|record| match record.kind() {
5113 RecordKind::RunAdmitted { policy_bundle, .. } => {
5114 Some(policy_bundle.as_deref().cloned())
5115 }
5116 _ => None,
5117 })
5118 .ok_or_else(|| {
5119 RuntimeError::PlanContract("journal has no RunAdmitted record".into())
5120 })?;
5121 let configured = self.policy.as_ref().map(|policy| policy.bundle());
5122 if recorded != configured {
5123 return Ok(Some(RuntimeError::PolicyBundleChanged {
5124 recorded: recorded.as_ref().map(PolicyBundleIdentity::digest),
5125 configured: configured.as_ref().map(PolicyBundleIdentity::digest),
5126 }));
5127 }
5128 Ok(None)
5129 }
5130
5131 /// The declaration half of [`resume_refusal`](Self::resume_refusal).
5132 ///
5133 /// For a declarative agent the manifest is the whole behaviour, and a resume
5134 /// under an edited one runs a different program over the first one's
5135 /// journal — which replay would eventually catch as a key mismatch, several
5136 /// effects in and without being able to say what changed. The digest is
5137 /// already on the record at admission, so refusing here costs a comparison
5138 /// and turns a late diagnosis into a named remedy.
5139 ///
5140 /// **Silent in two cases, and deliberately.** A run admitted with no
5141 /// declaration is a coded skill, whose behaviour is the embedder's binary:
5142 /// this crate cannot identify that and will not pretend to. And a plane that
5143 /// holds no manifest under the recorded name cannot run the agent
5144 /// declaratively at all, so the refusal it needs is the one plan resolution
5145 /// already gives.
5146 #[cfg(feature = "manifest")]
5147 fn declaration_refusal(
5148 &self,
5149 records: &[Record],
5150 ) -> Result<Option<RuntimeError>, RuntimeError> {
5151 let Some(recorded) = records.iter().find_map(|record| match record.kind() {
5152 RecordKind::RunAdmitted { governed_by, .. } => governed_by.as_deref().cloned(),
5153 _ => None,
5154 }) else {
5155 return Ok(None);
5156 };
5157 let Some(current) = self
5158 .governed_by
5159 .values()
5160 .find(|m| m.metadata.name == recorded.name)
5161 else {
5162 return Ok(None);
5163 };
5164 let configured = current
5165 .digest()
5166 .map_err(|e| RuntimeError::PlanContract(e.to_string()))?;
5167 if configured != recorded.digest {
5168 return Ok(Some(RuntimeError::DeclarationChanged {
5169 agent: recorded.name,
5170 recorded: recorded.digest,
5171 configured,
5172 }));
5173 }
5174 Ok(None)
5175 }
5176
5177 /// A plan left sealed is erased only where a key ring could have opened
5178 /// it.
5179 ///
5180 /// A wired ring opens every payload whose key still exists and fails the
5181 /// read when it cannot be reached, so a payload still sealed behind one was
5182 /// erased. Behind no ring it says nothing about the key at all: the data
5183 /// is intact, and telling a reader it is gone sends them to the wrong
5184 /// remedy.
5185 fn sealed_or_erased(&self, error: RuntimeError) -> RuntimeError {
5186 match error {
5187 RuntimeError::PayloadsErased { run } if !self.holds_key_ring() => {
5188 RuntimeError::PayloadsSealed { run }
5189 }
5190 other => other,
5191 }
5192 }
5193
5194 /// Whether this plane was handed a key ring to open sealed payloads with.
5195 #[cfg_attr(not(feature = "keyring"), allow(clippy::unused_self))]
5196 const fn holds_key_ring(&self) -> bool {
5197 #[cfg(feature = "keyring")]
5198 {
5199 self.keyring.is_some()
5200 }
5201 #[cfg(not(feature = "keyring"))]
5202 {
5203 false
5204 }
5205 }
5206
5207 /// Re-execute a recorded run from its journal.
5208 ///
5209 /// * [`Mode::Strict`] verifies determinism: every effect must match, and the
5210 /// run must not want any effect the journal lacks.
5211 /// * [`Mode::Resume`] recovers a crashed run: history is replayed, then
5212 /// execution continues live from wherever the record ends.
5213 ///
5214 /// Either way no external effect is performed for anything already in the
5215 /// journal. That is the whole point — a resumed run does not re-issue the
5216 /// invoice it already issued.
5217 pub async fn replay(&self, run: RunId, mode: Mode) -> Result<RunOutcome, RuntimeError> {
5218 let mode = Pass::from(mode);
5219 // ── Ownership before observation ───────────────────────────────────
5220 //
5221 // Resume writes, so it takes the lease *before* it reads. The old
5222 // order — snapshot the records, then acquire — left a window in which
5223 // a live owner could append past the snapshot; the resume then
5224 // replayed a prefix of the true history and continued live from a
5225 // frontier that was not the frontier, re-deriving keys the owner had
5226 // already written. Claiming first means the snapshot below is taken
5227 // under this instance's own fence. Strict verifies and never writes,
5228 // so it needs no lease — and must not take one, or every regression
5229 // check would fence a run somebody may be legitimately running.
5230 let lease = if mode == Pass::Resume {
5231 Some(
5232 self.store
5233 .acquire(run, &self.owner, self.lease_ttl)
5234 .await
5235 .map_err(RuntimeError::from_store)?,
5236 )
5237 } else {
5238 None
5239 };
5240
5241 self.replay_releasing(run, mode, lease, Claim::Standing)
5242 .await
5243 }
5244
5245 /// Replay a recorded run strictly and say what it found.
5246 ///
5247 /// [`replay`](Self::replay) with [`Mode::Strict`], answered as a
5248 /// [`Verdict`](super::Verdict) rather than a status: *verified* when every
5249 /// recorded effect was asked for again and the run reached the ending it
5250 /// recorded, whatever that ending was; *diverged* at the first effect this
5251 /// build asked for differently; *cannot replay* when the run cannot be
5252 /// re-derived here, for a reason that is not the declaration. Both
5253 /// revisions are named on every answer — the one the run was admitted
5254 /// under and the one this plane holds by that name.
5255 ///
5256 /// Writes nothing, takes no lease and dispatches nothing, as a strict
5257 /// replay never does. It is the question an author asks of an edited
5258 /// manifest: replayed under the edit, this run's history either reaches
5259 /// nothing it changed or names the first effect it did.
5260 ///
5261 /// # Errors
5262 ///
5263 /// If the run does not exist or its journal does not verify, or on a store
5264 /// failure — never for a divergence or a reason the verdict names.
5265 pub async fn verify(&self, run: RunId) -> Result<super::Verdict, RuntimeError> {
5266 use super::{CannotReplay, Finding};
5267
5268 let records = self
5269 .store
5270 .read(run, 1)
5271 .await
5272 .map_err(RuntimeError::from_store)?;
5273 if records.is_empty() {
5274 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
5275 run.to_string(),
5276 )));
5277 }
5278 let recorded = records.iter().find_map(|r| match r.kind() {
5279 RecordKind::RunAdmitted { governed_by, .. } => governed_by.as_deref().cloned(),
5280 _ => None,
5281 });
5282 let candidate = self.candidate_revision(recorded.as_ref());
5283 let verdict = |finding| super::Verdict {
5284 run,
5285 recorded: recorded.clone(),
5286 candidate: candidate.clone(),
5287 finding,
5288 };
5289
5290 // A property of the history rather than of how it ended, so it is
5291 // named whatever the run's state: every key moves under another rule.
5292 if let Err(RuntimeError::CanonicalizationChanged {
5293 recorded,
5294 implemented,
5295 }) = ensure_replayable_canon(&records)
5296 {
5297 return Ok(verdict(Finding::CannotReplay(
5298 CannotReplay::CanonicalizationChanged {
5299 recorded,
5300 implemented,
5301 },
5302 )));
5303 }
5304 let Some(ending) = recorded_ending(&records) else {
5305 return Ok(verdict(Finding::CannotReplay(CannotReplay::NotConcluded)));
5306 };
5307 if ended_by_an_act(&ending) {
5308 return Ok(verdict(Finding::CannotReplay(CannotReplay::EndedByAnAct {
5309 outcome: ending,
5310 })));
5311 }
5312
5313 let mut dispatched = false;
5314 let mut divergence = None;
5315 let replayed = self
5316 .replay_under(
5317 run,
5318 Pass::Strict,
5319 None,
5320 Claim::Standing,
5321 &mut dispatched,
5322 &mut divergence,
5323 )
5324 .await;
5325 let finding = match (replayed, divergence) {
5326 (Ok(_), Some(d)) => Finding::Diverged(d),
5327 (Ok(outcome), None) if outcome.status.as_str() == ending => {
5328 Finding::Verified { outcome: ending }
5329 }
5330 (Ok(outcome), None) => Finding::OutcomeDiffers {
5331 recorded: ending,
5332 replayed: outcome.status.as_str().to_owned(),
5333 },
5334 (Err(RuntimeError::PayloadsErased { .. }), _) => {
5335 Finding::CannotReplay(CannotReplay::Erased)
5336 }
5337 (Err(RuntimeError::PayloadsSealed { .. }), _) => {
5338 Finding::CannotReplay(CannotReplay::KeyAbsent)
5339 }
5340 (Err(RuntimeError::NoProvider { target, .. }), _) => {
5341 Finding::CannotReplay(CannotReplay::EntryPointRemoved { capability: target })
5342 }
5343 (Err(other), _) => return Err(other),
5344 };
5345 Ok(verdict(finding))
5346 }
5347
5348 /// The declaration this plane holds under the recorded agent's name.
5349 #[cfg(feature = "manifest")]
5350 fn candidate_revision(
5351 &self,
5352 recorded: Option<&crate::journal::AgentIdentity>,
5353 ) -> Option<crate::journal::AgentIdentity> {
5354 let name = &recorded?.name;
5355 let held = self
5356 .governed_by
5357 .values()
5358 .find(|m| &m.metadata.name == name)?;
5359 self.identity_of(held)
5360 }
5361
5362 /// Without the manifest there are no declarations to name.
5363 #[cfg(not(feature = "manifest"))]
5364 #[allow(clippy::unused_self)]
5365 const fn candidate_revision(
5366 &self,
5367 _recorded: Option<&crate::journal::AgentIdentity>,
5368 ) -> Option<crate::journal::AgentIdentity> {
5369 None
5370 }
5371
5372 /// Resume a run under a lease the caller already holds.
5373 ///
5374 /// The wake path's handover: a timer or event delivery records the wake
5375 /// under its own lease and then resumes the run. Releasing that lease and
5376 /// letting the resume acquire a fresh one opened a window — a crash
5377 /// between the two left a *released* lease over a run whose timer was
5378 /// disarmed and whose subscription was retired, which no queue in the
5379 /// system names: the abandonment sweep looks only for leases that expired
5380 /// while still holding an owner. Handing the lease over closes the window;
5381 /// the run is owned continuously from wake to conclusion, and a crash
5382 /// anywhere in between leaves an expired *owned* lease the sweep drains.
5383 ///
5384 /// A resume that fails does the same thing a crash does, and for the same
5385 /// reason: the recorded wake took the run off the waiting listing, so the
5386 /// lease is the only thing left that names it.
5387 pub(crate) async fn resume_holding(
5388 &self,
5389 run: RunId,
5390 lease: crate::journal::Lease,
5391 ) -> Result<RunOutcome, RuntimeError> {
5392 self.replay_releasing(run, Pass::Resume, Some(lease), Claim::Unlisted)
5393 .await
5394 }
5395
5396 /// Recover an owner-abandoned run without turning settlement repair into
5397 /// an operator-requested retry of a recorded conclusion.
5398 ///
5399 /// A recovery that fails for a reason no retry changes — the journal, not
5400 /// the moment, is what refuses — quarantines the run under the lease the
5401 /// failed pass kept, naming the reason. Left alone it would lapse owned
5402 /// and be recovered, and refused, every lease period for ever; a
5403 /// quarantine is listed, counted, and answered by a person.
5404 pub(crate) async fn recover_abandoned_run(
5405 &self,
5406 run: RunId,
5407 ) -> Result<RunOutcome, RuntimeError> {
5408 let lease = self
5409 .store
5410 .acquire(run, &self.owner, self.lease_ttl)
5411 .await
5412 .map_err(RuntimeError::from_store)?;
5413 match self
5414 .replay_releasing(run, Pass::Resume, Some(lease.clone()), Claim::Recovering)
5415 .await
5416 {
5417 Err(error) if recurs_on_every_resume(&error) => {
5418 let records = self
5419 .store
5420 .read(run, 1)
5421 .await
5422 .map_err(RuntimeError::from_store)?;
5423 let refusal = RuntimeError::PlanContract(format!(
5424 "recovery refused, and would be refused the same way on every attempt: \
5425 {error}"
5426 ));
5427 self.quarantine_refused(run, &records, &lease, &refusal)
5428 .await
5429 }
5430 other => other,
5431 }
5432 }
5433
5434 async fn replay_releasing(
5435 &self,
5436 run: RunId,
5437 mode: Pass,
5438 lease: Option<crate::journal::Lease>,
5439 claim: Claim,
5440 ) -> Result<RunOutcome, RuntimeError> {
5441 let mut dispatched = false;
5442 let outcome = self
5443 .replay_under(run, mode, lease.as_ref(), claim, &mut dispatched, &mut None)
5444 .await;
5445
5446 // Idempotent: a resume that reached a conclusion already handed its
5447 // lease back in `conclude`, and re-releasing the same epoch changes
5448 // nothing. Released on success, and on a failure only where the run
5449 // is still listed somewhere else: the caller claimed a run whose
5450 // journal already says where it stands, and the refusal came before
5451 // anything past the frontier was written. Every other failure keeps
5452 // the lease so it lapses owned, which is what puts the run on the
5453 // abandonment listing the recovery sweep drains. A wake or a delivery
5454 // has already recorded the awaited result, so the run is off the
5455 // waiting listing; a recovery holds a run nothing else lists; and a
5456 // pass that failed mid-flight wrote records past a conclusion that no
5457 // longer describes it. Releasing any of those strands the run with no
5458 // driver and on no listing.
5459 let release = match &outcome {
5460 Ok(_) => true,
5461 // Settlement is retried by recovery, which finds the run through
5462 // this lease.
5463 Err(RuntimeError::QuotaSettlementPending { .. }) => false,
5464 Err(_) => claim == Claim::Standing && !dispatched,
5465 };
5466 if release
5467 && let Some(lease) = &lease
5468 && let Err(e) = self.store.release_lease(run, lease.epoch).await
5469 {
5470 tracing::debug!(
5471 %run,
5472 error = %e,
5473 "could not hand back the resume lease; it will expire on its own"
5474 );
5475 }
5476 outcome
5477 }
5478
5479 #[allow(clippy::too_many_lines)]
5480 async fn replay_under(
5481 &self,
5482 run: RunId,
5483 mode: Pass,
5484 lease: Option<&crate::journal::Lease>,
5485 claim: Claim,
5486 dispatched: &mut bool,
5487 divergence: &mut Option<crate::journal::Divergence>,
5488 ) -> Result<RunOutcome, RuntimeError> {
5489 let records = self
5490 .store
5491 .read(run, 1)
5492 .await
5493 .map_err(RuntimeError::from_store)?;
5494 if records.is_empty() {
5495 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
5496 run.to_string(),
5497 )));
5498 }
5499
5500 // Never trust a journal that does not verify. A tampered or truncated
5501 // history would let replay "confirm" something that never happened.
5502 //
5503 // Per-record hashes are already checked on read, so a single altered
5504 // record fails before we get here; this catches the structural attacks
5505 // that survive individually-valid records — deletion, reordering, and
5506 // splicing history from another run.
5507 Record::verify_chain(&records, Digest::ZERO).map_err(RuntimeError::from_store)?;
5508
5509 ensure_replayable_canon(&records)?;
5510
5511 if mode == Pass::Resume
5512 && let Some(outcome) = self
5513 .recover_quota_before_resume(
5514 run,
5515 &records,
5516 lease.expect("resume holds a lease"),
5517 claim == Claim::Recovering,
5518 )
5519 .await?
5520 {
5521 return Ok(outcome);
5522 }
5523
5524 if mode == Pass::Resume
5525 && let Some(outcome) = self
5526 .closed_resume_outcome(run, &records, lease.expect("resume holds a lease"))
5527 .await?
5528 {
5529 return Ok(outcome);
5530 }
5531
5532 // Placed *before* the compensation guard below on purpose: a run that
5533 // quarantined partway through an unwind is exactly the one an operator
5534 // has no other way to close, and it must not be the one this refuses to
5535 // close.
5536 if mode == Pass::Resume
5537 && let Some(outcome) = self
5538 .abandon_if_decided(run, &records, lease.expect("resume holds a lease"))
5539 .await?
5540 {
5541 return Ok(outcome);
5542 }
5543
5544 if mode == Pass::Resume {
5545 refuse_resume_over_undone_work(run, &records)?;
5546 }
5547
5548 // **Read after the conclusions above, and that ordering is the rule.**
5549 // A plan is what a run needs to *execute*; concluding one needs no
5550 // plan, and a cancellation and an abandonment are recorded in the
5551 // clear. Reading here rather than at the top of this function is what
5552 // leaves an operator a verb when the plan can no longer be read at all
5553 // — which is an ordinary state, because erasing a case's key erases
5554 // the plan of every run in it.
5555 let input = records
5556 .iter()
5557 .find_map(Record::admitted_input)
5558 .ok_or_else(|| {
5559 RuntimeError::PlanContract("journal has no RunAdmitted record".into())
5560 })?;
5561 let (plan, successors) =
5562 frozen_plans(run, &records).map_err(|e| self.sealed_or_erased(e))?;
5563
5564 // Only a plane that can drive the run judges it. A plane holding no
5565 // provider for the run's steps — an operator's terminal, a fleet
5566 // member serving other agents — answers `NoProvider` before comparing
5567 // its policy or declarations with the run's: a mismatch there says
5568 // nothing about the run, and quarantining it would stop a run this
5569 // plane could never have continued.
5570 if mode == Pass::Resume {
5571 self.refuse_undrivable(std::iter::once(&plan).chain(&successors))?;
5572 self.refuse_caseless(run, &records)?;
5573 }
5574
5575 // Resume can dispatch new effects past the end of history, so it is
5576 // held to the bundle and the declaration recorded at admission. Strict
5577 // replay performs no effects and remains usable as an offline verifier
5578 // without loading the historical engine.
5579 //
5580 // A refusal here is journaled as a quarantine rather than returned. The
5581 // callers that reach this unattended — a wake, a delivery, the recovery
5582 // sweep — have already recorded the result that took the run off its
5583 // waiting listing, and an error there reaches nobody: the run would
5584 // sit on no listing, or be retried every tick into the same refusal.
5585 // A quarantine is the state that already means *the runtime cannot go
5586 // on and a person must decide*, it is listed and counted, and both of
5587 // its answers work here — reopen on a plane holding the recorded
5588 // revision, or abandon.
5589 if mode == Pass::Resume
5590 && let Some(refusal) = self.resume_refusal(&records)?
5591 {
5592 let lease = lease.expect("resume holds a lease");
5593 return self
5594 .quarantine_refused(run, &records, lease, &refusal)
5595 .await;
5596 }
5597
5598 // A resume executes the run, so it registers it first: what it adds is
5599 // whatever an admission that failed or died partway never wrote.
5600 if mode == Pass::Resume {
5601 self.register(run, recorded_case_id(&records))
5602 .await
5603 .map_err(RuntimeError::from_store)?;
5604 }
5605
5606 *dispatched = true;
5607 // The agent recorded at admission, so a replay is bounded by the
5608 // ceilings the run actually had — and dispatched no wider than the
5609 // width its reservation was sized for.
5610 let budget = self.budget_for(&recorded_agent(&records));
5611 let quota = self
5612 .start_replay_quota_pass(run, mode, reserved_width(&budget, &plan))
5613 .await?;
5614 let through = self.marking_pass(run, "a);
5615 let this = through.as_ref().unwrap_or(self);
5616
5617 let case_ctx = self.recorded_case(&records);
5618
5619 let mut cursor = ReplayCursor::from_records(&records);
5620
5621 // Strict verification must not write, and it does not: it holds no
5622 // lease, and appends happen only past the end of history, which
5623 // Strict mode refuses to reach. Resume writes under the lease it
5624 // claimed before reading.
5625 let epoch = lease.map_or_else(|| records.last().map_or(1, |r| r.body.epoch), |l| l.epoch);
5626
5627 // Strict verification never writes, so it holds no lease to renew.
5628 let _heartbeat = lease.map(|l| self.heartbeat(run, l.epoch));
5629 this.execute(
5630 Execution {
5631 run,
5632 epoch,
5633 quota,
5634 plan: &plan,
5635 input,
5636 mode,
5637 case: case_ctx,
5638 budget,
5639 agent: recorded_agent(&records),
5640 identity: recorded_chain(&records)?,
5641 subjects: recorded_subjects(run, &records),
5642 // A step-level refusal has no effect key, so it cannot ride the
5643 // replay cursor like an effect's does. It is lifted here from
5644 // the records `replay` has already read.
5645 refusal: recorded_step_refusal(&records),
5646 withheld_subject: standing_withholding(&records),
5647 successors,
5648 started: recorded_started_steps(&records),
5649 finished: recorded_finished_steps(&records),
5650 succeeded: recorded_succeeded_steps(&records),
5651 recorded_groups: recorded_groups(&records),
5652 },
5653 &mut cursor,
5654 )
5655 .await
5656 .inspect(|_| *divergence = cursor.divergence().cloned())
5657 }
5658
5659 /// The trace root. Every span below is a child, so "what did this run do"
5660 /// is one query rather than a correlation exercise across logs.
5661 ///
5662 /// Instrumented rather than entered: an `Entered` guard held across an
5663 /// `.await` belongs to the thread, and with concurrent dispatch that
5664 /// reparents whatever runs next onto this span.
5665 async fn execute(
5666 &self,
5667 plan: Execution<'_>,
5668 cursor: &mut ReplayCursor,
5669 ) -> Result<RunOutcome, RuntimeError> {
5670 let span = tracing::info_span!(
5671 telemetry::RUN_SPAN,
5672 { telemetry::GEN_AI_OPERATION } = telemetry::GEN_AI_INVOKE_AGENT,
5673 { telemetry::RUN_ID } = tracing::field::display(plan.run),
5674 // Which declaration this run is executing. A plane serving several
5675 // agents cannot otherwise separate them, and separating them is
5676 // the first thing an operator does.
5677 { telemetry::GEN_AI_AGENT_NAME } = plan.agent.as_str(),
5678 { telemetry::MODE } = telemetry::mode_str(plan.mode),
5679 { telemetry::GEN_AI_CONVERSATION_ID } = plan
5680 .case
5681 .as_ref()
5682 .map(|c| super::ctx::CaseContext::id(c).to_string()),
5683 // Empty under the default policy, which is what `Omitted` means.
5684 // Recorded rather than declared so the attribute is absent rather
5685 // than blank — a collector cannot read *not disclosed* as *no
5686 // tenant* if there is no field at all.
5687 { telemetry::TENANT } = tracing::field::Empty,
5688 { telemetry::OUTCOME } = tracing::field::Empty,
5689 { telemetry::SEMCONV } = telemetry::SEMCONV_VERSION,
5690 { telemetry::OTEL_NAME } = format!("{} {}", telemetry::GEN_AI_INVOKE_AGENT, plan.agent),
5691 { telemetry::OTEL_KIND } = "internal",
5692 );
5693 if !self.meter.tenant().is_empty() {
5694 span.record(telemetry::TENANT, self.meter.tenant());
5695 }
5696 self.execute_inner(plan, cursor).instrument(span).await
5697 }
5698
5699 /// The driver loop: ready set, admit, dispatch, apply, repeat.
5700 ///
5701 /// Long by line count and deliberately not split further. Every *step* is
5702 /// already its own method — `admit_ready`, `dispatch`, `collect`, `apply`,
5703 /// `adopt_successor`, `stop`. What is left is the order they happen in, and
5704 /// that order is the algorithm. Breaking it up again would scatter one
5705 /// readable sequence across functions that exist only to satisfy a line
5706 /// count, which is the opposite of the thing the lint is protecting.
5707 #[allow(clippy::too_many_lines)]
5708 async fn execute_inner(
5709 &self,
5710 plan: Execution<'_>,
5711 cursor: &mut ReplayCursor,
5712 ) -> Result<RunOutcome, RuntimeError> {
5713 let Execution {
5714 run,
5715 epoch,
5716 quota,
5717 plan: ir,
5718 input,
5719 mode,
5720 case,
5721 budget,
5722 agent,
5723 identity,
5724 subjects,
5725 refusal: recorded_refusal,
5726 withheld_subject,
5727 successors,
5728 started,
5729 finished,
5730 succeeded,
5731 recorded_groups,
5732 } = plan;
5733 // Every value derived from what the run took in carries its subjects.
5734 let input = input.attributed(&subjects);
5735 let writing = !matches!(mode, Pass::Strict);
5736 let case_id = case.as_ref().map(super::ctx::CaseContext::id);
5737 let stamp = |a: Append| match case_id {
5738 Some(c) => a.case(c),
5739 None => a,
5740 };
5741
5742 // The plan in force. Owned rather than borrowed, because a replan
5743 // replaces it and a reference into the version list could not
5744 // survive that. `recorded_successors` is empty on a live run and seeded
5745 // from the journal on a replay, so a successor is read back rather than
5746 // re-synthesised.
5747 let mut current: PlanIR = ir.clone();
5748 let mut replans: u32 = 0;
5749 // A withholding the recorded history left standing, if it did. Seeded
5750 // from the records for the same reason the step refusal is: it is a fact
5751 // about *this* run that a resume has to answer, and asking the journal
5752 // again at every boundary would be a read per ready set for a question
5753 // whose answer only this loop can change.
5754 let mut withheld: Option<String> = withheld_subject;
5755 let recorded_successors = successors;
5756
5757 // One ledger for the run. A step never gets its own allowance to blow.
5758 let ledger = Arc::new(std::sync::Mutex::new(Ledger::new(budget)));
5759
5760 // Steps already completed, and what they produced. Rebuilt from the
5761 // journal on replay so a resumed run knows where it got to.
5762 let mut done: BTreeSet<StepId> = BTreeSet::new();
5763
5764 // The same steps as `done`, in the order they finished, **with the
5765 // capability that actually ran**. Unwinding needs both: a set has no
5766 // order to reverse, and after a replan the current plan may have
5767 // different work — or nothing at all — at a completed step's id.
5768 // Resolving the compensation from the live plan then undoes something
5769 // that never ran, which is a refund for a charge nobody made.
5770 let mut completed: Vec<(StepId, Capability)> = Vec::new();
5771 // The node each completed step ran as. A successor may carry a
5772 // completed step over only as this exact node: `done` is by id, so a
5773 // redeclared node would count as done on the strength of other work.
5774 let mut ran: BTreeMap<StepId, PlanNode> = BTreeMap::new();
5775 let mut outputs: BTreeMap<StepId, Tainted<Value>> = BTreeMap::new();
5776
5777 // ── Ready-set scheduling ───────────────────────────────────────────
5778 //
5779 // Dispatch order is a deterministic total order (ascending step id), so
5780 // replay reproduces it exactly. A plan with parallelism that
5781 // dispatched in completion order would replay differently every time.
5782 // The new work of a ready set whose recorded successes replay first.
5783 let mut deferred: Option<Vec<StepId>> = None;
5784 loop {
5785 let mut ready = current.ready(&done);
5786 if let Some(rest) = deferred.take() {
5787 let held: Vec<StepId> =
5788 ready.iter().copied().filter(|s| rest.contains(s)).collect();
5789 if !held.is_empty() {
5790 ready = held;
5791 }
5792 }
5793 // A resume's ready set can hold a step the record shows succeeding
5794 // beside one it does not — parallel work cut off mid-flight. The
5795 // recorded steps replay as a set of their own first, so the checks
5796 // below meet the frontier with their outputs already restored; the
5797 // rest is the next set, with nothing their replay made ready
5798 // joining it.
5799 if writing
5800 && ready.iter().any(|s| succeeded.contains(s))
5801 && ready.iter().any(|s| !succeeded.contains(s))
5802 {
5803 let (recorded, rest) = ready.into_iter().partition(|s| succeeded.contains(s));
5804 ready = recorded;
5805 deferred = Some(rest);
5806 }
5807 // ── The stop check, at a step boundary and nowhere else ────────
5808 //
5809 // Between boundaries an effect may be announced and not yet
5810 // recorded, and interrupting there manufactures the in-doubt case
5811 // the whole protocol exists to avoid. Checking here costs one
5812 // store read per ready set and buys a cancellation that can never
5813 // strand an effect.
5814 //
5815 // Skipped while replaying: a recorded run's history already
5816 // contains whatever stop it received, and re-reading the live
5817 // request would let a cancellation arriving *today* rewrite what a
5818 // run did last year.
5819 //
5820 // A resume asks only at its frontier — once the ready set holds a
5821 // step the record does not show succeeding. Before that, the
5822 // replayed steps have not yet refilled `completed` and `outputs`,
5823 // and an unwind from there would hand each compensation nothing
5824 // and walk the steps in id order rather than in reverse of the
5825 // order they finished.
5826 let at_frontier = ready.is_empty() || ready.iter().any(|s| !succeeded.contains(s));
5827 if writing
5828 && at_frontier
5829 && let Some(c) = self
5830 .store
5831 .cancellation(run)
5832 .await
5833 .map_err(RuntimeError::from_store)?
5834 {
5835 let status = RunStatus::Cancelled {
5836 actor: c.actor.clone(),
5837 reason: c.reason.clone(),
5838 };
5839 // Journaled *before* unwinding, so the reason the run stopped is
5840 // in the chain even if compensation then fails and quarantines
5841 // it. An operator reading a half-unwound run must be able to see
5842 // that somebody asked for this.
5843 self.store
5844 .append(
5845 epoch,
5846 vec![stamp(Append::new(
5847 run,
5848 RecordKind::RunCancelled {
5849 actor: c.actor,
5850 reason: c.reason,
5851 },
5852 ))],
5853 )
5854 .await
5855 .map_err(RuntimeError::from_store)?;
5856 // A step that paused with its group open holds work only that
5857 // step can take back: its reversals are built from the members'
5858 // outputs as the step runs. Re-run it once to abandon the group
5859 // — members read back, the pause aborts instead of waiting —
5860 // and only then unwind the steps before it.
5861 let abandon: Vec<StepId> = ready
5862 .iter()
5863 .copied()
5864 .filter(|&s| {
5865 recorded_groups.iter().any(|((step, phase, _), g)| {
5866 *step == s && *phase == Phase::Forward && g.opened > g.settled
5867 })
5868 })
5869 .collect();
5870 if !abandon.is_empty() {
5871 let dispatched = self
5872 .dispatch(
5873 &abandon,
5874 cursor,
5875 Batch {
5876 agent: &agent,
5877 identity: identity.as_ref(),
5878 subjects: &subjects,
5879 run,
5880 epoch,
5881 ir: ¤t,
5882 mode,
5883 case: &case,
5884 ledger: &ledger,
5885 writing,
5886 stamp: &stamp,
5887 input: &input,
5888 outputs: &outputs,
5889 started: &started,
5890 finished: &finished,
5891 recorded_groups: &recorded_groups,
5892 parallelism: 1,
5893 abandoning: true,
5894 },
5895 )
5896 .await;
5897 collect(dispatched, &abandon, cursor)?;
5898 }
5899 return self
5900 .stop(
5901 Unwind {
5902 agent: &agent,
5903 identity: identity.as_ref(),
5904 subjects: &subjects,
5905 run,
5906 epoch,
5907 ir: ¤t,
5908 mode,
5909 case: case.clone(),
5910 ledger: &ledger,
5911 writing,
5912 stamp: &stamp,
5913 },
5914 status,
5915 &completed,
5916 &outputs,
5917 cursor,
5918 case_id,
5919 "a,
5920 )
5921 .await;
5922 }
5923
5924 // ── The withdrawal check, at the same boundary ─────────────────
5925 //
5926 // Beside the stop check and for its reasons. Unlike a cancellation
5927 // this is a **pause**: `stop` is reached with a status
5928 // `maybe_unwind` passes straight through, as exhaustion is.
5929 if writing && at_frontier {
5930 let standing = self
5931 .withdrawal_against(identity.as_ref())
5932 .await
5933 .map_err(RuntimeError::Store)?;
5934 match (standing, withheld.take()) {
5935 // Lifted since the run was withheld: that decision is a
5936 // fact about the run and goes on the record, beside the
5937 // withholding it supersedes rather than instead of it.
5938 // Without it a strict replay would stop at the old pause and
5939 // report `withheld` about a run whose own later records show
5940 // it finishing.
5941 (None, Some(prior)) => {
5942 self.store
5943 .append(
5944 epoch,
5945 vec![stamp(Append::new(
5946 run,
5947 RecordKind::AuthorityRestored { subject: prior },
5948 ))],
5949 )
5950 .await
5951 .map_err(RuntimeError::from_store)?;
5952 }
5953 // Still withdrawn. The standing record already says so — no
5954 // second one, for the reason a re-exhausted run writes no
5955 // second refusal.
5956 // The prior is dropped rather than restored: this pass
5957 // returns, and the *records* are what a later resume reads
5958 // its standing state from.
5959 (Some(withdrawal), Some(_prior)) => {
5960 return self
5961 .withhold(
5962 Unwind {
5963 agent: &agent,
5964 identity: identity.as_ref(),
5965 subjects: &subjects,
5966 run,
5967 epoch,
5968 ir: ¤t,
5969 mode,
5970 case: case.clone(),
5971 ledger: &ledger,
5972 writing,
5973 stamp: &stamp,
5974 },
5975 withdrawal,
5976 false,
5977 &completed,
5978 &outputs,
5979 cursor,
5980 case_id,
5981 "a,
5982 )
5983 .await;
5984 }
5985 (Some(withdrawal), None) => {
5986 return self
5987 .withhold(
5988 Unwind {
5989 agent: &agent,
5990 identity: identity.as_ref(),
5991 subjects: &subjects,
5992 run,
5993 epoch,
5994 ir: ¤t,
5995 mode,
5996 case: case.clone(),
5997 ledger: &ledger,
5998 writing,
5999 stamp: &stamp,
6000 },
6001 withdrawal,
6002 true,
6003 &completed,
6004 &outputs,
6005 cursor,
6006 case_id,
6007 "a,
6008 )
6009 .await;
6010 }
6011 (None, None) => {}
6012 }
6013 }
6014 if ready.is_empty() {
6015 break;
6016 }
6017
6018 // Admission first, and deliberately not concurrent: which step a
6019 // ceiling refuses must be a property of the plan, not of which
6020 // future happened to poll first.
6021 let (admitted, refused) = self
6022 .admit_ready(
6023 &ready,
6024 &ledger,
6025 mode,
6026 recorded_refusal.as_ref(),
6027 cursor,
6028 Journalling {
6029 run,
6030 epoch,
6031 writing,
6032 stamp: &stamp,
6033 },
6034 )
6035 .await?;
6036
6037 let dispatched = self
6038 .dispatch(
6039 &admitted,
6040 cursor,
6041 Batch {
6042 agent: &agent,
6043 identity: identity.as_ref(),
6044 subjects: &subjects,
6045 run,
6046 epoch,
6047 ir: ¤t,
6048 mode,
6049 case: &case,
6050 ledger: &ledger,
6051 writing,
6052 stamp: &stamp,
6053 input: &input,
6054 outputs: &outputs,
6055 started: &started,
6056 finished: &finished,
6057 recorded_groups: &recorded_groups,
6058 parallelism: quota.parallelism(&budget),
6059 abandoning: false,
6060 },
6061 )
6062 .await;
6063 let outcomes = collect(dispatched, &ready, cursor)?;
6064
6065 // ── Strict verification consumes everything, per step ──────────
6066 //
6067 // Divergence has two directions, and the ordered key comparison
6068 // only sees one of them. A build that asks for a *different*
6069 // effect fails at the mismatch; a build that asks for **fewer**
6070 // effects than the record sails past the remainder and would
6071 // report the step verified — a false confirmation about exactly
6072 // the history a verification pass exists to check. So a step that
6073 // finished with journaled effects still unread is the same
6074 // quarantine-grade finding as a key mismatch, named by the first
6075 // effect nothing requested.
6076 if mode == Pass::Strict {
6077 for &(step, ref status, _) in &outcomes {
6078 // A step that stopped — suspended, failed, refused — ended
6079 // where its history says it ended, and the stop itself is
6080 // the verified finding. Only a step that claims to have
6081 // *finished* owes a fully consumed slice.
6082 if !matches!(status, RunStatus::Succeeded) {
6083 continue;
6084 }
6085 if let Some(missing) = cursor.unconsumed_in(step, Phase::Forward) {
6086 cursor.note_divergence(crate::journal::Divergence::unconsumed(
6087 step,
6088 Phase::Forward,
6089 &missing,
6090 ));
6091 self.meter.count(metrics::DIVERGENCES, "");
6092 tracing::error!(
6093 target: telemetry::NONDETERMINISM,
6094 %step, key = %missing.key, unconsumed = true,
6095 );
6096 return self
6097 .conclude(
6098 run,
6099 epoch,
6100 RunStatus::Quarantined(format!(
6101 "strict replay verified less than the run recorded: \
6102 step {step} finished with journaled effect {missing} \
6103 never requested — this build performs fewer effects than the \
6104 recorded one"
6105 )),
6106 None,
6107 writing,
6108 case_id,
6109 Consumed::default(),
6110 Spend::default(),
6111 "a,
6112 )
6113 .await;
6114 }
6115 }
6116 }
6117
6118 // A step that stops for any reason stops the run: whatever remains
6119 // either depended on it, or will be dispatched when it resumes.
6120 // Anything already done may need undoing first.
6121 let mut stopped = apply(
6122 ¤t,
6123 outcomes,
6124 &mut done,
6125 &mut completed,
6126 &mut ran,
6127 &mut outputs,
6128 )
6129 .or_else(|| refused.map(RunStatus::Exhausted));
6130
6131 if let Some(RunStatus::Replanning(reason)) = &stopped {
6132 let recorded = recorded_successors.get(replans as usize);
6133 let journal = Journalling {
6134 run,
6135 epoch,
6136 writing: writing && recorded.is_none(),
6137 stamp: &stamp,
6138 };
6139 let cx = Replan {
6140 run,
6141 current: ¤t,
6142 ran: &ran,
6143 reason,
6144 already_replanned: replans,
6145 max_replans: budget.max_replans,
6146 recorded,
6147 };
6148 match self
6149 .adopt_successor(cx, journal, &outputs, &completed)
6150 .await?
6151 {
6152 Ok(next) => {
6153 replans += 1;
6154 current = next;
6155 continue;
6156 }
6157 Err(refusal) => stopped = Some(refusal),
6158 }
6159 }
6160
6161 if let Some(status) = stopped {
6162 return self
6163 .stop(
6164 Unwind {
6165 agent: &agent,
6166 identity: identity.as_ref(),
6167 subjects: &subjects,
6168 run,
6169 epoch,
6170 ir: ¤t,
6171 mode,
6172 case: case.clone(),
6173 ledger: &ledger,
6174 writing,
6175 stamp: &stamp,
6176 },
6177 status,
6178 &completed,
6179 &outputs,
6180 cursor,
6181 case_id,
6182 "a,
6183 )
6184 .await;
6185 }
6186 }
6187
6188 // ── Strict verification consumes everything, at the end ────────────
6189 //
6190 // The per-batch check above covers steps this build dispatched; this
6191 // covers the ones it never asked for at all — a step a changed plan no
6192 // longer reaches, a compensation slice nothing requested. Either way the
6193 // record holds work this build cannot account for, and "verified" must
6194 // not be the answer.
6195 if mode == Pass::Strict
6196 && let Some((step, phase, missing)) = cursor.first_unconsumed()
6197 {
6198 cursor.note_divergence(crate::journal::Divergence::unconsumed(
6199 step, phase, &missing,
6200 ));
6201 self.meter.count(metrics::DIVERGENCES, "");
6202 tracing::error!(
6203 target: telemetry::NONDETERMINISM,
6204 %step, key = %missing.key, unconsumed = true,
6205 );
6206 return self
6207 .conclude(
6208 run,
6209 epoch,
6210 RunStatus::Quarantined(format!(
6211 "strict replay verified less than the run recorded: journaled \
6212 effect {missing} at step {step}, {phase:?} phase, was never \
6213 requested — this build performs fewer effects than the recorded one"
6214 )),
6215 None,
6216 writing,
6217 case_id,
6218 Consumed::default(),
6219 Spend::default(),
6220 "a,
6221 )
6222 .await;
6223 }
6224
6225 // Read into a value first. A `MutexGuard` built inline as an argument
6226 // lives until the end of the full expression — which here is *after*
6227 // the await — so the lock would be held across a suspension. That is
6228 // the same shape as the `Span::enter()` bug: a guard whose
6229 // scope is wider than it looks, and invisible until something else
6230 // needs the lock.
6231 let (consumed, live_spend) = {
6232 let l = ledger.lock().expect("budget mutex");
6233 (l.consumed(), l.live_spend())
6234 };
6235 self.conclude(
6236 run,
6237 epoch,
6238 completion(¤t, &done),
6239 run_output(¤t, &outputs),
6240 writing,
6241 case_id,
6242 consumed,
6243 live_spend,
6244 "a,
6245 )
6246 .await
6247 }
6248
6249 /// Read the clock into the ledger, when a wall-clock ceiling is declared.
6250 ///
6251 /// Separate so the guard's scope is two statements that cannot span the
6252 /// await between them, and so the condition is asked once rather than
6253 /// spelled at the caller.
6254 async fn observe_clock(
6255 cx: &mut StepCtx<'_>,
6256 ledger: &Arc<std::sync::Mutex<Ledger>>,
6257 ) -> Result<(), crate::core::StepError> {
6258 if !ledger.lock().expect("budget mutex").tracks_wallclock() {
6259 return Ok(());
6260 }
6261 let at = cx.now().await?;
6262 ledger.lock().expect("budget mutex").observe_clock(at);
6263 Ok(())
6264 }
6265
6266 /// Run a ready set concurrently, at most `max_parallel_steps` at a time.
6267 ///
6268 /// The ready set is every node whose predecessors are done, so nothing in
6269 /// it depends on anything else in it — running them one at a time is a
6270 /// choice, and the wrong one when steps are waiting on models and networks.
6271 ///
6272 /// Each step takes its own slice of history, which is what makes this sound:
6273 /// a step touches only its own effects, so no shared mutable state is left
6274 /// between them, and the per-step replay cursor verifies each one's order
6275 /// independently of how the journal happened to interleave them.
6276 ///
6277 /// The width is bounded because it is not otherwise bounded by anything:
6278 /// a fan-out is as wide as its author wrote it, and every step in flight
6279 /// holds a connection, a provider's rate limit and one operation's worth
6280 /// of spend the metered ceilings have checked and not yet billed. Ordered
6281 /// rather than completion-ordered — `buffered`, not `buffer_unordered` —
6282 /// so the outcomes still apply in ready order and a plan's dispatch is a
6283 /// property of the plan rather than of which future finished first.
6284 async fn dispatch(
6285 &self,
6286 admitted: &[StepId],
6287 cursor: &mut ReplayCursor,
6288 batch: Batch<'_>,
6289 ) -> Vec<Dispatched> {
6290 use futures_util::StreamExt;
6291
6292 let slices: Vec<(StepId, StepCursor)> = admitted
6293 .iter()
6294 .map(|&s| (s, cursor.take(s, Phase::Forward)))
6295 .collect();
6296 // Floored at one because a width of zero would poll nothing forever.
6297 // A zero ceiling is refused at parse and at build, so this is the
6298 // second half of a rule rather than the only half — and the failure it
6299 // guards against is a hang, which is the one shape a later reader
6300 // cannot diagnose from the outside.
6301 let width = batch.parallelism.max(1);
6302
6303 futures_util::stream::iter(slices.into_iter().map(|(step, slice)| async move {
6304 let node = batch
6305 .ir
6306 .node(step)
6307 .ok_or_else(|| RuntimeError::PlanContract(format!("no node {step}")))?;
6308 let (status, out, slice) = self
6309 .run_step(
6310 StepRun {
6311 agent: batch.agent,
6312 identity: batch.identity,
6313 subjects: batch.subjects,
6314 run: batch.run,
6315 epoch: batch.epoch,
6316 node,
6317 phase: Phase::Forward,
6318 mode: batch.mode,
6319 case: batch.case.clone(),
6320 ledger: batch.ledger,
6321 writing: batch.writing,
6322 stamp: batch.stamp,
6323 already_started: batch.started.contains(&step),
6324 already_finished: batch.finished.contains(&step),
6325 recorded_groups: batch
6326 .recorded_groups
6327 .iter()
6328 .filter(|((s, p, _), _)| *s == step && *p == Phase::Forward)
6329 .map(|((_, _, name), n)| (name.clone(), *n))
6330 .collect(),
6331 abandoning: batch.abandoning,
6332 },
6333 batch.input,
6334 batch.outputs,
6335 slice,
6336 )
6337 .await?;
6338 Ok((step, status, out, slice))
6339 }))
6340 .buffered(width)
6341 .collect()
6342 .await
6343 }
6344
6345 /// Take the successor a step asked for: check it, record it, announce it.
6346 ///
6347 /// The nested result separates two failures. The outer `RuntimeError` is the
6348 /// runtime itself failing — the journal would not accept the plan — and is
6349 /// never recoverable. The inner `RunStatus` is the *request* being refused,
6350 /// which is an ordinary outcome the run reports.
6351 async fn adopt_successor(
6352 &self,
6353 cx: Replan<'_>,
6354 journal: Journalling<'_>,
6355 outputs: &BTreeMap<StepId, Tainted<Value>>,
6356 completed: &[(StepId, Capability)],
6357 ) -> Result<Result<PlanIR, RunStatus>, RuntimeError> {
6358 // What each step that started and journaled an effect ran as, read
6359 // from the journal only when a successor is about to be checked — a
6360 // replay reads the successor back instead.
6361 let started = if cx.recorded.is_none() {
6362 let records = self
6363 .store
6364 .read(cx.run, 1)
6365 .await
6366 .map_err(RuntimeError::from_store)?;
6367 let effectful: BTreeSet<StepId> = records
6368 .iter()
6369 .filter(|r| r.body.phase.is_forward())
6370 .filter_map(|r| match r.kind() {
6371 RecordKind::EffectStarted { .. } => r.body.step,
6372 _ => None,
6373 })
6374 .collect();
6375 let mut started = started_skills(&records);
6376 started.retain(|step, _| effectful.contains(step));
6377 started
6378 } else {
6379 BTreeMap::new()
6380 };
6381 let next = match self.successor(cx, outputs, completed, &started).await {
6382 Ok(next) => next,
6383 Err(refusal) => return Ok(Err(refusal)),
6384 };
6385 if journal.writing {
6386 self.freeze(journal.run, journal.epoch, &next, journal.stamp)
6387 .await?;
6388 }
6389 announce_replan(
6390 &self.meter,
6391 journal.run,
6392 &next,
6393 next.reason.as_deref().unwrap_or(""),
6394 );
6395 Ok(Ok(next))
6396 }
6397
6398 /// Record a plan version in the journal.
6399 ///
6400 /// A successor is frozen exactly as a first plan is, so replay reads it back
6401 /// rather than asking a planner that may since have changed its mind.
6402 async fn freeze(
6403 &self,
6404 run: RunId,
6405 epoch: u64,
6406 plan: &PlanIR,
6407 stamp: &(dyn Fn(Append) -> Append + Send + Sync),
6408 ) -> Result<(), RuntimeError> {
6409 self.store
6410 .append(
6411 epoch,
6412 vec![stamp(Append::new(
6413 run,
6414 RecordKind::PlanFrozen {
6415 steps: plan.nodes.iter().map(|n| n.capability.0.clone()).collect(),
6416 plan: serde_json::to_value(plan)?,
6417 },
6418 ))],
6419 )
6420 .await
6421 .map_err(RuntimeError::from_store)?;
6422 Ok(())
6423 }
6424
6425 /// Produce the successor plan a step asked for, or say why not.
6426 ///
6427 /// Three gates, and the first is not negotiable.
6428 ///
6429 /// **Provenance.** The frozen plan is an authorization graph compiled from
6430 /// trusted input only. A replan *changes that graph*, so once any
6431 /// untrusted value has reached working memory, anything shaping the new plan
6432 /// may be attacker-chosen — and choosing the authorization graph is the
6433 /// whole game. `plan-then-execute` is enforced here, structurally. A run
6434 /// that wants a different plan after reading untrusted input is describing
6435 /// exactly the attack.
6436 ///
6437 /// **Budget.** A run that replans without bound has stopped making progress
6438 /// and started thrashing.
6439 ///
6440 /// **On replay, the successor is read back, never re-synthesised.** A
6441 /// planner asked twice can answer differently — a changed router, a
6442 /// different model — and replay would then verify the run against a plan
6443 /// that never governed it. Same rule as the first plan, for the same
6444 /// reason.
6445 async fn successor(
6446 &self,
6447 cx: Replan<'_>,
6448 outputs: &BTreeMap<StepId, Tainted<Value>>,
6449 completed: &[(StepId, Capability)],
6450 started: &BTreeMap<StepId, String>,
6451 ) -> Result<PlanIR, RunStatus> {
6452 if let Some(source) = untrusted_in(outputs) {
6453 return Err(RunStatus::Failed(format!(
6454 "replanning refused: untrusted data from {source} is already in working memory, and the plan is an authorization graph — letting it change now would let that data choose what runs next ({})",
6455 cx.reason
6456 )));
6457 }
6458
6459 let spent = cx.already_replanned;
6460 if let Some(max) = cx.max_replans
6461 && spent >= max
6462 {
6463 return Err(RunStatus::Exhausted(crate::core::BudgetExceeded::Replans {
6464 allowed: max,
6465 }));
6466 }
6467
6468 // Replay: the successor is in the journal, at the position this replan
6469 // reached. Reading it back is what keeps a re-planned run replayable.
6470 if let Some(recorded) = cx.recorded {
6471 return Ok(recorded.clone());
6472 }
6473
6474 let replanner = self.replanner.as_ref().ok_or_else(|| {
6475 RunStatus::Failed(format!(
6476 "a step asked to replan and this runtime has no planner — build it with `.replanner(..)` ({})",
6477 cx.reason
6478 ))
6479 })?;
6480
6481 let next = replanner
6482 .replan(cx.current, cx.reason, completed)
6483 .await
6484 .map_err(|e| RunStatus::Failed(format!("replanning failed: {e}")))?;
6485
6486 // A successor faces the same contract a first plan does. One that fails
6487 // validation stops the run rather than half-applying.
6488 crate::plan::validate(&next, &self.contract())
6489 .map_err(|e| RunStatus::Failed(format!("the successor plan is invalid: {e}")))?;
6490
6491 // A completed step carries over only as the node it ran as, or not at
6492 // all. Other work at its id breaks replay — effect keys derive from the
6493 // step id — and has the saga undo something that never ran. The same
6494 // capability redeclared is refused too: `done` is by id, so a node with
6495 // new arguments, dependencies or flags would count as done on the
6496 // strength of work done before any of them held — a terminal verifier
6497 // satisfied without ever seeing what it verifies.
6498 for (step, node_ran) in cx.ran {
6499 let Some(node) = next.node(*step) else {
6500 continue;
6501 };
6502 if node.capability != node_ran.capability {
6503 return Err(RunStatus::Failed(format!(
6504 "the successor plan reuses step {step} — which already ran \
6505 as '{}' — for '{}'. Keep a completed step's node or leave \
6506 the step out; effect keys are derived from the step id, so \
6507 new work at a used id cannot be replayed",
6508 node_ran.capability.0, node.capability.0
6509 )));
6510 }
6511 if node != node_ran {
6512 return Err(RunStatus::Failed(format!(
6513 "the successor plan redeclares completed step {step} ('{}') \
6514 with different arguments, dependencies or flags. A \
6515 completed step counts as done by its id, so the new node \
6516 would be satisfied by work done under the old one — carry \
6517 it over unchanged, or give the new work a fresh id",
6518 node.capability.0
6519 )));
6520 }
6521 }
6522
6523 // A step that started, journaled an effect and did not complete holds
6524 // keys derived from its id just the same — it may have charged before
6525 // asking to replan — so its id may not be reused for work another
6526 // skill would do. One that journaled nothing, the step that asked for
6527 // a fallback before touching anything, leaves its id free.
6528 for (step, skill) in started {
6529 if cx.ran.contains_key(step) {
6530 continue;
6531 }
6532 let Some(node) = next.node(*step) else {
6533 continue;
6534 };
6535 let resolved = self
6536 .resolve(&node.capability.0)
6537 .map(|s| s.descriptor().name)
6538 .ok();
6539 if resolved.as_deref() != Some(skill.as_str()) {
6540 return Err(RunStatus::Failed(format!(
6541 "the successor plan reuses step {step} — which already \
6542 started as '{skill}' and journaled effects — for '{}'. Keep the step's \
6543 capability or leave the step out; effect keys are derived \
6544 from the step id, and the unwind undoes a started step as \
6545 the skill that ran",
6546 node.capability.0
6547 )));
6548 }
6549 }
6550
6551 if next.derived_from != Some(cx.current.digest()) {
6552 return Err(RunStatus::Failed(
6553 "the successor plan does not name its predecessor — use `PlanIR::succeed_with`, or the audit trail has a hole where the lineage should be"
6554 .into(),
6555 ));
6556 }
6557
6558 // The third leg of the lineage `succeed_with` sets. A successor with
6559 // no reason would freeze into the journal as a plan that replaced
6560 // another with nothing on the record saying why — the half of an
6561 // incident (*what changed the run's mind*) that versioned replanning
6562 // exists to keep.
6563 if next.reason.as_deref().is_none_or(str::is_empty) {
6564 return Err(RunStatus::Failed(
6565 "the successor plan carries no reason — use `PlanIR::succeed_with`, or the \
6566 journal freezes a plan that replaced another with nothing saying why"
6567 .into(),
6568 ));
6569 }
6570
6571 Ok(next)
6572 }
6573
6574 /// Decide which of a ready set may start, in ready order.
6575 ///
6576 /// Three regimes, one per mode, and the middle one is the subtle case:
6577 ///
6578 /// * **Strict** consumes recorded verdicts and recomputes none. A run
6579 /// replayed under a larger budget stopped where it stopped; recomputing
6580 /// made a step-limited run replay as *succeeded* — a false audit result.
6581 /// * **Resume** takes each verdict from wherever the truth lives. A step
6582 /// with unconsumed history was admitted by the run that recorded it,
6583 /// and the history *is* the verdict. A step at the frontier is about to
6584 /// run **live**, and every live step is metered — keying this on "is
6585 /// the mode a replay" instead of "will this step dispatch live" let a
6586 /// resumed run cross `max_steps` unmetered for as long as it kept
6587 /// running. A *recorded refusal* is re-evaluated against the ledger now
6588 /// in force: exhaustion is a pause, and raising the ceiling and
6589 /// resuming is how it un-pauses.
6590 /// * **Live** asks the ledger, which is the ordinary case.
6591 async fn admit_ready(
6592 &self,
6593 ready: &[StepId],
6594 ledger: &Arc<std::sync::Mutex<Ledger>>,
6595 mode: Pass,
6596 recorded: Option<&(StepId, String, String)>,
6597 cursor: &ReplayCursor,
6598 journal: Journalling<'_>,
6599 ) -> Result<(Vec<StepId>, Option<crate::core::BudgetExceeded>), RuntimeError> {
6600 let mut admitted = Vec::new();
6601
6602 for &step in ready {
6603 // ── A recorded step refusal at this step ───────────────────────
6604 if let Some((at, limit, used)) = recorded
6605 && *at == step
6606 {
6607 if mode == Pass::Strict {
6608 // Verbatim: the recorded run stopped here, and a
6609 // verification pass reaches the same conclusion without
6610 // consulting today's ledger.
6611 return Ok((
6612 admitted,
6613 Some(crate::core::BudgetExceeded::Recorded {
6614 limit: limit.clone(),
6615 used: used.clone(),
6616 }),
6617 ));
6618 }
6619 // Resume: the replayed prefix has already billed the journaled
6620 // historical usage into this ledger, so the question is "does
6621 // the budget now in force admit one more step over what the
6622 // run already spent". Still no → the run concludes exhausted
6623 // again, and the standing refusal already says so — no second
6624 // record. Yes → the re-admission is its own record, beside
6625 // the refusal it supersedes, so a strict replay of this
6626 // history reads refusal-then-continuation as a decision
6627 // somebody made rather than as divergence.
6628 if let Err(exceeded) = ledger
6629 .lock()
6630 .expect("budget mutex")
6631 .admit_step(admitted.len())
6632 {
6633 return Ok((admitted, Some(exceeded)));
6634 }
6635 if journal.writing {
6636 self.store
6637 .append(
6638 journal.epoch,
6639 vec![(journal.stamp)(
6640 Append::new(
6641 journal.run,
6642 RecordKind::BudgetReadmitted {
6643 limit: limit.clone(),
6644 },
6645 )
6646 .step(step),
6647 )],
6648 )
6649 .await
6650 .map_err(RuntimeError::from_store)?;
6651 }
6652 admitted.push(step);
6653 continue;
6654 }
6655
6656 let verdict = match mode {
6657 Pass::Strict => Ok(()),
6658 Pass::Resume if !cursor.exhausted(step, Phase::Forward) => Ok(()),
6659 Pass::Live | Pass::Resume => ledger
6660 .lock()
6661 .expect("budget mutex")
6662 .admit_step(admitted.len()),
6663 };
6664
6665 let Err(exceeded) = verdict else {
6666 admitted.push(step);
6667 continue;
6668 };
6669
6670 if journal.writing {
6671 // The refusal goes in the journal under the step it refused, so
6672 // replay reads the verdict rather than recomputing it.
6673 let used = format!("{:?}", ledger.lock().expect("budget mutex").consumed());
6674 self.store
6675 .append(
6676 journal.epoch,
6677 vec![(journal.stamp)(
6678 Append::new(
6679 journal.run,
6680 RecordKind::BudgetRefused {
6681 limit: exceeded.to_string(),
6682 used,
6683 },
6684 )
6685 .step(step),
6686 )],
6687 )
6688 .await
6689 .map_err(RuntimeError::from_store)?;
6690 }
6691 return Ok((admitted, Some(exceeded)));
6692 }
6693
6694 Ok((admitted, None))
6695 }
6696
6697 /// End a run that stopped early: undo what warrants undoing, then seal.
6698 ///
6699 /// Both places a run can stop short go through here, so the unwind can never
6700 /// be attached to one of them and forgotten on the other.
6701 #[allow(clippy::too_many_arguments)]
6702 async fn stop(
6703 &self,
6704 cx: Unwind<'_>,
6705 status: RunStatus,
6706 completed: &[(StepId, Capability)],
6707 outputs: &BTreeMap<StepId, Tainted<Value>>,
6708 cursor: &mut ReplayCursor,
6709 case_id: Option<crate::core::CaseId>,
6710 quota: &QuotaPass,
6711 ) -> Result<RunOutcome, RuntimeError> {
6712 let (run, epoch, writing, ir) = (cx.run, cx.epoch, cx.writing, cx.ir);
6713 let output = run_output(ir, outputs);
6714 // Cloned before `maybe_unwind` consumes `cx`; read *after* it, because an
6715 // item's cost is what the whole attempt consumed, compensation included.
6716 let ledger = cx.ledger.clone();
6717 let unwound = self
6718 .maybe_unwind(cx, status, completed, outputs, cursor)
6719 .await?;
6720 // A strict pass that stops early is still a verification, and its
6721 // verdict must not outrun what it consumed. The ready loop's
6722 // unconsumed-effects check runs only when the loop drains normally,
6723 // and every early conclusion — a failure, an unwind, a recorded
6724 // suspension — comes through here instead; without this check a build
6725 // that performs fewer effects than the record (a compensation that
6726 // shrank, a step that now suspends earlier) would return the recorded
6727 // status as "verified" over effects it never requested. Checked after
6728 // `maybe_unwind` because a strict pass consumes recorded compensation
6729 // slices there, and what remains after that is genuinely unclaimed.
6730 //
6731 // A run that already quarantined keeps its own reason. Divergence
6732 // leaves history unconsumed as a matter of course — the step stopped
6733 // where it disagreed — so reporting the leftovers here would replace
6734 // the precise finding ("expected this effect, recomputed that one")
6735 // with a vaguer restatement of its consequence, and an operator would
6736 // be told the build performs fewer effects when what it actually did
6737 // was perform a different one.
6738 if !writing
6739 && !matches!(unwound, RunStatus::Quarantined(_))
6740 && let Some((step, phase, missing)) = cursor.first_unconsumed()
6741 {
6742 cursor.note_divergence(crate::journal::Divergence::unconsumed(
6743 step, phase, &missing,
6744 ));
6745 self.meter.count(metrics::DIVERGENCES, "");
6746 tracing::error!(
6747 target: telemetry::NONDETERMINISM,
6748 %step, key = %missing.key, unconsumed = true,
6749 );
6750 return self
6751 .conclude(
6752 run,
6753 epoch,
6754 RunStatus::Quarantined(format!(
6755 "strict replay verified less than the run recorded: journaled \
6756 effect {missing} at step {step}, {phase:?} phase, was never \
6757 requested — this build performs fewer effects than the recorded one"
6758 )),
6759 None,
6760 writing,
6761 case_id,
6762 Consumed::default(),
6763 Spend::default(),
6764 quota,
6765 )
6766 .await;
6767 }
6768 // Scoped before the await — see `execute_inner` on why an inline guard
6769 // outlives the call it is passed to.
6770 let (consumed, live_spend) = {
6771 let l = ledger.lock().expect("budget mutex");
6772 (l.consumed(), l.live_spend())
6773 };
6774 self.conclude(
6775 run, epoch, unwound, output, writing, case_id, consumed, live_spend, quota,
6776 )
6777 .await
6778 }
6779
6780 /// Undo the completed steps, if the way the run stopped calls for it.
6781 ///
6782 /// # When a run does *not* unwind
6783 ///
6784 /// * **Quarantined.** The run holds an effect whose outcome is unknown, and
6785 /// you cannot safely undo around one: compensating a payment that may
6786 /// never have gone out creates a refund for money nobody took. Everything
6787 /// stays exactly where it is until a person supplies the fact the journal
6788 /// lacks — [`Runtime::reconcile_effect`](super::Runtime::reconcile_effect)
6789 /// and [`decide_quarantine`](super::Runtime::decide_quarantine) — and
6790 /// giving up
6791 /// ([`Abandon`](crate::core::QuarantineDecision::Abandon)) still unwinds
6792 /// nothing, because impatience is not evidence. This is the rule that
6793 /// separates a saga that is honest about distributed systems from one that
6794 /// tidies up and hopes.
6795 /// * **Suspended.** The run is healthy and waiting. Nothing has failed.
6796 ///
6797 /// A failure *after the pivot* also stops the unwind at the pivot: once the
6798 /// business has committed, reversing the decisions leading up to it would
6799 /// contradict something the outside world has already acted on.
6800 ///
6801 /// # Two rules every unwind takes
6802 ///
6803 /// **It must not unwind around an unknown outcome.** The same rule
6804 /// quarantine already enforces, applied at the unwind's own door. For a
6805 /// cancelled run the check is unconditional — cancellation gets no second
6806 /// pass, so an unresolved outcome stays unresolved forever. For a failed
6807 /// run it is conditional on the unwind list: an orphan under a failure is
6808 /// resolvable, because the announcement is journaled, the effect declared
6809 /// a `Recovery`, and resuming resolves it — but only while nothing was
6810 /// compensated, since compensation closes the run to resume. A failure
6811 /// that would compensate around doubt therefore quarantines instead; a
6812 /// failure with nothing to unwind stays a failure and keeps its resume.
6813 ///
6814 /// **It must undo the steps it interrupted.** Compensation walks
6815 /// *completed* steps, which covers the step that failed only if it never
6816 /// mutated. A stop arrives from outside while a step is typically
6817 /// suspended — holding effects it performed and never completing — and a
6818 /// failure in one concurrent sibling interrupts the others the same way:
6819 /// severity ordering can conclude the run while a suspended sibling holds
6820 /// a landed mutation. Unwinding only completed steps would leave exactly
6821 /// that work standing — a run that posted to a ledger and then suspended
6822 /// for approval would conclude with the posting still in the world and,
6823 /// once anything else was compensated, no resume left to finish it.
6824 async fn maybe_unwind(
6825 &self,
6826 cx: Unwind<'_>,
6827 status: RunStatus,
6828 completed: &[(StepId, Capability)],
6829 outputs: &BTreeMap<StepId, Tainted<Value>>,
6830 cursor: &mut ReplayCursor,
6831 ) -> Result<RunStatus, RuntimeError> {
6832 match status {
6833 // A cancelled run unwinds exactly as a failed one does. Stopping a
6834 // run that has already moved money and leaving the movement in
6835 // place is not stopping it — and the operator who asked is entitled
6836 // to assume "stop" means the world is put back, not that the
6837 // process merely exited.
6838 //
6839 // **Exhausted is deliberately not here.** Exhaustion is a pause,
6840 // not a fault: the run did what it was told, and what it was told
6841 // included a ceiling. Its mutations stand, because the operator's
6842 // two honest options both need them standing — raise the ceiling
6843 // and resume (which continues *over* the completed work), or
6844 // cancel (which unwinds through this same function under a status
6845 // that does). Unwinding on exhaustion would make the three ends of
6846 // an exhausted run contradict each other: the work reversed, the
6847 // run still resumable, and the resume reporting success over a
6848 // world where the work no longer stands.
6849 RunStatus::Failed(_) | RunStatus::Cancelled { .. } => {}
6850 other => return Ok(other),
6851 }
6852
6853 let (list, evidence) = match self.gated_unwind_list(&cx, &status, completed).await? {
6854 Ok(scope) => scope,
6855 Err(quarantine) => return Ok(quarantine),
6856 };
6857 let UnwindEvidence {
6858 mutated,
6859 started: _,
6860 undone: already_undone,
6861 recorded_groups,
6862 failed: _,
6863 } = evidence;
6864 let completed = &list[..];
6865
6866 for (step, capability) in completed.iter().rev().cloned() {
6867 // Resolved from what ran, not from the plan in force. After a replan
6868 // the two can differ, and undoing whatever now occupies that slot is
6869 // how a saga compensates work that never happened.
6870 let skill = self.resolve(&capability.0)?;
6871 let declared = skill.compensation();
6872
6873 match declared {
6874 // The point of no return. Everything from here back stays.
6875 crate::core::Compensation::Pivot => break,
6876 crate::core::Compensation::Unnecessary => continue,
6877 crate::core::Compensation::Undeclared => {
6878 if !mutated.contains(&step) {
6879 // Nothing to undo, and the journal proves it.
6880 continue;
6881 }
6882 return Ok(RunStatus::Quarantined(format!(
6883 "step {step} ('{}') changed external state and declares no \
6884 compensation, so the run cannot be safely unwound — \
6885 declare Compensation on it, or resolve this by hand",
6886 capability.0
6887 )));
6888 }
6889 crate::core::Compensation::Compensatable => {}
6890 }
6891
6892 let result = self
6893 .run_compensation(
6894 &cx,
6895 step,
6896 skill.as_ref(),
6897 outputs,
6898 cursor,
6899 recorded_groups
6900 .iter()
6901 .filter(|((s, p, _), _)| *s == step && *p == Phase::Compensating)
6902 .map(|((_, _, name), n)| (name.clone(), *n))
6903 .collect(),
6904 )
6905 .await;
6906
6907 // A compensation may legitimately need to wait — a refund that needs
6908 // four eyes is still a refund. Suspension is not failure: the run is
6909 // healthy, its frame is durable, and it will finish unwinding when
6910 // the answer arrives. Reporting it as a failure would quarantine a
6911 // run doing exactly the right thing and tell the operator the
6912 // compensation had broken.
6913 if let Err(crate::core::SkillError::Step(crate::core::StepError::Suspended(reason))) =
6914 &result
6915 {
6916 return Ok(RunStatus::Suspended(reason.clone()));
6917 }
6918
6919 let outcome = match &result {
6920 Ok(()) => "compensated".to_owned(),
6921 Err(e) => e.to_string(),
6922 };
6923
6924 // Re-run but do not re-record. On resume the compensation executes
6925 // again with every effect served from the journal, which is what
6926 // keeps strict verification meaningful — but a second
6927 // `StepCompensated` would report one compensation as two.
6928 if result.is_ok() {
6929 tracing::info!(target: telemetry::COMPENSATED, run = %cx.run, %step);
6930 self.meter.count(metrics::COMPENSATIONS, "done");
6931 }
6932 if cx.writing && !already_undone.contains(&step) {
6933 self.store
6934 .append(
6935 cx.epoch,
6936 vec![(cx.stamp)(
6937 Append::new(
6938 cx.run,
6939 RecordKind::StepCompensated {
6940 compensation: declared,
6941 outcome: outcome.clone(),
6942 },
6943 )
6944 .step(step)
6945 .phase(Phase::Compensating),
6946 )],
6947 )
6948 .await
6949 .map_err(RuntimeError::from_store)?;
6950 }
6951
6952 if let Err(fault) = &result {
6953 tracing::error!(
6954 target: telemetry::COMPENSATION_FAILED,
6955 run = %cx.run,
6956 %step,
6957 error_type = fault.class(),
6958 reason_digest = %telemetry::reason_digest(&outcome),
6959 );
6960 self.meter.count(metrics::COMPENSATIONS, "failed");
6961 // Not a problem more compensation solves. Unwinding further
6962 // would undo steps *before* one that is now in an unknown
6963 // state, which is strictly worse than stopping and saying so.
6964 return Ok(RunStatus::Quarantined(format!(
6965 "compensation failed for step {step} ('{}'): {outcome} — the run is \
6966 partially unwound and needs an operator",
6967 capability.0
6968 )));
6969 }
6970 }
6971
6972 Ok(status)
6973 }
6974
6975 /// Whether `run`'s journal holds a landed mutation no compensation undid.
6976 ///
6977 /// The attention roll-up's question about a failed run: `failed` alone may
6978 /// be cleared by a resume, but one standing on landed work leaves that
6979 /// work in the world until somebody finishes or unwinds it.
6980 pub(super) async fn holds_landed_work(&self, run: RunId) -> Result<bool, RuntimeError> {
6981 let evidence = self.unwind_evidence(run).await?;
6982 Ok(!evidence.mutated.is_subset(&evidence.undone))
6983 }
6984
6985 /// What the journal knows about an unwind before it starts:
6986 /// `(steps that changed something, steps already compensated)`.
6987 ///
6988 /// Evidence, not bookkeeping. The journal already knows both, it knows the
6989 /// same thing on replay, and nothing has to be threaded through the executor
6990 /// to keep a parallel copy honest.
6991 ///
6992 /// **A mutating effect that provably did not happen does not mark its
6993 /// step.** The announcement alone is not the evidence: an effect whose
6994 /// terminal record classifies it `DidNotHappen` — a peer that refused
6995 /// before acting, a call that never left — changed nothing, and
6996 /// compensating it is a refund for money nobody took. Everything else
6997 /// counts, and the defaults lean that way: an announcement with no
6998 /// terminal record is an orphan and therefore in doubt, and each new
6999 /// attempt starts from doubt again, so a retry that lands after a refusal
7000 /// is still evidence the step touched the world.
7001 ///
7002 /// **Nor does a member of a group that settled `Aborted`.** That settlement
7003 /// is written only once every landed member was reversed, so the group's
7004 /// effects — members and reversals alike — left nothing standing, and
7005 /// counting them marked a cleanly aborted step as mutated: a quarantine for
7006 /// a step declaring no compensation, a second undo for one that did.
7007 ///
7008 /// What this does **not** establish is that a `DidNotHappen` classification
7009 /// is true — that is the driver's claim, and a driver that mislabels a
7010 /// landed call leaves a mutation nothing here will undo.
7011 async fn unwind_evidence(&self, run: RunId) -> Result<UnwindEvidence, RuntimeError> {
7012 let records = self
7013 .store
7014 .read(run, 1)
7015 .await
7016 .map_err(RuntimeError::from_store)?;
7017
7018 // Per mutating forward effect: the step it belongs to, and whether the
7019 // record leaves it capable of having touched the world.
7020 let mut touching: BTreeMap<crate::core::EffectKey, (StepId, bool)> = BTreeMap::new();
7021 // The mutating effects of the group each step has open. A step holds
7022 // at most one open group, so the step is the key.
7023 let mut grouped: BTreeMap<StepId, Vec<crate::core::EffectKey>> = BTreeMap::new();
7024 let mut undone = BTreeSet::new();
7025 let mut failed = BTreeSet::new();
7026 for r in &records {
7027 let Some(step) = r.body.step else { continue };
7028 let key = r.effect_key();
7029 match r.kind() {
7030 RecordKind::StepFinished { outcome } if r.body.phase.is_forward() => {
7031 if outcome == RunStatus::Succeeded.as_str() {
7032 failed.remove(&step);
7033 } else {
7034 failed.insert(step);
7035 }
7036 }
7037 RecordKind::EffectStarted { mutates: true, .. } if r.body.phase.is_forward() => {
7038 if let Some(key) = key {
7039 // A fresh attempt is undecided until its own terminal
7040 // record says otherwise, so it supersedes whatever the
7041 // previous attempt concluded.
7042 touching.insert(key, (step, true));
7043 if let Some(members) = grouped.get_mut(&step) {
7044 members.push(key);
7045 }
7046 }
7047 }
7048 RecordKind::GroupOpened { .. } if r.body.phase.is_forward() => {
7049 grouped.insert(step, Vec::new());
7050 }
7051 RecordKind::GroupSettled { outcome, .. } if r.body.phase.is_forward() => {
7052 let members = grouped.remove(&step).unwrap_or_default();
7053 if *outcome == crate::core::GroupOutcome::Aborted {
7054 for member in members {
7055 touching.remove(&member);
7056 }
7057 }
7058 }
7059 RecordKind::EffectFailed { disposition, .. }
7060 | RecordKind::EffectReconciled { disposition, .. } => {
7061 if let Some(entry) = key.and_then(|k| touching.get_mut(&k)) {
7062 entry.1 = *disposition != crate::core::Disposition::DidNotHappen;
7063 }
7064 }
7065 RecordKind::StepCompensated { .. } => {
7066 undone.insert(step);
7067 }
7068 _ => {}
7069 }
7070 }
7071 let mutated = touching
7072 .into_values()
7073 .filter_map(|(step, touched)| touched.then_some(step))
7074 .collect();
7075 Ok(UnwindEvidence {
7076 mutated,
7077 started: started_skills(&records),
7078 undone,
7079 recorded_groups: recorded_groups(&records),
7080 failed,
7081 })
7082 }
7083
7084 /// The half of a step's frame that belongs to the **plane** rather than to
7085 /// the step.
7086 ///
7087 /// One place, for both a forward step and a compensation: a plane
7088 /// capability wired into one of the two and not the other is a control
7089 /// that applies going forward and not while unwinding.
7090 fn plane_frame(&self, step: StepFrame) -> super::ctx::Frame {
7091 let StepFrame {
7092 run,
7093 epoch,
7094 step,
7095 phase,
7096 mode,
7097 case,
7098 ledger,
7099 identity,
7100 subjects,
7101 agent,
7102 #[cfg(feature = "manifest")]
7103 manifest,
7104 recorded_groups,
7105 } = step;
7106 super::ctx::Frame {
7107 run,
7108 epoch,
7109 step,
7110 phase,
7111 mode,
7112 case,
7113 timers: self.timers.clone(),
7114 blobs: self.blobs.clone(),
7115 memories: self.memories.clone(),
7116 semantic: self.semantic.clone(),
7117 authorities: self.authorities.clone(),
7118 peers: self.peers.clone(),
7119 #[cfg(feature = "manifest")]
7120 tools: self.tools.clone(),
7121 journal_ceiling: self.journal_ceiling,
7122 #[cfg(feature = "manifest")]
7123 egress: self.egress.clone(),
7124 #[cfg(feature = "manifest")]
7125 checkers: self.checkers.clone(),
7126 streams: self.streams.clone(),
7127 #[cfg(feature = "manifest")]
7128 rates: self.rates.clone(),
7129 meter: self.meter.clone(),
7130 #[cfg(feature = "keyring")]
7131 keyring: self.keyring.clone(),
7132 tenant: self.tenant.clone(),
7133 ledger,
7134 policy: self.policy.clone(),
7135 identity,
7136 subjects,
7137 agent,
7138 plane: self.self_ref.clone(),
7139 #[cfg(feature = "manifest")]
7140 declaration: manifest.as_deref().and_then(|m| self.identity_of(m)),
7141 #[cfg(feature = "manifest")]
7142 manifest,
7143 signer: self.signer.clone(),
7144 recorded_groups,
7145 }
7146 }
7147
7148 /// Run one step's `compensate`, in its own phase and cursor slice.
7149 ///
7150 /// Split out so the unwind reads as the policy it is. The step gets a full
7151 /// `StepCtx` on purpose: compensating effects are journaled, retried,
7152 /// reconciled and replayed exactly like forward ones, and may suspend for a
7153 /// human — a refund that needs four eyes is still a refund.
7154 async fn run_compensation(
7155 &self,
7156 cx: &Unwind<'_>,
7157 step: StepId,
7158 skill: &dyn Skill,
7159 outputs: &BTreeMap<StepId, Tainted<Value>>,
7160 cursor: &mut ReplayCursor,
7161 recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
7162 ) -> Result<(), crate::core::SkillError> {
7163 // A step with no recorded output still gets compensated: the absence of
7164 // a result says nothing about whether it changed anything, and the
7165 // compensation is what knows.
7166 let output = outputs
7167 .get(&step)
7168 .cloned()
7169 .unwrap_or_else(|| Tainted::trusted(Value::Null));
7170
7171 let mut ctx = StepCtx::new(
7172 &self.store,
7173 cursor.take(step, Phase::Compensating),
7174 self.plane_frame(StepFrame {
7175 run: cx.run,
7176 epoch: cx.epoch,
7177 step,
7178 phase: Phase::Compensating,
7179 mode: cx.mode,
7180 case: cx.case.clone(),
7181 ledger: Arc::clone(cx.ledger),
7182 identity: cx.identity.cloned(),
7183 subjects: cx.subjects.clone(),
7184 agent: cx.agent.to_owned(),
7185 #[cfg(feature = "manifest")]
7186 manifest: self.governing(skill),
7187 recorded_groups,
7188 }),
7189 );
7190
7191 let result = skill.compensate(&mut ctx, &output).await;
7192 let result = moved_blob_wins(&ctx, result);
7193 let mut slice = ctx.into_cursor();
7194 if let Err(crate::core::SkillError::Step(error)) = &result
7195 && let Some(divergence) =
7196 crate::journal::Divergence::of(step, Phase::Compensating, error, &slice)
7197 {
7198 slice.note_divergence(divergence);
7199 }
7200 cursor.restore(step, Phase::Compensating, slice);
7201 result
7202 }
7203
7204 /// What the unwind walks, gated — or the quarantine that replaces it.
7205 ///
7206 /// Reads the journal's evidence and applies the stop gates. The gates run
7207 /// exactly when this conclusion closes the run to resume: always for a
7208 /// cancellation (it gets no second pass), and for a failure precisely
7209 /// when the unwind will compensate something — compensation is what
7210 /// closes resume, and a failure that compensates nothing stays open, so
7211 /// its orphans resolve and its suspended siblings re-register on the
7212 /// resume instead of being unwound around.
7213 async fn gated_unwind_list(
7214 &self,
7215 cx: &Unwind<'_>,
7216 status: &RunStatus,
7217 completed: &[(StepId, Capability)],
7218 ) -> Result<Result<(Vec<(StepId, Capability)>, UnwindEvidence), RunStatus>, RuntimeError> {
7219 // Which steps actually changed something outside. Read from the
7220 // journal rather than tracked in memory: it is the same evidence live
7221 // and on replay, and it is what lets an *undeclared* step be judged
7222 // on what it did instead of on what nobody said about it.
7223 let evidence = self.unwind_evidence(cx.run).await?;
7224 // The question is asked of the interrupted siblings too: a sibling
7225 // holding landed work the failure stranded is work the unwind must
7226 // reach, and asked of completed steps alone, a failure with nothing
7227 // completed to compensate left it standing. The failing step itself
7228 // is not asked — its own landed work leaves the run resumable.
7229 let candidates = Self::with_interrupted_siblings(completed, &evidence);
7230 let list = if status.is_cancelled() || self.will_compensate(&candidates, &evidence.mutated)
7231 {
7232 match self
7233 .stop_list(cx.run, completed, &evidence.started, &evidence.mutated)
7234 .await?
7235 {
7236 Ok(list) => list,
7237 Err(quarantine) => return Ok(Err(quarantine)),
7238 }
7239 } else {
7240 completed.to_vec()
7241 };
7242 Ok(Ok((list, evidence)))
7243 }
7244
7245 /// The unwind list for a stop, or the quarantine that replaces it.
7246 ///
7247 /// Both rules in `maybe_unwind`'s doc comment, applied in order: refuse to
7248 /// unwind while an outcome is unknown, and extend the list with the steps
7249 /// the stop interrupted. Called only for a conclusion that closes the run
7250 /// to resume — a cancellation, or a failure whose unwind will compensate
7251 /// — so doubt quarantines unconditionally here: there is no later pass on
7252 /// which an unresolved outcome could still resolve, and compensating
7253 /// around the one call nobody can account for would destroy the evidence
7254 /// an operator needs to resolve it by hand.
7255 async fn stop_list(
7256 &self,
7257 run: RunId,
7258 completed: &[(StepId, Capability)],
7259 started: &BTreeMap<StepId, String>,
7260 mutated: &BTreeSet<StepId>,
7261 ) -> Result<Result<Vec<(StepId, Capability)>, RunStatus>, RuntimeError> {
7262 if let Some(step) = self.undecided_effect(run).await? {
7263 return Ok(Err(RunStatus::Quarantined(format!(
7264 "step {step} holds a mutating effect whose outcome is unknown — it \
7265 was announced and never concluded, or concluded in doubt with no \
7266 reconciliation — so the run cannot be unwound: compensating around \
7267 it would undo everything except the one thing nobody can account for"
7268 ))));
7269 }
7270 Ok(Ok(Self::with_interrupted_steps(
7271 completed, started, mutated,
7272 )))
7273 }
7274
7275 /// Whether an unwind over these completed steps would compensate anything.
7276 ///
7277 /// The question that decides whether a failure closes its run: a declared
7278 /// `Compensatable` step will be compensated, and an `Undeclared` step
7279 /// that provably mutated quarantines inside the loop — either way the
7280 /// conclusion is not one a resume can reopen. `Pivot` and `Unnecessary`
7281 /// steps compensate nothing. Deliberately conservative on a capability
7282 /// that no longer resolves: an unresolvable skill is treated as
7283 /// compensating, so the unwind path — which will surface the resolution
7284 /// error loudly — is taken rather than silently skipped.
7285 fn will_compensate(
7286 &self,
7287 completed: &[(StepId, Capability)],
7288 mutated: &BTreeSet<StepId>,
7289 ) -> bool {
7290 completed.iter().any(|(step, capability)| {
7291 self.resolve(&capability.0)
7292 .map_or(true, |skill| match skill.compensation() {
7293 crate::core::Compensation::Compensatable => true,
7294 crate::core::Compensation::Undeclared => mutated.contains(step),
7295 crate::core::Compensation::Pivot | crate::core::Compensation::Unnecessary => {
7296 false
7297 }
7298 })
7299 })
7300 }
7301
7302 /// The completed steps, plus every sibling a failure interrupted holding
7303 /// landed work: a step that mutated, never completed, and did not itself
7304 /// fail. Undone as the skill its `StepStarted` names, as
7305 /// [`with_interrupted_steps`](Self::with_interrupted_steps) undoes it.
7306 fn with_interrupted_siblings(
7307 completed: &[(StepId, Capability)],
7308 evidence: &UnwindEvidence,
7309 ) -> Vec<(StepId, Capability)> {
7310 let mut out = completed.to_vec();
7311 let done: BTreeSet<StepId> = out.iter().map(|(s, _)| *s).collect();
7312 for step in evidence
7313 .mutated
7314 .iter()
7315 .filter(|s| !done.contains(s) && !evidence.failed.contains(s))
7316 {
7317 if let Some(skill) = evidence.started.get(step) {
7318 out.push((*step, Capability::new(skill.as_str())));
7319 }
7320 }
7321 out
7322 }
7323
7324 /// The unwind list for a stop: completed steps, plus any step that mutated
7325 /// without completing — the one a cancellation interrupted, or the sibling
7326 /// a failure stranded mid-suspension. The interrupted steps go last, so
7327 /// the caller's reverse walk undoes them first.
7328 ///
7329 /// An interrupted step is undone as the skill its `StepStarted` names —
7330 /// the capability that ran — never as whatever the plan in force has at
7331 /// its id. A step that charged and then asked to replan is interrupted
7332 /// under a successor that may drop it or put other work at its id, and
7333 /// either reading would leave the charge standing or refund it through a
7334 /// skill that never ran.
7335 ///
7336 /// A step whose compensation is already on the record stays in the list,
7337 /// exactly as a completed step does. The walk re-runs its compensation
7338 /// with every effect served from the journal and the `StepCompensated`
7339 /// dedup keeps the record single — while filtering it out here made a
7340 /// *replayed* unwind walk fewer steps than the recorded one, so a strict
7341 /// pass over an honestly-unwound run left the interrupted step's
7342 /// compensation slice unconsumed and quarantined a history that was
7343 /// telling the truth.
7344 fn with_interrupted_steps(
7345 completed: &[(StepId, Capability)],
7346 started: &BTreeMap<StepId, String>,
7347 mutated: &BTreeSet<StepId>,
7348 ) -> Vec<(StepId, Capability)> {
7349 let mut out = completed.to_vec();
7350 let done: BTreeSet<StepId> = out.iter().map(|(s, _)| *s).collect();
7351 for step in mutated.iter().filter(|s| !done.contains(s)) {
7352 if let Some(skill) = started.get(step) {
7353 out.push((*step, Capability::new(skill.as_str())));
7354 }
7355 }
7356 out
7357 }
7358
7359 /// The lowest step holding an effect this journal cannot decide, if any.
7360 ///
7361 /// Ordinarily the run is already `Quarantined` when this is true, and a
7362 /// quarantined run never unwinds. Cancellation opens a second door into the
7363 /// unwind, and it has to be shut the same way — otherwise an operator's
7364 /// stop compensates every step *around* the one nobody can account for,
7365 /// which is precisely the refund for money nobody took, arriving through a
7366 /// control meant to make things safer.
7367 ///
7368 /// The rule itself is [`undecided_effects`](crate::journal::undecided_effects),
7369 /// shared with the operator API and the offline audit. This adds only the
7370 /// question the unwind asks of it: *how far back does the doubt reach*.
7371 async fn undecided_effect(&self, run: RunId) -> Result<Option<StepId>, RuntimeError> {
7372 let records = self
7373 .store
7374 .read(run, 1)
7375 .await
7376 .map_err(RuntimeError::from_store)?;
7377 Ok(crate::journal::undecided_effects(&records)
7378 .into_iter()
7379 .map(|u| u.step)
7380 .min())
7381 }
7382
7383 /// Execute one plan node.
7384 async fn run_step(
7385 &self,
7386 ctx: StepRun<'_>,
7387 run_input: &Tainted<Value>,
7388 outputs: &BTreeMap<StepId, Tainted<Value>>,
7389 cursor: crate::journal::StepCursor,
7390 ) -> Result<
7391 (
7392 RunStatus,
7393 Option<Tainted<Value>>,
7394 crate::journal::StepCursor,
7395 ),
7396 RuntimeError,
7397 > {
7398 let span = tracing::info_span!(
7399 telemetry::STEP_SPAN,
7400 { telemetry::STEP } = tracing::field::display(ctx.node.id),
7401 { telemetry::CAPABILITY } = tracing::field::display(&ctx.node.capability.0),
7402 { telemetry::PHASE } = ctx.phase.as_str(),
7403 { telemetry::MODE } = telemetry::mode_str(ctx.mode),
7404 { telemetry::OUTCOME } = tracing::field::Empty,
7405 );
7406 self.run_step_inner(ctx, run_input, outputs, cursor)
7407 .instrument(span)
7408 .await
7409 }
7410
7411 #[allow(clippy::too_many_lines)]
7412 async fn run_step_inner(
7413 &self,
7414 ctx: StepRun<'_>,
7415 run_input: &Tainted<Value>,
7416 outputs: &BTreeMap<StepId, Tainted<Value>>,
7417 cursor: crate::journal::StepCursor,
7418 ) -> Result<
7419 (
7420 RunStatus,
7421 Option<Tainted<Value>>,
7422 crate::journal::StepCursor,
7423 ),
7424 RuntimeError,
7425 > {
7426 let StepRun {
7427 identity,
7428 subjects,
7429 run,
7430 epoch,
7431 node,
7432 phase,
7433 mode,
7434 case,
7435 ledger,
7436 writing,
7437 stamp,
7438 agent,
7439 already_started,
7440 already_finished,
7441 recorded_groups,
7442 abandoning,
7443 } = ctx;
7444 let step = node.id;
7445 let skill = self.resolve(&node.capability.0)?;
7446
7447 // Assemble this step's input from its declared sources. Labels join, so
7448 // provenance flows through the graph without anyone threading it by hand.
7449 let step_input = assemble(node, run_input, outputs)?;
7450
7451 // Announce a step doing new work, once: always in `Live`, and on a
7452 // resume only for a step the journal never announced — a frontier
7453 // step running for the first time. Keying this on the mode alone got
7454 // both directions wrong: resumed frontier steps went unannounced, and
7455 // the `!Strict` finish gate below duplicated `StepFinished` for every
7456 // fully replayed step on every resume.
7457 let announce = writes_step_record(mode, !already_started);
7458 if announce {
7459 self.store
7460 .append(
7461 epoch,
7462 vec![stamp(
7463 Append::new(
7464 run,
7465 RecordKind::StepStarted {
7466 skill: skill.descriptor().name,
7467 },
7468 )
7469 .step(step),
7470 )],
7471 )
7472 .await
7473 .map_err(RuntimeError::from_store)?;
7474 }
7475
7476 let mut cx = StepCtx::new(
7477 &self.store,
7478 cursor,
7479 self.plane_frame(StepFrame {
7480 run,
7481 epoch,
7482 step,
7483 phase,
7484 mode,
7485 case,
7486 ledger: Arc::clone(ledger),
7487 identity: identity.cloned(),
7488 subjects: subjects.clone(),
7489 agent: agent.to_owned(),
7490 #[cfg(feature = "manifest")]
7491 manifest: self.governing(skill.as_ref()),
7492 recorded_groups,
7493 }),
7494 );
7495 cx.abandoning = abandoning;
7496 // The one place elapsed time is measured, and it is measured with a
7497 // journaled clock read like every other observation — a wall-clock
7498 // ceiling decided from an ambient clock would give a replayed run a
7499 // different verdict every time it was looked at.
7500 //
7501 // Paid only by a run that asked for the ceiling, which is why the
7502 // step's first effect is this one when a wall-clock budget is declared
7503 // and the skill's own when it is not. The consequence is worth stating:
7504 // that history can be replayed under a *raised* wall-clock ceiling, and
7505 // not under a build that removed it, because the recorded run performs
7506 // a read this one would not request.
7507 let result = match Self::observe_clock(&mut cx, ledger).await {
7508 Ok(()) => skill.invoke(&mut cx, step_input).await,
7509 Err(e) => Err(crate::core::SkillError::Step(e)),
7510 };
7511 let result = settle_abandoned_group(&mut cx, result).await;
7512 let result = moved_blob_wins(&cx, result);
7513 let wrote = cx.wrote_records();
7514 let mut cursor = cx.into_cursor();
7515 if let Err(crate::core::SkillError::Step(error)) = &result
7516 && let Some(divergence) = crate::journal::Divergence::of(step, phase, error, &cursor)
7517 {
7518 cursor.note_divergence(divergence);
7519 }
7520 ledger.lock().expect("budget mutex").record_step();
7521
7522 let fault = result.as_ref().err().map(crate::core::SkillError::class);
7523 let (status, output) = classify(&self.meter, result);
7524 tracing::Span::current().record(telemetry::OUTCOME, status.as_str());
7525 if let RunStatus::Quarantined(why) = &status {
7526 tracing::error!(
7527 target: telemetry::QUARANTINED,
7528 %run,
7529 %step,
7530 error_type = fault.unwrap_or("quarantined"),
7531 reason_digest = %telemetry::reason_digest(why),
7532 );
7533 }
7534
7535 // Record the ending only where the journal does not already hold it:
7536 // every live step; a resumed step that did new work; and a resumed
7537 // step reaching an ending the record never captured — how a suspended
7538 // step that finishes purely from replayed history (its awaited event
7539 // recorded by the delivery) still gets its one `StepFinished`. What
7540 // this rules out is a duplicate ending for every fully replayed
7541 // completed step, appended on every resume of a run doing nothing new.
7542 let record_ending = writes_step_record(mode, announce || wrote || !already_finished);
7543 if writing && record_ending {
7544 // A suspended step has not finished, so it records why it stopped
7545 // rather than claiming an outcome.
7546 let record = match &status {
7547 RunStatus::Suspended(reason) => RecordKind::RunSuspended {
7548 reason: reason.clone(),
7549 },
7550 other => RecordKind::StepFinished {
7551 outcome: other.as_str().to_owned(),
7552 },
7553 };
7554 self.store
7555 .append(epoch, vec![stamp(Append::new(run, record).step(step))])
7556 .await
7557 .map_err(RuntimeError::from_store)?;
7558 }
7559
7560 Ok((status, output, cursor))
7561 }
7562
7563 /// Whether the chain's last record is already this conclusion. `head` is
7564 /// the chain's last sequence.
7565 async fn last_is_conclusion(
7566 &self,
7567 run: RunId,
7568 head: crate::core::Seq,
7569 status: &RunStatus,
7570 ) -> Result<bool, RuntimeError> {
7571 if head < 1 {
7572 return Ok(false);
7573 }
7574 let page = self
7575 .store
7576 .read_page(run, head, 1)
7577 .await
7578 .map_err(RuntimeError::from_store)?;
7579 Ok(matches!(
7580 page.first().map(Record::kind),
7581 Some(RecordKind::RunConcluded { outcome, .. }) if outcome == status.as_str()
7582 ))
7583 }
7584
7585 /// Seal the run and report.
7586 /// Eight arguments, and each is a distinct fact about how the run ended
7587 /// that the caller already holds. Bundling them into a struct would move the
7588 /// same fields one indirection away without removing a single one.
7589 #[allow(clippy::too_many_arguments)]
7590 async fn conclude(
7591 &self,
7592 run: RunId,
7593 epoch: u64,
7594 status: RunStatus,
7595 output: Option<Tainted<Value>>,
7596 writing: bool,
7597 case: Option<crate::core::CaseId>,
7598 consumed: Consumed,
7599 live_spend: Spend,
7600 quota: &QuotaPass,
7601 ) -> Result<RunOutcome, RuntimeError> {
7602 // Loud toward the operator, ordinary toward the caller. A failed run is
7603 // a conclusion a resume can honestly answer, so it is not an incident —
7604 // but without this event nothing says why one failed except the
7605 // journal and an index that needs the HTTP surface mounted.
7606 if let RunStatus::Failed(reason) = &status {
7607 tracing::warn!(
7608 target: telemetry::RUN_FAILED,
7609 %run,
7610 error_type = "failed",
7611 reason_digest = %telemetry::reason_digest(reason),
7612 );
7613 }
7614
7615 // A suspended run is not sealed: its chain is going to be extended the
7616 // moment whatever it waits for arrives.
7617 let chain_head = if writing && !status.is_suspended() {
7618 // The conclusion goes *in* the chain — before the chain is closed
7619 // over it, where the conclusion is one that closes it. Two things
7620 // follow, and both were missing while the outcome lived only in a
7621 // side table: tamper detection covers how the run ended, and a
7622 // resumed run can read that fact from the same history it verifies
7623 // rather than inferring it from the last step that happened to
7624 // finish. The stores also derive the outcome index from this
7625 // record, last conclusion wins — which is what keeps a
7626 // failed-then-resumed-then-succeeded run from being listed as
7627 // failed forever.
7628 let before = self
7629 .store
7630 .head(run)
7631 .await
7632 .map_err(RuntimeError::from_store)?;
7633 // A resume that did nothing new reaches the same open conclusion
7634 // the record already carries, and appending it again would grow
7635 // the journal by one identical conclusion per resume. The chain's
7636 // own last record is the authority: only when it already *is*
7637 // this conclusion is the append skipped. A resume that worked and then concluded the same way
7638 // has its work between the two conclusions, so both are kept.
7639 // Scoped to open conclusions, because a sealing one is
7640 // unreachable twice: the seal refuses further appends and a
7641 // sealed run's replay is a no-op long before here.
7642 let repeated =
7643 !status.seals() && self.last_is_conclusion(run, before.seq, &status).await?;
7644 if repeated {
7645 before.hash
7646 } else {
7647 let mut sealed = Append::new(
7648 run,
7649 RecordKind::RunConcluded {
7650 outcome: status.as_str().to_owned(),
7651 reason: status.own_account().map(std::borrow::Cow::into_owned),
7652 exhaustion: match &status {
7653 RunStatus::Exhausted(limit) => Some(limit.clone()),
7654 _ => None,
7655 },
7656 live_spend,
7657 chain_head: before.hash,
7658 },
7659 );
7660 if let Some(c) = case {
7661 sealed = sealed.case(c);
7662 }
7663 let concluded = self
7664 .store
7665 .append(epoch, vec![sealed])
7666 .await
7667 .map_err(RuntimeError::from_store)?;
7668
7669 concluded.last().map_or(before.hash, |r| r.hash)
7670 }
7671 } else {
7672 self.store
7673 .head(run)
7674 .await
7675 .map_err(RuntimeError::from_store)?
7676 .hash
7677 };
7678
7679 // Hand the lease back rather than letting it time out.
7680 //
7681 // Whatever the outcome — sealed, suspended, exhausted — this instance is
7682 // finished with the run. Holding the lease until expiry would make every
7683 // failover wait out the TTL for nothing.
7684 //
7685 // Best-effort on purpose. A release that fails costs a TTL of patience;
7686 // turning it into a run failure would convert a tidiness problem into a
7687 // correctness one, after the work is already done and journaled.
7688 //
7689 // Only when this pass held a lease at all. A strict pass acquires
7690 // nothing — it is a read — and both stores release on epoch match
7691 // alone, so a strict verification of a run whose owner is live under
7692 // that same epoch would mark the *owner's* lease released: its
7693 // heartbeat stops renewing and any delivery or sweep may fence a
7694 // healthy run mid-step. The same guard keeps a read-only pass out of
7695 // the quota ledger and the metrics: a verification is not work the
7696 // tenant did today, and accruing a historical run's spend into the
7697 // current period on every audit would bill the past once per look.
7698 self.forget_commissions_of(run);
7699 if writing {
7700 // Settlement precedes both physical sealing and lease release. If
7701 // it fails, the conclusion remains open under an owned lease; once
7702 // that lease expires, recovery derives this pass from the journal
7703 // and retries the idempotent receipt.
7704 //
7705 // Whatever the outcome this instance is finished with the run, so
7706 // a **suspended** run gives its slot back too — it costs a row,
7707 // not a thread, and holding the slot would mean a tenant waiting
7708 // on a hundred approvals could start nothing. The figure accrued
7709 // is what *this pass* dispatched, not the run's cumulative spend,
7710 // which would bill the replayed prefix once per resume.
7711 self.settle_quota(run, epoch, live_spend, quota, status.seals())
7712 .await?;
7713
7714 let chain_head = if status.seals() {
7715 let head = self
7716 .store
7717 .seal(run, epoch, status.as_str())
7718 .await
7719 .map_err(RuntimeError::from_store)?;
7720 self.retire_waits(run).await;
7721 head
7722 } else {
7723 chain_head
7724 };
7725
7726 if let Err(e) = self.store.release_lease(run, epoch).await {
7727 tracing::debug!(
7728 %run,
7729 error = %e,
7730 "could not hand back the lease; it will expire on its own"
7731 );
7732 }
7733
7734 announce(&self.meter, run, &status);
7735
7736 return Ok(RunOutcome {
7737 failed_effect: self.failed_effect(run, &status).await?,
7738 run_id: run,
7739 status,
7740 chain_head,
7741 consumed,
7742 output,
7743 });
7744 }
7745
7746 Ok(RunOutcome {
7747 failed_effect: self.failed_effect(run, &status).await?,
7748 run_id: run,
7749 status,
7750 chain_head,
7751 consumed,
7752 // The caller is outside the lattice, so the label is dropped at the
7753 // boundary rather than inside the graph.
7754 output,
7755 })
7756 }
7757}
7758
7759/// The effect a run ended on, when it failed on one.
7760fn failure_of(status: &RunStatus, records: &[Record]) -> Option<crate::journal::FailedEffect> {
7761 matches!(status, RunStatus::Failed(_))
7762 .then(|| crate::journal::failed_effect(records))
7763 .flatten()
7764}
7765
7766/// The run's result: the terminal step's output.
7767///
7768/// Not "whichever step finished last". That coincides with the terminal step
7769/// only while dispatch is sequential, and stops being well-defined the moment
7770/// two steps run at once. Lowest id wins when a plan has several terminals, so
7771/// the answer is a property of the plan rather than of the schedule.
7772fn run_output(ir: &PlanIR, outputs: &BTreeMap<StepId, Tainted<Value>>) -> Option<Tainted<Value>> {
7773 ir.nodes
7774 .iter()
7775 .filter(|n| n.terminal)
7776 .map(|n| n.id)
7777 .min()
7778 .and_then(|id| outputs.get(&id).cloned())
7779 // A run that stopped before any terminal step still has something to
7780 // report: the furthest output it did produce.
7781 .or_else(|| outputs.iter().next_back().map(|(_, v)| v.clone()))
7782}
7783
7784/// Effects a run announced and never recorded an outcome for — the waits a
7785/// closed run leaves behind.
7786fn unanswered_waits(records: &[Record]) -> Vec<crate::core::EffectKey> {
7787 let settled: BTreeSet<_> = records
7788 .iter()
7789 .filter(|r| {
7790 matches!(
7791 r.kind(),
7792 RecordKind::EffectDone { .. }
7793 | RecordKind::EffectFailed { .. }
7794 | RecordKind::EffectReconciled { .. }
7795 )
7796 })
7797 .filter_map(Record::effect_key)
7798 .collect();
7799 records
7800 .iter()
7801 .filter(|r| matches!(r.kind(), RecordKind::EffectStarted { .. }))
7802 .filter_map(Record::effect_key)
7803 .filter(|k| !settled.contains(k))
7804 .collect()
7805}
7806
7807/// Whether `epoch` holds the conclusion that sealed the run — the pass whose
7808/// settlement releases what the run still held.
7809fn concluded_in(records: &[Record], epoch: crate::core::Epoch) -> bool {
7810 records
7811 .iter()
7812 .any(|r| r.body.epoch == epoch && is_sealing_conclusion(r))
7813}
7814
7815fn is_sealing_conclusion(record: &Record) -> bool {
7816 matches!(
7817 record.kind(),
7818 RecordKind::RunConcluded { outcome, .. } if SEALED_OUTCOMES.contains(&outcome.as_str())
7819 )
7820}
7821
7822/// Spend durably recorded by effects written under one live pass epoch.
7823fn spend_recorded_in_epoch(records: &[Record], epoch: crate::core::Epoch) -> Spend {
7824 records
7825 .iter()
7826 .filter(|record| record.body.epoch == epoch)
7827 .fold(Spend::default(), |total, record| {
7828 let spend = match record.kind() {
7829 RecordKind::EffectDone { spend, .. }
7830 | RecordKind::EffectFailed { spend, .. }
7831 | RecordKind::EffectReconciled { spend, .. } => *spend,
7832 _ => Spend::default(),
7833 };
7834 total.plus(spend)
7835 })
7836}
7837
7838/// Whether the plan actually finished.
7839///
7840/// Structural, never self-reported: a workload asserting it is done is not
7841/// evidence, so the runtime checks that every terminal node ran.
7842fn completion(ir: &PlanIR, done: &BTreeSet<StepId>) -> RunStatus {
7843 if ir.is_complete(done) {
7844 return RunStatus::Succeeded;
7845 }
7846 let missing: Vec<String> = ir
7847 .nodes
7848 .iter()
7849 .filter(|n| n.terminal && !done.contains(&n.id))
7850 .map(|n| n.id.to_string())
7851 .collect();
7852 RunStatus::Failed(format!(
7853 "plan did not complete: terminal step(s) {} never ran",
7854 missing.join(", ")
7855 ))
7856}
7857
7858/// Say that a run changed its plan, and what it changed from.
7859fn announce_replan(meter: &super::metrics::Meter, run: RunId, next: &PlanIR, reason: &str) {
7860 tracing::info!(
7861 target: telemetry::REPLANNED,
7862 %run,
7863 from = next.derived_from.map(Digest::to_hex),
7864 version = next.version,
7865 reason_digest = %telemetry::reason_digest(reason),
7866 );
7867 meter.count(metrics::REPLANS, "");
7868}
7869
7870/// Say how a run ended, on the run span and — for the loud ones — as an event.
7871fn announce(meter: &super::metrics::Meter, run: RunId, status: &RunStatus) {
7872 // Counted here and nowhere else. A step that quarantines also fails its run,
7873 // so counting at both levels would report one incident as two — and the
7874 // terminal status is the fact an operator is counting.
7875 meter.count(metrics::RUNS, status.as_str());
7876 match status {
7877 RunStatus::Quarantined(why) => {
7878 tracing::error!(
7879 target: telemetry::QUARANTINED,
7880 %run,
7881 error_type = "quarantined",
7882 reason_digest = %telemetry::reason_digest(why),
7883 );
7884 meter.count(metrics::QUARANTINES, "");
7885 }
7886 // Loud, though a person asked for it. The operator who decided already
7887 // knows; this reaches whoever answers for what the run left standing.
7888 RunStatus::Abandoned { actor, reason } => {
7889 tracing::error!(
7890 target: telemetry::ABANDONED,
7891 %run,
7892 %actor,
7893 reason_digest = %telemetry::reason_digest(reason),
7894 );
7895 }
7896 RunStatus::Exhausted(limit) => {
7897 tracing::warn!(target: telemetry::BUDGET_REFUSED, %run, %limit);
7898 }
7899 _ => {}
7900 }
7901 tracing::Span::current().record(telemetry::OUTCOME, status.as_str());
7902}
7903
7904/// Record a batch's successes, then report the first step that stopped.
7905///
7906/// **Every** success is recorded, including those of siblings dispatched
7907/// alongside the one that stopped. Returning early on the first failure loses
7908/// them — and a sibling that already performed a mutating effect would then
7909/// never be compensated, because `completed` is what the unwind reverses. The
7910/// work happened; the saga has to know about it.
7911///
7912/// When siblings stop for different reasons, **severity wins over ready order**.
7913/// A suspension is the run working; a failure is the run over. Letting one
7914/// sibling's wait mask another's failure would defer the unwind until an event
7915/// that may never arrive — leaving the failed sibling's mutations in place
7916/// indefinitely. Within one severity, ready order decides, so the choice stays a
7917/// property of the plan rather than of the schedule.
7918fn apply(
7919 plan: &PlanIR,
7920 outcomes: Vec<StepOutcome>,
7921 done: &mut BTreeSet<StepId>,
7922 completed: &mut Vec<(StepId, Capability)>,
7923 ran: &mut BTreeMap<StepId, PlanNode>,
7924 outputs: &mut BTreeMap<StepId, Tainted<Value>>,
7925) -> Option<RunStatus> {
7926 let mut stopped: Option<RunStatus> = None;
7927 for (step, status, output) in outcomes {
7928 let RunStatus::Succeeded = status else {
7929 if stopped
7930 .as_ref()
7931 .is_none_or(|held| severity(&status) > severity(held))
7932 {
7933 stopped = Some(status);
7934 }
7935 continue;
7936 };
7937 if let Some(v) = output {
7938 outputs.insert(step, v);
7939 }
7940 done.insert(step);
7941 if let Some(node) = plan.node(step) {
7942 completed.push((step, node.capability.clone()));
7943 ran.insert(step, node.clone());
7944 }
7945 }
7946 stopped
7947}
7948
7949/// How much a stop reason dominates a competing one.
7950///
7951/// `Quarantined` is highest because it is the only one that must *not* unwind:
7952/// something is undecidable, and compensating around it can make the damage
7953/// worse. `Suspended` is lowest because it is not a stop at all — the run is
7954/// healthy and waiting.
7955fn severity(status: &RunStatus) -> u8 {
7956 match status {
7957 RunStatus::Quarantined(_) => 3,
7958 // A stop ranks with a failure, not above it: both end the run, both
7959 // unwind, and when they arrive together the run is over either way.
7960 // Abandonment is not produced by a step and never competes here; it is
7961 // named so a new stop reason cannot be given a rank by omission.
7962 RunStatus::Cancelled { .. } | RunStatus::Abandoned { .. } | RunStatus::Failed(_) => 2,
7963 // A withdrawal ranks with exhaustion for the same reason: both pause a
7964 // healthy run, neither unwinds, and a sibling that actually failed has
7965 // decided more about the run than either.
7966 RunStatus::Exhausted(_) | RunStatus::Withheld { .. } => 1,
7967 // A replan request is the weakest signal in a batch: a sibling that
7968 // failed outright has already decided the run, and re-planning around a
7969 // failure is not what the requesting step was asking for.
7970 // Neither is reachable here at all — a sweep, a crossing and a lift are
7971 // sealed at birth with no steps to compete — and both are named for the reason
7972 // abandonment is: a rank given by omission is a decision nobody made.
7973 RunStatus::Replanning(_)
7974 | RunStatus::Suspended(_)
7975 | RunStatus::Succeeded
7976 | RunStatus::Swept
7977 // Unreachable for the same reason, and named rather than left to a
7978 // wildcard: an observed session has no steps of this plane's to
7979 // compete, because this plane ran none of it.
7980 | RunStatus::Observed
7981 | RunStatus::BrokeGlass { .. }
7982 | RunStatus::HaltLifted { .. }
7983 | RunStatus::HoldReleased { .. } => 0,
7984 }
7985}
7986
7987/// Gather a dispatched batch back into ready order.
7988///
7989/// Not completion order. `completed` is what the unwind reverses, and a saga
7990/// whose compensation order depended on which future finished first would undo
7991/// a plan differently on every run.
7992type Dispatched = Result<(StepId, RunStatus, Option<Tainted<Value>>, StepCursor), RuntimeError>;
7993
7994/// One step's result, once its history has been handed back.
7995type StepOutcome = (StepId, RunStatus, Option<Tainted<Value>>);
7996
7997fn collect(
7998 dispatched: Vec<Dispatched>,
7999 ready: &[StepId],
8000 cursor: &mut ReplayCursor,
8001) -> Result<Vec<StepOutcome>, RuntimeError> {
8002 let mut outcomes = Vec::with_capacity(dispatched.len());
8003 for result in dispatched {
8004 let (step, status, out, slice) = result?;
8005 cursor.restore(step, Phase::Forward, slice);
8006 outcomes.push((step, status, out));
8007 }
8008 // The key is an `Option`, and `None` would sort *first* — so the invariant
8009 // that makes this total is worth naming rather than trusting: every
8010 // dispatched step came from `admit_ready(&ready, ..)`, which returns a
8011 // subset of `ready`. A future that dispatched a step outside the ready set
8012 // would silently re-order this batch rather than fail, and the order is what
8013 // an unwind reverses.
8014 outcomes.sort_by_key(|(step, _, _)| ready.iter().position(|r| r == step));
8015 Ok(outcomes)
8016}
8017
8018/// What one ready set's dispatch needs.
8019struct Batch<'a> {
8020 agent: &'a str,
8021 identity: Option<&'a crate::core::Delegation>,
8022 subjects: &'a BTreeSet<crate::core::SubjectRef>,
8023 run: RunId,
8024 epoch: u64,
8025 ir: &'a PlanIR,
8026 mode: Pass,
8027 case: &'a Option<CaseContext>,
8028 ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8029 writing: bool,
8030 stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8031 input: &'a Tainted<Value>,
8032 outputs: &'a BTreeMap<StepId, Tainted<Value>>,
8033 /// Steps whose `StepStarted` the journal already holds. Empty on a live
8034 /// run.
8035 started: &'a BTreeSet<StepId>,
8036 /// Steps whose `StepFinished` the journal already holds. Empty on a live
8037 /// run.
8038 finished: &'a BTreeSet<StepId>,
8039 /// Group records the journal already holds, by writing step and phase.
8040 /// Empty on a live run.
8041 recorded_groups: &'a BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
8042 /// How many of this ready set may be in flight at once.
8043 parallelism: usize,
8044 /// Re-running steps only to abandon groups a pause left open.
8045 abandoning: bool,
8046}
8047
8048/// A standing withdrawal, as the step boundary meets it.
8049///
8050/// Carries the operator from the halt rather than only its reason: the
8051/// withholding record is the one durable trace of this act on the run it
8052/// stopped, and a pause nobody is named on cannot be asked about afterwards.
8053pub(crate) struct Withdrawal {
8054 pub(crate) subject: String,
8055 pub(crate) reason: String,
8056 pub(crate) by: crate::core::Operator,
8057}
8058
8059/// What the journal proves about a run an unwind is deciding over.
8060struct UnwindEvidence {
8061 /// Steps that announced a mutating forward effect.
8062 mutated: BTreeSet<StepId>,
8063 /// The skill each step that started ran as, from its `StepStarted`.
8064 started: BTreeMap<StepId, String>,
8065 /// Steps whose compensation is already on the record.
8066 undone: BTreeSet<StepId>,
8067 /// Group records already on the journal, by writing step and phase.
8068 recorded_groups: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
8069 /// Steps whose last forward ending is not a success: the failing step,
8070 /// which stays resumable, as opposed to a sibling the failure interrupted.
8071 failed: BTreeSet<StepId>,
8072}
8073
8074/// What producing a successor plan needs.
8075struct Replan<'a> {
8076 run: RunId,
8077 current: &'a PlanIR,
8078 /// The node each completed step ran as.
8079 ran: &'a BTreeMap<StepId, PlanNode>,
8080 reason: &'a str,
8081 already_replanned: u32,
8082 max_replans: Option<u32>,
8083 /// The successor this run produced when it first ran, if this is a replay.
8084 recorded: Option<&'a PlanIR>,
8085}
8086
8087/// The first untrusted value in working memory, if any.
8088///
8089/// Returns the source so the refusal can name it: "replanning refused" without
8090/// saying *what* made it unsafe sends an operator looking through the whole run.
8091fn untrusted_in(outputs: &BTreeMap<StepId, Tainted<Value>>) -> Option<String> {
8092 outputs.values().find_map(|v| {
8093 let label = v.label();
8094 label.is_untrusted().then(|| {
8095 label
8096 .provenance
8097 .first()
8098 .map_or_else(|| "an untrusted source".to_owned(), ToString::to_string)
8099 })
8100 })
8101}
8102
8103/// Where a refusal is recorded, when one is.
8104struct Journalling<'a> {
8105 run: RunId,
8106 epoch: u64,
8107 writing: bool,
8108 stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8109}
8110
8111/// What unwinding a run needs.
8112struct Unwind<'a> {
8113 identity: Option<&'a crate::core::Delegation>,
8114 subjects: &'a BTreeSet<crate::core::SubjectRef>,
8115 run: RunId,
8116 epoch: u64,
8117 ir: &'a PlanIR,
8118 mode: Pass,
8119 case: Option<CaseContext>,
8120 ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8121 writing: bool,
8122 stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8123 agent: &'a str,
8124}
8125
8126/// What one step's execution needs.
8127// Each flag is an independent fact about this dispatch; no combination is
8128// invalid, so an enum over them would satisfy the lint at the reader's expense.
8129#[allow(clippy::struct_excessive_bools)]
8130struct StepRun<'a> {
8131 identity: Option<&'a crate::core::Delegation>,
8132 subjects: &'a BTreeSet<crate::core::SubjectRef>,
8133 run: RunId,
8134 epoch: u64,
8135 node: &'a PlanNode,
8136 phase: Phase,
8137 mode: Pass,
8138 case: Option<CaseContext>,
8139 ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8140 writing: bool,
8141 stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8142 agent: &'a str,
8143 /// Whether the journal already announced this step. Always `false` live.
8144 already_started: bool,
8145 /// Whether the journal already records this step finishing. Always
8146 /// `false` live.
8147 already_finished: bool,
8148 /// Group records this step and phase already wrote. Always empty live.
8149 recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
8150 /// Whether this step is re-run only to abandon a group a pause left
8151 /// open, because the run was cancelled while paused.
8152 abandoning: bool,
8153}
8154
8155/// The half of a `StepCtx`'s frame that belongs to the **step**.
8156///
8157/// A struct rather than a dozen positional arguments, for the reason
8158/// [`StepRun`] is one: the other half comes from the plane, and a caller who
8159/// transposed `phase` and `mode` positionally would compile.
8160struct StepFrame {
8161 run: RunId,
8162 epoch: crate::core::Epoch,
8163 step: StepId,
8164 phase: Phase,
8165 mode: Pass,
8166 case: Option<CaseContext>,
8167 ledger: Arc<std::sync::Mutex<Ledger>>,
8168 identity: Option<crate::core::Delegation>,
8169 subjects: BTreeSet<crate::core::SubjectRef>,
8170 agent: String,
8171 #[cfg(feature = "manifest")]
8172 manifest: Option<Arc<crate::manifest::Manifest>>,
8173 recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
8174}
8175
8176/// Where the recorded run was refused by a *step* limit, if the refusal still
8177/// stands.
8178///
8179/// A step-level refusal has no effect key, so it cannot ride the replay cursor
8180/// the way an effect's does; it is lifted from the records instead.
8181///
8182/// **The last word wins.** An exhausted run may be resumed under a raised
8183/// ceiling; when the resume re-admits the refused step it journals a
8184/// `BudgetReadmitted` beside the old refusal, and from then on the refusal is
8185/// history that was *superseded*, not a verdict to re-serve. Without this, a
8186/// strict replay of the resumed history would stop at the old refusal and
8187/// report `Exhausted` about a run whose own later records show it finishing —
8188/// the refusal followed by the continuation would read as divergence.
8189fn recorded_step_refusal(records: &[Record]) -> Option<(StepId, String, String)> {
8190 let mut standing: Option<(StepId, String, String)> = None;
8191 for r in records {
8192 match r.kind() {
8193 RecordKind::BudgetRefused { limit, used } if r.effect_key().is_none() => {
8194 if let Some(step) = r.body.step {
8195 standing = Some((step, limit.clone(), used.clone()));
8196 }
8197 }
8198 RecordKind::BudgetReadmitted { .. }
8199 if standing
8200 .as_ref()
8201 .is_some_and(|(s, _, _)| Some(*s) == r.body.step) =>
8202 {
8203 standing = None;
8204 }
8205 _ => {}
8206 }
8207 }
8208 standing
8209}
8210
8211/// The subject of a withholding the records left standing, if one stands.
8212///
8213/// **The last word wins**, exactly as it does for a step refusal: a withheld run
8214/// resumes once somebody lifts the halt, and the resume journals an
8215/// `AuthorityRestored` beside the withholding. From then on the withholding is
8216/// history that was *superseded* rather than a verdict to re-serve — without
8217/// this, a resumed run would be treated as still withheld and would conclude
8218/// that way forever, with each resume adding nothing.
8219fn standing_withholding(records: &[Record]) -> Option<String> {
8220 let mut standing: Option<String> = None;
8221 for r in records {
8222 match r.kind() {
8223 RecordKind::AuthorityWithheld { subject, .. } => standing = Some(subject.clone()),
8224 RecordKind::AuthorityRestored { .. } => standing = None,
8225 _ => {}
8226 }
8227 }
8228 standing
8229}
8230
8231/// The run's most recent recorded conclusion, if it has one.
8232///
8233/// `None` for a run that has only ever suspended — a suspension is not a
8234/// conclusion, and `conclude` writes no `RunSealed` for it.
8235fn recorded_conclusion(records: &[Record]) -> Option<String> {
8236 records.iter().rev().find_map(|r| match r.kind() {
8237 RecordKind::RunConcluded { outcome, .. } => Some(outcome.clone()),
8238 _ => None,
8239 })
8240}
8241
8242/// Whether this pass writes a step-level record: always live, never under
8243/// strict verification, and on a resume only when `new_fact` — the record is
8244/// not already in the journal, or the step did new work.
8245const fn writes_step_record(mode: Pass, new_fact: bool) -> bool {
8246 match mode {
8247 Pass::Live => true,
8248 Pass::Resume => new_fact,
8249 Pass::Strict => false,
8250 }
8251}
8252
8253/// The steps whose start is already on the record.
8254///
8255/// Read back so a resume neither re-announces a step history already announced
8256/// nor forgets to announce one it is starting for the first time.
8257fn recorded_started_steps(records: &[Record]) -> BTreeSet<StepId> {
8258 records
8259 .iter()
8260 .filter_map(|r| match r.kind() {
8261 RecordKind::StepStarted { .. } => r.body.step,
8262 _ => None,
8263 })
8264 .collect()
8265}
8266
8267/// The skill each step started as, from its forward `StepStarted`.
8268///
8269/// The capability that actually ran, which after a replan the plan in force
8270/// may no longer name at that id — or may name as other work.
8271fn started_skills(records: &[Record]) -> BTreeMap<StepId, String> {
8272 records
8273 .iter()
8274 .filter(|r| r.body.phase.is_forward())
8275 .filter_map(|r| match (r.kind(), r.body.step) {
8276 (RecordKind::StepStarted { skill }, Some(step)) => Some((step, skill.clone())),
8277 _ => None,
8278 })
8279 .collect()
8280}
8281
8282/// The steps whose ending is already on the record.
8283///
8284/// Read back so a resume records a `StepFinished` exactly for the steps that
8285/// reach an ending the journal does not yet hold: a fully replayed completed
8286/// step keeps its one record, and a suspended step that finally finishes —
8287/// even purely from replayed history, its awaited event having been recorded
8288/// by the delivery — gets the one it never had.
8289fn recorded_finished_steps(records: &[Record]) -> BTreeSet<StepId> {
8290 records
8291 .iter()
8292 .filter_map(|r| match r.kind() {
8293 RecordKind::StepFinished { .. } => r.body.step,
8294 _ => None,
8295 })
8296 .collect()
8297}
8298
8299/// The case the run is bound to, read from its `CaseBound` record.
8300fn recorded_case_id(records: &[Record]) -> Option<crate::core::CaseId> {
8301 records.iter().find_map(|r| match r.kind() {
8302 RecordKind::CaseBound { .. } => r.body.case,
8303 _ => None,
8304 })
8305}
8306
8307/// The steps whose latest forward ending on the record is a success.
8308fn recorded_succeeded_steps(records: &[Record]) -> BTreeSet<StepId> {
8309 let mut succeeded = BTreeSet::new();
8310 for r in records.iter().filter(|r| r.body.phase.is_forward()) {
8311 if let (RecordKind::StepFinished { outcome }, Some(step)) = (r.kind(), r.body.step) {
8312 if outcome == RunStatus::Succeeded.as_str() {
8313 succeeded.insert(step);
8314 } else {
8315 succeeded.remove(&step);
8316 }
8317 }
8318 }
8319 succeeded
8320}
8321
8322/// The group records already on the journal, by writing step and phase.
8323///
8324/// Read back for the same reason the step sets are: group records are not
8325/// effects, so the cursor cannot dedup them, and a resumed step that re-opens
8326/// its group or reaches its end at the frontier — cursor exhausted, writes
8327/// enabled — would otherwise write the same record a second time.
8328fn recorded_groups(
8329 records: &[Record],
8330) -> BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup> {
8331 let mut recorded: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup> =
8332 BTreeMap::new();
8333 for r in records {
8334 let (group, opened) = match r.kind() {
8335 RecordKind::GroupOpened { group, .. } => (group, true),
8336 RecordKind::GroupSettled { group, .. } => (group, false),
8337 _ => continue,
8338 };
8339 let Some(step) = r.body.step else { continue };
8340 let entry = recorded
8341 .entry((step, r.body.phase, group.clone()))
8342 .or_default();
8343 if opened {
8344 entry.opened += 1;
8345 } else {
8346 entry.settled += 1;
8347 }
8348 }
8349 recorded
8350}
8351
8352/// Every capability an agent advertises is provided by one of **its own**
8353/// skills.
8354///
8355/// An agent advertising a capability none of its skills provide is a card that
8356/// lies, and the caller who believed it finds out at dispatch — in production —
8357/// rather than here at startup.
8358///
8359/// This checked a plane-wide map, and the difference is not pedantry. A skill
8360/// registered on the *builder* rather than on the agent — `.agent(Agent::new(&m))`
8361/// followed by `.skill(s)` — satisfied a plane-wide check while being
8362/// **ungoverned**: `governed_by` is keyed from the agent's own skills, so that
8363/// skill gets no manifest. It runs under the plane's default budget instead of
8364/// the declared one, and `StepCtx::gate` never refuses a model or tool the file
8365/// did not list, because there is no file.
8366///
8367/// The plane built cleanly and the assertion's own message said "its skills",
8368/// so the only signal was a `None` from `cx.manifest()` that a skill has no
8369/// reason to check. That is a declaration reading as a control while governing
8370/// nothing, which is the one shape this codebase refuses everywhere.
8371#[cfg(feature = "manifest")]
8372fn check_declaration_matches_skills(
8373 m: &crate::manifest::Manifest,
8374 mine: &HashSet<Capability>,
8375) -> Result<(), BuildError> {
8376 let missing: Vec<String> = m
8377 .spec
8378 .capabilities
8379 .provides
8380 .iter()
8381 .filter(|c| !mine.contains(&Capability::new(c.as_str())))
8382 .cloned()
8383 .collect();
8384 if !missing.is_empty() {
8385 return Err(BuildError::AdvertisesWhatItCannotProvide {
8386 agent: m.metadata.name.clone(),
8387 missing,
8388 });
8389 }
8390
8391 // And the other direction, which is the one that leaves no trace. A skill
8392 // registered under a manifest is *governed* by it — the manifest's budget,
8393 // model grants, egress ceiling and policy identity all apply — so a
8394 // capability the skill answers and the declaration never names is a
8395 // reviewed surface with an unreviewed door in it. The declaration is the
8396 // artifact that gets read, digested and pinned, and the A2A card is built
8397 // from it, so the extra capability is served and advertised nowhere.
8398 //
8399 // It is the same argument the manifest already makes about prompts: a
8400 // system prompt composed in the deployer's code has no version, changes in
8401 // a deploy, and nothing connects the change to the runs it affected. A
8402 // capability added in code does the same to the agent's surface.
8403 let undeclared: Vec<String> = {
8404 let declared: HashSet<Capability> = m
8405 .spec
8406 .capabilities
8407 .provides
8408 .iter()
8409 .map(|c| Capability::new(c.as_str()))
8410 .collect();
8411 let mut extra: Vec<String> = mine
8412 .iter()
8413 .filter(|c| !declared.contains(c))
8414 .map(|c| c.0.clone())
8415 .collect();
8416 // Deterministic, so the message does not depend on hash order.
8417 extra.sort();
8418 extra
8419 };
8420 if !undeclared.is_empty() {
8421 return Err(BuildError::ProvidesWhatItDoesNotAdvertise {
8422 agent: m.metadata.name.clone(),
8423 undeclared,
8424 });
8425 }
8426 Ok(())
8427}
8428
8429/// Add one skill to the plane's two lookup tables, refusing a collision.
8430///
8431/// Both maps are plane-wide, and a bare `insert` would take a second
8432/// registration silently. That was tolerable when a plane was one agent and is
8433/// not now: dispatch resolves a capability to a skill *and to the manifest
8434/// governing it*, so a silent overwrite does not merely shadow the loser — it
8435/// moves work the loser still advertises out from under the loser's budget,
8436/// model grants and egress ceiling. Nothing in the journal would show it,
8437/// because the winner looks like the only claimant that ever existed.
8438///
8439/// Returns the skill's name, which is the key governance is recorded under.
8440///
8441/// # Errors
8442///
8443/// If another skill already holds this name, or another skill already claims one
8444/// of its capabilities.
8445fn register_skill(
8446 skill: Arc<dyn Skill>,
8447 caps: &mut HashMap<Capability, String>,
8448 skills: &mut HashMap<String, Arc<dyn Skill>>,
8449) -> Result<String, BuildError> {
8450 let d = skill.descriptor();
8451 if let Some(existing) = skills.get(&d.name)
8452 // Registering the *same* `Arc` twice is idempotent rather than a
8453 // mistake; two distinct skills under one name is the collision.
8454 && !Arc::ptr_eq(existing, &skill)
8455 {
8456 return Err(BuildError::DuplicateSkillName { name: d.name });
8457 }
8458 // `capabilities()`, not the raw field: a descriptor that declared nothing
8459 // answers its own name, and registering that here is what makes
8460 // `run("greet")` and an `agent/greet` grant resolve without a hello-world
8461 // program inventing a second name for its one skill.
8462 for cap in d.capabilities() {
8463 if let Some(first) = caps.get(&cap)
8464 && first != &d.name
8465 {
8466 return Err(BuildError::CapabilityClaimedTwice {
8467 capability: cap.0,
8468 first: first.clone(),
8469 second: d.name,
8470 });
8471 }
8472 caps.insert(cap, d.name.clone());
8473 }
8474 skills.insert(d.name.clone(), skill);
8475 Ok(d.name)
8476}
8477
8478/// The capability a bare `run(target, ..)` admits: the first declared, or the
8479/// skill's own name — [`SkillDescriptor::capabilities`] is never empty.
8480fn first_capability(descriptor: &SkillDescriptor) -> Capability {
8481 descriptor
8482 .capabilities()
8483 .into_iter()
8484 .next()
8485 .unwrap_or_else(|| Capability::new(descriptor.name.clone()))
8486}
8487
8488/// The business keys a case is identified by, for the binding record.
8489///
8490/// Read from the case rather than taken from the request, because a run joining
8491/// an existing case is admitted with whichever key its message carried while the
8492/// case may be identified by several — and a manifest binding naming any of them
8493/// must resolve.
8494///
8495/// Falls back to the admitted keys if the case cannot be read back. That is not
8496/// defensive padding: `correlate_or_open` has already committed the binding, so
8497/// a failure here is a read that lost a race with nothing, and the keys the run
8498/// was admitted with are a true subset of the case's. Recording them is strictly
8499/// better than recording none, and *nothing* is what would silently make a
8500/// binding unresolvable for a run whose case is perfectly well identified.
8501async fn case_correlation(
8502 cases: &dyn CaseStore,
8503 case_id: crate::core::CaseId,
8504 admitted: &[CorrelationKey],
8505) -> Result<Vec<CorrelationKey>, RuntimeError> {
8506 let stored = cases
8507 .case(case_id)
8508 .await
8509 .map_err(RuntimeError::from_store)?
8510 .map(|case| case.correlation)
8511 .unwrap_or_default();
8512 let mut keys = if stored.is_empty() {
8513 admitted.to_vec()
8514 } else {
8515 stored
8516 };
8517 // Canonical order, so the record is byte-stable for a given set and two
8518 // stores that answer in different orders produce the same journal.
8519 keys.sort();
8520 keys.dedup();
8521 Ok(keys)
8522}
8523
8524/// A lease owner that no other process will accidentally share.
8525///
8526/// The previous default was the constant `"agentplane"`, which every replica and
8527/// every restart used. Two consequences, both silent:
8528///
8529/// * Two replicas each saw the other's lease as their own and renewed it
8530/// without bumping the epoch — two writers on one run, which is the exact
8531/// situation fencing exists to make impossible.
8532/// * A process restarting after a crash "renewed" the dead process's lease
8533/// instead of waiting for expiry and fencing it, so a zombie still holding a
8534/// socket could keep writing under the same epoch as its replacement.
8535///
8536/// A per-process random identity turns both into the correct behaviour: a
8537/// different owner cannot renew, so it waits for expiry and takes over with
8538/// `epoch + 1`.
8539///
8540/// Not derived from a hostname or PID: containers reuse both. Randomness is the
8541/// property that matters; readability is what the `owner` override is for, and a
8542/// deployment with a real instance identity — a pod name — should pass it.
8543fn default_owner() -> String {
8544 use std::collections::hash_map::RandomState;
8545 use std::hash::{BuildHasher, Hasher};
8546 use std::sync::atomic::{AtomicU64, Ordering};
8547
8548 // OS entropy, and deliberately *not* the clock: this crate forbids reading
8549 // the wall clock outside a journaled effect, and rightly — a lease owner is
8550 // a poor reason to make an exception to a rule that keeps replay honest.
8551 // `RandomState` is seeded by the operating system, so two processes differ
8552 // even where a container has reused a PID.
8553 static SEED: std::sync::OnceLock<u64> = std::sync::OnceLock::new();
8554 // A counter beside it, so two runtimes built in one process — which tests do
8555 // constantly — never alias each other either.
8556 static SEQ: AtomicU64 = AtomicU64::new(0);
8557
8558 let seed = *SEED.get_or_init(|| RandomState::new().build_hasher().finish());
8559 let n = SEQ.fetch_add(1, Ordering::Relaxed);
8560 format!("agentplane-{seed:016x}-{n}")
8561}
8562
8563/// Resolve a run's data-subject bindings against its input, before the run
8564/// is bound to a case.
8565///
8566/// Declared bindings first, in declaration order, then the subjects the
8567/// embedder named; each entry's position is the index every reference to it
8568/// names. A binding that resolves to nothing refuses the run rather than
8569/// attributing its data to nobody. A `$case` binding is refused when the run
8570/// asked for no case, and otherwise left for [`fill_case_subjects`].
8571fn bind_subjects(
8572 declared: &[crate::journal::SubjectBinding],
8573 named: Vec<String>,
8574 input: &Tainted<Value>,
8575 in_case: bool,
8576) -> Result<Vec<crate::journal::BoundSubject>, RuntimeError> {
8577 use crate::journal::SubjectBinding;
8578 let refuse = |binding: &SubjectBinding, reason: &str| RuntimeError::SubjectUnbound {
8579 binding: binding.to_string(),
8580 reason: reason.to_owned(),
8581 };
8582 let mut resolved = Vec::new();
8583 let named = named
8584 .into_iter()
8585 .map(|subject| (SubjectBinding::Named, Some(subject)));
8586 for (position, (binding, given)) in declared
8587 .iter()
8588 .cloned()
8589 .map(|binding| (binding, None))
8590 .chain(named)
8591 .enumerate()
8592 {
8593 let (subject, trusted) = match (&binding, given) {
8594 (_, Some(subject)) => (subject, true),
8595 (SubjectBinding::Input { pointer }, None) => {
8596 let Some(selected) = input.project_pointer(pointer) else {
8597 return Err(refuse(&binding, "it selects nothing in the run's input"));
8598 };
8599 let subject = match selected.peek() {
8600 Value::String(text) => text.clone(),
8601 Value::Number(number) => number.to_string(),
8602 _ => {
8603 return Err(refuse(&binding, "it selects neither a string nor a number"));
8604 }
8605 };
8606 (
8607 subject,
8608 selected.label().trust == crate::core::Trust::Trusted,
8609 )
8610 }
8611 (SubjectBinding::Case, None) => {
8612 if !in_case {
8613 return Err(refuse(&binding, "the run belongs to no case"));
8614 }
8615 (CASE_PENDING.to_owned(), true)
8616 }
8617 (SubjectBinding::Named, None) => {
8618 return Err(refuse(&binding, "no subject was named"));
8619 }
8620 };
8621 if subject.trim().is_empty() {
8622 return Err(refuse(&binding, "it resolves to an empty subject"));
8623 }
8624 let index = u16::try_from(position)
8625 .map_err(|_| refuse(&binding, "a run binds at most 65536 subjects"))?;
8626 resolved.push(crate::journal::BoundSubject {
8627 index,
8628 binding,
8629 trusted,
8630 subject,
8631 });
8632 }
8633 Ok(resolved)
8634}
8635
8636/// What a `$case` binding holds between [`bind_subjects`] and
8637/// [`fill_case_subjects`]: never journaled.
8638const CASE_PENDING: &str = "$case";
8639
8640/// Give each `$case` binding the id of the case the run was bound to.
8641fn fill_case_subjects(
8642 mut bound: Vec<crate::journal::BoundSubject>,
8643 case: Option<&CaseContext>,
8644) -> Vec<crate::journal::BoundSubject> {
8645 if let Some(case) = case {
8646 for entry in &mut bound {
8647 if entry.binding == crate::journal::SubjectBinding::Case {
8648 entry.subject = case.case_id.to_string();
8649 }
8650 }
8651 }
8652 bound
8653}
8654
8655/// The references a run's `DataSubjectBound` record makes available to every
8656/// value it reads.
8657fn subject_refs(
8658 run: RunId,
8659 bound: &[crate::journal::BoundSubject],
8660) -> BTreeSet<crate::core::SubjectRef> {
8661 bound
8662 .iter()
8663 .map(|b| crate::core::SubjectRef {
8664 run,
8665 index: b.index,
8666 })
8667 .collect()
8668}
8669
8670/// The references a recorded run's `DataSubjectBound` names, read back so a
8671/// resume attributes exactly as the live run did.
8672fn recorded_subjects(run: RunId, records: &[Record]) -> BTreeSet<crate::core::SubjectRef> {
8673 records
8674 .iter()
8675 .find_map(|r| match r.kind() {
8676 RecordKind::DataSubjectBound { bindings } => Some(subject_refs(run, bindings)),
8677 _ => None,
8678 })
8679 .unwrap_or_default()
8680}
8681
8682/// Refuse a history this build cannot re-derive.
8683///
8684/// Before recomputing anything, because every effect key a replay derives comes
8685/// out of the canonicalizer: a run written under another rule recomputes
8686/// different keys and would be quarantined as *non-determinism* — the most
8687/// serious conclusion this runtime reaches, reported for a healthy run because
8688/// the rule moved underneath it.
8689///
8690/// The chain itself is fine and always was: it hashes the bytes it stored rather
8691/// than re-canonicalizing them. What moved is everything *derived*, which is
8692/// exactly the class replay compares.
8693fn ensure_replayable_canon(records: &[Record]) -> Result<(), RuntimeError> {
8694 if let Some(recorded) = records.iter().find_map(recorded_canon)
8695 && recorded != crate::core::canon::VERSION
8696 {
8697 return Err(RuntimeError::CanonicalizationChanged {
8698 recorded,
8699 implemented: crate::core::canon::VERSION,
8700 });
8701 }
8702 Ok(())
8703}
8704
8705/// Which canonicalization rule wrote this run's derived digests.
8706fn recorded_canon(r: &Record) -> Option<u16> {
8707 match r.kind() {
8708 RecordKind::RunAdmitted { canon, .. } => Some(*canon),
8709 _ => None,
8710 }
8711}
8712
8713/// The principal a run was admitted as.
8714///
8715/// Read back rather than recomputed, for the same reason the plan is: the
8716/// principal a run was authorized as is a fact *about that run*, and deriving it
8717/// again from a plan that may since have been edited would silently re-attribute
8718/// history.
8719/// The chain a recorded run acted under, re-checked structurally on the way
8720/// back — never re-verified as a credential, for the reason `core::identity`
8721/// gives. `None` for a run admitted without one.
8722fn recorded_chain(records: &[Record]) -> Result<Option<crate::core::Delegation>, RuntimeError> {
8723 records
8724 .iter()
8725 .find_map(|r| match r.kind() {
8726 RecordKind::IdentityBound { chain } => Some(chain.clone()),
8727 _ => None,
8728 })
8729 .map(crate::core::Delegation::rehydrate)
8730 .transpose()
8731 .map_err(RuntimeError::Delegation)
8732}
8733
8734/// Whether a resume refused with `error` would be refused the same way on
8735/// every attempt: the journal it read is what refuses, not the store, a lease,
8736/// a quota or this plane's configuration.
8737///
8738/// Named rather than negated: a failure not listed here is retried, so a
8739/// transient one added later costs a retry loop rather than a quarantine.
8740/// `PlanContract`, `NoProvider` and `NoCaseStore` are not listed because they
8741/// are about this plane — a quota store, an agent or a case store it lacks —
8742/// and another plane may settle what this one cannot.
8743const fn recurs_on_every_resume(error: &RuntimeError) -> bool {
8744 matches!(
8745 error,
8746 RuntimeError::Encoding(_)
8747 | RuntimeError::ChainBroken { .. }
8748 | RuntimeError::CanonicalizationChanged { .. }
8749 | RuntimeError::Delegation(_)
8750 )
8751}
8752
8753fn recorded_agent(records: &[Record]) -> String {
8754 records
8755 .iter()
8756 .find_map(|r| match r.kind() {
8757 RecordKind::RunAdmitted { capability, .. } => Some(capability.clone()),
8758 _ => None,
8759 })
8760 .unwrap_or_default()
8761}
8762
8763/// Build a step's input from its declared argument sources.
8764///
8765/// Labels join across sources, so a step reading anything untrusted produces an
8766/// untrusted input without the plan author having to say so.
8767fn assemble(
8768 node: &PlanNode,
8769 run_input: &Tainted<Value>,
8770 outputs: &BTreeMap<StepId, Tainted<Value>>,
8771) -> Result<Tainted<Value>, RuntimeError> {
8772 // The common case — a single argument — passes the value through rather than
8773 // wrapping it in a one-key object, so simple plans stay legible.
8774 if node.args.len() == 1
8775 && let Some((_, only)) = node.args.iter().next()
8776 {
8777 return resolve_arg(node, only, run_input, outputs);
8778 }
8779
8780 let mut fields = Vec::with_capacity(node.args.len());
8781 for (name, source) in &node.args {
8782 let v = resolve_arg(node, source, run_input, outputs)?;
8783 fields.push((name.clone(), v));
8784 }
8785 Ok(Tainted::object(fields))
8786}
8787
8788fn resolve_arg(
8789 node: &PlanNode,
8790 source: &ArgSource,
8791 run_input: &Tainted<Value>,
8792 outputs: &BTreeMap<StepId, Tainted<Value>>,
8793) -> Result<Tainted<Value>, RuntimeError> {
8794 Ok(match source {
8795 // Projected as a step's output is: a field keeps the label it was
8796 // admitted with, and one admitted without its own inherits the whole
8797 // value's — the parts of an untrusted document are untrusted.
8798 ArgSource::RunInput { field } => project(run_input, field.as_deref()),
8799 ArgSource::Const { value } => Tainted::trusted(value.clone()),
8800 ArgSource::Node { step, field } => {
8801 // The contract already proved this is upstream, so a miss here means
8802 // the scheduler dispatched out of order — a bug worth naming rather
8803 // than papering over with a null.
8804 let upstream = outputs.get(step).ok_or_else(|| {
8805 RuntimeError::PlanContract(format!(
8806 "step {} read step {step}, which has not produced a value",
8807 node.id
8808 ))
8809 })?;
8810 project(upstream, field.as_deref())
8811 }
8812 })
8813}
8814
8815/// A value, or one of its top-level fields with its descendants' labels; an
8816/// absent field is a null under the whole value's label.
8817fn project(value: &Tainted<Value>, field: Option<&str>) -> Tainted<Value> {
8818 match field {
8819 Some(field) => value
8820 .project_field(field)
8821 .unwrap_or_else(|| Tainted::with_label(Value::Null, value.label().clone())),
8822 None => value.clone(),
8823 }
8824}
8825
8826/// A step that read a blob rewritten under its address ends unreproducible,
8827/// whatever the skill returned: its history cannot be reproduced, whether the
8828/// skill propagated the error, retried it or swallowed it.
8829fn moved_blob_wins<T>(
8830 cx: &StepCtx<'_>,
8831 result: Result<T, crate::core::SkillError>,
8832) -> Result<T, crate::core::SkillError> {
8833 match cx.moved_blob() {
8834 Some(what) => Err(crate::core::SkillError::Step(
8835 crate::core::StepError::Unreproducible {
8836 what: what.to_owned(),
8837 detail: "a pinned read returned other bytes under its address".to_owned(),
8838 },
8839 )),
8840 None => result,
8841 }
8842}
8843
8844/// Settle a group the skill left open, because `Drop` cannot.
8845///
8846/// A skill that fails with `?` never reaches `commit` or `abort`, so the handle
8847/// is dropped with members standing. Reversing them is async and `Drop` is not,
8848/// which is why the group lives on the context and the executor finishes what
8849/// the handle abandoned — the same relationship the executor already has with a
8850/// step's compensation.
8851///
8852/// Three situations, and they are not the same:
8853///
8854/// * **suspended** — the step has not ended. Its frame is persisted and it will
8855/// re-run from the top, rebuilding the group from the journal as it replays
8856/// the members. Reversing here would undo a run that is merely waiting.
8857/// * **failed** — abort, unless the failure leaves the world in doubt. Doubt is
8858/// the one condition under which nothing may be reversed.
8859/// * **succeeded with a group still open** — an author bug, and the safe
8860/// reading is that the group was never meant to take. It is reversed and the
8861/// step fails loudly, because a group that commits by being forgotten is
8862/// worse than one that does not commit at all.
8863async fn settle_abandoned_group(
8864 cx: &mut StepCtx<'_>,
8865 result: Result<Outcome, crate::core::SkillError>,
8866) -> Result<Outcome, crate::core::SkillError> {
8867 use crate::core::{SkillError, StepError};
8868
8869 let Some(name) = cx.open_group().map(|g| g.name.clone()) else {
8870 return result;
8871 };
8872 // A pause leaves the group open for the resumed step to settle — unless
8873 // the run was cancelled while paused and this pass exists to abandon it.
8874 if !cx.abandoning
8875 && matches!(&result, Err(SkillError::Step(e)) if crate::runtime::group::is_pause(e))
8876 {
8877 return result;
8878 }
8879
8880 // A member whose failure may have reached the world travels no further.
8881 // Reversing around a call that may have — or did — happen leaves the world
8882 // holding a write no `Aborted` settlement can honestly claim to have undone.
8883 let doubt = match &result {
8884 Err(SkillError::Step(e)) => crate::runtime::group::may_have_externalised(e),
8885 _ => false,
8886 };
8887 if doubt {
8888 let detail = match &result {
8889 Err(e) => e.to_string(),
8890 Ok(_) => String::new(),
8891 };
8892 let settled = cx
8893 .settle_open_group(crate::core::GroupOutcome::Quarantined, Some(&detail))
8894 .await;
8895 return match settled {
8896 Ok(()) => result,
8897 Err(e) => Err(SkillError::Step(e)),
8898 };
8899 }
8900
8901 match cx
8902 .abort_open_group("the step ended without settling the group")
8903 .await
8904 {
8905 // The abort itself could not be completed. That outranks whatever the
8906 // step was reporting: a partly unwound group is the more dangerous fact.
8907 Err(e) => Err(SkillError::Step(e)),
8908 Ok(()) => match result {
8909 // A reported failure keeps its own reason. `Outcome::Fail` is an
8910 // `Ok` at the type level and a failure in fact: leaving the group
8911 // to the runtime is the ordinary path there, not an author bug, and
8912 // overwriting the reason would tell an operator the step "returned
8913 // successfully" while hiding why it actually stopped.
8914 Err(e) => Err(e),
8915 failed @ Ok(Outcome::Fail { .. }) => failed,
8916 // Anything else claimed to make progress while leaving a group
8917 // unsettled.
8918 Ok(_) => Err(SkillError::Step(StepError::GroupAborted {
8919 what: format!(
8920 "step made progress with group '{name}' still open — it was \
8921 reversed, because a group that commits by being forgotten is worse \
8922 than one that does not commit at all"
8923 ),
8924 })),
8925 },
8926 }
8927}
8928
8929/// Turn a step's result into a run status.
8930///
8931/// The distinction that matters is between an ordinary failure and a run whose
8932/// *history can no longer be trusted*. Divergence and orphaned effects are the
8933/// latter: they mean the journal no longer describes what this code does, so a
8934/// human has to look before anything else happens. Folding them into `Failed`
8935/// would put them in the same bucket as "the invoice was rejected", and they
8936/// would be retried like one.
8937fn classify(
8938 meter: &super::metrics::Meter,
8939 result: Result<Outcome, crate::core::SkillError>,
8940) -> (RunStatus, Option<Tainted<Value>>) {
8941 use crate::core::{SkillError, StepError};
8942
8943 match result {
8944 // The label travels with the value. Stripping it here would silently
8945 // launder provenance at every step boundary: a downstream step reading
8946 // an untrusted upstream output would receive it marked trusted, and the
8947 // taint gates further on would have nothing to act on.
8948 Ok(Outcome::Done(v)) => (RunStatus::Succeeded, Some(v)),
8949 Ok(Outcome::Fail { reason }) => (RunStatus::Failed(reason), None),
8950 // Not a failure: the executor decides whether a new plan is allowed,
8951 // because the answer depends on the run's provenance and budget, which
8952 // a step cannot see.
8953 Ok(Outcome::Replan { reason }) => (RunStatus::Replanning(reason), None),
8954 // Suspension is not a failure: the run is healthy and waiting. It
8955 // reaches here as an error only because that is how control leaves a
8956 // skill.
8957 Err(SkillError::Step(StepError::Suspended(reason))) => (RunStatus::Suspended(reason), None),
8958 // The ceiling did its job. Reporting this as a failure would have
8959 // operators debugging a system that behaved exactly as instructed.
8960 Err(SkillError::Step(StepError::Budget(exceeded))) => {
8961 (RunStatus::Exhausted(exceeded), None)
8962 }
8963 // A withdrawal found at a hop is the boundary's pause, reached
8964 // mid-step: the hop recorded the withholding before refusing.
8965 Err(SkillError::Step(StepError::Withheld { subject, reason })) => {
8966 (RunStatus::Withheld { subject, reason }, None)
8967 }
8968 Err(e) => {
8969 let msg = e.to_string();
8970 // Matched structurally, never on the message: a match on the word
8971 // "quarantined" would let a reworded error silently downgrade a run
8972 // to `Failed` — history kept, the flag that said not to trust it
8973 // lost.
8974 // Each of these is a failure P7 exists to make loud, and each gets
8975 // its own event so "did this happen" is a query rather than a grep.
8976 match &e {
8977 SkillError::Step(StepError::NonDeterminism {
8978 seq,
8979 expected,
8980 actual,
8981 detail,
8982 }) => {
8983 tracing::error!(
8984 target: telemetry::NONDETERMINISM,
8985 %seq, %expected, %actual,
8986 error_type = "nondeterminism",
8987 reason_digest = %telemetry::reason_digest(detail),
8988 );
8989 meter.count(metrics::DIVERGENCES, "");
8990 }
8991 SkillError::Step(StepError::ReplayOverrun { actual, kind }) => {
8992 tracing::error!(
8993 target: telemetry::NONDETERMINISM,
8994 %actual, %kind, overrun = true,
8995 );
8996 meter.count(metrics::DIVERGENCES, "");
8997 }
8998 SkillError::Step(StepError::Undecidable { key, detail, .. }) => {
8999 tracing::error!(
9000 target: telemetry::UNDECIDABLE,
9001 %key,
9002 error_type = "undecidable",
9003 reason_digest = %telemetry::reason_digest(detail),
9004 );
9005 meter.count(metrics::UNDECIDABLE, "");
9006 }
9007 SkillError::Step(StepError::Unreproducible { what, detail }) => {
9008 tracing::error!(
9009 target: telemetry::UNREPRODUCIBLE,
9010 error_type = "unreproducible",
9011 reason_digest = %telemetry::reason_digest(&format!("{what}: {detail}")),
9012 );
9013 meter.count(metrics::UNREPRODUCIBLE, "");
9014 }
9015 _ => {}
9016 }
9017
9018 let untrustworthy = matches!(
9019 e,
9020 SkillError::Step(
9021 StepError::NonDeterminism { .. }
9022 | StepError::ReplayOverrun { .. }
9023 | StepError::Undecidable { .. }
9024 | StepError::GroupUnsettled { .. }
9025 | StepError::Unreproducible { .. }
9026 )
9027 );
9028 if untrustworthy {
9029 (RunStatus::Quarantined(msg), None)
9030 } else {
9031 (RunStatus::Failed(msg), None)
9032 }
9033 }
9034 }
9035}
9036
9037/// Everything one execution needs, gathered so the executor is not called with
9038/// eight positional arguments — two of which are `u64`-shaped and would swap
9039/// silently.
9040struct Execution<'a> {
9041 /// Where the recorded run was refused by a step limit, if it was. `None`
9042 /// for a live run, which has no history to consult.
9043 refusal: Option<(StepId, String, String)>,
9044 /// The subject of a withholding the recorded history left standing. `None`
9045 /// for a live run, and for a resumed one whose withdrawal was already
9046 /// lifted and recorded.
9047 withheld_subject: Option<String>,
9048 /// Successor plans the recorded run produced, oldest first. Empty on a live
9049 /// run. Read back rather than re-synthesised, because a planner asked twice
9050 /// can answer differently.
9051 successors: Vec<PlanIR>,
9052 /// Steps whose `StepStarted` is already on the record. Empty on a live
9053 /// run. Read back so a resume neither re-announces a step nor forgets to
9054 /// announce one it starts for the first time.
9055 started: BTreeSet<StepId>,
9056 /// Steps whose `StepFinished` is already on the record. Empty on a live
9057 /// run.
9058 finished: BTreeSet<StepId>,
9059 /// Steps whose latest forward ending on the record is a success. Empty on
9060 /// a live run. A resume replays these before it asks whether to stop, so
9061 /// an unwind sees their outputs and the order they finished in.
9062 succeeded: BTreeSet<StepId>,
9063 /// Group records already on the journal, keyed by the step and phase
9064 /// that wrote them. Empty on a live run. Read back for the same reason
9065 /// `finished` is: group records are not effects, so the cursor cannot
9066 /// dedup them, and a resumed step that re-opens or re-settles its group
9067 /// at the frontier — where its cursor is exhausted and writes are
9068 /// enabled — would report one group as two.
9069 recorded_groups: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
9070 run: RunId,
9071 epoch: u64,
9072 quota: QuotaPass,
9073 plan: &'a PlanIR,
9074 input: Tainted<Value>,
9075 mode: Pass,
9076 case: Option<CaseContext>,
9077 budget: Budget,
9078 /// The run's data-subject references: from the `DataSubjectBound` just
9079 /// written on a live run, read back from it on a replay.
9080 subjects: BTreeSet<crate::core::SubjectRef>,
9081 /// Who is acting, for the policy principal. Read back from `RunAdmitted` on
9082 /// a replay rather than recomputed, for the same reason the plan is: the
9083 /// principal a run was authorized as is a fact about that run.
9084 agent: String,
9085 /// On whose behalf, for the policy context and for what this run
9086 /// commissions. Read back from `IdentityBound` on a replay for the same
9087 /// reason: a resumed run acting under the plane's chain instead of the one
9088 /// recorded for it would reach different verdicts than the live run did.
9089 identity: Option<crate::core::Delegation>,
9090}
9091
9092/// What a conclusion says when nothing on the chain says who ended the run.
9093///
9094/// Quarantine rather than a placeholder name, for the reason the exhausted arm
9095/// beside it quarantines when the typed ceiling verdict is missing: the record
9096/// is not trustworthy, and *not trustworthy* is a different bucket from *try
9097/// again*. A name invented here would be served through the operator API as the
9098/// person who did it.
9099const UNATTRIBUTED: &str = "the chain records this ending and not who asked for it — an \
9100 operator act with nobody on it cannot be answered by reading further";
9101
9102/// The recorded status of a run that must not be resumed.
9103///
9104/// The outcomes that close a run to recovery:
9105///
9106/// * **Succeeded** — there is nothing outstanding. Re-executing would repeat
9107/// work that is not an effect (a case-state write, say), which is the same
9108/// class of bug the effect protocol prevents, arriving through a side door.
9109/// * **Cancelled** — a stopped run stays stopped, or the next inbound event
9110/// resumes it and it carries on doing the thing somebody intervened to
9111/// prevent.
9112/// * **Abandoned** — a person already decided there will never be an answer.
9113/// * **Quarantined, until a person answers it.** Resuming an unanswered
9114/// quarantine would re-hit whatever could not be decided, and burying that in
9115/// a retry loop is exactly how an undecidable situation becomes an unnoticed
9116/// one. A recorded [`QuarantineDecided`](RecordKind::QuarantineDecided)
9117/// lifts it — the *precondition* is that a named person looked, and the
9118/// verdict is still the runtime's: a reopened run re-decides from a history
9119/// that now holds whatever they established, and reaches the same quarantine
9120/// again if they were wrong or incomplete.
9121///
9122/// A **failed** run is deliberately *not* terminal here: a process that died
9123/// mid-flight records a failure, and recovering it is the entire point.
9124///
9125/// # Why the seal, and not the last step
9126///
9127/// Scanning backwards for a `StepFinished` and reading its outcome is wrong
9128/// twice, and the second is severe:
9129///
9130/// * A step's outcome is not the run's. They coincide only in a one-step plan.
9131/// * `find_map` **skips** a record it does not recognise and keeps looking. A
9132/// run whose last step failed after earlier steps succeeded would match an
9133/// *earlier* `StepFinished { outcome: "succeeded" }` and be reported
9134/// closed-and-succeeded: every multi-step run suspended after a failure —
9135/// every saga waiting on an approval to finish unwinding — could never be
9136/// resumed, and would report success while stuck.
9137///
9138/// `RunSealed` is written by `conclude` for exactly the runs that reached a
9139/// conclusion, and never for a suspended one. That is the fact this needs, so
9140/// it is the fact it reads.
9141pub(crate) fn resume_is_closed(records: &[Record]) -> Option<RunStatus> {
9142 let outcome = records.iter().rev().find_map(|r| match r.kind() {
9143 RecordKind::RunConcluded { outcome, .. } => Some(outcome.as_str()),
9144 _ => None,
9145 })?;
9146
9147 match outcome {
9148 "succeeded" => Some(RunStatus::Succeeded),
9149 "quarantined" => quarantine_decision(records).is_none().then(|| {
9150 RunStatus::Quarantined(
9151 "recorded as quarantined; a named person has to answer it before it can run \
9152 again — reopen it once the doubt is resolved, or abandon it"
9153 .into(),
9154 )
9155 }),
9156 // A conclusion whose attribution record is missing is **quarantined**,
9157 // not attributed to a person called "unknown". The arm for an exhausted
9158 // run with no typed ceiling verdict already reads this way; these three
9159 // fabricated a name instead, and a fabricated one is worse than a
9160 // missing one because it is indistinguishable from a real operator with
9161 // that name and it is served through the operator API as fact.
9162 //
9163 // The reason is the one the person gave, from the record that holds
9164 // it, rather than a sentence written here: an operator refused a
9165 // resume is asking *why this run is closed*, and a canned restatement
9166 // of the outcome answers a question they did not ask.
9167 "abandoned" => Some(recorded_abandonment(records).map_or_else(
9168 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9169 |(actor, reason)| RunStatus::Abandoned { actor, reason },
9170 )),
9171 // A stopped run stays stopped. Otherwise the next inbound event resumes
9172 // it and it carries on doing the thing somebody intervened to prevent —
9173 // and the intervention would look, from the journal, like it worked.
9174 "cancelled" => Some(recorded_cancellation(records).map_or_else(
9175 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9176 |(actor, reason)| RunStatus::Cancelled { actor, reason },
9177 )),
9178 // The two conclusions that deliberately do not close a run: a failed
9179 // run resumes with its completed effects read back from history, and an
9180 // exhausted one continues once somebody raises the ceiling.
9181 //
9182 // Their relationship to [`RunStatus::seals`] is the load-bearing part
9183 // and is pinned by `a_sealing_conclusion_is_never_resumable`. The
9184 // direction that matters is *no sealing status may be resumable*: a
9185 // status that seals froze the journal and published a Merkle leaf, so
9186 // resuming it would grow the history past the leaf every later
9187 // checkpoint attests. The reverse direction is deliberately **not** an
9188 // equality — `Suspended` and `Replanning` also do not seal, and never
9189 // reach here at all, because neither is ever a recorded conclusion.
9190 // Stating that as "the two that do not seal" was wrong; there are four,
9191 // and only two of them can be a `RunSealed` outcome.
9192 // A withdrawal joins them, and is the clearest case of the three: the
9193 // run did nothing wrong, its mutations stand, and lifting the halt is
9194 // what continues it. Closing it would turn *stop acting under this
9195 // credential* into *destroy this work*, which is not what an operator
9196 // withdrawing a credential asked for — the operator who wants that
9197 // cancels, and cancelling unwinds.
9198 "failed" | "exhausted" | WITHHELD_OUTCOME => None,
9199 // Fail closed. An outcome this build does not recognise — a sweep's
9200 // `swept`, a future variant, a corrupted string — is not permission to
9201 // resume; it is a run whose recorded ending this code cannot interpret,
9202 // and continuing it would graft new behaviour onto a history that says
9203 // it ended.
9204 other => Some(RunStatus::Quarantined(format!(
9205 "recorded as '{other}', which this build does not recognise as resumable"
9206 ))),
9207 }
9208}
9209
9210/// Refuse a resume that would report success over work its own history reversed.
9211///
9212/// A run whose journal holds compensation-phase records is not resumable once it
9213/// stands concluded. The unwind *reversed* work — a hold released, a refund
9214/// issued — and a resume that replays the forward history and continues would
9215/// conclude success over a world where the work was undone: the journal would
9216/// say "done" about effects its own later records say were taken back.
9217///
9218/// Scoped to runs with a standing conclusion, deliberately: a run that is
9219/// mid-unwind (a compensation suspended for four eyes, a crash between
9220/// reversals) has concluded nothing, and resuming it is how the unwind finishes.
9221fn refuse_resume_over_undone_work(run: RunId, records: &[Record]) -> Result<(), RuntimeError> {
9222 let Some(outcome) = recorded_conclusion(records) else {
9223 return Ok(());
9224 };
9225 if !records
9226 .iter()
9227 .any(|r| matches!(r.kind(), RecordKind::StepCompensated { .. }))
9228 {
9229 return Ok(());
9230 }
9231 Err(RuntimeError::PlanContract(format!(
9232 "run {run} concluded '{outcome}' after compensating completed steps — the work was \
9233 reversed, and resuming over undone work would report success about a world where it no \
9234 longer stands. Start a fresh run"
9235 )))
9236}
9237
9238/// The person's answer standing against this run's quarantine, if there is one.
9239///
9240/// Scoped to what follows the **last** conclusion, so an answer is spent by the
9241/// pass it authorized. A reopened run that quarantines again is quarantined
9242/// again: the earlier decision was made about a history that has since moved,
9243/// and reading it as standing would turn one person's judgement into a
9244/// permanent licence to retry.
9245fn quarantine_decision(records: &[Record]) -> Option<(crate::core::QuarantineDecision, &Record)> {
9246 records
9247 .iter()
9248 .rev()
9249 .take_while(|r| !matches!(r.kind(), RecordKind::RunConcluded { .. }))
9250 .find_map(|r| match r.kind() {
9251 RecordKind::QuarantineDecided { decision, .. } => Some((*decision, r)),
9252 _ => None,
9253 })
9254}
9255
9256/// The abandonment this run's history asks for but has not yet recorded.
9257///
9258/// A decision is written before it is acted on, so a crash in between leaves
9259/// the instruction standing rather than lost. This is what the next resume
9260/// reads to finish the job — and it is why abandonment is not simply performed
9261/// inline by the verb that requests it.
9262fn pending_abandonment(records: &[Record]) -> Option<RunStatus> {
9263 let (decision, record) = quarantine_decision(records)?;
9264 if decision != crate::core::QuarantineDecision::Abandon {
9265 return None;
9266 }
9267 match record.kind() {
9268 RecordKind::QuarantineDecided {
9269 decider, reason, ..
9270 } => Some(RunStatus::Abandoned {
9271 actor: decider.clone(),
9272 reason: reason.clone(),
9273 }),
9274 _ => None,
9275 }
9276}
9277
9278/// The longest admission key this runtime accepts.
9279///
9280/// Bounded because the key is a **counterparty-controlled** string that becomes
9281/// part of a storage key: unbounded, it is an index entry an emitter chooses the
9282/// size of. 512 bytes is far above what an identity needs — a `CloudEvents`
9283/// `source` and `id` together are tens of bytes — and far below what a btree
9284/// index refuses, so the refusal is this crate's with a reason rather than the
9285/// backend's with an error code.
9286pub const MAX_ADMISSION_KEY_BYTES: usize = 512;
9287
9288/// Refuse a key that cannot mean what the caller thinks it means.
9289///
9290/// An **empty** key is the important one, and it is refused rather than
9291/// accepted as a value. A missing header or an unset variable arrives here as
9292/// `""`, and `""` is a perfectly good key: the first message claims it and every
9293/// later message — a *different* message — is answered with the first one's run.
9294/// That is silent, total message loss produced by a configuration mistake, and
9295/// it is exactly what an at-most-once control must not do quietly.
9296fn validate_admission_key(key: &str) -> Result<(), RuntimeError> {
9297 if key.trim().is_empty() {
9298 return Err(RuntimeError::PlanContract(
9299 "an admission key is empty — an unset header or variable arrives here as \
9300 an empty string, and accepting it would answer every later message with \
9301 the first one's run. Use the message's own identity, such as \
9302 `InboundEvent::dedup_key()`"
9303 .into(),
9304 ));
9305 }
9306 if key.len() > MAX_ADMISSION_KEY_BYTES {
9307 return Err(RuntimeError::PlanContract(format!(
9308 "an admission key of {} bytes exceeds the {MAX_ADMISSION_KEY_BYTES}-byte \
9309 limit — the key is chosen by the emitter and becomes part of a storage \
9310 key, so its size is not theirs to decide",
9311 key.len()
9312 )));
9313 }
9314 Ok(())
9315}
9316
9317/// The run id a store named as an admission key's holder.
9318///
9319/// An unparseable holder is corruption in a derived index. Refused rather than
9320/// admitted around: "the index is unreadable" is not evidence the key is free.
9321fn parse_holder(run: &str) -> Result<RunId, RuntimeError> {
9322 RunId::parse(run).map_err(|e| {
9323 RuntimeError::PlanContract(format!(
9324 "an admission key is held by run '{run}', which does not parse ({e}) — refusing rather than admitting a second run under a key whose holder this build cannot read"
9325 ))
9326 })
9327}
9328
9329/// What a run's history says its state is, if it says anything.
9330///
9331/// **One reader, for every surface that answers the question.** Idempotent
9332/// admission, the operator API and the A2A projection are three views of one
9333/// fact, and three copies of this match agree right up until somebody adds a
9334/// record kind or a conclusion arrives that this build cannot read — after
9335/// which a caller polling and the same caller streaming are told different
9336/// things about one run, which is worse than either answer being wrong. The
9337/// laxest copy is the one that decides what an operator sees, so there is one.
9338///
9339/// `None` means the run is still going: the last record is neither a suspension
9340/// nor a conclusion. It is not this function's business whether that means
9341/// *running* or *never existed* — an empty history answers `None` too, and the
9342/// caller knows which it asked for.
9343///
9344/// **The last record, not any record.** A run that waited, was resumed and
9345/// carried on has a suspension in its history and is not suspended now.
9346pub(crate) fn observed_status(records: &[Record]) -> Option<RunStatus> {
9347 Some(match records.last()?.kind() {
9348 RecordKind::RunSuspended { reason } => RunStatus::Suspended(reason.clone()),
9349 RecordKind::RunConcluded {
9350 outcome,
9351 reason,
9352 exhaustion,
9353 ..
9354 } => recorded_status(outcome, reason.as_deref(), exhaustion.as_ref(), records),
9355 _ => return None,
9356 })
9357}
9358
9359/// A sealed run's status, rebuilt from its recorded conclusion.
9360///
9361/// Distinct from `resume_is_closed`, which asks the narrower *may this run be
9362/// resumed* and returns `None` for the conclusions that stay resumable. Every
9363/// conclusion has an answer here: a failed run reported as "still working"
9364/// would send a redelivery into a second admission.
9365fn recorded_status(
9366 outcome: &str,
9367 reason: Option<&str>,
9368 exhaustion: Option<&crate::core::BudgetExceeded>,
9369 records: &[Record],
9370) -> RunStatus {
9371 // A conclusion that carries no reason gets a statement of that fact rather
9372 // than an empty string: the empty summary is what `RunStatus::reason`
9373 // exists to prevent, and this is the surface a retried delivery reads.
9374 let said = || {
9375 reason.map_or_else(
9376 || "concluded without a recorded reason".to_owned(),
9377 ToOwned::to_owned,
9378 )
9379 };
9380 match outcome {
9381 "succeeded" => RunStatus::Succeeded,
9382 "failed" => RunStatus::Failed(said()),
9383 "replanning" => RunStatus::Replanning(said()),
9384 "exhausted" => exhaustion.cloned().map_or_else(
9385 || {
9386 RunStatus::Quarantined(
9387 "recorded as exhausted without the typed ceiling verdict".to_owned(),
9388 )
9389 },
9390 RunStatus::Exhausted,
9391 ),
9392 "quarantined" => RunStatus::Quarantined(said()),
9393 "cancelled" => recorded_cancellation(records).map_or_else(
9394 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9395 |(actor, reason)| RunStatus::Cancelled { actor, reason },
9396 ),
9397 "abandoned" => recorded_abandonment(records).map_or_else(
9398 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9399 |(actor, reason)| RunStatus::Abandoned { actor, reason },
9400 ),
9401 // A withdrawal, read from the record that holds both halves. Without
9402 // this arm the catch-all answered *quarantined, and this build does
9403 // not recognise 'withheld'* — the plane calling its own record
9404 // foreign, on the surface an operator uses to decide whether to lift a
9405 // halt or to cancel.
9406 WITHHELD_OUTCOME => recorded_withholding(records).map_or_else(
9407 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9408 |(subject, reason)| RunStatus::Withheld { subject, reason },
9409 ),
9410 super::sweeper::SWEEP_OUTCOME => RunStatus::Swept,
9411 // Read back as itself, like every other ending this build writes. The
9412 // catch-all below would answer `Quarantined` — this plane calling its
9413 // own record foreign, on a run whose whole point is that somebody else
9414 // ran it.
9415 OBSERVED_OUTCOME => RunStatus::Observed,
9416 BREAK_GLASS_OUTCOME => recorded_crossing(records).map_or_else(
9417 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9418 |(actor, reason)| RunStatus::BrokeGlass { actor, reason },
9419 ),
9420 HALT_LIFTED_OUTCOME => recorded_halt_lift(records).map_or_else(
9421 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9422 |actor| RunStatus::HaltLifted { actor },
9423 ),
9424 HOLD_RELEASED_OUTCOME => recorded_hold_release(records).map_or_else(
9425 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9426 |actor| RunStatus::HoldReleased { actor },
9427 ),
9428 // Fail closed, as the resume path does: a conclusion this build cannot
9429 // interpret is not permission to treat the run as ordinary. Every
9430 // outcome this build *writes* has an arm above, and a guard holds the
9431 // two lists together — an ending answered here as unrecognised would be
9432 // this plane calling its own record foreign, and the surface it reaches
9433 // is the one an operator uses to read a break-glass crossing.
9434 other => RunStatus::Quarantined(format!(
9435 "recorded as '{other}', which this build does not recognise"
9436 )),
9437 }
9438}
9439
9440/// Whether this value is a sealed payload rather than a readable one.
9441///
9442/// Always false without `keyring`: nothing seals, so nothing can be sealed, and
9443/// a build with no key ring must not carry a dependency on the module that
9444/// defines the wrapper.
9445fn sealed_payload(value: &serde_json::Value) -> bool {
9446 #[cfg(feature = "keyring")]
9447 {
9448 crate::journal::payload::is_sealed(value)
9449 }
9450 #[cfg(not(feature = "keyring"))]
9451 {
9452 let _ = value;
9453 false
9454 }
9455}
9456
9457/// How the run's history says it last ended, or that it is waiting.
9458///
9459/// The last conclusion or suspension on the record, as the outcome word a
9460/// replay's own status reports, so the two compare directly. `None` for a run
9461/// that has neither: it is still in flight, and a strict replay of it would
9462/// stop at its frontier and call that divergence.
9463fn recorded_ending(records: &[Record]) -> Option<String> {
9464 records.iter().rev().find_map(|r| match r.kind() {
9465 RecordKind::RunConcluded { outcome, .. } => Some(outcome.clone()),
9466 RecordKind::RunSuspended { .. } => Some("suspended".to_owned()),
9467 _ => None,
9468 })
9469}
9470
9471/// Whether an ending was an act rather than something the run's steps reached.
9472///
9473/// A cancellation, an abandonment, a sweep, a break-glass crossing, a lift, a
9474/// withdrawal and an observation are each recorded by whoever took them; a
9475/// strict replay re-derives steps, and no step decided these.
9476fn ended_by_an_act(outcome: &str) -> bool {
9477 matches!(outcome, "cancelled" | "abandoned")
9478 || [
9479 super::sweeper::SWEEP_OUTCOME,
9480 BREAK_GLASS_OUTCOME,
9481 HALT_LIFTED_OUTCOME,
9482 HOLD_RELEASED_OUTCOME,
9483 WITHHELD_OUTCOME,
9484 OBSERVED_OUTCOME,
9485 ]
9486 .contains(&outcome)
9487}
9488
9489/// The plan the run was admitted under, and each successor a replan produced,
9490/// oldest first. Read back rather than recompiled, which could produce a graph
9491/// that never governed the run.
9492///
9493/// All or nothing. Successors are walked by position, so skipping one this
9494/// build cannot read would put every later replan on the wrong plan.
9495///
9496/// # Errors
9497///
9498/// [`RuntimeError::PayloadsErased`] when a plan is still sealed, named before
9499/// it is parsed: a sealed payload deserialises into a *missing field*, and that
9500/// sentence sends an operator looking for a corrupt journal rather than telling
9501/// them the data was erased on request.
9502fn frozen_plans(run: RunId, records: &[Record]) -> Result<(PlanIR, Vec<PlanIR>), RuntimeError> {
9503 let mut plans = records
9504 .iter()
9505 .filter_map(|r| match r.kind() {
9506 RecordKind::PlanFrozen { plan, .. } => Some(plan),
9507 _ => None,
9508 })
9509 .map(|plan| {
9510 if sealed_payload(plan) {
9511 return Err(RuntimeError::PayloadsErased {
9512 run: run.to_string(),
9513 });
9514 }
9515 serde_json::from_value(plan.clone()).map_err(RuntimeError::Encoding)
9516 })
9517 .collect::<Result<Vec<PlanIR>, _>>()?
9518 .into_iter();
9519 let admitted = plans
9520 .next()
9521 .ok_or_else(|| RuntimeError::PlanContract("journal has no PlanFrozen record".into()))?;
9522 Ok((admitted, plans.collect()))
9523}
9524
9525/// Whether this run's history is sealed to a key that no longer exists.
9526///
9527/// Read from the frozen plan, which every admitted run has and which is sealed
9528/// under the run's own erasure scope — so it is gone exactly when the run's
9529/// data is gone. Checking a payload rather than asking the key ring keeps this
9530/// a property of the records in hand: the same answer offline, on a restored
9531/// copy, and in a build with no ring wired.
9532fn payloads_erased(records: &[Record]) -> bool {
9533 records.iter().any(|r| match r.kind() {
9534 RecordKind::PlanFrozen { plan, .. } => sealed_payload(plan),
9535 _ => false,
9536 })
9537}
9538
9539/// Who asked for the stop, read back from the chain.
9540///
9541/// Read rather than remembered, for the same reason every other fact about a run
9542/// is: the journal gives the same answer on every subsequent read, and an
9543/// operator asking "who stopped this?" six weeks later is asking history.
9544///
9545/// **Both halves, from the one record that holds them.** The conclusion does
9546/// not repeat the reason — see [`RecordKind::RunConcluded`]'s `reason` for why
9547/// a second copy of it would be the worse copy.
9548fn recorded_cancellation(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9549 records.iter().rev().find_map(|r| match r.kind() {
9550 RecordKind::RunCancelled { actor, reason } => Some((actor.clone(), reason.clone())),
9551 _ => None,
9552 })
9553}
9554
9555/// Who answered the quarantine and what they said, read back from the chain.
9556fn recorded_abandonment(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9557 records.iter().rev().find_map(|r| match r.kind() {
9558 RecordKind::QuarantineDecided {
9559 decider, reason, ..
9560 } => Some((decider.clone(), reason.clone())),
9561 _ => None,
9562 })
9563}
9564
9565/// Who crossed into this tenant and why, read back from the chain.
9566fn recorded_crossing(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9567 records.iter().rev().find_map(|r| match r.kind() {
9568 RecordKind::BreakGlass { actor, reason, .. } => Some((actor.clone(), reason.clone())),
9569 _ => None,
9570 })
9571}
9572
9573/// Who lifted the halt this run records, read back from its lift record.
9574fn recorded_halt_lift(records: &[Record]) -> Option<crate::core::Operator> {
9575 records.iter().rev().find_map(|r| match r.kind() {
9576 RecordKind::HaltLifted { by, .. } => Some(by.clone()),
9577 _ => None,
9578 })
9579}
9580
9581/// Who released the hold this run records, read back from its release record.
9582fn recorded_hold_release(records: &[Record]) -> Option<crate::core::Operator> {
9583 records.iter().rev().find_map(|r| match r.kind() {
9584 RecordKind::HoldReleased { by, .. } => Some(by.clone()),
9585 _ => None,
9586 })
9587}
9588
9589/// Whose authority was withdrawn, and what the operator said when withdrawing
9590/// it — read back from the withholding record.
9591///
9592/// The last one standing: a withdrawal may be lifted and thrown again within a
9593/// run's history, and the conclusion is about the one that stopped it.
9594fn recorded_withholding(records: &[Record]) -> Option<(String, String)> {
9595 records.iter().rev().find_map(|r| match r.kind() {
9596 RecordKind::AuthorityWithheld {
9597 subject, reason, ..
9598 } => Some((subject.clone(), reason.clone())),
9599 _ => None,
9600 })
9601}
9602
9603/// Wall-clock read for the case's `opened_at` stamp, and for a rate
9604/// reservation's instant.
9605///
9606/// Admission happens before any step exists, so there is no `StepCtx` to
9607/// journal through. Both values are store metadata and never participate in
9608/// replay: a rate refusal is journaled on the run and read back, never
9609/// re-counted — run-visible time still goes through `StepCtx::now`, which
9610/// journals it.
9611#[allow(clippy::disallowed_methods)]
9612pub(super) fn now_for_admission() -> crate::core::Timestamp {
9613 crate::core::Timestamp::now_utc()
9614}
9615
9616/// A lift was recorded in `run` and the register's row was not removed.
9617///
9618/// The control still stands; the operator is told the lift failed and which
9619/// run recorded the attempt.
9620fn removal_failed(run: RunId, e: &crate::core::StoreError) -> RuntimeError {
9621 RuntimeError::ControlStands {
9622 run: run.to_string(),
9623 detail: e.to_string(),
9624 }
9625}
9626
9627/// An operator's lift or release that found a control standing and recorded
9628/// ending it.
9629#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9630pub struct ControlLifted {
9631 /// The run holding the record.
9632 pub record: RunId,
9633 /// Whether this call removed the register row. `false` when another lift
9634 /// or release removed it between this call's read and its removal: the
9635 /// control is gone either way, and the record still stands.
9636 pub removed: bool,
9637}
9638
9639/// The epoch an operator act's own run is written under.
9640///
9641/// Such a run has no competing writer to fence against — it is created,
9642/// written and sealed in one call — so a constant is honest here for the same
9643/// reason it is in the sweeper.
9644const BREAK_GLASS_EPOCH: crate::core::Epoch = LEASE_FREE_EPOCH;
9645
9646/// How a break-glass run ends. Not a run status: it neither succeeded nor
9647/// failed at a goal, which is why a sweep seals as `swept` rather than
9648/// borrowing one.
9649const BREAK_GLASS_OUTCOME: &str = "broke-glass";
9650
9651/// How a run recording an operator lifting a halt ends.
9652pub const HALT_LIFTED_OUTCOME: &str = "halt-lifted";
9653
9654/// How a run recording an operator releasing a legal hold ends.
9655pub const HOLD_RELEASED_OUTCOME: &str = "hold-released";
9656
9657/// How a run ends when the authority it acts under is withdrawn under it.
9658///
9659/// Resumable, so it is deliberately absent from both
9660/// [`SEALED_OUTCOMES`] and [`OUTCOMES_OF_RECORD`]: the run has not ended, it is
9661/// waiting for somebody to lift a halt or to cancel it properly.
9662pub const WITHHELD_OUTCOME: &str = "withheld";
9663
9664/// The outcome a session this plane only **observed** is sealed under.
9665///
9666/// Its own word, and not `succeeded`: a session this runtime did not execute
9667/// has no success to report, and answering with one would put somebody else's
9668/// agent's work in the column an operator reads as *this plane did it*. The
9669/// distinction is the whole reason the status exists.
9670pub const OBSERVED_OUTCOME: &str = "observed";
9671
9672/// The epoch every run written without a lease is written under: a sweep's
9673/// record, an operator act's, an observed session's. Nothing takes such a run
9674/// over, so there is no ownership for the epoch to arbitrate, and one constant
9675/// lets the pass that finishes their seals use the epoch they were written at.
9676pub(crate) const LEASE_FREE_EPOCH: crate::core::Epoch = 1;
9677
9678/// The sealing outcomes whose runs are written without a lease.
9679///
9680/// Recovery selects a run by its lapsed lease, so it never finds one of these
9681/// whose writer died between conclusion and seal;
9682/// [`Runtime::seal_unsealed_conclusions`] is what finishes them.
9683pub(crate) const LEASE_FREE_OUTCOMES: &[&str] = &[
9684 super::sweeper::SWEEP_OUTCOME,
9685 BREAK_GLASS_OUTCOME,
9686 HALT_LIFTED_OUTCOME,
9687 HOLD_RELEASED_OUTCOME,
9688 OBSERVED_OUTCOME,
9689];
9690
9691/// One governed identity: a declaration and the skills that serve it.
9692///
9693/// A runtime **runs** agents; it is not one. It owns the journal, the stores,
9694/// the model drivers and the policy engine — infrastructure, shared. An agent
9695/// owns a manifest and its skills — governance, per-identity. Several agents on
9696/// one plane share a journal and are still separately declared, separately
9697/// bounded, and separately answerable.
9698///
9699/// Conflating the two forced a runtime per agent, which meant a lease owner per
9700/// agent for what is one process, a model driver registered once per agent, and
9701/// nowhere in the journal to record *which* agent governed a run.
9702#[cfg(feature = "manifest")]
9703#[derive(Debug, Default)]
9704pub struct Agent {
9705 manifest: Option<Arc<crate::manifest::Manifest>>,
9706 /// Who vouched for the declaration, when it came from a verified resolution.
9707 publisher: Option<crate::core::KeyId>,
9708 skills: Vec<Arc<dyn Skill>>,
9709}
9710
9711#[cfg(feature = "manifest")]
9712impl Agent {
9713 /// An agent governed by this declaration.
9714 ///
9715 /// Nobody has vouched for it. Prefer [`Agent::published_by`] where the
9716 /// manifest came from a verified registry resolution.
9717 #[must_use]
9718 pub fn new(manifest: &crate::manifest::Manifest) -> Self {
9719 Self {
9720 manifest: Some(Arc::new(manifest.clone())),
9721 publisher: None,
9722 skills: Vec::new(),
9723 }
9724 }
9725
9726 /// Record who vouched for this declaration.
9727 ///
9728 /// Takes the [`KeyId`](crate::core::KeyId) that
9729 /// [`Registry::resolve_verified`](crate::manifest::Registry::resolve_verified)
9730 /// returned beside the manifest — which is otherwise dropped on the floor,
9731 /// so a verified resolution and a parsed file become indistinguishable the
9732 /// moment they reach the runtime.
9733 ///
9734 /// # Why this is the grouping a policy wants
9735 ///
9736 /// A rule has to name *a set of agents*, and the obvious candidates do not
9737 /// survive contact with a deployment:
9738 ///
9739 /// * the **workload identity** is per-instance, so a rule naming one is a
9740 /// rule rewritten on every deploy;
9741 /// * the agent **name**, its **role**, or any group label in the manifest is
9742 /// self-asserted — a file claims it, so a rule granting authority to one
9743 /// grants it to any file that types the same string;
9744 /// * the **digest** is unforgeable but names exactly one revision, so every
9745 /// edit is a policy change.
9746 ///
9747 /// A publisher key is the only one that is both a group — many agents, many
9748 /// versions — and impossible to claim without holding the key. Bind the rule
9749 /// to the publisher, keep the digest for "this exact revision", and leave
9750 /// the name for humans reading logs.
9751 #[must_use]
9752 pub fn published_by(mut self, key_id: impl Into<crate::core::KeyId>) -> Self {
9753 self.publisher = Some(key_id.into());
9754 self
9755 }
9756
9757 /// Give it a skill.
9758 #[must_use]
9759 pub fn skill(mut self, skill: impl Skill + 'static) -> Self {
9760 self.skills.push(Arc::new(skill));
9761 self
9762 }
9763}
9764
9765/// Receives every model call's live output, with the run it belongs to.
9766///
9767/// What [`ModelStreamObserver`](crate::model::ModelStreamObserver) receives for
9768/// one call, for every call a plane makes — so a surface can show a run's model
9769/// writing while the journal holds only the finished answer.
9770pub trait RunStreamObserver: Send + Sync + std::fmt::Debug {
9771 /// Delivery is advisory and must not block the call.
9772 fn event(&self, run: RunId, event: Tainted<crate::model::ModelStreamEvent>);
9773}
9774
9775/// Assembles a [`Runtime`].
9776#[derive(Debug)]
9777pub struct RuntimeBuilder {
9778 store: Arc<dyn JournalStore>,
9779 /// The witnesses this plane submits checkpoints to, and the bar a round
9780 /// is held to. Both or neither: a witness list with no declared quorum is
9781 /// witnessing whose evidence is whatever happened to arrive.
9782 witnesses: Vec<Arc<dyn crate::journal::Witness>>,
9783 quorum: Option<crate::journal::WitnessQuorum>,
9784 witness_interval: Option<std::time::Duration>,
9785 signer: Option<Arc<dyn crate::core::Signer>>,
9786 skills: Vec<Arc<dyn Skill>>,
9787 #[cfg(feature = "manifest")]
9788 tools: Option<(
9789 Arc<crate::tools::ToolCatalog>,
9790 Arc<dyn crate::tools::ToolClient>,
9791 )>,
9792 #[cfg(feature = "manifest")]
9793 egress: Option<Arc<crate::core::Egress>>,
9794 #[cfg(feature = "manifest")]
9795 checkers: crate::content::Checkers,
9796 streams: Option<Arc<dyn RunStreamObserver>>,
9797 journal_ceiling: Option<crate::core::Sensitivity>,
9798 tenant: crate::core::TenantId,
9799 owner: Option<String>,
9800 lease_ttl: Duration,
9801 memories: Option<Arc<dyn crate::memory::MemoryStore>>,
9802 semantic: Option<Arc<super::SemanticMemory>>,
9803 authorities: Option<Arc<dyn crate::authority::AuthorityStore>>,
9804 peers: Option<Arc<PeerWiring>>,
9805 tenant_label: super::telemetry::TenantLabel,
9806 quotas: Option<Arc<dyn crate::quota::QuotaStore>>,
9807 quota: crate::quota::TenantQuota,
9808 budget: Budget,
9809 require_verifier: bool,
9810 /// Typed tools whose coherence with every agent is checked at `build`.
9811 #[cfg(feature = "manifest")]
9812 toolbox: Option<crate::tools::ToolBox>,
9813 /// Tool servers reached by some transport other than the box, by name.
9814 #[cfg(feature = "manifest")]
9815 tool_servers: Vec<(String, Arc<dyn crate::tools::ToolClient>)>,
9816 /// Agents registered on this plane, each with its own declaration.
9817 #[cfg(feature = "manifest")]
9818 agents: Vec<Agent>,
9819 /// Drivers by the name a manifest calls them.
9820 #[cfg(feature = "manifest")]
9821 providers: HashMap<String, Arc<dyn crate::model::ModelProvider>>,
9822 cases: Option<Arc<dyn CaseStore>>,
9823 events: Option<Arc<dyn EventStore>>,
9824 tasks: Option<Arc<dyn TaskStore>>,
9825 timers: Option<Arc<dyn TimerStore>>,
9826 blobs: Option<Arc<dyn crate::blob::BlobStore>>,
9827 disclosures: Option<Arc<dyn crate::disclosure::DisclosureRegister>>,
9828 #[cfg(feature = "push")]
9829 push: Option<Arc<dyn crate::push::PushStore>>,
9830 #[cfg(feature = "keyring")]
9831 keyring: Option<Arc<dyn crate::keyring::KeyRing>>,
9832 batches: Option<Arc<dyn crate::batch::BatchStore>>,
9833 policy: Option<Arc<dyn crate::core::PolicyEngine>>,
9834 identity: Option<crate::core::Delegation>,
9835 replanner: Option<Arc<dyn crate::plan::Replanner>>,
9836 calendar: Option<Arc<dyn Calendar>>,
9837 /// Destinations this deployment sends its own run events to.
9838 #[cfg(feature = "push")]
9839 outbox: Option<Arc<crate::push::Outbox>>,
9840}
9841
9842impl RuntimeBuilder {
9843 #[must_use]
9844 pub fn skill(mut self, s: impl Skill) -> Self {
9845 self.skills.push(Arc::new(s));
9846 self
9847 }
9848
9849 /// The workload identity this plane signs its outward claims with.
9850 ///
9851 /// What it buys is that a tool or peer can *check* who called it. Without a
9852 /// signer the provenance block still travels — a server can correlate on it
9853 /// — but it is an assertion any intermediary could have written, and a
9854 /// callee must not authorize on it.
9855 ///
9856 /// Give the store the same signer ([`signing_as`] there) so records and
9857 /// outward claims carry one identity. They are separate settings because a
9858 /// plane can legitimately have one without the other.
9859 ///
9860 /// [`signing_as`]: crate::store::RedbStore::signing_as
9861 #[must_use]
9862 pub fn signing_as(mut self, signer: Arc<dyn crate::core::Signer>) -> Self {
9863 self.signer = Some(signer);
9864 self
9865 }
9866
9867 /// How long this plane's run leases last.
9868 ///
9869 /// The trade is recovery speed against tolerance for a slow instance: a
9870 /// crashed owner's runs stay unclaimable for this long, and a live owner
9871 /// must renew within it. The runtime heartbeats while a run executes, so
9872 /// this bounds *crash* detection rather than how long a run may take.
9873 ///
9874 /// A TTL below [`MIN_LEASE_TTL`] is refused at
9875 /// [`build`](Self::build)/[`try_build`](Self::try_build) as
9876 /// [`BuildError::LeaseUnrenewable`](crate::runtime::BuildError::LeaseUnrenewable).
9877 /// Both stores keep lease expiry in **whole seconds** and treat
9878 /// `expires_at <= now` as lapsed, so a one-second lease expires the moment
9879 /// the clock ticks past the second it was written in — no matter how often
9880 /// it is renewed. Such a lease cannot be held by a live run, and a run that
9881 /// cannot hold its lease is one any instance may take away mid-flight.
9882 /// Refused where a manifest-serving plane can report it as a diagnostic
9883 /// rather than left as a panic in a setter, which `try_build`'s callers
9884 /// could never see as a value.
9885 #[must_use]
9886 pub fn lease_ttl(mut self, ttl: Duration) -> Self {
9887 self.lease_ttl = ttl;
9888 self
9889 }
9890
9891 /// Put this plane's tenant on what it reports about itself.
9892 ///
9893 /// Off by default, and one policy for both signals: the `tenant` field of
9894 /// every metric event, and `agentplane.tenant` on the run span. Read
9895 /// [`telemetry::TenantLabel`](super::telemetry::TenantLabel) before turning
9896 /// it on — a tenant name is often a customer name, and an observability
9897 /// backend is usually the least protected system in a deployment.
9898 ///
9899 /// Cardinality is bounded by construction — the label is *this plane's*
9900 /// tenant, so the number of streams is the number of planes configured, and
9901 /// no request can grow it.
9902 #[must_use]
9903 pub const fn tenant_label(mut self, label: super::telemetry::TenantLabel) -> Self {
9904 self.tenant_label = label;
9905 self
9906 }
9907
9908 /// Give this plane's agents a memory.
9909 ///
9910 /// Optional, and absent by default: an agent with no memory is a normal
9911 /// agent, and one that quietly gained persistent state because a store was
9912 /// wired for something else would be a surprise.
9913 ///
9914 /// Read [`crate::memory`] before wiring one. Writable memory is delayed
9915 /// code: what is written today is read into a context window tomorrow, where
9916 /// a model treats it as established fact.
9917 #[must_use]
9918 pub fn memory(mut self, memories: Arc<dyn crate::memory::MemoryStore>) -> Self {
9919 self.memories = Some(memories);
9920 self
9921 }
9922
9923 /// Give this plane a semantic index, and the embedder that speaks its
9924 /// language.
9925 ///
9926 /// Both together, never one at a time: `build` holds
9927 /// [`Embedder::revision`] to the [`IndexIdentity::query_revision`] the
9928 /// index declares it accepts, and refuses the pair otherwise
9929 /// ([`BuildError::EmbeddingSpaceMismatch`] says what a mismatch costs).
9930 ///
9931 /// Also needs [`memory`](Self::memory): the index holds only commitments,
9932 /// and every hit is materialised from the authoritative store before its
9933 /// content is exposed.
9934 ///
9935 /// [`Embedder::revision`]: crate::memory::Embedder::revision
9936 /// [`IndexIdentity::query_revision`]: crate::memory::IndexIdentity::query_revision
9937 /// [`BuildError::EmbeddingSpaceMismatch`]: crate::runtime::BuildError::EmbeddingSpaceMismatch
9938 #[must_use]
9939 pub fn semantic_memory(
9940 mut self,
9941 embedder: Arc<dyn crate::memory::Embedder>,
9942 retriever: Arc<dyn crate::memory::SemanticRetriever>,
9943 ) -> Self {
9944 self.semantic = Some(Arc::new(super::SemanticMemory {
9945 embedder,
9946 retriever,
9947 }));
9948 self
9949 }
9950
9951 /// Whether the plane's budget, or any registered agent's, caps money.
9952 fn states_a_money_ceiling(&self) -> bool {
9953 #[cfg(feature = "manifest")]
9954 if self
9955 .agents
9956 .iter()
9957 .filter_map(|agent| agent.manifest.as_ref())
9958 .any(|manifest| manifest.budget().max_minor_units.is_some())
9959 {
9960 return true;
9961 }
9962 self.budget.max_minor_units.is_some()
9963 }
9964
9965 /// Attach durable standing-authority accounting.
9966 ///
9967 /// The ceiling neither of the other two can express. A budget bounds one
9968 /// run; a quota bounds a tenant over a billing period. A standing authority
9969 /// bounds *an authorization* — what one customer approved, spanning as many
9970 /// runs as it takes, revocable when they change their mind.
9971 ///
9972 /// Without one, [`StepCtx::draw`](crate::runtime::StepCtx::draw) refuses
9973 /// rather than falling back to an in-process counter. That fallback would
9974 /// fail **open** the moment a second instance started, which is exactly when
9975 /// a shared ceiling was needed.
9976 #[must_use]
9977 pub fn authorities(mut self, authorities: Arc<dyn crate::authority::AuthorityStore>) -> Self {
9978 self.authorities = Some(authorities);
9979 self
9980 }
9981
9982 /// Bound what this tenant may consume, durably.
9983 ///
9984 /// Budgets bound one run; this bounds the tenant. Both are needed: a caller
9985 /// that can start runs can start a thousand, each within its own ceiling.
9986 ///
9987 /// The accounting lives in the store, so the ceiling survives a second
9988 /// instance — an in-process counter would silently double the moment
9989 /// somebody scales out, which is exactly when it was needed.
9990 ///
9991 /// Read [`crate::quota`] for what each ceiling does and does not bound; a
9992 /// limit believed to bound something it does not is worse than none.
9993 #[must_use]
9994 pub fn quota(
9995 mut self,
9996 quotas: Arc<dyn crate::quota::QuotaStore>,
9997 quota: crate::quota::TenantQuota,
9998 ) -> Self {
9999 self.quotas = Some(quotas);
10000 self.quota = quota;
10001 self
10002 }
10003
10004 /// This **process instance's** identity, as it appears in run leases.
10005 ///
10006 /// Not the agent's name. Fencing rests on the epoch — a claim never renews,
10007 /// and a renewal names the exact epoch it holds — so a shared string costs
10008 /// no safety; it costs the answer to *which process holds this run*, which
10009 /// is what an operator asks of a stuck one and what `LeaseHeld` reports.
10010 ///
10011 /// So keep it unique per running process, which is what the default is —
10012 /// override it only with a better instance identity than a random one,
10013 /// such as a pod name. An agent's *name* is
10014 /// [`Manifest::metadata`](crate::manifest::Metadata::name).
10015 ///
10016 /// The owner lives in the lease table and never in the chain, so it has no
10017 /// bearing on replay.
10018 #[must_use]
10019 pub fn owner(mut self, o: impl Into<String>) -> Self {
10020 self.owner = Some(o.into());
10021 self
10022 }
10023
10024 /// Cap what a run may consume.
10025 ///
10026 /// Defaults to [`Budget::unlimited`], which is right for a runtime whose
10027 /// effects are all free and local, and wrong the moment one of them calls a
10028 /// metered API.
10029 #[must_use]
10030 pub fn budget(mut self, budget: Budget) -> Self {
10031 self.budget = budget;
10032 self
10033 }
10034
10035 /// Refuse a plan in which nothing checks the work.
10036 ///
10037 /// A node that declares [`verifies`](crate::core::PlanNode::verifies) must
10038 /// already depend on what it checks; this makes carrying one a condition of
10039 /// admission. It binds every plan this runtime validates, and the one that
10040 /// matters is the one the embedder did not write: a
10041 /// [`Replanner`](crate::plan::Replanner)'s successor is a graph proposed
10042 /// mid-run, and a control that held for the first plan and not for its
10043 /// replacement is a control a replan removes.
10044 #[must_use]
10045 pub fn require_verifier(mut self) -> Self {
10046 self.require_verifier = true;
10047 self
10048 }
10049
10050 /// Attach long-lived case storage, enabling correlation and deadlines.
10051 #[must_use]
10052 pub fn cases(mut self, cases: Arc<dyn CaseStore>) -> Self {
10053 self.cases = Some(cases);
10054 self
10055 }
10056
10057 /// Send this plane's own events to destinations the deployment configured.
10058 ///
10059 /// Every run registers each destination **at admission**, so there is no
10060 /// window in which a run exists and nothing is watching it. Delivery is a
10061 /// separate, operator-scheduled sweep — see
10062 /// [`DeliveryWorker`](crate::push::DeliveryWorker) — reading each run's
10063 /// journal past a cursor that advances only on 2xx.
10064 ///
10065 /// This is the mirror image of A2A push, and the mirror is the point: there
10066 /// the *caller* names a URL and three controls exist because of it. Here the
10067 /// deployment names it, and the run's own history is the outbox rather than
10068 /// a queue that can fall out of sync with it.
10069 #[cfg(feature = "push")]
10070 #[must_use]
10071 pub fn outbox(mut self, outbox: Arc<crate::push::Outbox>) -> Self {
10072 self.outbox = Some(outbox);
10073 self
10074 }
10075
10076 /// Attach the webhook registration store.
10077 ///
10078 /// The same handle the A2A server and the [`Outbox`](crate::push::Outbox)
10079 /// are given: this is the *durable* half of push, and it belongs to the
10080 /// plane because its failures do. A receiver that answers permanently, or
10081 /// stays silent past the retry ceiling, has its registration
10082 /// [parked](crate::push::PushStore::park) rather than deleted — the cursor
10083 /// is the only record of how far that receiver got — and a parked row is a
10084 /// backlog an operator reads and re-arms on the operator API. Without this
10085 /// the plane serves the delivery and cannot answer for it.
10086 #[cfg(feature = "push")]
10087 #[must_use]
10088 pub fn push(mut self, push: Arc<dyn crate::push::PushStore>) -> Self {
10089 self.push = Some(push);
10090 self
10091 }
10092
10093 /// Attach inbound-event storage, enabling durable waits.
10094 #[must_use]
10095 pub fn events(mut self, events: Arc<dyn EventStore>) -> Self {
10096 self.events = Some(events);
10097 self
10098 }
10099
10100 /// Attach a worklist, enabling human tasks.
10101 #[must_use]
10102 pub fn tasks(mut self, tasks: Arc<dyn TaskStore>) -> Self {
10103 self.tasks = Some(tasks);
10104 self
10105 }
10106
10107 /// Supply the planner that produces successor plans.
10108 ///
10109 /// Without one, a step asking to replan fails with that as the reason —
10110 /// which is the honest outcome, not a silent no-op.
10111 #[must_use]
10112 pub fn replanner(mut self, r: Arc<dyn crate::plan::Replanner>) -> Self {
10113 self.replanner = Some(r);
10114 self
10115 }
10116
10117 /// Supply the durable-timer store.
10118 ///
10119 /// Needed by `StepCtx::sleep` and `sleep_until`, and by the sweep that wakes
10120 /// them. A runtime without one refuses to sleep rather than falling back to
10121 /// an in-process wait that a restart would forget.
10122 #[must_use]
10123 pub fn timers(mut self, timers: Arc<dyn TimerStore>) -> Self {
10124 self.timers = Some(timers);
10125 self
10126 }
10127
10128 /// Register an agent on this plane.
10129 ///
10130 /// A runtime runs agents; it is not one. This is where a declaration and
10131 /// its skills arrive together, so several agents can share one journal, one
10132 /// set of drivers and one process identity while each stays separately
10133 /// governed.
10134 ///
10135 /// The declaration **binds** for that agent's steps: an effect naming a
10136 /// model or tool its manifest never listed is refused before dispatch and
10137 /// journaled, and the egress and delegation ceilings combine with the sink's
10138 /// own — the stricter wins. Its budget bounds its runs. Architectural
10139 /// injection patterns are deliberately absent from the schema, because this
10140 /// runtime cannot prove that arbitrary skill code follows one.
10141 ///
10142 /// An agent declaring `spec.execution` needs no skill: the runtime supplies
10143 /// the behaviour. See [`provider`](Self::provider) for the driver mapping it
10144 /// needs.
10145 ///
10146 /// It does **not** set the lease owner. That identifies a *process*, and one
10147 /// plane running four agents is still one process — see
10148 /// [`owner`](Self::owner).
10149 ///
10150 /// # This call never fails
10151 ///
10152 /// It records the agent and returns. Every refusal an agent can cause — a
10153 /// capability none of its skills provide, a provider no driver is registered
10154 /// for, oversight with nowhere to put a decision — is raised at
10155 /// [`build`](Self::build), which panics, or returned by
10156 /// [`try_build`](Self::try_build), which does not.
10157 ///
10158 /// That split matters for a daemon assembling a plane from files it did not
10159 /// write: a bad manifest is an *input* there, and a panic takes down every
10160 /// other tenant in the process to report it. A `# Panics` section here,
10161 /// describing `build`'s refusals, would send readers looking for a
10162 /// fallible variant of the wrong call — the refusals are documented where
10163 /// they fire.
10164 #[cfg(feature = "manifest")]
10165 #[must_use]
10166 pub fn agent(mut self, agent: Agent) -> Self {
10167 self.agents.push(agent);
10168 self
10169 }
10170
10171 /// Register a model driver under the name a manifest uses for it.
10172 ///
10173 /// The seam a declarative agent needs. A manifest says `provider: anthropic`
10174 /// — a string a reviewer can read — and something has to map that to a
10175 /// driver holding a credential. That mapping is deployment wiring, not a
10176 /// property of the agent, which is exactly why it lives here and not in the
10177 /// file: an agent's declaration should not change when its API key does.
10178 ///
10179 /// Required only for [`ExecutionKind::Completion`] and the other declarative
10180 /// kinds. A hand-written skill constructs its own `ModelCall` and never
10181 /// consults this.
10182 ///
10183 /// [`ExecutionKind::Completion`]: crate::manifest::ExecutionKind::Completion
10184 #[cfg(feature = "manifest")]
10185 #[must_use]
10186 pub fn provider(
10187 mut self,
10188 name: impl Into<String>,
10189 provider: Arc<dyn crate::model::ModelProvider>,
10190 ) -> Self {
10191 self.providers.insert(name.into(), provider);
10192 self
10193 }
10194
10195 /// Supply content-addressed blob storage.
10196 ///
10197 /// Needed by `StepCtx::store_blob`, which is how bytes too large for a
10198 /// journal record get somewhere durable while the chain keeps only their
10199 /// digest. A runtime without one refuses rather than silently inlining
10200 /// megabytes into an append-only chain that can never take them back.
10201 #[must_use]
10202 pub fn blobs(mut self, blobs: Arc<dyn crate::blob::BlobStore>) -> Self {
10203 self.blobs = Some(blobs);
10204 self
10205 }
10206
10207 /// Where disclosures of a matter are recorded, so a retention pass names
10208 /// the copies of what it erased.
10209 #[must_use]
10210 pub fn disclosures(mut self, register: Arc<dyn crate::disclosure::DisclosureRegister>) -> Self {
10211 self.disclosures = Some(register);
10212 self
10213 }
10214
10215 /// The operator's tool catalogue, and the client that reaches those tools.
10216 ///
10217 /// Required by a `tool-calling` agent and by nothing else: a skill that
10218 /// calls tools builds its own [`ToolCall`](crate::tools::ToolCall), because
10219 /// it knows which client it means. A declarative agent has no code to make
10220 /// that choice, so the plane makes it once.
10221 ///
10222 /// The catalogue is the authority. A manifest grants a subset of it, the
10223 /// model is offered exactly that subset, and a name the model returns is
10224 /// matched against it byte for byte.
10225 #[cfg(feature = "manifest")]
10226 #[must_use]
10227 pub fn tools(
10228 mut self,
10229 catalog: Arc<crate::tools::ToolCatalog>,
10230 client: Arc<dyn crate::tools::ToolClient>,
10231 ) -> Self {
10232 self.tools = Some((catalog, client));
10233 self
10234 }
10235
10236 /// Typed tools, with their catalogue derived and their coherence enforced.
10237 ///
10238 /// The one-call form, and the reason it exists is not brevity. Deriving the
10239 /// catalogue and checking it against every agent's manifest were both
10240 /// possible before and both **optional**, and a control a caller may forget
10241 /// is not a control — it is advice that reads like one.
10242 ///
10243 /// So this does three things that were three things:
10244 ///
10245 /// * derives the catalogue from each agent's declaration, so a grant, its
10246 /// ceiling and its protected fields are stated once;
10247 /// * refuses to build if the tools this binary implements and the manifests
10248 /// a reviewer approved have drifted apart;
10249 /// * wires the box as the client.
10250 ///
10251 /// The work happens in [`build`](Self::build) rather than here, and that is
10252 /// the whole reason it is trustworthy: checking on this call would check
10253 /// against the agents registered *so far*, so `.toolbox(..).agent(..)` would
10254 /// pass by having nothing to disagree with. An enforcement that depends on
10255 /// the order a builder was written is not one.
10256 #[cfg(feature = "manifest")]
10257 #[must_use]
10258 pub fn toolbox(mut self, tools: crate::tools::ToolBox) -> Self {
10259 self.toolbox = Some(tools);
10260 self
10261 }
10262
10263 /// A tool server this plane reaches over some transport of its own.
10264 ///
10265 /// An MCP connection is the usual one. Registering it does three things:
10266 ///
10267 /// * a plane may reach **several** servers, because the router resolves the
10268 /// `tool://server/name` a grant carries rather than handing every id to one
10269 /// client;
10270 /// * typed in-process tools and remote servers can be used by the *same*
10271 /// agent, which is the ordinary shape;
10272 /// * a grant naming a server nobody wired is refused at build, in the same
10273 /// breath as a grant nothing implements — both mean the model would be
10274 /// offered a tool that fails when chosen.
10275 ///
10276 /// Composes with [`toolbox`](Self::toolbox); the box answers for the servers
10277 /// its own tools name and these answer for theirs. A server claimed twice is
10278 /// a panic, because registration order deciding which transport carries a
10279 /// call is the defect [`ToolRouter`](crate::tools::ToolRouter) exists to
10280 /// remove.
10281 #[cfg(feature = "manifest")]
10282 #[must_use]
10283 pub fn tool_server(
10284 mut self,
10285 name: impl Into<String>,
10286 client: Arc<dyn crate::tools::ToolClient>,
10287 ) -> Self {
10288 self.tool_servers.push((name.into(), client));
10289 self
10290 }
10291
10292 /// Where this plane's **tool transports** may connect.
10293 ///
10294 /// A `ToolClient` is the embedder's code, and the split is the one every
10295 /// other outbound path here makes: **the client owns the connection, the
10296 /// plane owns the destination.** A transport says where it goes with
10297 /// [`ToolClient::destination`](crate::tools::ToolClient::destination), and
10298 /// this decides whether that is somewhere it may go — before the effect
10299 /// exists, so nothing leaves, nothing is journaled and nothing is metered.
10300 ///
10301 /// A [`Destination::Local`](crate::tools::Destination::Local) transport is
10302 /// not judged: tools compiled into the binary and a stdio child process
10303 /// contact no host of this plane's choosing, so there is nothing for an
10304 /// allowlist to decide. What that does not claim is that the far side
10305 /// reaches nothing — the same residual a compromised allowlisted endpoint
10306 /// carries.
10307 ///
10308 /// Unset means no egress control on the tool path, spelled the way an
10309 /// absent policy engine is: by configuring nothing, rather than by
10310 /// configuring something that looks like a control and is not.
10311 ///
10312 /// ```ignore
10313 /// let plane = Runtime::builder(store)
10314 /// .tool_server("tickets", mcp_over_https)
10315 /// .egress(Egress::new().allow("mcp.tickets.example"))
10316 /// .build();
10317 /// ```
10318 #[cfg(feature = "manifest")]
10319 #[must_use]
10320 pub fn egress(mut self, egress: crate::core::Egress) -> Self {
10321 self.egress = Some(Arc::new(egress));
10322 self
10323 }
10324
10325 /// Forward every model call's live output, with the run it belongs to.
10326 ///
10327 /// Advisory and live-only: the journal's completion stays the canonical
10328 /// answer, a replay performs no call and so streams nothing, and an
10329 /// observer that drops events changes nothing a run does.
10330 #[must_use]
10331 pub fn observe_model_streams(mut self, observer: Arc<dyn RunStreamObserver>) -> Self {
10332 self.streams = Some(observer);
10333 self
10334 }
10335
10336 /// Register a content checker, by its [`name`](crate::content::ContentChecker::name).
10337 ///
10338 /// A manifest check naming a checker no builder registered, or mapping a
10339 /// category the checker does not declare, refuses the build: a declared
10340 /// check nothing could run is a control a reviewer approved that does not
10341 /// exist.
10342 #[cfg(feature = "manifest")]
10343 #[must_use]
10344 pub fn content_checker(mut self, checker: Arc<dyn crate::content::ContentChecker>) -> Self {
10345 Arc::make_mut(&mut self.checkers).insert(checker.name().to_owned(), checker);
10346 self
10347 }
10348
10349 /// The highest sensitivity this plane will write into an append-only record.
10350 ///
10351 /// # What this is, and what the key ring is
10352 ///
10353 /// Two different answers to *this data must be erasable*, and they compose.
10354 /// [`keyring`](Self::keyring) is the **seal it** half: payloads become
10355 /// ciphertext whose key an erasure destroys, which reaches every replica
10356 /// and every backup at once. This is the **refuse it** half: an argument
10357 /// more sensitive than the deployment is willing to make permanent is
10358 /// refused at dispatch, before it reaches the chain.
10359 ///
10360 /// The plane-level twin of `spec.security.max_sensitivity_journaled`, for
10361 /// a plane of hand-written skills that runs under no manifest. The default
10362 /// is the unsafe one — a journal keeps everything, indefinitely — so a
10363 /// plane must be able to state the ceiling whether or not it is
10364 /// declarative. It is an enforcement point at the same gate the manifest's
10365 /// field reaches, not a warning; where both are present **the stricter
10366 /// wins**, as a reviewed tool grant may only tighten what the operator's
10367 /// catalogue allows.
10368 ///
10369 /// Absent means no ceiling, spelled the way an absent policy engine is.
10370 ///
10371 /// ```ignore
10372 /// let plane = Runtime::builder(store)
10373 /// // Personal data goes in a blob and the chain gets the digest.
10374 /// .max_sensitivity_journaled(Sensitivity::Internal)
10375 /// .skill(Triage)
10376 /// .build();
10377 /// ```
10378 #[must_use]
10379 pub const fn max_sensitivity_journaled(mut self, ceiling: crate::core::Sensitivity) -> Self {
10380 self.journal_ceiling = Some(ceiling);
10381 self
10382 }
10383
10384 /// Which tenant this plane runs as.
10385 ///
10386 /// **One plane, one tenant — but one process, many planes.** A plane is the
10387 /// unit that is bound to a tenant; serving several is
10388 /// [`Planes`](crate::api::Planes)' job, and it resolves the plane from the
10389 /// authenticated caller's tenant rather than from the request, so a handler
10390 /// cannot reach a store it did not resolve. An unregistered tenant is
10391 /// refused rather than defaulted.
10392 ///
10393 /// The name scopes **data keys**, so one tenant's cryptographic erasure
10394 /// cannot reach another's bytes, and it reaches the **policy request**, so a
10395 /// rule can be written per tenant.
10396 ///
10397 /// It does **not** scope the store — that is a separate handle, scoped by
10398 /// `RedbStore::for_tenant` or `PostgresStore::for_tenant`. Two tenants may
10399 /// share one store, because the tenant is a key component of every row on
10400 /// both backends rather than a filter. Setting one and not the other is
10401 /// refused at [`build`](Self::build) rather than discovered later: a plane
10402 /// whose store is scoped elsewhere works perfectly and writes its runs into
10403 /// somebody else's keyspace.
10404 ///
10405 /// Defaults to `default`, which is a real tenant rather than an absence: the
10406 /// single-tenant path is then the same code as the multi-tenant one, and a
10407 /// special "no tenant" case is a second path that would not get tested.
10408 #[must_use]
10409 pub fn tenant(mut self, tenant: crate::core::TenantId) -> Self {
10410 self.tenant = tenant;
10411 self
10412 }
10413
10414 /// Seal payload bytes, and make erasure reach copies deletion cannot.
10415 ///
10416 /// With a key ring configured, everything written through
10417 /// [`StepCtx::blobs`](crate::runtime::StepCtx::blobs) — including
10418 /// [`store_blob`](crate::runtime::StepCtx::store_blob) and governed media —
10419 /// is encrypted under a data key belonging to the run's **case**. Erasing
10420 /// that case destroys the key, so every copy of those bytes becomes
10421 /// unreadable at once: the live store, the replicas, and every backup ever
10422 /// taken. Expiring blobs only reaches the first of those.
10423 ///
10424 /// The case is the erasure unit because it is already the retention unit —
10425 /// bytes are linked to their case at write time, and a second, differently
10426 /// shaped unit for keys would let the two disagree about what an erasure
10427 /// covered.
10428 ///
10429 /// It also seals the stores the plane holds — the journal's payloads, case
10430 /// state, task proposals and buffered event payloads — at
10431 /// [`build`](Self::build), so the order they were registered in cannot lose
10432 /// the guarantee — a store registered after this call is sealed just the
10433 /// same.
10434 ///
10435 /// **Governed memory is the one store this does not reach.**
10436 /// [`EncryptedMemoryStore`](crate::keyring::EncryptedMemoryStore)
10437 /// serialises subject erasure against writes and legal-hold changes with a
10438 /// process-local mutex, so it holds its contract on a single-writer
10439 /// deployment and nowhere else; wrapping it here would hand that adapter to
10440 /// an active-active `PostgreSQL` plane, where the mutex coordinates nothing
10441 /// and the hold race it exists to prevent is the result. Its erasure unit
10442 /// is one item version (`memory-item/<id>@<version>`) and outlives every
10443 /// case, so `erase_case` is not the act that reaches it either. Wrap it
10444 /// yourself, where the deployment's topology is visible:
10445 ///
10446 /// ```ignore
10447 /// let memories = EncryptedMemoryStore::new(inner, keys.clone(), tenant.clone());
10448 /// Runtime::builder(store).memory(Arc::new(memories)).keyring(keys).build()
10449 /// ```
10450 ///
10451 /// Without one, bytes are stored as given and erasure remains deletion.
10452 #[cfg(feature = "keyring")]
10453 #[must_use]
10454 pub fn keyring(mut self, keyring: Arc<dyn crate::keyring::KeyRing>) -> Self {
10455 self.keyring = Some(keyring);
10456 self
10457 }
10458
10459 /// Wrap every store a key ring can seal, at **build** time.
10460 ///
10461 /// One call, one guarantee. A deployer who configures a key ring reads it
10462 /// as *this plane is encrypted*, so the journal, the case store, the
10463 /// worklist, the event buffer and the blobs are sealed together. Five
10464 /// independent wrapping calls would be a control that can be forgotten
10465 /// four times, and forgetting looks exactly like remembering.
10466 ///
10467 /// Wrapped here rather than inside `keyring()` for the reason the tool
10468 /// catalogue is checked here: a wrap applied when the ring is supplied
10469 /// would cover only the stores registered so far, so `keyring(..)` before
10470 /// `cases(..)` would silently seal less than `keyring(..)` after it. An
10471 /// enforcement a reordering can lose is one a reformatter can delete.
10472 ///
10473 /// Every decorator is handed `self.tenant` rather than reading a name back
10474 /// out of the store it wraps, so all four derive the erasure scope from one
10475 /// value. `keyring()`'s own documentation names the one store deliberately
10476 /// left out, and why.
10477 #[cfg(feature = "keyring")]
10478 fn seal_stores(&mut self) {
10479 let Some(keys) = self.keyring.clone() else {
10480 return;
10481 };
10482 let tenant = self.tenant.clone();
10483 self.store = crate::keyring::SealedJournal::wrap(
10484 Arc::clone(&self.store),
10485 Arc::clone(&keys),
10486 tenant.clone(),
10487 );
10488 if let Some(cases) = self.cases.take() {
10489 self.cases = Some(crate::keyring::SealedCases::wrap(
10490 cases,
10491 Arc::clone(&keys),
10492 tenant.clone(),
10493 ));
10494 }
10495 if let Some(events) = self.events.take() {
10496 self.events = Some(crate::keyring::SealedEvents::wrap(
10497 events,
10498 Arc::clone(&keys),
10499 tenant.clone(),
10500 ));
10501 }
10502 // The outbox's store keeps the operator destinations' bearer tokens —
10503 // credentials like any caller's, and the one store here reached through
10504 // a handle the embedder built rather than registered. Sealing it in the
10505 // same sweep is what keeps `keyring()` one decision rather than five
10506 // and a sixth.
10507 #[cfg(feature = "push")]
10508 if let Some(outbox) = self.outbox.take() {
10509 self.outbox = Some(Arc::new(
10510 outbox
10511 .as_ref()
10512 .clone()
10513 .sealed(Arc::clone(&keys), tenant.clone()),
10514 ));
10515 }
10516 if let Some(tasks) = self.tasks.take() {
10517 self.tasks = Some(crate::keyring::SealedTasks::wrap(tasks, keys, tenant));
10518 }
10519 }
10520
10521 /// Without the `keyring` feature there is nothing to seal.
10522 ///
10523 /// `&mut self` matches the sealing arm above, which does mutate. Taking
10524 /// `&self` here would make the signature depend on a feature flag, and
10525 /// every caller with it.
10526 #[cfg(not(feature = "keyring"))]
10527 #[allow(clippy::unused_self, clippy::needless_pass_by_ref_mut)]
10528 fn seal_stores(&mut self) {}
10529
10530 /// Supply the store that tracks batch items.
10531 ///
10532 /// Only needed for [`Runtime::run_batch`]; a plane that runs no batches does
10533 /// not need one, and asking for it unconditionally would make the common
10534 /// case carry the uncommon one's setup.
10535 #[must_use]
10536 pub fn batches(mut self, batches: Arc<dyn crate::batch::BatchStore>) -> Self {
10537 self.batches = Some(batches);
10538 self
10539 }
10540
10541 /// Supply the authorization engine.
10542 ///
10543 /// Without one there is no policy layer — the information-flow gates still
10544 /// apply, but nothing asks whether the principal was allowed. That is a
10545 /// deliberate absence rather than a permissive default: see `core::policy`
10546 /// on why there is no `AllowAll` to configure by mistake.
10547 ///
10548 /// The engine's complete immutable bundle identity is recorded at admission,
10549 /// so both whether policy was on and exactly which executable semantics
10550 /// governed the run are answerable from the journal. An open run may resume
10551 /// only under that same identity.
10552 #[must_use]
10553 pub fn policy(mut self, policy: Arc<dyn crate::core::PolicyEngine>) -> Self {
10554 self.policy = Some(policy);
10555 self
10556 }
10557
10558 /// The chain this plane's own runs act under.
10559 ///
10560 /// The identity of a run the embedder starts in-process — checked against
10561 /// the plan at admission (the plan is the authorization graph, so one that
10562 /// exceeds the chain's authority never starts), held to the chain's
10563 /// audience and validity, and journaled, so "on whose behalf" is
10564 /// answerable from history rather than reconstructed from timestamps.
10565 ///
10566 /// It is **not** the identity of a run a served surface admits for a
10567 /// caller: those act under the chain the caller presented, or under none
10568 /// when it presented none ([`RunTerms::served`]). A plane that bound its own
10569 /// chain to every peer's run would be an ambient credential.
10570 ///
10571 /// Verification of the *credential* belongs to whatever authenticates the
10572 /// caller; what arrives here is already a chain, and its attenuation is
10573 /// guaranteed by its own constructors however it was obtained.
10574 #[must_use]
10575 pub fn acting_as(mut self, chain: crate::core::Delegation) -> Self {
10576 self.identity = Some(chain);
10577 self
10578 }
10579
10580 /// The peers this plane may call, and the transport that reaches them.
10581 ///
10582 /// The registry is wiring — what each peer is granted, which credential
10583 /// reaches it, how a failed call recovers. What a run may *ask* a peer
10584 /// for is a manifest grant, `tool://<peer>/<capability>`, dispatched
10585 /// through [`StepCtx::call_peer`](crate::runtime::StepCtx::call_peer).
10586 /// Refused at build: a peer named `agent` or like a wired tool server,
10587 /// and a grant naming a capability outside the peer's registry scope.
10588 #[must_use]
10589 pub fn peers(
10590 mut self,
10591 registry: crate::peers::PeerRegistry,
10592 client: Arc<dyn crate::peers::PeerClient>,
10593 ) -> Self {
10594 self.peers = Some(Arc::new(PeerWiring { registry, client }));
10595 self
10596 }
10597
10598 /// Submit this plane's checkpoints to witnesses, and hold each round to a
10599 /// quorum.
10600 ///
10601 /// This is the only control here whose value comes from *not* being this
10602 /// plane. The hash chain proves no record was edited, the signatures say
10603 /// which workload wrote them, and the Merkle log proves no sealed run was
10604 /// removed — but every input to all three comes from the party an auditor
10605 /// is being asked to trust. None of them detects a whole run deleted
10606 /// before anyone took a checkpoint, or two internally perfect histories
10607 /// shown to two auditors. A witness keeps the last checkpoint it saw and
10608 /// cosigns only a checkpoint that provably extends it, so those two stop
10609 /// being invisible.
10610 ///
10611 /// Availability never waits on it: submission runs on
10612 /// [`sweep`](Runtime::sweep), after sealing and off the run path, and a
10613 /// round that falls short is a report rather than a refusal. A run whose
10614 /// witnesses are unreachable finished long ago, and making the plane's
10615 /// availability depend on a third party is the wrong trade for evidence
10616 /// that is read after the fact.
10617 ///
10618 /// Point it at witnesses somebody else runs. A witness hosted beside the
10619 /// history it vouches for is an anchor one compromise removes, and
10620 /// [`MemoryWitness`](crate::journal::MemoryWitness) says so at its own
10621 /// definition.
10622 ///
10623 /// # Errors
10624 ///
10625 /// [`build`](RuntimeBuilder::build) refuses a quorum larger than the
10626 /// number of witnesses configured, and a witness list with no quorum.
10627 #[must_use]
10628 pub fn witnesses(
10629 mut self,
10630 witnesses: Vec<Arc<dyn crate::journal::Witness>>,
10631 quorum: crate::journal::WitnessQuorum,
10632 ) -> Self {
10633 self.witnesses = witnesses;
10634 self.quorum = Some(quorum);
10635 self
10636 }
10637
10638 /// Declare the maximum time between checkpoint submissions to each
10639 /// witness: [`sweep`](Runtime::sweep) re-submits an unchanged checkpoint
10640 /// once it has passed since the last round that met quorum, so an idle
10641 /// plane still carries a fresh witness timestamp. Without it an unchanged
10642 /// log is never re-submitted.
10643 ///
10644 /// The interval is kept only as often as `sweep` runs; a deployment
10645 /// sweeping on its own scheduler owns that cadence.
10646 ///
10647 /// # Errors
10648 ///
10649 /// [`build`](RuntimeBuilder::build) refuses an interval with no witnesses.
10650 #[must_use]
10651 pub fn witness_interval(mut self, interval: std::time::Duration) -> Self {
10652 self.witness_interval = Some(interval);
10653 self
10654 }
10655
10656 /// Supply the calendar that resolves deadline descriptions to instants.
10657 ///
10658 /// Defaults to [`WallClock`], which understands plain offsets and refuses
10659 /// anything it does not know rather than approximating it. Domain calendars
10660 /// — working days, holidays, cut-off hours — are the adapter's job.
10661 #[must_use]
10662 pub fn calendar(mut self, calendar: Arc<dyn Calendar>) -> Self {
10663 self.calendar = Some(calendar);
10664 self
10665 }
10666
10667 /// Derive the tool catalogue from the agents and refuse a disagreement.
10668 ///
10669 /// Every agent, not the first: a plane may host several, and a tool granted
10670 /// to none of them is still a tool this binary can be asked for.
10671 ///
10672 /// # Errors
10673 ///
10674 /// If the box and any agent's manifest disagree. A build-time
10675 /// misconfiguration has a fix and no recovery, and it is refused the way the
10676 /// tenant mismatch beside it is: before anything runs.
10677 #[cfg(feature = "manifest")]
10678 fn settle_toolbox(&mut self) -> Result<(), BuildError> {
10679 let servers = std::mem::take(&mut self.tool_servers);
10680 let tools = self.toolbox.take();
10681 // A peer's name is a server name a grant can carry, so it must mean
10682 // exactly one thing on this plane: not `agent`, and not a name some
10683 // tool transport also answers for.
10684 let peer_names: std::collections::BTreeSet<String> = self
10685 .peers
10686 .as_ref()
10687 .map(|wiring| wiring.registry.peers().map(ToString::to_string).collect())
10688 .unwrap_or_default();
10689 if peer_names.contains(crate::tools::AGENT_SERVER) {
10690 return Err(BuildError::ReservedToolServer);
10691 }
10692 for name in &peer_names {
10693 if servers.iter().any(|(server, _)| server == name)
10694 || tools
10695 .as_ref()
10696 .is_some_and(|t| t.servers().any(|s| s == name))
10697 {
10698 return Err(BuildError::PeerIsAlsoAToolServer {
10699 server: name.clone(),
10700 });
10701 }
10702 }
10703 if tools.is_none() && servers.is_empty() {
10704 return Ok(());
10705 }
10706 let tools = tools.unwrap_or_default();
10707 let remote_servers: std::collections::BTreeSet<String> =
10708 servers.iter().map(|(name, _)| name.clone()).collect();
10709 // What a grant may name without this box implementing it: a wired
10710 // transport, or a registered peer.
10711 let reachable_elsewhere: std::collections::BTreeSet<String> =
10712 remote_servers.union(&peer_names).cloned().collect();
10713 // `agent` names agents on this plane, and only them. A transport or a
10714 // typed tool under that name would let a deployment decide whether a
10715 // reviewed grant means "an agent here" or "somebody's server".
10716 if remote_servers.contains(crate::tools::AGENT_SERVER)
10717 || tools
10718 .servers()
10719 .any(|server| server == crate::tools::AGENT_SERVER)
10720 {
10721 return Err(BuildError::ReservedToolServer);
10722 }
10723 if remote_servers.len() != servers.len() {
10724 // The set lost an entry, so some name appears twice. Naming it beats
10725 // reporting a count a reader then has to go and diff by hand.
10726 let mut seen = std::collections::BTreeSet::new();
10727 let duplicate = servers
10728 .iter()
10729 .map(|(name, _)| name)
10730 .find(|name| !seen.insert((*name).clone()))
10731 .cloned()
10732 .unwrap_or_default();
10733 return Err(BuildError::DuplicateToolServer { server: duplicate });
10734 }
10735 // Both forms wired is not a merge and must not silently be one. The
10736 // hand-built catalogue is the operator saying something deliberate; the
10737 // derived one is the agent's declaration. Overwriting either with the
10738 // other would run a plane under grants nobody chose.
10739 if self.tools.is_some() {
10740 return Err(BuildError::ToolsWiredTwice);
10741 }
10742 let mut catalog = crate::tools::ToolCatalog::new();
10743 let mut declared = 0usize;
10744 // Which agent's declaration a tool's catalogue entry came from, so a
10745 // second agent declaring the same tool *differently* is a build-time
10746 // refusal rather than a silent overwrite.
10747 //
10748 // A plane has one catalogue and its agents have one manifest each, so
10749 // two agents granting `tool://ledger/read` with different protected
10750 // fields cannot both be satisfied. Merging by last-writer would resolve
10751 // it by **registration order** — the one thing this builder already
10752 // says an enforcement must never depend on — and it fails at a
10753 // distance: `declared` compares each agent's manifest against the
10754 // catalogue-derived descriptor exactly, so the agent that lost the race
10755 // is refused *every* call to that tool, in production, with a message
10756 // blaming a code-versus-manifest drift that neither file exhibits.
10757 let mut source: BTreeMap<crate::tools::ToolId, (String, crate::tools::ToolSafety)> =
10758 BTreeMap::new();
10759 for agent in &self.agents {
10760 let Some(manifest) = agent.manifest.as_ref() else {
10761 continue;
10762 };
10763 declared += 1;
10764 tools
10765 .check_against(manifest, &reachable_elsewhere)
10766 .map_err(|problems| BuildError::ToolDrift {
10767 agent: manifest.metadata.name.clone(),
10768 problems,
10769 })?;
10770 for (id, safety) in crate::tools::ToolCatalog::from_manifest(manifest).entries() {
10771 if let Some((first, existing)) = source.get(&id) {
10772 if existing != &safety {
10773 return Err(BuildError::ToolDeclaredTwoWays {
10774 tool: id.reference(),
10775 first: first.clone(),
10776 second: manifest.metadata.name.clone(),
10777 });
10778 }
10779 continue;
10780 }
10781 source.insert(id.clone(), (manifest.metadata.name.clone(), safety.clone()));
10782 catalog = catalog.allow(id, safety);
10783 }
10784 }
10785 // A box with nothing to be coherent *with* is the same defect one step
10786 // earlier: tools wired to a plane where no declaration admits them, so
10787 // nothing a reviewer reads describes what this binary can reach.
10788 if declared == 0 {
10789 return Err(BuildError::ToolsWithoutDeclaration);
10790 }
10791 // The typed argument type is the schema source. Overlay its
10792 // presentation only after every manifest has been checked, so the
10793 // model sees exactly what the body will deserialize rather than the
10794 // old permissive `{ type: object }` fallback.
10795 for id in tools.ids() {
10796 let (description, schema, _) = tools
10797 .declared(id)
10798 .expect("every registered typed tool has a declaration");
10799 let reviewed_description = catalog
10800 .declaration(id)
10801 .map_or_else(|| description.to_owned(), |(text, _)| text.to_owned());
10802 catalog = catalog.declare(id.clone(), reviewed_description, schema.clone());
10803 }
10804 // One client per server, resolved by the name a grant carries. A single
10805 // client handed every id could not tell `tool://ledger/read` from
10806 // `tool://tickets/read`, and a transport that never reads the server
10807 // component answers both — from whichever server it happens to hold.
10808 let router = servers.into_iter().fold(
10809 crate::tools::ToolRouter::new().toolbox(&Arc::new(tools)),
10810 |router, (name, client)| router.server(name, client),
10811 );
10812 self.tools = Some((
10813 Arc::new(catalog),
10814 Arc::new(router) as Arc<dyn crate::tools::ToolClient>,
10815 ));
10816 Ok(())
10817 }
10818
10819 /// Settle the tool catalogue: derive it if asked, then hold it to the
10820 /// declarations.
10821 ///
10822 /// One call because the two halves are one decision seen from either side.
10823 /// [`settle_toolbox`](Self::settle_toolbox) covers the derived catalogue,
10824 /// where code and manifest could disagree; the check after it covers the
10825 /// stated one, where operator and manifest could. Whichever way the
10826 /// catalogue arrived, it is checked before anything runs.
10827 #[cfg(feature = "manifest")]
10828 fn settle_tools(&mut self) -> Result<(), BuildError> {
10829 self.settle_toolbox()?;
10830 self.check_catalogue_not_laxer_than_grants()
10831 }
10832
10833 /// Refuse a stated catalogue that is **laxer** than a reviewed grant.
10834 ///
10835 /// `toolbox(..)` derives the catalogue from the manifests, so the two
10836 /// cannot drift. `tools(..)` states it by hand, and there the operator's
10837 /// entry and the agent's declaration are two copies of one decision — with
10838 /// nothing, until this, that noticed them disagreeing.
10839 ///
10840 /// Only one direction is a defect, the same one
10841 /// [`ToolBox::check_against`](crate::tools::ToolBox::check_against) refuses.
10842 /// An operator being **more** cautious than the declaration is fine and
10843 /// often right. An operator being **less** cautious is not, and it changes
10844 /// two things at once:
10845 ///
10846 /// * the whole-value taint gate stops firing, so model-chosen arguments
10847 /// reach something that changes the world;
10848 /// * `ToolSafety::read_only` carries `Recovery::Retry`, so a timed-out call
10849 /// to a money-moving tool is sent a second time.
10850 ///
10851 /// The dispatch gates already take the stricter `mutates` of the two, so
10852 /// the first is contained at runtime. The second is not — recovery is read
10853 /// from the catalogue alone — and neither should have to be, because this
10854 /// is a wiring mistake with a fix and no recovery. It is refused here,
10855 /// beside the rest of them.
10856 #[cfg(feature = "manifest")]
10857 fn check_catalogue_not_laxer_than_grants(&self) -> Result<(), BuildError> {
10858 let Some((catalog, _)) = self.tools.as_ref() else {
10859 return Ok(());
10860 };
10861 let mut problems = Vec::new();
10862 for agent in &self.agents {
10863 let Some(manifest) = agent.manifest.as_ref() else {
10864 continue;
10865 };
10866 for grant in &manifest.spec.tools {
10867 if !grant.mutates {
10868 continue;
10869 }
10870 let Some(id) = crate::tools::ToolId::parse(&grant.reference) else {
10871 continue;
10872 };
10873 if catalog.safety(&id).is_some_and(|s| !s.mutates) {
10874 problems.push(format!(
10875 "agent '{}' grants '{}' as mutating and the stated catalogue \
10876 calls it read-only",
10877 manifest.metadata.name, grant.reference
10878 ));
10879 }
10880 }
10881 }
10882 if problems.is_empty() {
10883 Ok(())
10884 } else {
10885 Err(BuildError::CatalogueLaxerThanGrant { problems })
10886 }
10887 }
10888
10889 /// Assemble the runtime, or panic naming the wiring mistake.
10890 ///
10891 /// The ordinary entry point. Every refusal below is a bug in code the author
10892 /// is looking at, so propagating it through `?` to a `main` that prints it
10893 /// is ceremony around an abort — and each is caught here at startup rather
10894 /// than at dispatch, in production, where the cost is a run that has already
10895 /// begun.
10896 ///
10897 /// Use [`try_build`](Self::try_build) where a manifest arrives at *runtime*
10898 /// — read from disk, pinned by a registry, or supplied per tenant. There a
10899 /// bad declaration is an input rather than a bug, and a panic would take
10900 /// every other tenant in the process down to report it.
10901 ///
10902 /// # Panics
10903 ///
10904 /// On any [`BuildError`]:
10905 ///
10906 /// * A manifest declares a capability in `spec.capabilities.provides` that
10907 /// no registered skill provides. **An agent has skills**, so a declaration
10908 /// advertising one it cannot perform is a card that lies.
10909 /// * Two agents claim the same capability. Dispatch resolves a capability to
10910 /// one skill *and to the manifest governing it*, so a second claim would
10911 /// silently take the first's work out from under the first's budget,
10912 /// model grants and egress ceiling.
10913 /// * Two skills share a name. A name is what a capability resolves to and
10914 /// what governance is keyed on; two of them make both lookups arbitrary.
10915 /// * A stated catalogue calls a tool read-only that a reviewed manifest
10916 /// grants as mutating. That exemption drops the whole-value taint gate
10917 /// and makes a timed-out money-moving call retryable — the one direction
10918 /// an operator cannot be right about.
10919 /// * A declarative agent names a provider no driver is registered for, or
10920 /// declares `spec.execution` without a privileged model to call.
10921 /// * A plane and its store — or its blob store — are scoped to different
10922 /// tenants. The two are set
10923 /// separately — this builder's tenant scopes data keys and the policy
10924 /// request, `for_tenant` scopes the store's keys — and the mismatch does
10925 /// not show up at runtime. It *works*, and writes this tenant's runs into
10926 /// another's keyspace while every erasure and every policy request names
10927 /// the right one.
10928 ///
10929 /// # Long-running services want [`try_build`](Self::try_build)
10930 ///
10931 /// This panics, which is the honest answer for a binary wiring its own
10932 /// skills: every variant above is a bug in code the author is looking at,
10933 /// and aborting reports it at the moment it can be fixed. A daemon is a
10934 /// different case — it wants to exit with a diagnostic, and a plane
10935 /// assembled from a manifest that arrived at runtime is handling an *input*,
10936 /// where a panic reports one tenant's typo by killing every other tenant's
10937 /// in-flight run. `try_build` returns the same `BuildError` instead. One
10938 /// implementation underneath both, so they cannot disagree about what is
10939 /// refused.
10940 #[must_use]
10941 pub fn build(self) -> Arc<Runtime> {
10942 // Not `expect`. That formats the error with `Debug`, which would print
10943 // `AdvertisesWhatItCannotProvide { agent: "…", missing: [...] }` — the
10944 // variant's *shape* — while the sentence explaining what to do about it
10945 // lives in `Display`. Panicking with `{error}` keeps the two entry
10946 // points telling one story, which is the whole point of `build` being
10947 // `try_build` underneath.
10948 match self.try_build() {
10949 Ok(runtime) => runtime,
10950 Err(error) => panic!("{error}"),
10951 }
10952 }
10953
10954 /// The driver for a declarative agent's privileged model — named rather
10955 /// than defaulted, because falling back to some other registered driver
10956 /// would run the agent on a model its own declaration does not name.
10957 /// Every declared content check names a registered checker, and maps only
10958 /// categories that checker declares.
10959 #[cfg(feature = "manifest")]
10960 fn check_content_checkers(
10961 checkers: &crate::content::Checkers,
10962 m: &crate::manifest::Manifest,
10963 ) -> Result<(), BuildError> {
10964 let Some(content) = m.spec.security.content.as_ref() else {
10965 return Ok(());
10966 };
10967 for check in &content.checks {
10968 let refused = |detail: String| BuildError::ContentCheck {
10969 agent: m.metadata.name.clone(),
10970 check: check.id.clone(),
10971 detail,
10972 };
10973 let Some(checker) = checkers.get(&check.checker) else {
10974 return Err(refused(format!(
10975 "no checker named '{}' is registered on this plane",
10976 check.checker
10977 )));
10978 };
10979 if let Some(unknown) = check
10980 .on
10981 .keys()
10982 .find(|category| !checker.categories().contains(*category))
10983 {
10984 return Err(refused(format!(
10985 "'{unknown}' is not a category checker '{}' declares",
10986 check.checker
10987 )));
10988 }
10989 }
10990 Ok(())
10991 }
10992
10993 #[cfg(feature = "manifest")]
10994 fn declared_provider(
10995 providers: &HashMap<String, Arc<dyn crate::model::ModelProvider>>,
10996 m: &crate::manifest::Manifest,
10997 ) -> Result<Arc<dyn crate::model::ModelProvider>, BuildError> {
10998 let model = m
10999 .spec
11000 .models
11001 .as_ref()
11002 .and_then(|x| x.privileged.as_ref())
11003 .ok_or_else(|| BuildError::DeclarativeWithoutModel {
11004 agent: m.metadata.name.clone(),
11005 })?;
11006 providers
11007 .get(&model.provider)
11008 .map(Arc::clone)
11009 .ok_or_else(|| BuildError::UnknownProvider {
11010 agent: m.metadata.name.clone(),
11011 provider: model.provider.clone(),
11012 })
11013 }
11014
11015 /// Assemble the runtime, or say why it cannot be.
11016 ///
11017 /// The same checks as [`build`](Self::build), returned rather than raised.
11018 /// One implementation behind both, so they cannot come to disagree about
11019 /// what is refused.
11020 ///
11021 /// # Errors
11022 ///
11023 /// Any [`BuildError`] — see [`build`](Self::build) for what each means.
11024 // `mut` is for `settle_toolbox`, which only exists when manifests do.
11025 #[cfg_attr(not(feature = "manifest"), allow(unused_mut))]
11026 #[allow(clippy::too_many_lines)]
11027 pub fn try_build(mut self) -> Result<Arc<Runtime>, BuildError> {
11028 if self.lease_ttl < MIN_LEASE_TTL {
11029 return Err(BuildError::LeaseUnrenewable {
11030 ttl: self.lease_ttl,
11031 minimum: MIN_LEASE_TTL,
11032 });
11033 }
11034 // A ceiling of zero is a budget already spent, and it refuses the
11035 // first effect of every run this plane will ever make. The manifest
11036 // refuses one at parse; a plane wired in Rust reaches the same `Budget`
11037 // without passing a parser, so the rule is applied at both doors from
11038 // the one definition in `Budget::bricked_ceiling`.
11039 if let Some(field) = self.budget.bricked_ceiling() {
11040 return Err(BuildError::BudgetPermitsNothing { field });
11041 }
11042 if let Some(engine) = self.policy.as_ref() {
11043 preflight_policy(engine.as_ref(), self.identity.as_ref())?;
11044 }
11045 // Before `seal_stores`, necessarily: sealing wraps each handle in one
11046 // scoped to the *plane's* tenant, after which every store agrees with
11047 // the plane and the question cannot be asked.
11048 // Both halves of push, because either can be wired to another
11049 // tenant's rows and the consequence is the same: registrations,
11050 // cursors and credentials filed under a scope this plane's erasure
11051 // never names.
11052 #[cfg(feature = "push")]
11053 let push = self
11054 .push
11055 .as_ref()
11056 .map(|s| s.tenant())
11057 .or_else(|| self.outbox.as_ref().map(|o| o.store_tenant()));
11058 #[cfg(not(feature = "push"))]
11059 let push: Option<&str> = None;
11060 check_same_tenant(
11061 self.store.as_ref(),
11062 self.blobs.as_ref(),
11063 self.memories.as_ref(),
11064 &[
11065 ("case", self.cases.as_ref().map(|s| s.tenant())),
11066 ("event", self.events.as_ref().map(|s| s.tenant())),
11067 ("task", self.tasks.as_ref().map(|s| s.tenant())),
11068 ("memory", self.memories.as_ref().map(|s| s.tenant())),
11069 ("quota", self.quotas.as_ref().map(|s| s.tenant())),
11070 ("timer", self.timers.as_ref().map(|s| s.tenant())),
11071 ("batch", self.batches.as_ref().map(|s| s.tenant())),
11072 ("authority", self.authorities.as_ref().map(|s| s.tenant())),
11073 ("push", push),
11074 ],
11075 &self.tenant,
11076 )?;
11077 // The one wiring mistake here that would otherwise never fail: a
11078 // cross-space query ranks rather than refusing.
11079 if let Some(semantic) = &self.semantic {
11080 let embedder = semantic.embedder.revision();
11081 let index = semantic.retriever.index().query_revision;
11082 if embedder != index {
11083 return Err(BuildError::EmbeddingSpaceMismatch { embedder, index });
11084 }
11085 if semantic.embedder.pricing().is_none() && self.states_a_money_ceiling() {
11086 return Err(BuildError::UnpricedEmbedder { embedder });
11087 }
11088 let Some(memories) = self.memories.take() else {
11089 return Err(BuildError::SemanticMemoryWithoutStore);
11090 };
11091 // Every erasure through the plane's store — and through a sealed
11092 // store's own subject erasure — reaches the index built from it.
11093 let reached = memories.erasure_index();
11094 let indexed: Arc<dyn crate::memory::MemoryStore> = match reached {
11095 Some(index) if Arc::ptr_eq(&index, &semantic.retriever) => memories,
11096 Some(_) => return Err(BuildError::MemoryIndexedElsewhere),
11097 None if memories.seals() => {
11098 return Err(BuildError::SealedMemoryMissesIndex);
11099 }
11100 None => Arc::new(crate::memory::IndexedMemoryStore::new(
11101 Arc::clone(&memories),
11102 Arc::clone(&semantic.retriever),
11103 )),
11104 };
11105 self.memories = Some(indexed);
11106 }
11107 self.seal_stores();
11108 #[cfg(feature = "manifest")]
11109 self.settle_tools()?;
11110
11111 let mut skills = HashMap::new();
11112 let mut by_capability = HashMap::new();
11113 #[cfg(feature = "manifest")]
11114 let mut governed_by: HashMap<String, Arc<crate::manifest::Manifest>> = HashMap::new();
11115 #[cfg(feature = "manifest")]
11116 let mut published_by: HashMap<String, crate::core::KeyId> = HashMap::new();
11117
11118 // Skills registered directly belong to the plane's anonymous agent: no
11119 // declaration, so nothing to enforce against them beyond the runtime's
11120 // own budget. That is a legitimate shape — not every agent needs a
11121 // manifest — and it is why `skill()` still exists beside `agent()`.
11122 for s in self.skills {
11123 register_skill(s, &mut by_capability, &mut skills)?;
11124 }
11125
11126 #[cfg(feature = "manifest")]
11127 for agent in self.agents {
11128 if let (Some(m), Some(key)) = (agent.manifest.as_ref(), agent.publisher.clone()) {
11129 published_by.insert(m.metadata.name.clone(), key);
11130 }
11131 let Some(m) = agent.manifest.clone() else {
11132 for s in agent.skills {
11133 register_skill(s, &mut by_capability, &mut skills)?;
11134 }
11135 continue;
11136 };
11137
11138 // The capabilities this agent's *own* skills provide, which is what
11139 // its declaration is checked against below.
11140 let mut mine: HashSet<Capability> = HashSet::new();
11141 for s in agent.skills {
11142 mine.extend(s.descriptor().capabilities());
11143 let name = register_skill(s, &mut by_capability, &mut skills)?;
11144 governed_by.insert(name, Arc::clone(&m));
11145 }
11146
11147 // A declarative agent needs no skill: the runtime supplies the
11148 // behaviour its manifest asked for.
11149 if let Some(execution) = &m.spec.execution {
11150 // A `call` calls no model, so it needs no provider.
11151 let provider = if execution.kind == crate::manifest::ExecutionKind::Call {
11152 None
11153 } else {
11154 Some(Self::declared_provider(&self.providers, &m)?)
11155 };
11156 if m.spec.capabilities.provides.is_empty() {
11157 return Err(BuildError::DeclarativeProvidesNothing {
11158 agent: m.metadata.name.clone(),
11159 });
11160 }
11161 // A plane whose only tools are agents needs no toolbox and no
11162 // transport: the catalogue is derived from the declaration and
11163 // dispatch is `commission`. The empty router is deliberate —
11164 // an agent-server call never reaches it, and anything else
11165 // arriving there is refused as unreachable rather than
11166 // silently absorbed.
11167 let tools = self.tools.clone().or_else(|| {
11168 // Agents on this plane and registered peers: both
11169 // dispatch through the runtime's own effects rather than
11170 // a tool transport, so a declaration made only of them
11171 // needs no box and no router.
11172 let all_agent = !m.spec.tools.is_empty()
11173 && m.spec.tools.iter().all(|g| {
11174 crate::tools::ToolId::parse(&g.reference).is_some_and(|id| {
11175 id.server == crate::tools::AGENT_SERVER
11176 || self.peers.as_ref().is_some_and(|wiring| {
11177 wiring
11178 .registry
11179 .grant(&crate::peers::PeerId::new(&id.server))
11180 .is_some()
11181 })
11182 })
11183 });
11184 all_agent.then(|| {
11185 (
11186 Arc::new(crate::tools::ToolCatalog::from_manifest(&m)),
11187 Arc::new(crate::tools::ToolRouter::new())
11188 as Arc<dyn crate::tools::ToolClient>,
11189 )
11190 })
11191 });
11192 // A tool loop with nothing to reach fails on every run with the
11193 // same sentence, and the manifest that says so was read at
11194 // build. `planned` only needs a catalogue when it has grants;
11195 // `tool-calling` needs one to exist at all, since offering a
11196 // model no tools is not a tool loop.
11197 if tools.is_none() {
11198 let needs = match execution.kind {
11199 crate::manifest::ExecutionKind::ToolCalling => {
11200 Some(if m.spec.tools.is_empty() {
11201 "no tool grants".to_owned()
11202 } else {
11203 format!("{} tool grant(s)", m.spec.tools.len())
11204 })
11205 }
11206 crate::manifest::ExecutionKind::Planned
11207 | crate::manifest::ExecutionKind::Call
11208 if !m.spec.tools.is_empty() =>
11209 {
11210 Some(format!("{} tool grant(s)", m.spec.tools.len()))
11211 }
11212 _ => None,
11213 };
11214 if let Some(grants) = needs {
11215 return Err(BuildError::DeclarativeToolsUnreachable {
11216 agent: m.metadata.name.clone(),
11217 kind: execution.kind.as_str(),
11218 grants,
11219 });
11220 }
11221 }
11222
11223 // Oversight needs somewhere to put the decision, and both
11224 // halves are known here. Left to run time it surfaces at the
11225 // first real approval — a person already waiting, on the code
11226 // path a test suite is least likely to reach.
11227 let declared = if m.spec.oversight.is_some() {
11228 Some("`spec.oversight`".to_owned())
11229 } else if m.spec.tools.iter().any(|g| g.requires_approval) {
11230 Some("a grant with `requires_approval: true`".to_owned())
11231 } else {
11232 None
11233 };
11234 if let Some(declared) = declared {
11235 // Ordered by what a reader fixes first: without a case
11236 // there is nothing for a task to hang off.
11237 //
11238 // Timers are deliberately **not** required, and the test
11239 // suite is why: four approval tests wire a case store and a
11240 // worklist, no timers, and run an approval end to end. A
11241 // task is opened, the run suspends, a person decides. What
11242 // a timer store adds is the sweeper firing `on_expiry`, and
11243 // refusing a configuration that works would be this check
11244 // asserting more than it knows — the failure mode it exists
11245 // to prevent, arriving from the other side.
11246 let missing = if self.cases.is_none() {
11247 Some(("case store", "cases"))
11248 } else if self.tasks.is_none() {
11249 Some(("worklist", "tasks"))
11250 } else {
11251 None
11252 };
11253 if let Some((missing, remedy)) = missing {
11254 return Err(BuildError::OversightUnreachable {
11255 agent: m.metadata.name.clone(),
11256 declared,
11257 missing,
11258 remedy,
11259 });
11260 }
11261 }
11262
11263 // A recall reads before the first model call and formation
11264 // writes after the answer, so a missing store costs a run its
11265 // first effect or its whole budget depending on which half was
11266 // declared. Both facts are here: the file says memories are
11267 // reached, the plane says there is nowhere to reach.
11268 if let Some(memory) = &m.spec.memory {
11269 let declared: [(&'static str, Option<&crate::manifest::MemorySubject>); 2] = [
11270 (
11271 "spec.memory.recall",
11272 memory.recall.as_ref().map(|r| &r.subject),
11273 ),
11274 (
11275 "spec.memory.formation",
11276 memory.formation.as_ref().map(|f| &f.subject),
11277 ),
11278 ];
11279 for (field, subject) in declared {
11280 let Some(subject) = subject else { continue };
11281 if self.memories.is_none() {
11282 return Err(BuildError::MemoryWithoutStore {
11283 agent: m.metadata.name.clone(),
11284 declared: field,
11285 });
11286 }
11287 if subject.needs_case() && self.cases.is_none() {
11288 return Err(BuildError::MemorySubjectUnbindable {
11289 agent: m.metadata.name.clone(),
11290 subject: subject.as_written(),
11291 });
11292 }
11293 }
11294 }
11295
11296 for cap in &m.spec.capabilities.provides {
11297 let skill: Arc<dyn Skill> = Arc::new(super::declarative::Declarative::new(
11298 execution.kind,
11299 cap.clone(),
11300 m.metadata.name.clone(),
11301 provider.clone(),
11302 tools.clone(),
11303 execution.max_turns,
11304 ));
11305 mine.insert(Capability::new(cap.as_str()));
11306 let name = register_skill(skill, &mut by_capability, &mut skills)?;
11307 governed_by.insert(name, Arc::clone(&m));
11308 }
11309 }
11310
11311 check_declaration_matches_skills(&m, &mine)?;
11312 }
11313
11314 // Agent grants are validated against the finished plane, because the
11315 // capability they name may belong to an agent registered *later* — a
11316 // check inside the loop would pass or fail on registration order.
11317 #[cfg(feature = "manifest")]
11318 {
11319 let mut checked = std::collections::BTreeSet::new();
11320 for m in governed_by.values() {
11321 if !checked.insert(m.metadata.name.clone()) {
11322 continue;
11323 }
11324 Self::check_content_checkers(&self.checkers, m)?;
11325 for grant in &m.spec.tools {
11326 let Some(id) = crate::tools::ToolId::parse(&grant.reference) else {
11327 continue;
11328 };
11329 // A grant naming a registered peer is a delegating hop,
11330 // and both things that would refuse it on every run are
11331 // knowable here: a capability the registry never gave the
11332 // peer, and a granting agent whose own ceiling is zero.
11333 if let Some(wiring) = self.peers.as_ref()
11334 && let Some(peer_grant) = wiring
11335 .registry
11336 .grant(&crate::peers::PeerId::new(&id.server))
11337 {
11338 if !peer_grant.scope.permits(&Capability::new(id.tool.as_str())) {
11339 return Err(BuildError::PeerGrantOutsideScope {
11340 agent: m.metadata.name.clone(),
11341 peer: id.server,
11342 capability: id.tool,
11343 });
11344 }
11345 let cannot_delegate = m
11346 .spec
11347 .topology
11348 .as_ref()
11349 .is_some_and(|t| t.role == crate::manifest::Role::Specialist)
11350 || m.spec.security.max_delegation_depth == Some(0);
11351 if cannot_delegate {
11352 return Err(BuildError::PeerGrantOnASpecialist {
11353 agent: m.metadata.name.clone(),
11354 peer: id.server,
11355 });
11356 }
11357 continue;
11358 }
11359 if id.server != crate::tools::AGENT_SERVER {
11360 continue;
11361 }
11362 if m.spec.capabilities.provides.contains(&id.tool) {
11363 return Err(BuildError::AgentToolSelfReference {
11364 agent: m.metadata.name.clone(),
11365 capability: id.tool,
11366 });
11367 }
11368 if !by_capability.contains_key(&Capability::new(id.tool.as_str())) {
11369 return Err(BuildError::AgentToolUnknownCapability {
11370 agent: m.metadata.name.clone(),
11371 capability: id.tool,
11372 });
11373 }
11374 }
11375 }
11376 }
11377
11378 // Every ceiling any declaration on this plane states, by tool: the
11379 // count is per tool, so the tightest binds every agent calling it.
11380 #[cfg(feature = "manifest")]
11381 let rates = {
11382 let mut declarations: Vec<&Arc<crate::manifest::Manifest>> =
11383 governed_by.values().collect();
11384 declarations.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
11385 let mut ceilings: BTreeMap<String, Vec<crate::quota::RateCeiling>> = BTreeMap::new();
11386 let mut first = None;
11387 for m in declarations {
11388 for grant in &m.spec.tools {
11389 let Some(rate) = grant.rate_limit else {
11390 continue;
11391 };
11392 first.get_or_insert_with(|| (m.metadata.name.clone(), grant.reference.clone()));
11393 let stated = ceilings.entry(grant.reference.clone()).or_default();
11394 let ceiling = crate::quota::RateCeiling::from(rate);
11395 if !stated.contains(&ceiling) {
11396 stated.push(ceiling);
11397 stated.sort();
11398 }
11399 }
11400 }
11401 if ceilings.is_empty() {
11402 None
11403 } else if let Some(store) = self.quotas.clone() {
11404 Some(Arc::new(super::ctx::Rates { store, ceilings }))
11405 } else {
11406 let (agent, grant) = first.unwrap_or_default();
11407 return Err(BuildError::RateLimitWithoutQuotaStore { agent, grant });
11408 }
11409 };
11410
11411 // Both or neither, checked here because a builder cannot see the other
11412 // half at the moment either is set.
11413 if self.witness_interval.is_some() && self.witnesses.is_empty() {
11414 return Err(BuildError::IntervalWithoutWitnesses);
11415 }
11416 let witnesses = match (self.witnesses.is_empty(), self.quorum) {
11417 (true, None) => None,
11418 (false, Some(quorum)) if quorum.required() <= self.witnesses.len() => {
11419 Some(Arc::new(Witnessing {
11420 witnesses: self.witnesses,
11421 quorum,
11422 submitted: std::sync::atomic::AtomicU64::new(0),
11423 interval: self.witness_interval,
11424 last_met: std::sync::Mutex::new(None),
11425 }))
11426 }
11427 (empty, quorum) => {
11428 return Err(BuildError::WitnessQuorumUnreachable {
11429 declared: quorum.map_or(0, |q| q.required()),
11430 configured: if empty { 0 } else { self.witnesses.len() },
11431 });
11432 }
11433 };
11434
11435 Ok(Arc::new_cyclic(|self_ref| Runtime {
11436 self_ref: self_ref.clone(),
11437 witnesses,
11438 inflight: Arc::default(),
11439 commissions: Arc::default(),
11440 signer: self.signer,
11441 store: self.store,
11442 skills,
11443 by_capability,
11444 #[cfg(feature = "manifest")]
11445 published_by,
11446 #[cfg(feature = "manifest")]
11447 providers: self.providers,
11448 meter: super::metrics::Meter::new(self.tenant_label, &self.tenant),
11449 tenant: self.tenant,
11450 owner: self.owner.unwrap_or_else(default_owner),
11451 lease_ttl: self.lease_ttl,
11452 memories: self.memories,
11453 semantic: self.semantic,
11454 authorities: self.authorities,
11455 peers: self.peers,
11456 #[cfg(feature = "manifest")]
11457 tools: self.tools,
11458 journal_ceiling: self.journal_ceiling,
11459 #[cfg(feature = "manifest")]
11460 egress: self.egress,
11461 #[cfg(feature = "manifest")]
11462 checkers: self.checkers,
11463 streams: self.streams,
11464 quotas: self.quotas,
11465 #[cfg(feature = "manifest")]
11466 rates,
11467 quota: self.quota,
11468 budget: self.budget,
11469 require_verifier: self.require_verifier,
11470 cases: self.cases,
11471 events: self.events,
11472 tasks: self.tasks,
11473 timers: self.timers,
11474 blobs: self.blobs,
11475 disclosures: self.disclosures,
11476 #[cfg(feature = "keyring")]
11477 keyring: self.keyring,
11478 batches: self.batches,
11479 policy: self.policy,
11480 identity: self.identity,
11481 replanner: self.replanner,
11482 calendar: self.calendar.unwrap_or_else(|| Arc::new(WallClock)),
11483 #[cfg(feature = "push")]
11484 push: self.push,
11485 #[cfg(feature = "push")]
11486 outbox: self.outbox,
11487 #[cfg(feature = "manifest")]
11488 governed_by,
11489 }))
11490 }
11491}
11492
11493/// Ask the policy set the questions this plane will ask, before it asks them
11494/// for real.
11495///
11496/// Cedar evaluates **every** rule against **every** request, so a rule reading
11497/// an attribute a request does not carry does not quietly fail to match — it
11498/// errors, and an unevaluable rule may be the `forbid` that would have stopped
11499/// the call, so the gate refuses. One unguarded rule therefore denies every
11500/// effect of every run, from a policy set that compiled cleanly and validated
11501/// against its schema.
11502///
11503/// The probes cover every shape a runtime request takes: an effect that reads
11504/// and one that mutates, each with and without a label (only `sink` carries
11505/// one), a release, and an admission. `mutates` matters on its own: `&&`
11506/// short-circuits, so `context.mutates && context.label.trust == ..` evaluates
11507/// cleanly on a read and errors on every mutating call that carries no label.
11508///
11509/// The delegation attributes follow the plane. A plane with a chain carries
11510/// `owner`, `scope` and `delegation_depth` on every request, and a rule reading
11511/// them unguarded is correct there. A plane without one never carries them on
11512/// its own runs — and still admits runs under a caller's chain from a served
11513/// surface, so both shapes are asked.
11514///
11515/// What this does **not** establish: that the rules are *right*. A set that
11516/// permits everything passes here, as does one whose rules error only on a
11517/// shape carrying an attribute of an unexpected type. It answers one question
11518/// — can this plane's own requests be evaluated at all — which is the question
11519/// whose wrong answer looks like a working plane that refuses everything. The
11520/// served surfaces ask their own actions with their own contexts, and check
11521/// those where they are built.
11522#[cfg_attr(not(feature = "manifest"), allow(dead_code))]
11523fn preflight_policy(
11524 engine: &dyn crate::core::PolicyEngine,
11525 identity: Option<&crate::core::Delegation>,
11526) -> Result<(), BuildError> {
11527 let problems = runtime_policy_problems(engine, identity);
11528 if problems.is_empty() {
11529 return Ok(());
11530 }
11531 Err(BuildError::PolicyUnevaluable {
11532 problems: problems.join("; "),
11533 })
11534}
11535
11536/// The runtime tier's probes, answered by `engine`.
11537fn runtime_policy_problems(
11538 engine: &dyn crate::core::PolicyEngine,
11539 identity: Option<&crate::core::Delegation>,
11540) -> Vec<String> {
11541 use crate::policy::requests;
11542
11543 // Values are placeholders shaped like the real ones: a rule reading
11544 // `context.label.trust` on a release must find a label there, or the probe
11545 // reports a defect the real request would never hit. Nulls are absent for
11546 // the same reason — no gate sends one, and the adapter strips them.
11547 let run = RunId(ulid::Ulid::nil());
11548 let step = StepId(0);
11549 let label = crate::core::Label::untrusted(crate::core::SourceId::new("preflight"));
11550 let release = crate::core::Release::whole(
11551 crate::core::ReleaseScope::trust(),
11552 "preflight",
11553 "tool://preflight/probe",
11554 ["preflight"],
11555 );
11556 // A tool call's arguments are the one shape the runtime fixes for a kind,
11557 // so a rule scoped to `tool.call` may read `context.args.server` as-is.
11558 let tool_args = serde_json::json!({
11559 "server": "preflight",
11560 "tool": "preflight",
11561 "arguments": {},
11562 "protected_fields": [],
11563 });
11564
11565 // The chain shapes this plane's requests take. A chain of the plane's own
11566 // is on every request; without one, a run may still carry its caller's.
11567 let caller = crate::core::Delegation::root(crate::core::Principal::new(
11568 "preflight",
11569 crate::core::Scope::root(),
11570 ));
11571 let chains: Vec<Option<&crate::core::Delegation>> = match identity {
11572 Some(chain) => vec![Some(chain)],
11573 None => vec![None, Some(&caller)],
11574 };
11575
11576 // Built by the gates' own builders, so a probe cannot drift from the
11577 // shape it stands in for: an effect that reads and one that mutates, each
11578 // with and without a label (only `sink` carries one), a release, and an
11579 // admission.
11580 let mut built = Vec::new();
11581 for chain in chains {
11582 let acting = requests::Acting {
11583 tenant: "preflight",
11584 capability: "preflight.capability",
11585 agent: None,
11586 chain,
11587 };
11588 for (kind, args) in [
11589 ("preflight.effect", serde_json::json!({})),
11590 ("tool.call", tool_args.clone()),
11591 ] {
11592 for (mutates, labelled) in [(false, false), (true, false), (false, true), (true, true)]
11593 {
11594 built.push(requests::effect(
11595 &acting,
11596 run,
11597 step,
11598 kind,
11599 &args,
11600 mutates,
11601 labelled.then_some(&label),
11602 ));
11603 }
11604 }
11605 built.push(requests::release(&acting, run, step, &release, &label));
11606 built.push(requests::admission(&acting, &serde_json::json!({})));
11607 }
11608 let requests: Vec<crate::core::PolicyRequest<'_>> = built
11609 .iter()
11610 .map(requests::GatedRequest::as_request)
11611 .collect();
11612
11613 // Asked through the engine's own preflight rather than by calling
11614 // `authorize` here. An engine written as Rust code has no missing-attribute
11615 // trap and nothing to report; the engine that has the trap implements the
11616 // method, and knows which resources its own rules name.
11617 let mut problems = engine.preflight(&requests);
11618 problems.sort();
11619 problems.dedup();
11620 problems
11621}
11622
11623/// Refuse a plane whose stores answer for somebody else.
11624///
11625/// The plane's tenant scopes its data keys and names its policy requests; each
11626/// store handle is scoped separately. Nothing at runtime looks wrong when the
11627/// two disagree — the work is written into another tenant's keyspace while
11628/// every erasure and every policy request names the right one — so the mismatch
11629/// is easy to make and invisible once made. Asking each store who it serves is
11630/// what turns it into a startup refusal.
11631///
11632/// **A store that does not override its accessor answers `default`.** That is
11633/// the honest answer for a single-tenant deployment and the reason the check
11634/// can be unconditional, but it is also why this catches a wiring mistake
11635/// rather than proving isolation: an implementation that is scoped to `acme`
11636/// and never overrode the accessor reports `default` and is refused, which is
11637/// the safe direction — the unsafe one would be an implementation that claims a
11638/// tenant it does not enforce, and no build-time question can settle that.
11639fn check_same_tenant(
11640 store: &dyn JournalStore,
11641 blobs: Option<&Arc<dyn crate::blob::BlobStore>>,
11642 memories: Option<&Arc<dyn crate::memory::MemoryStore>>,
11643 state: &[(&'static str, Option<&str>)],
11644 tenant: &crate::core::TenantId,
11645) -> Result<(), BuildError> {
11646 if let Some(blobs) = blobs
11647 && blobs.tenant() != tenant.as_str()
11648 {
11649 return Err(BuildError::BlobStoreTenant {
11650 plane: tenant.to_string(),
11651 store: blobs.tenant().to_owned(),
11652 });
11653 }
11654 if store.tenant() != tenant.as_str() {
11655 return Err(BuildError::JournalStoreTenant {
11656 plane: tenant.to_string(),
11657 store: store.tenant().to_owned(),
11658 });
11659 }
11660
11661 // The stores whose state a key ring seals, in the caller's fixed order so a
11662 // plane with two mismatches reports the same one every build — an error
11663 // that varies between runs of the same wiring sends an operator looking for
11664 // a race that is not there.
11665 for &(store, serves) in state {
11666 if let Some(serves) = serves
11667 && serves != tenant.as_str()
11668 {
11669 return Err(BuildError::StateStoreTenant {
11670 store,
11671 plane: tenant.to_string(),
11672 tenant: serves.to_owned(),
11673 });
11674 }
11675 }
11676
11677 // After the tenant checks, deliberately: a store scoped to the wrong tenant
11678 // is the more basic fault, and reporting the erasure lock first would send
11679 // an operator to fix the second-most-wrong thing.
11680 //
11681 // A process-local erasure lock beside a store two instances can write is a
11682 // control that reads as present and is not. The window it fails to close is
11683 // between an erasure's legal-hold check and its key destruction: the other
11684 // instance writes an item, that item is sealed under a scope about to stop
11685 // existing, and the erasure reports success. Both facts are here, so the
11686 // refusal is here.
11687 if store.is_shared() && memories.is_some_and(|m| m.erasure_is_distributed() == Some(false)) {
11688 return Err(BuildError::ErasureCoordinatorNotShared);
11689 }
11690 Ok(())
11691}
11692
11693/// Refuse an event from outside whose kind this plane mints for itself.
11694fn refuse_reserved_kind(event: &InboundEvent) -> Result<(), RuntimeError> {
11695 if crate::core::is_reserved_kind(&event.kind) {
11696 return Err(RuntimeError::ReservedEventKind {
11697 kind: event.kind.clone(),
11698 });
11699 }
11700 Ok(())
11701}
11702
11703/// Inbound event delivery.
11704impl Runtime {
11705 /// Deliver an inbound event, resuming whichever run was waiting for it.
11706 ///
11707 /// # Ordering
11708 ///
11709 /// The event is **stored before** anyone looks for a waiter. That ordering
11710 /// is the whole reason this works: a message can arrive before its run
11711 /// reaches the wait, and one that is matched-then-discarded leaves that run
11712 /// waiting forever for something that already happened.
11713 ///
11714 /// A [`Delivery::Buffered`] result is therefore normal and not an error —
11715 /// it means "held until someone asks". Only the sweep
11716 /// ([`EventStore::sweep_unclaimed`](crate::case::EventStore::sweep_unclaimed))
11717 /// decides an event is genuinely unroutable, because that is a claim about
11718 /// the future rather than about this instant.
11719 ///
11720 /// # Errors
11721 ///
11722 /// [`RuntimeError::ReservedEventKind`] for a kind in the `agentplane.`
11723 /// namespace, which only this plane mints; [`RuntimeError`] for a store
11724 /// failure or a refused resume.
11725 pub async fn deliver(&self, event: &InboundEvent) -> Result<Delivery, RuntimeError> {
11726 refuse_reserved_kind(event)?;
11727 self.deliver_buffered(event, true).await
11728 }
11729
11730 /// Deliver an event this plane minted itself — a worklist decision.
11731 ///
11732 /// The one path into the reserved namespace [`deliver`](Self::deliver)
11733 /// refuses from outside.
11734 pub(crate) async fn deliver_minted(
11735 &self,
11736 event: &InboundEvent,
11737 ) -> Result<Delivery, RuntimeError> {
11738 self.deliver_buffered(event, false).await
11739 }
11740
11741 /// Store `event`, then offer it to the oldest eligible waiter.
11742 ///
11743 /// `rematch` offers a duplicate again: the buffer and the match are two
11744 /// writes, and a counterparty's retry is the one chance to repair a crash
11745 /// between them. A minted decision is never re-offered, because its
11746 /// duplicate is the answer *already on record*, and offering that stored
11747 /// answer under a second decider's call would report it as theirs.
11748 async fn deliver_buffered(
11749 &self,
11750 event: &InboundEvent,
11751 rematch: bool,
11752 ) -> Result<Delivery, RuntimeError> {
11753 let events = self.events.as_ref().ok_or_else(|| {
11754 RuntimeError::PlanContract(
11755 "this runtime has no event store — build it with `.events(store)`".into(),
11756 )
11757 })?;
11758
11759 let now = now_for_admission();
11760
11761 // Durable first. Deduplication by event id makes a counterparty's retry
11762 // — and they all retry — harmless.
11763 let fresh = events
11764 .buffer(event, now)
11765 .await
11766 .map_err(RuntimeError::from_store)?;
11767
11768 // The match claims only an unclaimed, live row, so a retry of a
11769 // message already consumed matches nothing.
11770 if !fresh && !rematch {
11771 return Ok(Delivery::Duplicate);
11772 }
11773 let Some(sub) = events
11774 .match_waiter(event, now)
11775 .await
11776 .map_err(RuntimeError::from_store)?
11777 else {
11778 return Ok(if fresh {
11779 Delivery::Buffered
11780 } else {
11781 Delivery::Duplicate
11782 });
11783 };
11784
11785 self.resume_subscription(events, sub, event).await
11786 }
11787
11788 /// Deliver an inbound event to exactly `run`.
11789 ///
11790 /// This is the task-addressed counterpart to [`Runtime::deliver`]. It is
11791 /// used by protocols such as A2A where a follow-up carries a concrete task
11792 /// id. Correlation alone is insufficient there: two tasks may wait on the
11793 /// same business key, and resuming the oldest would violate the request.
11794 ///
11795 /// The event store atomically inserts and claims the event for this run. A
11796 /// run that is not waiting leaves no buffered event behind for another run.
11797 ///
11798 /// # Errors
11799 ///
11800 /// As [`deliver`](Self::deliver), and [`RuntimeError::PlanContract`] when
11801 /// `run` is not waiting for this input. Addressing a run by id is no way
11802 /// around the reserved namespace: a run waiting on a human task is waiting
11803 /// for exactly the kind this refuses.
11804 pub async fn deliver_to(
11805 &self,
11806 run: RunId,
11807 event: &InboundEvent,
11808 ) -> Result<Delivery, RuntimeError> {
11809 refuse_reserved_kind(event)?;
11810 let events = self.events.as_ref().ok_or_else(|| {
11811 RuntimeError::PlanContract(
11812 "this runtime has no event store — build it with `.events(store)`".into(),
11813 )
11814 })?;
11815 match events
11816 .deliver_to(run, event, now_for_admission())
11817 .await
11818 .map_err(RuntimeError::from_store)?
11819 {
11820 crate::case::TargetedDelivery::Duplicate => Ok(Delivery::Duplicate),
11821 crate::case::TargetedDelivery::NotWaiting => Err(RuntimeError::PlanContract(format!(
11822 "run {run} is not waiting for this input"
11823 ))),
11824 crate::case::TargetedDelivery::Matched(sub) => {
11825 self.resume_subscription(events, sub, event).await
11826 }
11827 }
11828 }
11829
11830 async fn resume_subscription(
11831 &self,
11832 events: &Arc<dyn crate::case::EventStore>,
11833 sub: crate::core::Subscription,
11834 event: &InboundEvent,
11835 ) -> Result<Delivery, RuntimeError> {
11836 // Record the event as the awaited effect's result, then let replay do
11837 // the rest: the resumed run reads it back like any other completed
11838 // effect, and none of the suspension machinery exists twice.
11839 //
11840 // The lease may be briefly held: an event can arrive in the window
11841 // between the run registering its subscription and its owner finishing
11842 // the suspension bookkeeping — the owner is *concluding*, and will
11843 // release within milliseconds. Failing that race would be fatal in the
11844 // quietest possible way: the delivery fails `LeaseHeld` **after** the
11845 // event is durably claimed for this run, the counterparty's retry
11846 // deduplicates against the claimed event, and nothing ever resumes the
11847 // run — a message that arrived in time, parked forever. So the claim
11848 // is retried under a bounded backoff; if the owner is genuinely
11849 // stalled past the bound, the event is *left claimed* for this run
11850 // and reported as buffered — the sweep's redelivery pass finds a
11851 // waiting subscription with a claimed event and finishes the job on a
11852 // later tick, so the bound bounds latency, never delivery.
11853 let mut backoff = Duration::from_millis(25);
11854 let lease = loop {
11855 match self
11856 .store
11857 .acquire(sub.run, &self.owner, self.lease_ttl)
11858 .await
11859 {
11860 Ok(lease) => break lease,
11861 Err(crate::core::StoreError::LeaseHeld { .. })
11862 if backoff < Duration::from_secs(1) =>
11863 {
11864 tokio::time::sleep(backoff).await;
11865 backoff *= 2;
11866 }
11867 Err(crate::core::StoreError::LeaseHeld { .. }) => {
11868 // The claim retired the subscription, so nothing lists
11869 // this run as waiting any more — and the claimed event
11870 // blocks every dedup'd retry of itself. Parking the pair
11871 // is what keeps it findable: the sweep's redelivery pass
11872 // walks parked waits, re-claims (a claim by the same run
11873 // is returned, not filtered), and finishes the delivery
11874 // once the owner concludes.
11875 events
11876 .park_wait(&sub, now_for_admission())
11877 .await
11878 .map_err(RuntimeError::from_store)?;
11879 tracing::warn!(
11880 run = %sub.run,
11881 event = %event.id,
11882 "an event was claimed for a run whose owner did not conclude \
11883 within the retry window; the sweep will deliver it"
11884 );
11885 return Ok(Delivery::Buffered);
11886 }
11887 Err(e) => return Err(RuntimeError::from_store(e)),
11888 }
11889 };
11890
11891 let history = self
11892 .store
11893 .read(sub.run, 1)
11894 .await
11895 .map_err(RuntimeError::from_store)?;
11896 let already_recorded = history.iter().any(|record| {
11897 record.effect_key() == Some(sub.effect)
11898 && matches!(record.kind(), RecordKind::EffectDone { .. })
11899 });
11900 // A run whose conclusion is durable consumes nothing. An answer
11901 // appended after `RunConcluded` is one the run never acted on, and it
11902 // leaves a run its seal refuses forever. The resume below finishes it
11903 // instead — seals it and retires its waits, which hands this message,
11904 // still unconsumed, to the next waiter on its key.
11905 let closed = resume_is_closed(&history).is_some();
11906 if !already_recorded && !closed {
11907 // Judged here, where the output is recorded, under the run's own
11908 // declaration: the step that resumes reads the verdict back.
11909 let content = self.delivered_verdict(&history, &event.payload);
11910 self.store
11911 .append(
11912 lease.epoch,
11913 vec![{
11914 let mut a = Append::new(
11915 sub.run,
11916 RecordKind::EffectDone {
11917 output: event.payload.clone(),
11918 // The sender, so a replayed run rebuilds the same
11919 // provenance this delivery gave the value.
11920 source: Some(event.source.clone()),
11921 // And the operator who minted it, where one
11922 // did: the same record on the same path, so a
11923 // decision that resumes a suspended run is
11924 // attributed exactly as one consumed inline.
11925 by: event.by.clone(),
11926 spend: crate::core::Spend::default(),
11927 // An inbound payload is another party's data,
11928 // which is exactly the point this lattice value
11929 // names — the same one the run's own delivery
11930 // path records.
11931 declared: crate::core::DeclaredOutput::untrusted(),
11932 content,
11933 elapsed_ms: None,
11934 },
11935 )
11936 .effect(sub.effect)
11937 .step(sub.step)
11938 .phase(sub.phase);
11939 // Every record of a case-bound run carries its case, and a
11940 // record written from outside the run is no exception.
11941 if let Some(c) = sub.case {
11942 a = a.case(c);
11943 }
11944 a
11945 }],
11946 )
11947 .await
11948 .map_err(RuntimeError::from_store)?;
11949 }
11950
11951 if !closed {
11952 events
11953 .unsubscribe(sub.run, sub.effect)
11954 .await
11955 .map_err(RuntimeError::from_store)?;
11956 }
11957
11958 // The event is recorded as the awaited effect's result, and the resume
11959 // continues under the *same* lease. Releasing here and letting the
11960 // resume re-acquire opened the window this function's own doc names —
11961 // a crash between the release and the acquire left a released lease
11962 // over a run whose subscription was just retired: no driver, and
11963 // invisible to the abandonment queue, which lists only leases that
11964 // expired while still naming an owner. The handover keeps the run
11965 // owned continuously from delivery to conclusion.
11966 match self.resume_holding(sub.run, lease).await {
11967 // A `Fenced` means this lease lapsed mid-resume and somebody
11968 // claimed the run — a concurrent delivery, a recovery sweep. The
11969 // event is durably recorded, so whoever holds the lease replays
11970 // it; this delivery still delivered, and a sender told otherwise
11971 // retries into a duplicate.
11972 Ok(_) | Err(RuntimeError::Fenced { .. }) => {}
11973 // This plane holds no agent for the run — an operator's terminal
11974 // recording a decision. The answer is durable, and the lease is
11975 // kept so it lapses owned: that puts the run on the abandonment
11976 // listing, where the plane that runs the agent recovers it from
11977 // the recorded answer. Failing here would leave a recorded
11978 // decision reported as lost and the task it answers still claimed.
11979 // A plane holding no key ring for a sealed run, or no case store
11980 // for a case-bound one, is the same terminal: the plane that holds
11981 // it continues the run.
11982 Err(
11983 RuntimeError::NoProvider { .. }
11984 | RuntimeError::NoCaseStore { .. }
11985 | RuntimeError::PayloadsSealed { .. },
11986 ) => {
11987 return Ok(Delivery::Buffered);
11988 }
11989 Err(e) => return Err(e),
11990 }
11991 if closed {
11992 return Ok(Delivery::Buffered);
11993 }
11994 Ok(Delivery::Resumed { run: sub.run })
11995 }
11996
11997 /// Retire everything a run that concluded closed was still waiting on:
11998 /// its timers, its event subscriptions and its unanswered tasks.
11999 ///
12000 /// A sealed run can record nothing, so each of these left behind is work
12001 /// the plane keeps doing for nobody — a timer claimed and failed every
12002 /// lease period, the oldest waiter on a key taking the next event from a
12003 /// live run behind it, a task offered for a decision no answer reaches.
12004 ///
12005 /// Called after every seal of a concluded run — the conclusion's own and
12006 /// the one resume writes to repair a crash between conclusion and seal —
12007 /// and idempotent, so retiring twice is harmless.
12008 ///
12009 /// After the seal, and best-effort: the conclusion is already durable, and
12010 /// the stores pass over a sealed run's timers and subscriptions on their
12011 /// own, so a retirement that fails here costs clutter rather than a
12012 /// delivery. A task left pending by that failure is withdrawn by nothing
12013 /// else, which is why the failure is logged loudly.
12014 async fn retire_waits(&self, run: RunId) {
12015 if let Some(timers) = &self.timers
12016 && let Err(error) = timers.disarm_run(run).await
12017 {
12018 tracing::error!(%run, %error, "a closed run's timers could not be retired");
12019 }
12020 // The waits the run announced and never had answered: a message
12021 // claimed for one of them reached nobody and goes to the next waiter,
12022 // and a task opened for one is withdrawn. A task opened beside an
12023 // answer completes its own effect and is not among them: it is a
12024 // finding for a desk, and outlives the run by design.
12025 let unanswered = match self.store.read(run, 1).await {
12026 Ok(records) => unanswered_waits(&records),
12027 Err(error) => {
12028 tracing::error!(%run, %error, "a closed run's waits could not be read to retire them");
12029 return;
12030 }
12031 };
12032 if let Some(events) = &self.events {
12033 match events.unsubscribe_run(run, &unanswered).await {
12034 Ok(retired) => {
12035 for event in retired.released {
12036 self.offer_released(events, &event).await;
12037 }
12038 }
12039 Err(error) => {
12040 tracing::error!(%run, %error, "a closed run's subscriptions could not be retired");
12041 }
12042 }
12043 }
12044 if let Some(tasks) = &self.tasks {
12045 let awaited: Vec<_> = unanswered
12046 .iter()
12047 .map(|effect| crate::core::TaskId::derive(run, *effect))
12048 .collect();
12049 if let Err(error) = tasks.withdraw_run(run, &awaited).await {
12050 tracing::error!(%run, %error, "a closed run's open tasks could not be withdrawn");
12051 }
12052 }
12053 }
12054
12055 /// Offer a message a closed run held, and never consumed, to the next
12056 /// waiter on its key — the delivery it would have had if that run had not
12057 /// concluded first. Best effort, like the retirement that released it: a
12058 /// message no waiter takes now stays buffered for the next one.
12059 async fn offer_released(
12060 &self,
12061 events: &Arc<dyn crate::case::EventStore>,
12062 event: &InboundEvent,
12063 ) {
12064 let sub = match events.match_waiter(event, now_for_admission()).await {
12065 Ok(Some(sub)) => sub,
12066 Ok(None) => return,
12067 Err(error) => {
12068 tracing::error!(event = %event.id, %error, "a released message could not be offered");
12069 return;
12070 }
12071 };
12072 if let Err(error) = Box::pin(self.resume_subscription(events, sub, event)).await {
12073 tracing::error!(event = %event.id, %error, "a released message could not be delivered");
12074 }
12075 }
12076
12077 /// Finish deliveries that died between the claim and the resume.
12078 ///
12079 /// A delivery that raced a live owner leaves an event durably claimed for
12080 /// a run nothing resumed, and the counterparty's retries deduplicate
12081 /// against that claim — so no future delivery drives the run, and a
12082 /// message that arrived in time is stranded. The giving-up delivery parks
12083 /// the pair precisely so it stays findable; this pass walks the parked
12084 /// waits — never every registered wait, whose long legitimate members
12085 /// would fill the page ahead of a pair parked after them — re-claims
12086 /// (idempotently: a claim by the same run is returned, not filtered), and
12087 /// finishes the delivery. Runs whose owner is *still* live are skipped and
12088 /// found again next tick, so the delivery's retry bound bounds latency,
12089 /// never delivery.
12090 ///
12091 /// What this does **not** cover, honestly: a crash inside the store
12092 /// commit between an event's claim and the resume retires the
12093 /// subscription with nothing left to re-register it, and that window
12094 /// stays what it always was — recorded rather than closed, with later
12095 /// events for the same correlation dead-lettering as its symptom.
12096 pub(crate) async fn redeliver_claimed(
12097 &self,
12098 limit: usize,
12099 ) -> Result<crate::runtime::Redelivered, RuntimeError> {
12100 let Some(events) = self.events.as_ref() else {
12101 return Ok(crate::runtime::Redelivered::default());
12102 };
12103 let parked = events
12104 .parked_waits(limit)
12105 .await
12106 .map_err(RuntimeError::from_store)?;
12107 let examined = parked.len();
12108 let mut delivered = 0usize;
12109 for sub in parked {
12110 let Some(buffered) = events
12111 .claim_for(&sub, now_for_admission())
12112 .await
12113 .map_err(RuntimeError::from_store)?
12114 else {
12115 continue;
12116 };
12117 match self
12118 .resume_subscription(events, sub.clone(), &buffered.event)
12119 .await
12120 {
12121 Ok(Delivery::Resumed { .. }) => delivered += 1,
12122 // Still held live: the owner is working, and will either
12123 // consume the claim itself or conclude and be found next tick.
12124 Ok(_) => {}
12125 Err(error) => {
12126 tracing::error!(
12127 run = %sub.run,
12128 %error,
12129 "a claimed event's redelivery failed; retried next tick",
12130 );
12131 }
12132 }
12133 }
12134 Ok(crate::runtime::Redelivered {
12135 finished: delivered,
12136 examined,
12137 })
12138 }
12139
12140 /// Retire events that nobody claimed within `grace`.
12141 ///
12142 /// A non-empty dead-letter list means a correlation key is wrong somewhere:
12143 /// the message arrived, was held, and no run ever asked for it. That is the
12144 /// failure which otherwise presents as a process silently never completing,
12145 /// so it is worth alerting on rather than logging.
12146 /// `grace` is a `std::time::Duration` for the reason
12147 /// [`StepCtx::deadline`](crate::runtime::StepCtx::deadline)'s `warn_before`
12148 /// is: a negative grace window is meaningless, and the signed type could
12149 /// express it — a cutoff moved *forward* of now, retiring events that had
12150 /// not yet had their chance. It is also the `Duration` the caller has.
12151 pub async fn sweep_events(&self, grace: std::time::Duration) -> Result<usize, RuntimeError> {
12152 let events = self
12153 .events
12154 .as_ref()
12155 .ok_or_else(|| RuntimeError::PlanContract("this runtime has no event store".into()))?;
12156 // Saturating rather than fallible, at both steps: a grace window
12157 // beyond what the calendar can hold means "retire nothing", and
12158 // refusing the call would be a worse answer to a caller asking for a
12159 // longer hold. The subtraction has to saturate too — `time` panics on
12160 // underflow, so converting the window to `Duration::MAX` and then
12161 // subtracting it aborts the tick that also breaches obligations and
12162 // recovers abandoned runs. The first instant the calendar can name is
12163 // a cutoff nothing is older than, which is exactly "retire nothing".
12164 let grace = time::Duration::try_from(grace).unwrap_or(time::Duration::MAX);
12165 let cutoff = now_for_admission()
12166 .checked_sub(grace)
12167 .unwrap_or_else(crate::core::first_instant);
12168 let retired = events
12169 .sweep_unclaimed(cutoff, "no run claimed this event within the grace window")
12170 .await
12171 .map_err(RuntimeError::from_store)?;
12172
12173 if retired > 0 {
12174 // A non-empty dead-letter list means a correlation key is wrong
12175 // somewhere: the message arrived, was held, and no run ever asked
12176 // for it. That is the failure which otherwise presents as a process
12177 // silently never completing.
12178 tracing::error!(target: telemetry::DEAD_LETTERED, count = retired, %cutoff);
12179 self.meter
12180 .count_by(metrics::DEAD_LETTERS, "", retired as u64);
12181 }
12182 Ok(retired)
12183 }
12184}
12185
12186/// Every `RunStatus`, so adding one forces a decision everywhere one is owed.
12187///
12188/// Written out rather than derived, because Rust cannot enumerate a
12189/// data-carrying enum. What keeps it honest is **not** the count assertions in
12190/// its consumers: those fire when somebody edits this list, which is the case
12191/// that needs no help. A variant added to the enum and forgotten here changes
12192/// nothing they can see, and every agreement test then walks a list that no
12193/// longer describes the type — passing over the one variant nobody decided
12194/// about.
12195///
12196/// `every_run_status_variant_is_listed` in `tests/guards` is what closes that:
12197/// it reads the enum's variants and this function's arms out of the source and
12198/// holds them to each other, which is the only check available for a list a
12199/// language cannot derive.
12200///
12201/// Crate-visible so those consumers share **one** list. Two copies would be two
12202/// places to remember, and the second would be the one that went stale — which
12203/// is the same failure the tests using it exist to catch.
12204#[cfg(test)]
12205pub(crate) fn every_status() -> Vec<RunStatus> {
12206 use crate::core::{BudgetExceeded, CorrelationKey, SuspendReason, Timestamp};
12207 vec![
12208 RunStatus::Succeeded,
12209 RunStatus::Failed("because".into()),
12210 RunStatus::Suspended(SuspendReason::AwaitingEvent {
12211 kind: "reply".into(),
12212 correlation: vec![CorrelationKey::new("claim", "CLM-1")],
12213 until: Timestamp::from_unix_timestamp(1_760_000_000).expect("time"),
12214 }),
12215 RunStatus::Exhausted(BudgetExceeded::Steps { allowed: 1 }),
12216 RunStatus::Quarantined("unknown outcome".into()),
12217 RunStatus::Replanning("try again".into()),
12218 RunStatus::Cancelled {
12219 actor: crate::core::Operator::asserted("ops").expect("a name"),
12220 reason: "stop".into(),
12221 },
12222 RunStatus::Abandoned {
12223 actor: crate::core::Operator::asserted("ops").expect("a name"),
12224 reason: "nobody could establish what happened".into(),
12225 },
12226 RunStatus::Swept,
12227 RunStatus::BrokeGlass {
12228 actor: crate::core::Operator::asserted("ops").expect("a name"),
12229 reason: "INC-42".into(),
12230 },
12231 RunStatus::HaltLifted {
12232 actor: crate::core::Operator::asserted("ops").expect("a name"),
12233 },
12234 RunStatus::HoldReleased {
12235 actor: crate::core::Operator::asserted("ops").expect("a name"),
12236 },
12237 RunStatus::Withheld {
12238 subject: "alice".into(),
12239 reason: "credential withdrawn".into(),
12240 },
12241 RunStatus::Observed,
12242 ]
12243}
12244
12245#[cfg(test)]
12246mod resume_agreement_tests {
12247 use super::{QuotaPass, RunStatus, resume_is_closed};
12248 use crate::journal::RecordKind;
12249
12250 // Through the crate re-export rather than `super::`, so the one path both
12251 // consumers use is exercised in every build. Reaching it directly here
12252 // would leave the re-export unused whenever `a2a-server` is off — which is
12253 // a lint failure in exactly the feature configuration nobody runs locally.
12254 use crate::runtime::every_status;
12255
12256 /// **A conclusion that sealed the journal may never be resumed.**
12257 ///
12258 /// `RunStatus::seals` decides whether a conclusion freezes the run and
12259 /// publishes a Merkle leaf; `resume_is_closed` decides whether a recorded
12260 /// conclusion may be continued. A status that did the first and permits the
12261 /// second would grow the history past the leaf every later checkpoint
12262 /// attests — the failure the "a conclusion is not a closure" work removed
12263 /// for `failed`, reachable again the moment the two disagree.
12264 ///
12265 /// The executor's own comment claimed "a test pins the agreement" while no
12266 /// such test existed, which is the unfalsifiable-guarantee shape this
12267 /// project treats as a defect in itself: the rule was real, checked by
12268 /// nobody, and deletable in silence.
12269 #[test]
12270 fn a_sealing_conclusion_is_never_resumable() {
12271 let statuses = every_status();
12272 assert_eq!(
12273 statuses.len(),
12274 14,
12275 "a RunStatus variant was added or removed — decide whether it seals \
12276 and whether a resume may continue from it, then update this list"
12277 );
12278
12279 for status in &statuses {
12280 let records = sealed_as(status.as_str());
12281 let verdict = resume_is_closed(&records);
12282 if status.seals() {
12283 assert!(
12284 verdict.is_some(),
12285 "'{}' seals the journal and enters the Merkle log, yet a resume \
12286 is permitted from it — the resume would grow the history past \
12287 the leaf every later checkpoint attests",
12288 status.as_str()
12289 );
12290 }
12291 }
12292 }
12293
12294 /// The availability half: the conclusions that stay open really do.
12295 ///
12296 /// Asserted separately and by name rather than as the converse of the rule
12297 /// above, because the converse is false — `Suspended` and `Replanning` do
12298 /// not seal either, and neither is ever a recorded conclusion.
12299 ///
12300 /// `withheld` is here because it is a **pause**: the authority a run acts
12301 /// under was withdrawn, its mutations stand, and lifting the halt is what
12302 /// lets it continue. A withheld run that could not be resumed would make the
12303 /// withdrawal a destruction rather than a stop.
12304 #[test]
12305 fn a_failed_or_exhausted_run_may_still_be_resumed() {
12306 for outcome in ["failed", "exhausted", super::WITHHELD_OUTCOME] {
12307 assert!(
12308 resume_is_closed(&sealed_as(outcome)).is_none(),
12309 "'{outcome}' is a conclusion a resume must be able to continue \
12310 from — its completed effects are read back from history, which \
12311 is the point of having a journal"
12312 );
12313 }
12314 }
12315
12316 /// `SEALED_OUTCOMES` and `RunStatus::seals` are one rule in two spellings,
12317 /// and this holds them together variant by variant.
12318 ///
12319 /// The list exists so a caller wanting every sealed run — the export CLI —
12320 /// does not restate the rule as string literals in a binary, which is where
12321 /// a new sealing outcome would have been silently dropped from exactly the
12322 /// artifact an auditor asks for. Both directions matter: a sealing status
12323 /// missing from the list loses runs, and a non-sealing status present in it
12324 /// makes the export ask the outcome index about a backlog that drains.
12325 #[test]
12326 fn the_sealed_outcome_list_agrees_with_the_sealing_rule() {
12327 for status in super::every_status() {
12328 assert_eq!(
12329 super::SEALED_OUTCOMES.contains(&status.as_str()),
12330 status.seals(),
12331 "'{}' disagrees between RunStatus::seals and SEALED_OUTCOMES — \
12332 one rule, two spellings, and the export reads the list",
12333 status.as_str()
12334 );
12335 }
12336 }
12337
12338 /// **Every outcome this build writes is one its own reader recognises.**
12339 ///
12340 /// The reader is a `match` over `&str` with a catch-all, so it goes on
12341 /// compiling forever while the writer's list grows — and the catch-all is
12342 /// not benign: it answers `Quarantined`, so a run this plane sealed itself
12343 /// reads back as *the runtime could not decide what happened*. Two endings
12344 /// sat there, `swept` and `broke-glass`, and a break-glass crossing — the
12345 /// one control whose entire content is who crossed and why — answered the
12346 /// operator surface with the words "this build does not recognise it" in
12347 /// place of the reason.
12348 ///
12349 /// Held from the writer's side, and the writer's side is
12350 /// [`every_status`](super::every_status) — every variant the type can
12351 /// take. `OUTCOMES_OF_RECORD` is the *export's* list and deliberately
12352 /// omits the endings that stay resumable, so a guard reading it asks about
12353 /// a subset of what the writer writes and passes while the rest fall
12354 /// through the catch-all. `withheld` sat there: the one ending whose whole
12355 /// content is *whose credential was withdrawn*, answering the operator API
12356 /// with "this build does not recognise it".
12357 #[test]
12358 fn every_outcome_this_build_writes_is_one_it_can_read_back() {
12359 for written in super::every_status() {
12360 // A suspension is not a conclusion; it has its own record and its
12361 // own arm in `observed_status`.
12362 if written.is_suspended() {
12363 continue;
12364 }
12365 let outcome = written.as_str();
12366 let status = super::recorded_status(
12367 outcome,
12368 written.own_account().as_deref(),
12369 match &written {
12370 RunStatus::Exhausted(limit) => Some(limit),
12371 _ => None,
12372 },
12373 &sealed_as(outcome),
12374 );
12375 assert_eq!(
12376 status.as_str(),
12377 outcome,
12378 "this build concludes runs as '{outcome}' and reads that back as \
12379 '{}' — a catch-all answering `Quarantined` about a record this \
12380 plane wrote itself",
12381 status.as_str()
12382 );
12383 }
12384 }
12385
12386 /// **An ending a person caused reports the words that person gave.**
12387 ///
12388 /// The conclusion does not carry them: one sentence, one record. So the
12389 /// reader has to reach the attribution record for the reason exactly as it
12390 /// already does for the actor, and a reader that only learned the second
12391 /// half would answer an incident review with a blank where the operator
12392 /// wrote why.
12393 #[test]
12394 fn an_ending_a_person_caused_reports_their_own_words() {
12395 for (outcome, expected) in [
12396 ("cancelled", "stop"),
12397 ("abandoned", "nobody could establish what happened"),
12398 (super::BREAK_GLASS_OUTCOME, "INC-42"),
12399 (super::WITHHELD_OUTCOME, "credential withdrawn"),
12400 ] {
12401 // `None`, as the conclusion this build writes now carries.
12402 let status = super::recorded_status(outcome, None, None, &sealed_as(outcome));
12403 assert_eq!(
12404 status.reason().as_deref(),
12405 Some(expected),
12406 "'{outcome}' lost the reason the person gave: {status:?}"
12407 );
12408 }
12409 }
12410
12411 /// **An ending nobody is named on is quarantined, not given a name.**
12412 ///
12413 /// The chain this build writes always carries the attribution record before
12414 /// the conclusion; a chain that carries only the conclusion is a crash
12415 /// between the two appends, or a chain somebody edited. Either way the
12416 /// record is not trustworthy, and *not trustworthy* is a different bucket
12417 /// from *an operator called unknown did this*.
12418 ///
12419 /// A fabricated actor would be worse than a missing one: indistinguishable
12420 /// from a real operator with that name, and served through the operator API
12421 /// as fact.
12422 #[test]
12423 fn an_ending_with_no_attribution_record_is_not_given_a_name() {
12424 for outcome in ["cancelled", "abandoned", super::BREAK_GLASS_OUTCOME] {
12425 let status =
12426 super::recorded_status(outcome, Some("why"), None, &concluded_only(outcome));
12427 assert!(
12428 matches!(status, RunStatus::Quarantined(_)),
12429 "'{outcome}' with no record of who asked read back as {status:?} — \
12430 an operator act with nobody on it was given a name"
12431 );
12432 }
12433 }
12434
12435 /// The offline sweep covers every ending, plus the backlog that is not one.
12436 ///
12437 /// Held to `SEALED_OUTCOMES` rather than restated, because the failure this
12438 /// prevents is silent in exactly the artifact nobody re-reads: an ending
12439 /// added to the runtime and not to the export drops those runs from the
12440 /// file an auditor is handed, and the file looks complete.
12441 #[test]
12442 fn the_offline_sweep_covers_every_ending_and_the_quarantine_backlog() {
12443 for outcome in super::SEALED_OUTCOMES {
12444 assert!(
12445 super::OUTCOMES_OF_RECORD.contains(outcome),
12446 "'{outcome}' is an ending this plane can reach and the export's \
12447 default sweep would silently omit it"
12448 );
12449 }
12450 assert!(
12451 super::OUTCOMES_OF_RECORD.contains(&"quarantined"),
12452 "a quarantine is the plane's outstanding unresolved risk and the one \
12453 thing an auditor is most likely to have come for"
12454 );
12455 for open in ["failed", "exhausted", "suspended"] {
12456 assert!(
12457 !super::OUTCOMES_OF_RECORD.contains(&open),
12458 "'{open}' is work in progress that a resume moves on its own — \
12459 exporting it is a snapshot of a moving target, not a record"
12460 );
12461 }
12462 }
12463
12464 /// An outcome this build cannot interpret is never permission to resume.
12465 #[test]
12466 fn an_unrecognised_conclusion_fails_closed() {
12467 let verdict = resume_is_closed(&sealed_as("swept"));
12468 assert!(
12469 matches!(verdict, Some(RunStatus::Quarantined(_))),
12470 "an unrecognised conclusion must quarantine rather than resume: {verdict:?}"
12471 );
12472 }
12473
12474 /// A chain carrying the conclusion and **not** the attribution record.
12475 ///
12476 /// The damaged shape, which no healthy write produces: a crash between the
12477 /// two appends, or an edit. Built by taking the real chain and dropping
12478 /// everything before the conclusion, so it cannot drift from what
12479 /// [`sealed_as`] produces.
12480 fn concluded_only(outcome: &str) -> Vec<crate::journal::Record> {
12481 let full = sealed_as(outcome);
12482 full.into_iter()
12483 .filter(|r| matches!(r.kind(), RecordKind::RunConcluded { .. }))
12484 .collect()
12485 }
12486
12487 /// The chain this build actually writes for an ending.
12488 ///
12489 /// **Including the attribution record**, where the ending has one. A
12490 /// conclusion is never the only record of a cancellation, an abandonment or
12491 /// a crossing — the runtime appends who asked first — so a fixture holding
12492 /// the conclusion alone asserts a round trip over a chain this plane does
12493 /// not produce, and would keep passing if the reader began inventing a
12494 /// name for the missing half.
12495 fn sealed_as(outcome: &str) -> Vec<crate::journal::Record> {
12496 use crate::core::{Digest, Epoch};
12497 use crate::journal::{Record, RecordBody};
12498
12499 let run = crate::core::RunId::generate();
12500 let who = || crate::core::Operator::asserted("ops").expect("a name");
12501 let mut kinds: Vec<RecordKind> = match outcome {
12502 "cancelled" => vec![RecordKind::RunCancelled {
12503 actor: who(),
12504 reason: "stop".into(),
12505 }],
12506 "abandoned" => vec![RecordKind::QuarantineDecided {
12507 decider: who(),
12508 reason: "nobody could establish what happened".into(),
12509 decision: crate::core::QuarantineDecision::Abandon,
12510 }],
12511 super::BREAK_GLASS_OUTCOME => vec![RecordKind::BreakGlass {
12512 actor: who(),
12513 roles: vec!["oncall".into()],
12514 reason: "INC-42".into(),
12515 }],
12516 super::HALT_LIFTED_OUTCOME => vec![RecordKind::HaltLifted {
12517 scope: "tenant".into(),
12518 by: who(),
12519 at: crate::core::Timestamp::UNIX_EPOCH,
12520 reason: "INC-42".into(),
12521 thrown_by: who(),
12522 thrown_at: crate::core::Timestamp::UNIX_EPOCH,
12523 }],
12524 super::HOLD_RELEASED_OUTCOME => vec![RecordKind::HoldReleased {
12525 by: who(),
12526 at: crate::core::Timestamp::UNIX_EPOCH,
12527 placed_by: who(),
12528 placed_at: crate::core::Timestamp::UNIX_EPOCH,
12529 }],
12530 super::WITHHELD_OUTCOME => vec![RecordKind::AuthorityWithheld {
12531 subject: "alice".into(),
12532 reason: "credential withdrawn".into(),
12533 by: who(),
12534 }],
12535 _ => Vec::new(),
12536 };
12537 kinds.push(RecordKind::RunConcluded {
12538 outcome: outcome.to_owned(),
12539 chain_head: Digest::of(b""),
12540 reason: None,
12541 exhaustion: None,
12542 live_spend: crate::core::Spend::default(),
12543 });
12544
12545 let mut prev = Digest::of(b"");
12546 let mut out = Vec::new();
12547 for (i, kind) in kinds.into_iter().enumerate() {
12548 let body = RecordBody {
12549 seq: u64::try_from(i + 1).expect("a small fixture"),
12550 run,
12551 case: None,
12552 step: None,
12553 phase: super::Phase::Forward,
12554 epoch: Epoch::default(),
12555 v: 1,
12556 effect_key: None,
12557 kind,
12558 };
12559 let record = Record::seal(body, prev).expect("a sealed record");
12560 prev = record.hash;
12561 out.push(record);
12562 }
12563 out
12564 }
12565
12566 /// One execution pass is authorized and accrued against one period even
12567 /// when it finishes after a billing boundary.
12568 #[test]
12569 fn a_quota_pass_keeps_the_period_it_started_in() {
12570 let quota = crate::quota::TenantQuota {
12571 max_tokens_per_period: Some(1_000),
12572 period: crate::quota::Period::Daily,
12573 ..crate::quota::TenantQuota::default()
12574 };
12575 let before_midnight = crate::core::Timestamp::from_unix_timestamp(86_399).unwrap();
12576 let after_midnight = crate::core::Timestamp::from_unix_timestamp(86_400).unwrap();
12577
12578 let pass = QuotaPass::at("a, before_midnight, true);
12579 assert_eq!(
12580 pass.period(),
12581 Some(quota.period.key_for(before_midnight).as_str())
12582 );
12583 assert_ne!(
12584 pass.period(),
12585 Some(quota.period.key_for(after_midnight).as_str()),
12586 "settlement recomputed the period at completion, so this pass was \
12587 authorized against one ledger and charged to another"
12588 );
12589 }
12590}