agentplane/runtime/executor.rs
1//! The executor: admission, step dispatch, sealing, and replay.
2//!
3//! M0 executes a single step. The DAG scheduler, plan contract, and topology
4//! checks slot in above this without changing the effect protocol below it —
5//! which is the point of putting the determinism boundary at the effect rather
6//! than at the plan.
7
8#[cfg(feature = "manifest")]
9use std::collections::HashSet;
10use std::collections::{BTreeMap, BTreeSet, HashMap};
11use std::sync::Arc;
12use std::time::Duration;
13
14use serde_json::Value;
15
16use crate::case::{CaseStore, EventStore, TaskStore, TimerStore};
17use crate::core::{
18 ArgSource, Budget, Calendar, Capability, Consumed, CorrelationKey, Delivery, Digest,
19 InboundEvent, Ledger, Outcome, Phase, PlanIR, PlanNode, PolicyBundleIdentity, RunId,
20 RuntimeError, Skill, SkillDescriptor, Spend, StepId, Tainted, WallClock,
21};
22use crate::journal::{Append, JournalStore, Record, RecordKind, ReplayCursor, StepCursor};
23use crate::runtime::BuildError;
24
25use super::ctx::{CaseContext, Mode, Pass, StepCtx};
26use super::metrics;
27use super::telemetry;
28use tracing::Instrument;
29
30#[derive(Debug, Clone)]
31pub(crate) enum CaseBinding {
32 Correlate {
33 kind: String,
34 keys: Vec<CorrelationKey>,
35 },
36 Existing(crate::core::CaseId),
37}
38
39/// What a run is bound to, beyond its target and input: who it acts for,
40/// which matter it belongs to, who asked, which declaration it was reviewed
41/// against. Handed to [`RunRequest::terms`].
42///
43/// The value a front door computes per caller — above all a chain **per run**:
44/// a door that authenticates many callers admits each one's run under the
45/// chain that caller presented. Whether the run happens at most once is not a
46/// term; it is [`RunRequest::once`], because it changes what the answer is.
47///
48/// **Authority is chosen once, by constructor**: [`default`](Self::default)
49/// acts as the plane, [`acting_as`](Self::acting_as) under a caller's chain,
50/// [`served`](Self::served) for a served caller that may have presented none.
51///
52/// ```no_run
53/// # use agentplane::runtime::RunTerms;
54/// # fn f(chain: agentplane::core::Delegation, keys: &[agentplane::core::CorrelationKey]) {
55/// let terms = RunTerms::acting_as(chain).correlated("order", keys);
56/// # }
57/// ```
58///
59/// ```compile_fail
60/// # use agentplane::runtime::RunTerms;
61/// # fn f(chain: agentplane::core::Delegation) {
62/// let terms = RunTerms::served(None).acting_as(chain);
63/// # }
64/// ```
65#[derive(Debug, Clone, Default)]
66pub struct RunTerms {
67 /// Which matter this run belongs to, if any.
68 case: Option<CaseBinding>,
69 /// The admission key to claim. Travels to the `RunAdmitted` record, because
70 /// that is where the store claims it — see
71 /// [`JournalStore::admitted_as`](crate::journal::JournalStore::admitted_as).
72 idempotency_key: Option<String>,
73 /// On whose authority this run acts.
74 acting_as: Acting,
75 /// Who asked for this run, when a person or peer did. Journaled on
76 /// `RunAdmitted`, and barred from deciding the run's own tasks.
77 admitted_by: Option<String>,
78 /// The declaration revision the caller reviewed, when it pinned one.
79 expected_declaration: Option<Digest>,
80 /// Data subjects the embedder names for this run, beside any its
81 /// declaration binds.
82 subjects: Vec<String>,
83}
84
85/// On whose authority a run acts.
86///
87/// Three answers, not two, because "no chain was presented" means different
88/// things at different doors. An in-process embedder that names no chain is
89/// the plane itself, and acts under the plane's
90/// ([`RuntimeBuilder::acting_as`]). A served caller whose credential carried
91/// none is **somebody else** — and falling back to the plane's chain for them
92/// would make the operator's authority an ambient credential every
93/// authenticated peer holds.
94///
95/// Acting under a chain and being bounded by one are separate questions. A
96/// served caller with no chain acts under none, and is still bounded by the
97/// plane's: that chain is the most this plane may do for anybody, so its
98/// scope and its admissibility gate the plan
99/// ([`ceiling`](Self::ceiling)) while nothing of it reaches the policy
100/// context, the journal or a withdrawal lookup.
101#[derive(Debug, Clone, Default)]
102enum Acting {
103 /// The plane's own chain, if it has one: a run the embedder started.
104 #[default]
105 Plane,
106 /// The chain an authenticated caller presented.
107 Caller(crate::core::Delegation),
108 /// A served caller that presented no chain: this run acts under none.
109 Nobody,
110 /// A chain delegated from the plane's own, for a run the plane's run
111 /// commissioned: it acts under that chain, and as the plane, so a peer
112 /// it calls is presented the plane's credential rather than one naming
113 /// the plane's owner as if a person had asked.
114 PlaneDelegate(crate::core::Delegation),
115}
116
117impl Acting {
118 /// The chain this run acts under, given the plane's own.
119 const fn resolve<'a>(
120 &'a self,
121 plane: Option<&'a crate::core::Delegation>,
122 ) -> Option<&'a crate::core::Delegation> {
123 match self {
124 Self::Plane => plane,
125 Self::Caller(chain) | Self::PlaneDelegate(chain) => Some(chain),
126 Self::Nobody => None,
127 }
128 }
129
130 /// The chain that bounds what this run may be admitted for: the one it
131 /// acts under, or — for a served caller that presented none — the
132 /// plane's, which it does not act under.
133 const fn ceiling<'a>(
134 &'a self,
135 plane: Option<&'a crate::core::Delegation>,
136 ) -> Option<&'a crate::core::Delegation> {
137 match self {
138 Self::Plane | Self::Nobody => plane,
139 Self::Caller(chain) | Self::PlaneDelegate(chain) => Some(chain),
140 }
141 }
142}
143
144impl RunTerms {
145 /// Run inside a case that already exists.
146 #[must_use]
147 pub fn in_case(mut self, case: crate::core::CaseId) -> Self {
148 self.case = Some(CaseBinding::Existing(case));
149 self
150 }
151
152 /// Join or open a case by business key before running.
153 #[must_use]
154 pub fn correlated(mut self, kind: &str, keys: &[CorrelationKey]) -> Self {
155 self.case = Some(CaseBinding::Correlate {
156 kind: kind.to_owned(),
157 keys: keys.to_vec(),
158 });
159 self
160 }
161
162 /// Act under this chain rather than the plane's own.
163 ///
164 /// Checked at admission exactly as the plane's chain is: the plan must be
165 /// inside its scope, and [`Delegation::admissible`] must accept this
166 /// plane at this instant. Journaled as the run's `IdentityBound`, so
167 /// "on whose behalf" names the caller and not the plane.
168 ///
169 /// [`Delegation::admissible`]: crate::core::Delegation::admissible
170 #[must_use]
171 pub fn acting_as(chain: crate::core::Delegation) -> Self {
172 Self {
173 acting_as: Acting::Caller(chain),
174 ..Self::default()
175 }
176 }
177
178 /// Act under `chain` as the plane: a commission from a run the plane
179 /// itself admitted, whose sub-run is the plane's work one link down.
180 pub(crate) fn acting_as_plane(mut self, chain: crate::core::Delegation) -> Self {
181 self.acting_as = Acting::PlaneDelegate(chain);
182 self
183 }
184
185 /// Act for a served caller: under the chain its credential carried, or
186 /// under **none** — never the plane's.
187 ///
188 /// The door every surface that admits work for somebody else goes through.
189 /// [`acting_as`](Self::acting_as) alone cannot say "this caller presented
190 /// nothing", and leaving it unset means the plane's own chain.
191 #[must_use]
192 pub fn served(chain: Option<crate::core::Delegation>) -> Self {
193 Self {
194 acting_as: chain.map_or(Acting::Nobody, Acting::Caller),
195 ..Self::default()
196 }
197 }
198
199 /// Record who asked for this run: the authenticated caller a served
200 /// surface admitted it for, or the operator an embedder started it on
201 /// behalf of.
202 ///
203 /// Journaled as `RunAdmitted.admitted_by`, and excluded from deciding
204 /// every task the run opens — the four-eyes control, applied to the one
205 /// actor a declarative agent's approvals could not otherwise name. A
206 /// caller who could approve the call their own request produced would be
207 /// one person on both sides of the review.
208 #[must_use]
209 pub fn admitted_by(mut self, actor: impl Into<String>) -> Self {
210 self.admitted_by = Some(actor.into());
211 self
212 }
213
214 /// Attribute what this run takes in to a data subject.
215 ///
216 /// Recorded in the run's `DataSubjectBound` after any subjects its
217 /// declaration binds, so `agentplane subject` follows the run's input,
218 /// events and case state to the effects they reached. Attribution only:
219 /// no gate reads it.
220 #[must_use]
221 pub fn subject(mut self, subject: impl Into<String>) -> Self {
222 self.subjects.push(subject.into());
223 self
224 }
225
226 /// Admit only if this declaration revision governs the capability.
227 ///
228 /// For a caller that reviewed one revision and schedules runs against a
229 /// plane it does not control: another revision — or no declaration at
230 /// all, including on a build without the `manifest` feature — is refused
231 /// as [`RuntimeError::DeclarationPinMismatch`] before authorization, and
232 /// nothing is recorded.
233 #[must_use]
234 pub const fn expect_declaration(mut self, digest: Digest) -> Self {
235 self.expected_declaration = Some(digest);
236 self
237 }
238
239 fn keyed(mut self, key: &str) -> Self {
240 self.idempotency_key = Some(key.to_owned());
241 self
242 }
243}
244
245/// Default lease duration. A crashed owner's runs become claimable this long
246/// after its last heartbeat.
247///
248/// This bounds how long a *dead* owner's runs are stranded, not how long a run
249/// may take: a live run renews while it executes. Set per plane with
250/// [`RuntimeBuilder::lease_ttl`].
251pub const LEASE_TTL: Duration = Duration::from_secs(30);
252
253/// The shortest lease a live run can actually hold.
254///
255/// Both stores keep expiry in whole seconds and lapse on `expires_at <= now`, so
256/// a one-second lease is expired for part of every second it exists. Two is the
257/// smallest value a renewal can stay ahead of.
258pub const MIN_LEASE_TTL: Duration = Duration::from_secs(2);
259
260/// What a run produced.
261#[derive(Debug, Clone)]
262pub struct RunOutcome {
263 pub run_id: RunId,
264 pub status: RunStatus,
265 /// What this run consumed — steps, effects, spend, elapsed, refusals.
266 ///
267 /// Reported rather than left in the ledger because "what did the settlement
268 /// run cost" has to be answerable per item, and a batch sums its items. A
269 /// figure that only exists inside a dropped `Ledger` is a figure nobody can
270 /// bill against.
271 ///
272 /// The whole tally rather than only the money: the counts are what a
273 /// ceiling is sized against, and an operator raising `max_effects` after a
274 /// run stopped short is asking exactly the question this answers.
275 pub consumed: Consumed,
276 /// Terminal hash of the run's chain — what a signature would cover.
277 pub chain_head: Digest,
278 /// What the run produced, **with its label**.
279 ///
280 /// Labelled rather than bare, and the difference is not cosmetic: a
281 /// caller acting on a run's answer needs to know whether a model, a peer
282 /// or a person wrote it. The label was stripped here until an A2A reply
283 /// projection read a marker key out of an untrusted answer and let a
284 /// remote peer choose the envelope its own reply arrived in — a
285 /// confused-deputy reachable because the one fact that would have refused
286 /// it had been dropped at the boundary.
287 pub output: Option<Tainted<Value>>,
288}
289
290impl RunOutcome {
291 /// What this run spent, in metered units — the money half of
292 /// [`consumed`](Self::consumed).
293 #[must_use]
294 pub const fn spend(&self) -> Spend {
295 self.consumed.spend
296 }
297
298 /// Why this run ended — [`RunStatus::reason`] without the match.
299 ///
300 /// `None` for a success. The accessor is here as well as on the status
301 /// because this is the type an embedder holds after `run(..)`, and a
302 /// summary field that silently stays empty is the shape this exists to
303 /// prevent.
304 #[must_use]
305 pub fn reason(&self) -> Option<std::borrow::Cow<'_, str>> {
306 self.status.reason()
307 }
308
309 /// The output if the run succeeded, or a [`RunFailure`] saying why not.
310 ///
311 /// Every caller that acts on a run's answer asks the same two questions —
312 /// did it work, and what did it say — and pattern-matching
313 /// [`status`](Self::status) to learn the first is how the second gets read
314 /// off a run that quarantined. This folds the pair into one `?`:
315 ///
316 /// ```ignore
317 /// let answer = runtime.start("greet", input).run().await?.success()?;
318 /// ```
319 ///
320 /// `status` stays public for the callers whose branches genuinely differ —
321 /// a suspended run is not a failed one, and an operator surface treats
322 /// them differently. This is for the caller to whom anything short of an
323 /// answer is an error.
324 ///
325 /// # Errors
326 ///
327 /// [`RunFailure`] for every non-succeeded status, carrying the run id and
328 /// the status itself.
329 pub fn success(self) -> Result<Tainted<Value>, RunFailure> {
330 match self.status {
331 // A succeeded run's answer; `Null` for the degenerate run that
332 // finished without producing one, which is still a success.
333 RunStatus::Succeeded => {
334 Ok(self.output.unwrap_or_else(|| Tainted::trusted(Value::Null)))
335 }
336 status => Err(RunFailure {
337 run_id: self.run_id,
338 status,
339 }),
340 }
341 }
342}
343
344/// What an idempotent admission did.
345///
346/// A duplicate comes back as an answer rather than an error: a caller that
347/// retried wants the original run, not a conflict it has to interpret.
348///
349/// Exhaustive, unlike the error types: a key is unused, or held by a run that
350/// has rested, or held by one that has not. There is no fourth state to leave
351/// room for, and `#[non_exhaustive]` would cost every caller a `_` arm that
352/// means nothing.
353#[derive(Debug, Clone)]
354pub enum Admission {
355 /// The key was unused. This run is the one it admitted.
356 Fresh(RunOutcome),
357 /// The key already admitted a run that has reached a resting point —
358 /// concluded, or suspended waiting for something. Its answer, again.
359 ///
360 /// A suspension counts, and that is the case this exists for: a run parked
361 /// on a four-eyes decision has already opened the task, so a redelivery
362 /// that admitted would put a second identical approval in front of a
363 /// reviewer.
364 ///
365 /// Carries no `output`. A step's result is reconstructed by replay rather
366 /// than stored, so what the journal answers is *what happened* — status,
367 /// reason, chain head. Re-derive the value with [`replay`](Runtime::replay)
368 /// under [`Mode::Strict`], which performs no external effect.
369 Replayed(RunOutcome),
370 /// The key already admitted a run that has not rested yet — still
371 /// executing, or abandoned and awaiting the recovery sweep.
372 ///
373 /// The honest answer to an emitter is *accepted, already in progress* — a
374 /// 200 or 202 naming this run, never a retry-provoking failure.
375 InFlight(RunId),
376}
377
378impl Admission {
379 /// The run this key admitted, whichever of the three this is.
380 #[must_use]
381 pub const fn run_id(&self) -> RunId {
382 match self {
383 Self::Fresh(outcome) | Self::Replayed(outcome) => outcome.run_id,
384 Self::InFlight(run) => *run,
385 }
386 }
387
388 /// Whether this call is the one that admitted the run.
389 #[must_use]
390 pub const fn is_fresh(&self) -> bool {
391 matches!(self, Self::Fresh(_))
392 }
393
394 /// What the run produced, if it has produced anything yet.
395 #[must_use]
396 pub const fn outcome(&self) -> Option<&RunOutcome> {
397 match self {
398 Self::Fresh(outcome) | Self::Replayed(outcome) => Some(outcome),
399 Self::InFlight(_) => None,
400 }
401 }
402}
403
404/// What a keyed [`spawn`](RunRequest::spawn) did.
405///
406/// Separate from [`Admission`] because a spawn returns before the work happens,
407/// so it has no outcome to report.
408#[derive(Debug, Clone, Copy, PartialEq, Eq)]
409pub struct Spawned {
410 /// The run holding the key: the one just started, or the one that already
411 /// held it.
412 pub run: RunId,
413 /// False when the key was already held, so nothing new was started.
414 pub fresh: bool,
415}
416
417/// What a run starts from.
418#[derive(Debug, Clone)]
419pub enum Target {
420 /// A capability a registered skill answers — the degenerate one-node plan.
421 Capability(String),
422 /// A plan, validated and frozen before its first step: one that would
423 /// fail at step seven must not begin at step one.
424 Plan(PlanIR),
425}
426
427impl From<&str> for Target {
428 fn from(capability: &str) -> Self {
429 Self::Capability(capability.to_owned())
430 }
431}
432
433impl From<String> for Target {
434 fn from(capability: String) -> Self {
435 Self::Capability(capability)
436 }
437}
438
439impl From<&String> for Target {
440 fn from(capability: &String) -> Self {
441 Self::Capability(capability.clone())
442 }
443}
444
445impl From<PlanIR> for Target {
446 fn from(plan: PlanIR) -> Self {
447 Self::Plan(plan)
448 }
449}
450
451/// A request with no admission key: answered by what the run did.
452#[derive(Debug, Clone, Copy)]
453pub struct Unkeyed;
454
455/// A request under an admission key: answered by what the key admitted.
456#[derive(Debug, Clone)]
457pub struct Keyed(String);
458
459/// A run about to start — see [`Runtime::start`].
460///
461/// The answer's type follows the key. Unkeyed, a run answers with its
462/// [`RunOutcome`] and a spawn with its [`RunId`]; under
463/// [`once`](Self::once), with an [`Admission`] or a [`Spawned`], because a
464/// key already held is answered by the run that holds it, and a caller must
465/// say what it does then.
466///
467/// ```compile_fail
468/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
469/// let _: agentplane::runtime::RunOutcome = rt.start("c", input).once("k").run().await.unwrap();
470/// # }
471/// ```
472///
473/// ```compile_fail
474/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
475/// rt.start("c", input).once("a").once("b");
476/// # }
477/// ```
478///
479/// ```no_run
480/// # async fn f(rt: std::sync::Arc<agentplane::runtime::Runtime>, input: agentplane::core::Tainted<serde_json::Value>) {
481/// let _: agentplane::runtime::RunOutcome = rt.start("c", input.clone()).run().await.unwrap();
482/// let _: agentplane::runtime::Admission = rt.start("c", input).once("k").run().await.unwrap();
483/// # }
484/// ```
485#[must_use = "a run request does nothing until it is run, spawned or settled"]
486#[derive(Debug)]
487pub struct RunRequest<'a, K> {
488 runtime: &'a Runtime,
489 target: Target,
490 input: Tainted<Value>,
491 terms: RunTerms,
492 key: K,
493}
494
495impl<K> RunRequest<'_, K> {
496 /// Bind the run: a case, a caller's chain, who asked, a pinned
497 /// declaration. A second call replaces the first, as one value.
498 pub fn terms(mut self, terms: RunTerms) -> Self {
499 self.terms = terms;
500 self
501 }
502}
503
504impl<'a> RunRequest<'a, Unkeyed> {
505 /// Admit at most once per tenant under this key.
506 ///
507 /// The door for an at-least-once emitter: a bus that retries until it sees
508 /// a 2xx, a webhook fan-out, a queue with redelivery. The store claims the
509 /// key in the same transaction that writes the run's first record, so a
510 /// second request under it cannot admit a second run — not on a retry, not
511 /// across a restart, not from another instance sharing the store.
512 ///
513 /// **The key must carry its producer.** An id is unique only within one
514 /// emitter, so a bare message id lets two counterparties swallow each
515 /// other's messages as apparent retries.
516 /// [`InboundEvent::dedup_key`](crate::core::InboundEvent::dedup_key) is the
517 /// ready-made spelling.
518 ///
519 /// Not the event buffer: an event nobody subscribes to is dead-lettered by
520 /// [`sweep`](Runtime::sweep), so using the buffer as a ledger would spend
521 /// that signal to buy deduplication.
522 pub fn once(self, key: impl Into<String>) -> RunRequest<'a, Keyed> {
523 RunRequest {
524 runtime: self.runtime,
525 target: self.target,
526 input: self.input,
527 terms: self.terms,
528 key: Keyed(key.into()),
529 }
530 }
531
532 /// Run, and wait until it rests.
533 ///
534 /// # Errors
535 ///
536 /// [`RuntimeError::NoProvider`] when no skill provides the capability, and
537 /// whatever admission refuses — policy, quota, a halted tenant, an
538 /// instance draining, a chain this plane may not act under.
539 pub async fn run(self) -> Result<RunOutcome, RuntimeError> {
540 self.runtime
541 .admit_target(self.target, self.input, self.terms)
542 .await
543 }
544
545 /// Admit, and let the run proceed in the background, returning its id.
546 ///
547 /// **Admission happens before this returns.** The policy gate, the lease
548 /// and the admission records are written first, so a refusal is an error
549 /// here and not a task that never appears, and the id can be read at once.
550 /// The run is durable: a process that dies mid-flight leaves a journal
551 /// another instance resumes.
552 ///
553 /// # Panics
554 ///
555 /// Outside a Tokio runtime, as the rest of this crate's timing does.
556 ///
557 /// # Errors
558 ///
559 /// As [`run`](Self::run).
560 pub async fn spawn(self) -> Result<RunId, RuntimeError> {
561 self.runtime
562 .owned()
563 .spawn_target(self.target, self.input, self.terms)
564 .await
565 }
566}
567
568impl RunRequest<'_, Keyed> {
569 /// Run under the key. A key already held comes back as
570 /// [`Admission::Replayed`] or [`Admission::InFlight`], not an error.
571 ///
572 /// # Errors
573 ///
574 /// As [`RunRequest::run`], and [`RuntimeError::PlanContract`] for an empty
575 /// key or one over [`MAX_ADMISSION_KEY_BYTES`].
576 pub async fn run(self) -> Result<Admission, RuntimeError> {
577 let runtime = self.runtime;
578 let terms = self.terms.keyed(&self.key.0);
579 // Boxed: admission is the largest future in the crate, and the claim
580 // would otherwise carry it inline into every keyed caller's future.
581 let admit = Box::pin(runtime.admit_target(self.target, self.input, terms));
582 match runtime.claimed(&self.key.0, admit).await? {
583 Claimed::Fresh(outcome) => Ok(Admission::Fresh(outcome)),
584 Claimed::Held(run) => runtime.answer_with(run).await,
585 }
586 }
587
588 /// Spawn under the key. A key already held names its run, with `fresh`
589 /// false.
590 ///
591 /// # Panics
592 ///
593 /// Outside a Tokio runtime.
594 ///
595 /// # Errors
596 ///
597 /// As [`run`](Self::run).
598 pub async fn spawn(self) -> Result<Spawned, RuntimeError> {
599 let runtime = self.runtime.owned();
600 let terms = self.terms.keyed(&self.key.0);
601 let admit = Box::pin(runtime.spawn_target(self.target, self.input, terms));
602 Ok(match runtime.claimed(&self.key.0, admit).await? {
603 Claimed::Fresh(run) => Spawned { run, fresh: true },
604 Claimed::Held(run) => Spawned { run, fresh: false },
605 })
606 }
607
608 /// Run under the key until the run it admitted rests, whoever admitted it,
609 /// and answer with its outcome.
610 ///
611 /// The shape a process that may have been killed mid-run wants on restart:
612 /// the same request again. An unused key runs. A key whose run has rested
613 /// answers with that run, its value re-derived by strict replay, which
614 /// performs nothing. A key whose run is still working is resumed when
615 /// nobody holds its lease — its holder has gone — and otherwise followed
616 /// on the journal until it rests or its lease lapses.
617 ///
618 /// # Errors
619 ///
620 /// As [`run`](Self::run), and whatever resuming or replaying the held run
621 /// refuses.
622 pub async fn settle(self) -> Result<RunOutcome, RuntimeError> {
623 let runtime = self.runtime;
624 match self.run().await? {
625 Admission::Fresh(outcome) => Ok(outcome),
626 Admission::Replayed(rested) => runtime.with_output(rested).await,
627 Admission::InFlight(run) => runtime.settled(run).await,
628 }
629 }
630}
631
632/// What a keyed claim found.
633enum Claimed<T> {
634 /// The key was unused, and this admitted.
635 Fresh(T),
636 /// The key already admitted this run.
637 Held(RunId),
638}
639
640/// The stores a full plane runs on, as handles rather than as one type.
641///
642/// [`Runtime::builder_on`] is the shorthand for one backend implementing all
643/// of them, and is the wrong shape for a caller that picks its backend at **run
644/// time** — an operator naming either a file or a connection string holds
645/// `Arc<dyn JournalStore>`, and unsizing a generic parameter needs it `Sized`,
646/// so the wiring is a value that `builder_on` builds.
647///
648/// Here rather than in [`store`](crate::store) because that module is the
649/// backends, gated on `redb` or `postgres`, while this exists whether or not
650/// either is compiled in.
651#[derive(Clone)]
652pub struct Stores {
653 pub journal: std::sync::Arc<dyn crate::journal::JournalStore>,
654 pub cases: std::sync::Arc<dyn crate::case::CaseStore>,
655 pub tasks: std::sync::Arc<dyn crate::case::TaskStore>,
656 pub events: std::sync::Arc<dyn crate::case::EventStore>,
657 pub timers: std::sync::Arc<dyn crate::case::TimerStore>,
658 pub memory: std::sync::Arc<dyn crate::memory::MemoryStore>,
659 /// Where halts, subject withdrawals, rate windows and tenant ceilings live.
660 pub quotas: std::sync::Arc<dyn crate::quota::QuotaStore>,
661 /// Where disclosures of a matter are recorded.
662 pub disclosures: std::sync::Arc<dyn crate::disclosure::DisclosureRegister>,
663}
664
665impl std::fmt::Debug for Stores {
666 /// The handles are trait objects with no identity to print, so what a reader
667 /// wants here is *that the set is complete*, which it is by construction.
668 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
669 f.write_str("Stores { journal, cases, tasks, events, timers, memory, quotas, disclosures }")
670 }
671}
672
673impl Stores {
674 /// Every store from one backend that implements all of them.
675 #[must_use]
676 pub fn on<B: FullBackend + 'static>(store: std::sync::Arc<B>) -> Self {
677 Self {
678 journal: std::sync::Arc::clone(&store) as _,
679 cases: std::sync::Arc::clone(&store) as _,
680 tasks: std::sync::Arc::clone(&store) as _,
681 events: std::sync::Arc::clone(&store) as _,
682 timers: std::sync::Arc::clone(&store) as _,
683 memory: std::sync::Arc::clone(&store) as _,
684 quotas: std::sync::Arc::clone(&store) as _,
685 disclosures: store as _,
686 }
687 }
688}
689
690/// How far into a run's history [`Runtime::running_runs`] reads to attribute it.
691///
692/// `RunAdmitted` is seq 1 and `IdentityBound` follows it when the caller
693/// presented a chain, with the quota pass and the frozen plan between them — so
694/// a handful of records carries both and a long-running agent's history is not
695/// scanned to answer *who is this*.
696const ATTRIBUTION_RECORDS: usize = 8;
697
698/// One run this tenant currently holds an admission slot for.
699///
700/// What [`Runtime::running_runs`] answers with. The ids alone would be a listing
701/// an operator reads out; the attribution is what makes it one they can act on.
702#[derive(Debug, Clone)]
703pub struct LiveRun {
704 pub run: RunId,
705 /// The declared agent governing it, when a declared one does.
706 ///
707 /// `None` for a skill registered directly on the plane — a legitimate shape,
708 /// and one an incident about *a bad revision* can therefore rule out.
709 pub governed_by: Option<crate::journal::AgentIdentity>,
710 /// Who this run acts for: the root of the delegation chain bound at
711 /// admission, when the caller presented one.
712 pub subject: Option<String>,
713 /// The slot is held by a run whose lease has lapsed, so no process is
714 /// executing it.
715 ///
716 /// Not a cancellation candidate: the recovery sweep resumes these, and
717 /// cancelling one unwinds work that was about to continue.
718 pub stranded: bool,
719}
720
721/// A run that stopped short of an answer, as an error a caller can `?`.
722///
723/// Carries the [`RunStatus`] whole rather than a flattened string, so a caller
724/// that started with [`RunOutcome::success`] and later needs to tell a
725/// suspension from a quarantine can match on [`status`](Self::status) without
726/// re-plumbing the call site.
727#[derive(Debug, Clone)]
728pub struct RunFailure {
729 pub run_id: RunId,
730 pub status: RunStatus,
731}
732
733/// **Total, including over the status this type is not supposed to hold.**
734///
735/// `status` is public and `RunStatus` carries `Succeeded`, so a
736/// `RunFailure { status: Succeeded }` is a value any caller can build. Asserting
737/// that away would make *formatting an error* the operation that panics, and a
738/// `Display` impl runs wherever an error is logged — disproportionately during
739/// an incident. So it prints the contradiction instead.
740impl std::fmt::Display for RunFailure {
741 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
742 if matches!(self.status, RunStatus::Succeeded) {
743 return write!(
744 f,
745 "run {} is recorded as a failure carrying a success — \
746 whatever built this value did so wrongly",
747 self.run_id
748 );
749 }
750 write!(f, "run {} did not succeed: ", self.run_id)?;
751 match &self.status {
752 RunStatus::Succeeded => write!(f, "it succeeded"),
753 RunStatus::Failed(reason) => write!(f, "it failed — {reason}"),
754 RunStatus::Suspended(reason) => write!(f, "it is suspended — {reason:?}"),
755 RunStatus::Exhausted(limit) => write!(f, "a budget stopped it — {limit}"),
756 RunStatus::Quarantined(reason) => write!(f, "it is quarantined — {reason}"),
757 RunStatus::Replanning(reason) => write!(f, "it asked to replan — {reason}"),
758 RunStatus::Observed => write!(
759 f,
760 "it is a session this plane observed rather than executed, so it \
761 neither succeeded nor failed"
762 ),
763 RunStatus::Cancelled { actor, reason } => {
764 write!(f, "{actor} cancelled it — {reason}")
765 }
766 RunStatus::Swept => write!(f, "the plane recorded a pass over its own state"),
767 RunStatus::BrokeGlass { actor, reason } => {
768 write!(f, "{actor} crossed into this tenant — {reason}")
769 }
770 RunStatus::HaltLifted { actor } => write!(f, "{actor} lifted an emergency stop"),
771 RunStatus::HoldReleased { actor } => write!(f, "{actor} released a legal hold"),
772 RunStatus::Abandoned { actor, reason } => {
773 write!(
774 f,
775 "{actor} abandoned it, unresolved and not unwound — {reason}"
776 )
777 }
778 RunStatus::Withheld { subject, reason } => {
779 write!(
780 f,
781 "the authority it acts for — '{subject}' — was withdrawn: {reason}"
782 )
783 }
784 }
785 }
786}
787
788impl std::error::Error for RunFailure {}
789
790#[derive(Debug, Clone, PartialEq)]
791pub enum RunStatus {
792 /// All terminal work completed.
793 ///
794 /// Structural, never self-reported: a skill saying "done" is not what makes
795 /// a run succeed. Agents confidently announce success on unmet objectives,
796 /// so completion is determined by the runtime, not claimed by the workload.
797 Succeeded,
798 Failed(String),
799 /// Waiting for something that has not happened. The frame is persisted and
800 /// the task is gone: a suspended run costs disk, not a thread.
801 Suspended(crate::core::SuspendReason),
802 /// A limit stopped it. Not a fault — the run did what it was told, and what
803 /// it was told included a ceiling.
804 Exhausted(crate::core::BudgetExceeded),
805 /// Needs human resolution before anything else may happen to it.
806 Quarantined(String),
807 /// A step asked for a different plan.
808 ///
809 /// Never observed by a caller: the executor either produces a successor and
810 /// keeps going, or turns this into a failure with the reason it refused.
811 /// It exists as a status so a step's request travels the same path every
812 /// other outcome does.
813 Replanning(String),
814 /// An operator stopped it.
815 ///
816 /// Distinct from `Failed` on purpose. A failure is the run discovering it
817 /// cannot proceed; a cancellation is a human deciding it should not. They
818 /// call for different responses — one is investigated, the other was
819 /// intended — and an operator scanning for failures should not have to
820 /// mentally subtract their own interventions.
821 ///
822 /// Completed steps are unwound exactly as they are for a failure: stopping a
823 /// run that has moved money and leaving the movement in place is not
824 /// stopping it.
825 Cancelled {
826 actor: crate::core::Operator,
827 reason: String,
828 },
829 /// An operator closed a run whose outcome could never be established.
830 ///
831 /// The end of the road for a quarantine nobody can answer. Distinct from
832 /// `Cancelled` in the one way that matters to whoever reads it next:
833 /// **nothing was unwound**. A cancellation puts the world back; this leaves
834 /// it exactly as the run left it, because compensating around an unknown
835 /// outcome is the single thing quarantine exists to forbid, and an
836 /// operator's patience running out is not new evidence.
837 ///
838 /// So the doubt does not go away when the run does. It stops being a
839 /// *status* — which is a mutable thing a later action can overwrite — and
840 /// becomes an `agentplane audit` finding, which is derived from the journal
841 /// and outlives every status the run will ever have.
842 Abandoned {
843 actor: crate::core::Operator,
844 reason: String,
845 },
846 /// A pass this plane made over its own state, journaled as a run of its own.
847 ///
848 /// The sweep records what it decided — which obligations breached, which
849 /// tasks it escalated, which runs it took over — rather than logging it, so
850 /// *who decided what* is answerable from the chain six weeks later. It is
851 /// not `Succeeded` because nothing asked for it and there is no answer to
852 /// hand back: a sweep is the plane reporting on itself.
853 Swept,
854 /// An operator crossed into this tenant deliberately, and the reason they
855 /// gave for it.
856 ///
857 /// The one exception to tenant isolation, and the exception's entire value
858 /// is this record. Both halves are carried because an incident review asks
859 /// two questions and neither is worth less than the other: `actor` is read
860 /// back from the crossing record, `reason` from the conclusion, and a
861 /// crossing cannot be recorded without one.
862 BrokeGlass {
863 actor: crate::core::Operator,
864 reason: String,
865 },
866 /// An operator lifted an emergency stop, recorded as a run of its own.
867 ///
868 /// `actor` is read back from the run's
869 /// [`HaltLifted`](crate::journal::RecordKind::HaltLifted) record, which
870 /// also holds the stop it ended.
871 HaltLifted {
872 actor: crate::core::Operator,
873 },
874 /// An operator released a legal hold, recorded as a run of its own.
875 ///
876 /// `actor` is read back from the run's
877 /// [`HoldReleased`](crate::journal::RecordKind::HoldReleased) record.
878 HoldReleased {
879 actor: crate::core::Operator,
880 },
881 /// The authority this run acts under was withdrawn while it was running.
882 ///
883 /// **A pause, not a fault**, for the reason exhaustion is one: the run did
884 /// nothing wrong. Its mutations stand and it stays resumable — lift the halt
885 /// and it continues. Unwinding would reverse correct work for a reason
886 /// unrelated to it; cancelling is the verb for that.
887 ///
888 /// Carries the subject, because *whose* authority is what an operator
889 /// matches against the withdrawal they made.
890 Withheld {
891 subject: String,
892 reason: String,
893 },
894 /// A session this plane **observed** rather than executed, closed.
895 ///
896 /// The one status that is not about work this runtime did. A seat beside
897 /// somebody else's agent produces records at the *asserted* rung
898 /// ([`RecordKind::Observed`](crate::journal::RecordKind::Observed)), in a
899 /// run of their own with no admission — the shape a sweep's run already
900 /// takes — so that they share the chain, the Merkle log and the witness
901 /// while sharing no record kind with a dispatched effect.
902 ///
903 /// **It carries nothing, and the absence is the design.** There is no
904 /// reason to give: the session ended because the party running it ended
905 /// it, and this plane is in no position to say why. The session's own
906 /// identifier is on every observation record rather than here, so a plane
907 /// that stopped mid-session still leaves evidence that says which session
908 /// it describes.
909 ///
910 /// Sealed like any other ending, so an external verifier checks it without
911 /// being taught what an observation is. Never resumable: there is nothing
912 /// to resume onto, because nothing here was executing.
913 Observed,
914}
915
916impl RunStatus {
917 /// Why the run ended, whatever kind of ending it was.
918 ///
919 /// `None` only for [`Succeeded`](Self::Succeeded), which has no reason to
920 /// give. Every other conclusion carries one, and the point of gathering
921 /// them here is that an embedder mapping outcomes onto its own wire type
922 /// should not have to match every variant to find the sentence, and an
923 /// embedder that reads only the status ships an empty summary on failure.
924 ///
925 /// Borrowed where the variant already holds a string and formatted where
926 /// it holds a typed value: a suspension names what it waits for and an
927 /// exhaustion names the ceiling it hit, and both are worth more than the
928 /// word "suspended". `Cancelled` yields the operator's reason; the actor
929 /// who gave it stays on the variant, because *why* and *who* are two
930 /// questions and only one of them is the reason.
931 #[must_use]
932 pub fn reason(&self) -> Option<std::borrow::Cow<'_, str>> {
933 use std::borrow::Cow;
934 match self {
935 // Nothing to say, for three different reasons and one answer: a
936 // success has no why, a sweep's records carry theirs, and an
937 // observed session ended because somebody else ended it.
938 // A lift's record carries what it ended; the lifter gave no reason.
939 Self::Succeeded
940 | Self::Swept
941 | Self::Observed
942 | Self::HaltLifted { .. }
943 | Self::HoldReleased { .. } => None,
944 Self::Failed(reason)
945 | Self::Quarantined(reason)
946 | Self::Replanning(reason)
947 | Self::Cancelled { reason, .. }
948 | Self::Abandoned { reason, .. }
949 | Self::BrokeGlass { reason, .. }
950 | Self::Withheld { reason, .. } => Some(Cow::Borrowed(reason.as_str())),
951 Self::Suspended(reason) => Some(Cow::Owned(reason.to_string())),
952 Self::Exhausted(exceeded) => Some(Cow::Owned(exceeded.to_string())),
953 }
954 }
955
956 /// The run's **own** account of why it ended, for the journal.
957 ///
958 /// [`reason`](Self::reason) answers *what should a person be shown*, which
959 /// includes the words an operator typed when they stopped the run. This
960 /// answers the narrower question the conclusion record asks: *is there a
961 /// sentence here that nothing else on the chain already records?*
962 ///
963 /// `None` for every ending a person caused. Those write their own record —
964 /// [`RunCancelled`](crate::journal::RecordKind::RunCancelled),
965 /// [`QuarantineDecided`](crate::journal::RecordKind::QuarantineDecided),
966 /// [`BreakGlass`](crate::journal::RecordKind::BreakGlass),
967 /// [`AuthorityWithheld`](crate::journal::RecordKind::AuthorityWithheld) —
968 /// carrying the actor and the reason together, in the clear. Repeating the
969 /// reason on the conclusion would make one sentence two facts, and seal the
970 /// copy that an erasure then destroys.
971 ///
972 /// `None` for an exhaustion too: the typed ceiling verdict is the record,
973 /// and the prose was a rendering of it.
974 #[must_use]
975 pub fn own_account(&self) -> Option<std::borrow::Cow<'_, str>> {
976 use std::borrow::Cow;
977 match self {
978 // The runtime's own endings: a skill's words, a provider's words,
979 // a divergence this build found. Nothing else on the chain says it.
980 Self::Failed(reason) | Self::Quarantined(reason) | Self::Replanning(reason) => {
981 Some(Cow::Borrowed(reason.as_str()))
982 }
983 // Nothing to add, for two different reasons and one answer. The
984 // first four are somebody's doing and their own record already
985 // carries both halves. The rest either write no conclusion with a
986 // why — a success, a sweep, a suspension — or carry it typed
987 // instead, which is what `exhaustion` is.
988 //
989 // Enumerated rather than left to a wildcard: a status added later
990 // must be made to answer this question, and `_ => None` would
991 // answer it silently and wrongly for anything a person causes.
992 Self::Cancelled { .. }
993 | Self::Abandoned { .. }
994 | Self::BrokeGlass { .. }
995 | Self::HaltLifted { .. }
996 | Self::HoldReleased { .. }
997 | Self::Withheld { .. }
998 | Self::Succeeded
999 | Self::Swept
1000 // The account is the observation records themselves, and this plane
1001 // has none of its own to add: it did not run the session.
1002 | Self::Observed
1003 | Self::Suspended(_)
1004 | Self::Exhausted(_) => None,
1005 }
1006 }
1007
1008 /// Whether the run stopped without reaching a conclusion.
1009 #[must_use]
1010 pub fn is_suspended(&self) -> bool {
1011 matches!(self, Self::Suspended(_))
1012 }
1013
1014 /// Whether the recorded history can no longer be trusted to describe this
1015 /// code, so a human must look before anything else happens.
1016 #[must_use]
1017 pub fn is_quarantined(&self) -> bool {
1018 matches!(self, Self::Quarantined(_))
1019 }
1020
1021 #[must_use]
1022 pub fn as_str(&self) -> &'static str {
1023 match self {
1024 Self::Succeeded => "succeeded",
1025 Self::Failed(_) => "failed",
1026 Self::Suspended(_) => "suspended",
1027 Self::Exhausted(_) => "exhausted",
1028 Self::Quarantined(_) => "quarantined",
1029 Self::Replanning(_) => "replanning",
1030 Self::Cancelled { .. } => "cancelled",
1031 Self::Abandoned { .. } => "abandoned",
1032 Self::Swept => super::sweeper::SWEEP_OUTCOME,
1033 Self::BrokeGlass { .. } => BREAK_GLASS_OUTCOME,
1034 Self::HaltLifted { .. } => HALT_LIFTED_OUTCOME,
1035 Self::HoldReleased { .. } => HOLD_RELEASED_OUTCOME,
1036 Self::Withheld { .. } => WITHHELD_OUTCOME,
1037 Self::Observed => OBSERVED_OUTCOME,
1038 }
1039 }
1040
1041 /// Who brought this run to this state, where a person did.
1042 ///
1043 /// `None` for every ending the runtime reached on its own, which is most of
1044 /// them. One accessor rather than three `matches!` at each surface, because
1045 /// *who* is the question an operator asks immediately after *what* — and a
1046 /// surface that reads the reason and drops the actor answers half of it.
1047 #[must_use]
1048 pub fn actor(&self) -> Option<&crate::core::Operator> {
1049 match self {
1050 Self::Cancelled { actor, .. }
1051 | Self::Abandoned { actor, .. }
1052 | Self::BrokeGlass { actor, .. }
1053 | Self::HaltLifted { actor }
1054 | Self::HoldReleased { actor } => Some(actor),
1055 // A withdrawal names a *subject*, not an actor: the operator who
1056 // threw the halt and the credential it covers are different
1057 // parties, and answering "who" with the subject would report the
1058 // person whose authority was taken as the one who took it.
1059 Self::Succeeded
1060 | Self::Failed(_)
1061 | Self::Suspended(_)
1062 | Self::Exhausted(_)
1063 | Self::Quarantined(_)
1064 | Self::Replanning(_)
1065 | Self::Withheld { .. }
1066 | Self::Swept
1067 // Somebody was at the keyboard, and it was not this plane's
1068 // operator. Naming the observed party's user here would report a
1069 // person this runtime never authenticated as one of its own.
1070 | Self::Observed => None,
1071 }
1072 }
1073
1074 /// Whether an operator stopped this run.
1075 #[must_use]
1076 pub fn is_cancelled(&self) -> bool {
1077 matches!(self, Self::Cancelled { .. })
1078 }
1079
1080 /// Whether this conclusion freezes the journal and enters the Merkle log.
1081 ///
1082 /// Only conclusions that nothing may resume seal. A failed run may be
1083 /// resumed — its completed effects are read back from history rather than
1084 /// performed again, which is the point of having a journal — and an
1085 /// exhausted run may continue once somebody raises the ceiling. Sealing
1086 /// either would commit the Merkle log to a leaf its own resume is then
1087 /// permitted to grow past: the checkpoint would attest a prefix of a
1088 /// history that kept moving, and the store's refusal of appends after a
1089 /// seal would turn every legitimate resume into an error.
1090 ///
1091 /// **A quarantine does not seal**, and that follows from the same rule
1092 /// rather than being an exception to it. A quarantine is the runtime saying
1093 /// it does not know, which is a story that is not over; a Merkle leaf is a
1094 /// claim that a history is complete. Sealing one published a proof of
1095 /// completeness over an admittedly incomplete record, and — the practical
1096 /// half — froze the journal against the one record that answers it, so
1097 /// "a human must resolve it before it can run again" named a resolution the
1098 /// format made impossible. It enters the log when it reaches a real ending,
1099 /// including [`Abandoned`](Self::Abandoned), which is a person deciding
1100 /// there will never be one.
1101 ///
1102 /// This is the **one** implementation of that rule. `resume_is_closed`
1103 /// refuses exactly the outcomes this method seals — a test pins the
1104 /// agreement — because two copies of one rule agree everywhere except the
1105 /// boundary nobody probed.
1106 #[must_use]
1107 pub fn seals(&self) -> bool {
1108 matches!(
1109 self,
1110 Self::Succeeded
1111 | Self::Cancelled { .. }
1112 | Self::Abandoned { .. }
1113 | Self::Swept
1114 | Self::BrokeGlass { .. }
1115 | Self::HaltLifted { .. }
1116 | Self::HoldReleased { .. }
1117 | Self::Observed
1118 )
1119 }
1120}
1121
1122/// Every outcome a sealed run can carry, as the outcome index spells them.
1123///
1124/// The store indexes runs *by* outcome and deliberately has no "all runs"
1125/// query — a backlog listing is a range read rather than a table scan — so any
1126/// caller that wants every sealed run has to name the outcomes one by one. This
1127/// is the list to name them from, and it lives beside [`RunStatus::seals`]
1128/// because it is the same rule in its other spelling: exactly the statuses that
1129/// seal, with no exception clause. A sweep (`swept`) and a break-glass crossing
1130/// (`broke-glass`) are sealed at birth with no goal to have succeeded or failed
1131/// at, and they are statuses all the same — an ending this plane writes and its
1132/// own reader cannot name is an ending it reports as unrecognised.
1133///
1134/// It lived in the CLI first, as string literals — two implementations of one
1135/// rule, where a new sealing outcome would have been exported by nobody and
1136/// the export that silently omitted it would have dropped exactly the runs an
1137/// auditor came for. A test holds this list to `RunStatus::seals` variant by
1138/// variant, which the literals in a binary could never be held to.
1139pub const SEALED_OUTCOMES: &[&str] = &[
1140 "succeeded",
1141 "cancelled",
1142 "abandoned",
1143 super::sweeper::SWEEP_OUTCOME,
1144 BREAK_GLASS_OUTCOME,
1145 HALT_LIFTED_OUTCOME,
1146 HOLD_RELEASED_OUTCOME,
1147 OBSERVED_OUTCOME,
1148];
1149
1150/// What an offline verb sweeps when the operator names no outcome.
1151///
1152/// Every ending — [`SEALED_OUTCOMES`] — **plus `quarantined`**, which is not an
1153/// ending and is exactly why it belongs: a run set aside for a person is the
1154/// plane's outstanding unresolved risk, and an artifact holding everything that
1155/// finished and nothing that did not is not the one an auditor came for. It is
1156/// also the outcome most likely to be the reason they came.
1157///
1158/// Failed and exhausted runs are deliberately absent, and the distinction is
1159/// not sealing but *whether anything is waiting on a person*. Those two are
1160/// work in progress that a resume moves on its own; a quarantine moves only
1161/// when somebody decides, so it can sit for weeks and still be current.
1162///
1163/// A test holds this to [`SEALED_OUTCOMES`] rather than restating it, so a new
1164/// ending is swept here the day it exists.
1165pub const OUTCOMES_OF_RECORD: &[&str] = &[
1166 "succeeded",
1167 "cancelled",
1168 "abandoned",
1169 "quarantined",
1170 super::sweeper::SWEEP_OUTCOME,
1171 BREAK_GLASS_OUTCOME,
1172 HALT_LIFTED_OUTCOME,
1173 HOLD_RELEASED_OUTCOME,
1174 OBSERVED_OUTCOME,
1175];
1176
1177/// How a resume came to hold its lease, which decides what a failure does
1178/// with it.
1179///
1180/// A lease that lapses while still naming an owner is what the recovery sweep
1181/// lists, so keeping one on failure is how a run stays findable; releasing it
1182/// is right only where the run is findable without it.
1183#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1184enum Claim {
1185 /// Claimed by the resume itself, over a run whose journal already lists
1186 /// it — failed, exhausted, withheld, quarantined, waiting.
1187 Standing,
1188 /// Handed over by a wake or a delivery that recorded the awaited result
1189 /// under it, which took the run off the waiting listing.
1190 Unlisted,
1191 /// Claimed by the recovery sweep over a lease its owner died holding.
1192 Recovering,
1193}
1194
1195/// A run that exists but has not run.
1196///
1197/// Produced by admission and consumed by execution, so the two can happen in
1198/// different places — the same request for a blocking call, and a background
1199/// task for a non-blocking one.
1200struct Admitted {
1201 run: RunId,
1202 epoch: crate::core::Epoch,
1203 quota: QuotaPass,
1204 budget: Budget,
1205 agent: String,
1206 plan: PlanIR,
1207 input: Tainted<Value>,
1208 case: Option<CaseContext>,
1209 /// The chain admission bound — the caller's where one was presented,
1210 /// the plane's otherwise — and what every step of this run acts under.
1211 identity: Option<crate::core::Delegation>,
1212 /// The references its `DataSubjectBound` record makes available.
1213 subjects: BTreeSet<crate::core::SubjectRef>,
1214}
1215
1216/// The quota identity of one live execution pass.
1217///
1218/// Captured once at admission or resume and carried to settlement. Computing
1219/// the period again after the work finishes can cross a day or month boundary,
1220/// authorizing against one ledger and charging another.
1221#[derive(Debug, Clone, Default)]
1222struct QuotaPass {
1223 period: Option<String>,
1224 enabled: bool,
1225 release_slot: bool,
1226 /// How many steps this pass may dispatch at once, when the tenant bounds
1227 /// spend: the width the run's reservation was sized for. `None` leaves
1228 /// the run's own budget as the only bound.
1229 width: Option<usize>,
1230}
1231
1232impl QuotaPass {
1233 fn at(
1234 quota: &crate::quota::TenantQuota,
1235 at: crate::core::Timestamp,
1236 release_slot: bool,
1237 ) -> Self {
1238 Self {
1239 period: quota.bounds_spend().then(|| quota.period.key_for(at)),
1240 enabled: true,
1241 release_slot,
1242 width: None,
1243 }
1244 }
1245
1246 const fn disabled() -> Self {
1247 Self {
1248 period: None,
1249 enabled: false,
1250 release_slot: false,
1251 width: None,
1252 }
1253 }
1254
1255 /// Held to the width a spend reservation was sized for, whenever the
1256 /// tenant bounds spend.
1257 fn sized_for(mut self, width: usize) -> Self {
1258 if self.period.is_some() {
1259 self.width = Some(width);
1260 }
1261 self
1262 }
1263
1264 /// How wide this pass may dispatch: the budget's own width, narrowed to
1265 /// the one its reservation was sized for. A replan may widen the plan; it
1266 /// does not widen what the period holds for it.
1267 fn parallelism(&self, budget: &Budget) -> usize {
1268 self.width
1269 .map_or_else(|| budget.parallelism(), |w| budget.parallelism().min(w))
1270 }
1271
1272 fn period(&self) -> Option<&str> {
1273 self.period.as_deref()
1274 }
1275
1276 fn started(&self) -> Option<RecordKind> {
1277 self.enabled.then(|| RecordKind::QuotaPassStarted {
1278 period: self.period.clone(),
1279 release_slot: self.release_slot,
1280 })
1281 }
1282
1283 fn settlement(
1284 &self,
1285 run: RunId,
1286 epoch: crate::core::Epoch,
1287 spend: Spend,
1288 concludes: bool,
1289 ) -> Option<crate::quota::QuotaSettlement> {
1290 self.enabled.then(|| crate::quota::QuotaSettlement {
1291 run,
1292 epoch,
1293 period: self.period.clone(),
1294 spend,
1295 release_slot: self.release_slot,
1296 concludes,
1297 })
1298 }
1299}
1300
1301/// How many steps a run may have in flight: its budget's width, or its
1302/// admitted plan's node count when that is narrower or nothing narrows it.
1303///
1304/// What a spend reservation multiplies the per-call bound by, since each step
1305/// in flight may hold one operation the ledger admitted and has not billed.
1306fn reserved_width(budget: &Budget, plan: &PlanIR) -> usize {
1307 budget.parallelism().min(plan.nodes.len()).max(1)
1308}
1309
1310/// The peers a plane may call, and how it reaches them.
1311///
1312/// One value because the two are one decision: a registry says what each
1313/// peer is granted and which credential reaches it, and the client is the
1314/// transport those grants are spent over. Held by the plane and handed to
1315/// every step, so a skill calling a peer reaches it through the same checked
1316/// registry a declarative agent does — never one it carries itself.
1317#[derive(Debug)]
1318pub(crate) struct PeerWiring {
1319 pub(crate) registry: crate::peers::PeerRegistry,
1320 pub(crate) client: Arc<dyn crate::peers::PeerClient>,
1321}
1322
1323/// Keeps a run's lease alive for as long as it is executing.
1324///
1325/// A lease answers one question — *is this owner dead?* — and it answers by
1326/// expiry. Without renewal it also answers a question it was never asked: a
1327/// healthy run that outlives its TTL looks exactly like a crashed one, and agent
1328/// runs routinely outlive a lease because a single model call can. Another
1329/// instance then takes the run over, bumps the epoch, and the original is fenced
1330/// on its next append: killed mid-flight, having already done real work.
1331///
1332/// Renewing while executing separates the two. The TTL then bounds how long a
1333/// *crashed* owner strands its runs, which is what it is for, and stops bounding
1334/// how long a run may take, which it should never have bounded.
1335///
1336/// Aborted on drop, so the renewal stops the moment execution returns — by any
1337/// path, including a panic unwinding through it. A heartbeat that outlived its
1338/// run would hold a lease nobody is using and strand it for a full TTL after a
1339/// crash, which is the failure this exists to prevent, arriving late.
1340struct Heartbeat(tokio::task::JoinHandle<()>);
1341
1342impl Drop for Heartbeat {
1343 fn drop(&mut self) {
1344 self.0.abort();
1345 }
1346}
1347
1348/// The runtime.
1349#[derive(Debug, Clone)]
1350pub struct Runtime {
1351 store: Arc<dyn JournalStore>,
1352 skills: HashMap<String, Arc<dyn Skill>>,
1353 by_capability: HashMap<Capability, String>,
1354 /// A handle to this plane, for steps that commission other agents on it.
1355 ///
1356 /// Weak, and that is not an optimisation: a strong self-reference would
1357 /// leak every runtime ever built. It exists because commissioning belongs to
1358 /// the *runtime* — a skill holding an `Arc<Runtime>` cannot work, since the
1359 /// runtime needs the skill before the skill can have the runtime.
1360 self_ref: std::sync::Weak<Self>,
1361 /// The declaration governing each skill, by skill name.
1362 ///
1363 /// Per skill rather than per runtime, because a plane runs several agents
1364 /// and a step must be judged against *its own* agent's manifest.
1365 #[cfg(feature = "manifest")]
1366 governed_by: HashMap<String, Arc<crate::manifest::Manifest>>,
1367 /// Which tenant this plane runs as.
1368 ///
1369 /// One plane serves one tenant; a **process** may run several and serve
1370 /// them all, because `api::Planes` resolves a plane from the authenticated
1371 /// caller's tenant. That is what the composite keys on both backends bought:
1372 /// the tenant leads every key, lease, correlation and blob path, so a query
1373 /// that forgets it misses rather than returning somebody else's rows.
1374 tenant: crate::core::TenantId,
1375 /// Who vouched for each declaration, by agent name.
1376 ///
1377 /// Separate from the manifest because it is *not* in it: a document cannot
1378 /// state who signed it. It arrives beside the manifest from a verified
1379 /// registry resolution, and an absent entry means nobody vouched.
1380 #[cfg(feature = "manifest")]
1381 published_by: HashMap<String, crate::core::KeyId>,
1382 /// Model drivers by the name a manifest calls them.
1383 ///
1384 /// Kept after build — not only consumed by it — so a *hand-written* skill
1385 /// can reach the driver its manifest's model role names through
1386 /// [`StepCtx::complete`], instead of smuggling an `Arc<dyn ModelProvider>`
1387 /// field past the declaration.
1388 #[cfg(feature = "manifest")]
1389 providers: HashMap<String, Arc<dyn crate::model::ModelProvider>>,
1390 owner: String,
1391 /// How long a run's lease lasts, and how long a crashed owner's runs stay
1392 /// unclaimable.
1393 lease_ttl: Duration,
1394 /// Where this plane's agents remember things, when a deployment wires one.
1395 memories: Option<Arc<dyn crate::memory::MemoryStore>>,
1396 semantic: Option<Arc<super::SemanticMemory>>,
1397 authorities: Option<Arc<dyn crate::authority::AuthorityStore>>,
1398 /// The peers this plane may call, and the transport that reaches them.
1399 peers: Option<Arc<PeerWiring>>,
1400 /// The catalogue and transport a **skill** reaches tools through.
1401 ///
1402 /// Held by the plane rather than handed to each skill, and that is the
1403 /// point. A skill that built its own [`ToolCatalog`] could declare a tool
1404 /// read-only that its manifest calls mutating — and `ToolSafety::read_only`
1405 /// carries `Recovery::Retry`, so a timed-out money-moving call gets sent
1406 /// again. `check_catalogue_not_laxer_than_grants` refuses exactly that for
1407 /// the plane's catalogue at build; a hand-built one never passed under it.
1408 /// `StepCtx::call_tool` is the path that cannot.
1409 #[cfg(feature = "manifest")]
1410 tools: Option<(
1411 Arc<crate::tools::ToolCatalog>,
1412 Arc<dyn crate::tools::ToolClient>,
1413 )>,
1414 /// The highest sensitivity this plane will write into an append-only
1415 /// record, when it says.
1416 journal_ceiling: Option<crate::core::Sensitivity>,
1417 /// Where this plane's tool transports may connect.
1418 ///
1419 /// Behind `manifest` because the tool path is.
1420 ///
1421 /// Here rather than on each `ToolClient` because it is a **plane**
1422 /// decision: the client owns the connection, the plane owns the
1423 /// destination, exactly as it does for a model driver's base URL. Absent
1424 /// means no egress control on the tool path, spelled the same way an
1425 /// absent policy engine is.
1426 #[cfg(feature = "manifest")]
1427 egress: Option<Arc<crate::core::Egress>>,
1428 #[cfg(feature = "manifest")]
1429 checkers: crate::content::Checkers,
1430 /// Where live model output is forwarded, when anything listens.
1431 streams: Option<Arc<dyn RunStreamObserver>>,
1432 /// How this plane attributes its metrics.
1433 meter: super::metrics::Meter,
1434 /// Durable per-tenant ceilings, when a deployment wires them.
1435 quotas: Option<Arc<dyn crate::quota::QuotaStore>>,
1436 /// The rate ceilings this plane's declarations state, and the quota store
1437 /// that counts them. `None` when no grant declares one.
1438 #[cfg(feature = "manifest")]
1439 rates: Option<Arc<super::ctx::Rates>>,
1440 quota: crate::quota::TenantQuota,
1441 budget: Budget,
1442 /// Whether every plan this runtime admits must carry a verifier.
1443 require_verifier: bool,
1444
1445 cases: Option<Arc<dyn CaseStore>>,
1446 events: Option<Arc<dyn EventStore>>,
1447 tasks: Option<Arc<dyn TaskStore>>,
1448 timers: Option<Arc<dyn TimerStore>>,
1449 blobs: Option<Arc<dyn crate::blob::BlobStore>>,
1450 /// Where disclosures are recorded, read by a retention pass so it names
1451 /// the copies of what it erased.
1452 disclosures: Option<Arc<dyn crate::disclosure::DisclosureRegister>>,
1453 /// Where webhook registrations and their cursors live.
1454 ///
1455 /// Held by the plane rather than only by whoever delivers, because the
1456 /// registrations that stop being delivered are a **backlog**, and a backlog
1457 /// is answered by an operator on the plane's own surface — not by the
1458 /// worker that gave up on it.
1459 #[cfg(feature = "push")]
1460 push: Option<Arc<dyn crate::push::PushStore>>,
1461 /// Where data keys live, when payload bytes are sealed.
1462 #[cfg(feature = "keyring")]
1463 keyring: Option<Arc<dyn crate::keyring::KeyRing>>,
1464 batches: Option<Arc<dyn crate::batch::BatchStore>>,
1465 policy: Option<Arc<dyn crate::core::PolicyEngine>>,
1466 identity: Option<crate::core::Delegation>,
1467 replanner: Option<Arc<dyn crate::plan::Replanner>>,
1468 calendar: Arc<dyn Calendar>,
1469 signer: Option<Arc<dyn crate::core::Signer>>,
1470 /// Where this plane sends its own events, when a deployment configures it.
1471 ///
1472 /// Registered per run **at admission**, and again by every resume that
1473 /// executes: a destination attached after the first record would silently
1474 /// miss it, and "the journal is the outbox" only holds if the cursor starts
1475 /// at sequence one.
1476 #[cfg(feature = "push")]
1477 outbox: Option<Arc<crate::push::Outbox>>,
1478 /// The parties this plane submits its checkpoints to, and how many must
1479 /// answer.
1480 ///
1481 /// The one control here whose whole value is that it is **not** this
1482 /// plane: the chain, the signatures and the Merkle log all verify against
1483 /// inputs the operator supplies, so none of them detects a run removed or
1484 /// two histories shown to two auditors. Absent means the plane's history
1485 /// is anchored nowhere outside itself, which is a deployment decision and
1486 /// is reported as one by `audit`.
1487 witnesses: Option<Arc<Witnessing>>,
1488 /// The runs this instance put on a background task, and the gate that
1489 /// closes admission when it is asked to stop.
1490 ///
1491 /// Process-local on purpose: it answers *what is this instance executing
1492 /// right now*, which no other instance can be told and no store can hold.
1493 /// The durable cousin is [`Runtime::set_halt`], which stops a tenant
1494 /// everywhere and survives a restart — a different question with a
1495 /// different audience.
1496 inflight: Arc<super::drain::InFlight>,
1497}
1498
1499/// The witness set and the bar a submission round is held to.
1500#[derive(Debug)]
1501pub(crate) struct Witnessing {
1502 pub(crate) witnesses: Vec<Arc<dyn crate::journal::Witness>>,
1503 pub(crate) quorum: crate::journal::WitnessQuorum,
1504 /// The size last submitted under a **met** quorum.
1505 ///
1506 /// In memory, deliberately. It exists only to keep a periodic pass from
1507 /// re-submitting a log that has not grown, and the authority on where a
1508 /// witness actually is has always been the witness — which answers `409`
1509 /// with its own size and gets a proof built from there. So a restart
1510 /// re-submits once and the protocol absorbs it; a durable copy would be a
1511 /// second answer to a question the counterparty already answers.
1512 pub(crate) submitted: std::sync::atomic::AtomicU64,
1513 /// The declared maximum time between submissions of an unchanged log;
1514 /// `None` never re-submits one.
1515 pub(crate) interval: Option<std::time::Duration>,
1516 /// The sweep instant of the last round that met quorum, in memory for the
1517 /// reason `submitted` is.
1518 pub(crate) last_met: std::sync::Mutex<Option<crate::core::Timestamp>>,
1519}
1520
1521/// Where a run is entering this plane from.
1522///
1523/// The one thing the drain gate needs to know, and it cannot be derived at the
1524/// gate: by the time admission runs, an agent commissioning another one and a
1525/// peer sending a message look identical. The difference is whose work it is —
1526/// a commission is a step of a run this process is *already* executing and has
1527/// promised to finish, so refusing it would make the drain manufacture the
1528/// undecided effects it exists to prevent.
1529#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1530pub(crate) enum Entry {
1531 /// A caller outside this process: an embedder, an operator, a peer, a batch
1532 /// pass. Refused while draining.
1533 Outside,
1534 /// A run already executing here, ordering work it needs in order to
1535 /// conclude. Never refused for draining — the parent is what the drain is
1536 /// waiting for.
1537 Nested,
1538}
1539
1540/// A backend that implements every store a full plane runs on.
1541///
1542/// Blanket-implemented, so it is a *description* rather than an obligation:
1543/// any type providing the store traits — journal, cases, tasks, events,
1544/// timers, memory, quotas, disclosures — is a `FullBackend` without saying so, and both
1545/// shipped backends are. The quota store is in the set because halts live
1546/// there: a plane without it would admit past every emergency stop. What it buys is [`Runtime::builder_on`], where the
1547/// alternative was a cast of one `Arc` per store that every deployment spelled out
1548/// and none could get differently.
1549pub trait FullBackend:
1550 JournalStore
1551 + CaseStore
1552 + TaskStore
1553 + EventStore
1554 + TimerStore
1555 + crate::memory::MemoryStore
1556 + crate::quota::QuotaStore
1557 + crate::disclosure::DisclosureRegister
1558{
1559}
1560
1561impl<B> FullBackend for B where
1562 B: JournalStore
1563 + CaseStore
1564 + TaskStore
1565 + EventStore
1566 + TimerStore
1567 + crate::memory::MemoryStore
1568 + crate::quota::QuotaStore
1569 + crate::disclosure::DisclosureRegister
1570{
1571}
1572
1573impl Runtime {
1574 /// A builder with the whole case layer wired to one backend.
1575 ///
1576 /// The one-call form of the seven-cast litany: journal, cases, tasks,
1577 /// events, timers, memory and quotas all backed by `store`, which is what a
1578 /// deployment on a single `RedbStore` or `PostgresStore` means anyway. The
1579 /// à-la-carte methods remain — [`cases`](RuntimeBuilder::cases) and
1580 /// friends override an individual store afterwards, and
1581 /// [`builder`](Self::builder) still starts from the journal alone for a
1582 /// plane that wants nothing else.
1583 ///
1584 /// Blob storage is deliberately not included: bytes routinely live in a
1585 /// different system than rows, so [`blobs`](RuntimeBuilder::blobs) stays
1586 /// an explicit decision.
1587 #[must_use]
1588 pub fn builder_on<B: FullBackend + 'static>(store: Arc<B>) -> RuntimeBuilder {
1589 Self::builder_with(Stores::on(store))
1590 }
1591
1592 /// A builder with the whole case layer wired from handles.
1593 ///
1594 /// What [`builder_on`](Self::builder_on) is in terms of: the seven casts live
1595 /// in one place, so a deployment that chose its backend at run time — an
1596 /// operator naming either a file or a connection string — wires exactly what
1597 /// a deployment on one concrete type wires, rather than a hand-written
1598 /// approximation of it that drifts the day an eighth store is added.
1599 #[must_use]
1600 pub fn builder_with(stores: Stores) -> RuntimeBuilder {
1601 Self::builder(stores.journal)
1602 .cases(stores.cases)
1603 .tasks(stores.tasks)
1604 .events(stores.events)
1605 .timers(stores.timers)
1606 .memory(stores.memory)
1607 .quota(stores.quotas, crate::quota::TenantQuota::default())
1608 .disclosures(stores.disclosures)
1609 }
1610
1611 #[must_use]
1612 pub fn builder(store: Arc<dyn JournalStore>) -> RuntimeBuilder {
1613 RuntimeBuilder {
1614 store,
1615 signer: None,
1616 skills: Vec::new(),
1617 owner: None,
1618 lease_ttl: LEASE_TTL,
1619 memories: None,
1620 semantic: None,
1621 authorities: None,
1622 peers: None,
1623 tenant_label: super::telemetry::TenantLabel::default(),
1624 quotas: None,
1625 quota: crate::quota::TenantQuota::default(),
1626 budget: Budget::unlimited(),
1627 require_verifier: false,
1628 cases: None,
1629 events: None,
1630 tasks: None,
1631 timers: None,
1632 blobs: None,
1633 disclosures: None,
1634 #[cfg(feature = "push")]
1635 push: None,
1636 #[cfg(feature = "keyring")]
1637 keyring: None,
1638 #[cfg(feature = "manifest")]
1639 tools: None,
1640 journal_ceiling: None,
1641 #[cfg(feature = "manifest")]
1642 egress: None,
1643 #[cfg(feature = "manifest")]
1644 checkers: crate::content::Checkers::default(),
1645 streams: None,
1646 tenant: crate::core::TenantId::default(),
1647 batches: None,
1648 policy: None,
1649 identity: None,
1650 replanner: None,
1651 calendar: None,
1652 #[cfg(feature = "manifest")]
1653 toolbox: None,
1654 #[cfg(feature = "manifest")]
1655 tool_servers: Vec::new(),
1656 #[cfg(feature = "manifest")]
1657 agents: Vec::new(),
1658 #[cfg(feature = "manifest")]
1659 providers: HashMap::new(),
1660 #[cfg(feature = "push")]
1661 outbox: None,
1662 witnesses: Vec::new(),
1663 quorum: None,
1664 witness_interval: None,
1665 }
1666 }
1667
1668 /// The driver registered under a manifest's name for it, if any.
1669 #[cfg(feature = "manifest")]
1670 pub(crate) fn model_provider(
1671 &self,
1672 name: &str,
1673 ) -> Option<Arc<dyn crate::model::ModelProvider>> {
1674 self.providers.get(name).map(Arc::clone)
1675 }
1676
1677 /// The case store, if this runtime has one.
1678 #[must_use]
1679 pub fn cases(&self) -> Option<&Arc<dyn CaseStore>> {
1680 self.cases.as_ref()
1681 }
1682
1683 /// The worklist, if this runtime has one.
1684 #[must_use]
1685 pub fn tasks(&self) -> Option<&Arc<dyn TaskStore>> {
1686 self.tasks.as_ref()
1687 }
1688
1689 /// The inbound-event store, if this runtime has one.
1690 #[must_use]
1691 pub fn events(&self) -> Option<&Arc<dyn EventStore>> {
1692 self.events.as_ref()
1693 }
1694
1695 /// The ceilings every run under this runtime starts with.
1696 ///
1697 /// Readable because "what is this plane allowed to spend" is a question an
1698 /// operator asks of a running system, and answering it by re-reading the
1699 /// config that *should* have been applied is how a misapplied budget stays
1700 /// invisible.
1701 #[must_use]
1702 pub const fn budget(&self) -> &Budget {
1703 &self.budget
1704 }
1705
1706 /// The blob store, if this runtime has one.
1707 #[must_use]
1708 pub fn blobs(&self) -> Option<&Arc<dyn crate::blob::BlobStore>> {
1709 self.blobs.as_ref()
1710 }
1711
1712 /// Run the live half of the case-layer drill with this plane's own stores.
1713 ///
1714 /// The wiring is the point of this method existing: the drill's value is
1715 /// holding each case's references against the stores the plane *actually
1716 /// runs with*, and an embedder assembling `drill::Stores` by hand could
1717 /// hand it a different blob store than the one their runs write to —
1718 /// a drill that passes over the wrong bucket. Stores this runtime does
1719 /// not have are reported as unchecked by the drill itself, which is a
1720 /// different and better answer than an error: a plane with no blob store
1721 /// has no bytes to lose.
1722 ///
1723 /// **The clock is the caller's**, the same decision
1724 /// [`attention`](Self::attention) and the push sweep make. A rehearsal's
1725 /// instant is a fact about the outside world rather than a journaled
1726 /// observation, and taking it here would be a fourth ambient-clock escape
1727 /// inside the deterministic zone for a value the caller already has. A
1728 /// fixed instant also makes the recorded verdict testable.
1729 ///
1730 /// # Errors
1731 ///
1732 /// If this runtime has no case store — the drill walks cases, so there is
1733 /// nothing to drill — or if the case layer cannot be enumerated.
1734 pub async fn drill(
1735 &self,
1736 at: crate::core::Timestamp,
1737 ) -> Result<crate::drill::DrillReport, RuntimeError> {
1738 let cases = self.cases().ok_or_else(|| {
1739 RuntimeError::PlanContract(
1740 "this runtime has no case store — the drill walks cases, so there is \
1741 nothing to drill; build it with `.cases(store)`"
1742 .into(),
1743 )
1744 })?;
1745 let stores = crate::drill::Stores {
1746 cases,
1747 blobs: self.blobs(),
1748 #[cfg(feature = "keyring")]
1749 keys: self.keyring.as_ref(),
1750 tenant: &self.tenant,
1751 };
1752 let report = crate::drill::drill(&stores)
1753 .await
1754 .map_err(RuntimeError::from_store)?;
1755
1756 // **The verdict is recorded, not only returned.** What an audit asks
1757 // is *when did you last rehearse, and did it pass* — and left to the
1758 // caller that answer lives in whatever ran the verb, which is a CI log
1759 // or a wiki page. The checkpoint pins which store it ran against, so a
1760 // drill over a restored copy cannot be read as one over production.
1761 let checkpoint = self
1762 .store
1763 .checkpoint()
1764 .await
1765 .map_err(RuntimeError::from_store)?;
1766 cases
1767 .record_drill(&crate::case::DrillRecord {
1768 at,
1769 sound: report.is_sound(),
1770 cases: report.cases as u64,
1771 findings: report.findings.len() as u64,
1772 not_checked: report.not_checked.len() as u64,
1773 origin: checkpoint.origin.clone(),
1774 size: checkpoint.size,
1775 })
1776 .await
1777 .map_err(RuntimeError::from_store)?;
1778 Ok(report)
1779 }
1780
1781 /// The last recovery rehearsal this plane ran, or `None` if it never has.
1782 ///
1783 /// The half that makes `drill` evidence rather than a command: an audit
1784 /// asks *when did you last rehearse and did it pass*, and that answer has
1785 /// to come from the plane rather than from the thing that scheduled it.
1786 ///
1787 /// # Errors
1788 ///
1789 /// [`RuntimeError::PlanContract`] when this plane has no case store — the
1790 /// same condition that makes a drill impossible in the first place — and
1791 /// [`RuntimeError`] for a store failure.
1792 pub async fn last_drill(&self) -> Result<Option<crate::case::DrillRecord>, RuntimeError> {
1793 self.cases()
1794 .ok_or_else(|| {
1795 RuntimeError::PlanContract(
1796 "this runtime has no case store, so it has never drilled one".into(),
1797 )
1798 })?
1799 .last_drill()
1800 .await
1801 .map_err(RuntimeError::from_store)
1802 }
1803
1804 /// Erase every closed case opened before `older_than`. The retention verb.
1805 ///
1806 /// The wiring is the point, exactly as it is for [`drill`](Self::drill):
1807 /// erasure derives blob addresses and key scopes from the tenant and the
1808 /// case, so a caller assembling the stores by hand could tombstone
1809 /// addresses no run ever wrote and report a clean pass over nothing.
1810 ///
1811 /// `reason` lands on every tombstone and on each key destruction, so a
1812 /// later read says *expired, on this date, for this reason* rather than
1813 /// *missing* — the distinction the drill's three-way verdict turns on.
1814 ///
1815 /// `at` is a parameter rather than a clock read, for the reason the
1816 /// sweeper's is: a pass that read the clock could not be tested against a
1817 /// year of ageing cases.
1818 ///
1819 /// Read [`RetentionReport::not_erasable`](crate::retention::RetentionReport::not_erasable)
1820 /// before concluding anything. Without a key ring this expires blob
1821 /// tombstones in the live store and **journal payloads stay verbatim**,
1822 /// which is the residual the erasure page names and the one an erasure
1823 /// request actually turns on.
1824 ///
1825 /// # Errors
1826 ///
1827 /// If this runtime has no case store, or the case layer cannot be walked.
1828 /// A store that fails on one case is a report entry, not an error.
1829 pub async fn retain(
1830 &self,
1831 older_than: crate::core::Timestamp,
1832 at: crate::core::Timestamp,
1833 reason: &str,
1834 ) -> Result<crate::retention::RetentionReport, RuntimeError> {
1835 let cases = self.cases().ok_or_else(|| {
1836 RuntimeError::PlanContract(
1837 "this runtime has no case store — retention walks cases, so there is \
1838 nothing to retain; build it with `.cases(store)`"
1839 .into(),
1840 )
1841 })?;
1842 let stores = crate::retention::Stores {
1843 cases,
1844 blobs: self.blobs(),
1845 #[cfg(feature = "keyring")]
1846 keys: self.keyring.as_ref(),
1847 tenant: &self.tenant,
1848 disclosures: self.disclosures.as_ref(),
1849 };
1850 crate::retention::retain(&stores, self.store.as_ref(), older_than, at, reason)
1851 .await
1852 .map_err(RuntimeError::from_store)
1853 }
1854
1855 /// The timer store, if this runtime has one.
1856 #[must_use]
1857 pub fn timers(&self) -> Option<&Arc<dyn TimerStore>> {
1858 self.timers.as_ref()
1859 }
1860
1861 #[must_use]
1862 pub fn batches(&self) -> Option<&Arc<dyn crate::batch::BatchStore>> {
1863 self.batches.as_ref()
1864 }
1865
1866 #[cfg(feature = "push")]
1867 #[must_use]
1868 pub fn push(&self) -> Option<&Arc<dyn crate::push::PushStore>> {
1869 self.push.as_ref()
1870 }
1871
1872 /// Re-arm a webhook registration a delivery worker gave up on.
1873 ///
1874 /// A parked registration is not delivered and not deleted: it keeps its
1875 /// cursor, so re-arming resumes at the first record the receiver never
1876 /// acknowledged rather than at the head of the run. That is the whole
1877 /// reason parking exists, and the reason this verb is on the plane — a
1878 /// backlog nobody can empty is a warning line in yesterday's logs.
1879 ///
1880 /// The clock is read here rather than taken from the caller, unlike the
1881 /// delivery worker's: a worker's schedule has to be assertable in a test,
1882 /// and an operator asking for *now* means now.
1883 ///
1884 /// Returns whether a parked registration was found. `false` is an answer —
1885 /// already live, or never existed — and is told rather than swallowed,
1886 /// because an operator left waiting for a sweep with nothing to do has no
1887 /// way to tell the difference.
1888 ///
1889 /// # Errors
1890 ///
1891 /// [`RuntimeError::PlanContract`] if this plane has no push store, and
1892 /// [`RuntimeError::Store`] if it is unreachable.
1893 #[cfg(feature = "push")]
1894 pub async fn rearm_push(&self, task: RunId, id: &str) -> Result<bool, RuntimeError> {
1895 let push = self
1896 .push
1897 .as_ref()
1898 .ok_or_else(|| RuntimeError::PlanContract("this plane has no push store".into()))?;
1899 let at = now_for_admission().unix_timestamp().max(0).unsigned_abs();
1900 push.unpark(task, id, at)
1901 .await
1902 .map_err(RuntimeError::from_store)
1903 }
1904
1905 /// Ask a run to stop, and drive the stop if nothing else will.
1906 ///
1907 /// The request is durable before this returns, so an operator who gets an
1908 /// acknowledgement has one whether or not the run was reachable. What
1909 /// happens next depends on where the run is:
1910 ///
1911 /// * **Suspended** — nothing is executing, so this resumes the run itself.
1912 /// It observes the request at its first step boundary and unwinds.
1913 /// * **Running here or elsewhere** — the owner observes the request at its
1914 /// next step boundary. Nothing is interrupted mid-effect, deliberately:
1915 /// stopping between "announced" and "recorded" manufactures the in-doubt
1916 /// case the effect protocol exists to avoid.
1917 /// * **Quarantined** — refused with [`RuntimeError::CannotUnwind`]: the
1918 /// unwind a stop promises is what a quarantine forbids, and reopening or
1919 /// abandoning it are the verbs that apply.
1920 /// * **Concluded and closed** (succeeded, abandoned, cancelled) — refused
1921 /// with [`RuntimeError::AlreadyConcluded`] and nothing is recorded; a run
1922 /// somebody already stopped answers `false`. A failed or exhausted run is
1923 /// resumable, and stopping it unwinds its work.
1924 /// * **Held by no provider on this plane** — an operator's terminal: the
1925 /// request is recorded and not driven, and the plane that holds the run's
1926 /// agent observes it on its next resume.
1927 ///
1928 /// Returns whether *this* call recorded the request. A second caller gets
1929 /// `false`: the first asker stays on the record, because "who intervened"
1930 /// must not be rewritten by a retry.
1931 ///
1932 /// # Errors
1933 ///
1934 /// [`RuntimeError::PlanContract`] for a blank reason,
1935 /// [`RuntimeError::CannotUnwind`] for a quarantined run,
1936 /// [`RuntimeError::AlreadyConcluded`] for a closed one, and
1937 /// [`RuntimeError`] if the store is unreachable or resuming a suspended run
1938 /// fails.
1939 pub async fn request_cancel(
1940 &self,
1941 run: RunId,
1942 actor: &crate::core::Operator,
1943 reason: &str,
1944 ) -> Result<bool, RuntimeError> {
1945 // An intervention with no stated reason documents nobody's judgement,
1946 // and it is the one field of the record the requester writes.
1947 if reason.trim().is_empty() {
1948 return Err(RuntimeError::PlanContract(
1949 "stopping a run needs a reason — what the person stopping it saw".to_owned(),
1950 ));
1951 }
1952 // Checked before recording. Writing first and failing afterwards leaves a
1953 // request standing against an id that does not exist, and the operator's
1954 // retry then comes back "somebody else already asked" — which is a
1955 // confusing way to say "you mistyped".
1956 let records = self
1957 .store
1958 .read(run, 1)
1959 .await
1960 .map_err(RuntimeError::from_store)?;
1961 if records.is_empty() {
1962 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
1963 run.to_string(),
1964 )));
1965 }
1966
1967 // A quarantine is refused here rather than recorded and ignored. The
1968 // stop this verb promises is *unwind and put the world back*, and that
1969 // is the one thing a quarantined run may not do — so the request landed
1970 // durably, answered the operator `recorded: true`, and the resume then
1971 // walked straight past it. An operator who believes they stopped a run
1972 // holding an unresolved payment is worse off than one who was told no.
1973 //
1974 // Read from the recorded conclusion, not from a live status: a run that
1975 // quarantines *after* this call is a genuine race the standing request
1976 // loses, and that is correct — the request was made about a run nobody
1977 // had given up on.
1978 if recorded_conclusion(&records).as_deref() == Some("quarantined") {
1979 return Err(RuntimeError::CannotUnwind {
1980 run: run.to_string(),
1981 });
1982 }
1983 // Refused only where the conclusion closes the run. A failed or an
1984 // exhausted run is resumable and its work stands, so stopping it is the
1985 // operator's remedy — the unwind it asks for is exactly what those
1986 // conclusions left undone. Read from the latest conclusion, not the
1987 // last record: an answer to a doubt may follow a conclusion, and the
1988 // run is no less closed for it. A run already stopped answers `false`
1989 // rather than an error, so a second asker is told the intervention of
1990 // record is somebody else's.
1991 if let Some(status) = resume_is_closed(&records) {
1992 if self.cancellation(run).await?.is_some() {
1993 return Ok(false);
1994 }
1995 return Err(RuntimeError::AlreadyConcluded {
1996 run: run.to_string(),
1997 outcome: recorded_conclusion(&records)
1998 .unwrap_or_else(|| status.as_str().to_owned()),
1999 });
2000 }
2001
2002 let fresh = self
2003 .store
2004 .request_cancel(run, actor, reason)
2005 .await
2006 .map_err(RuntimeError::from_store)?;
2007
2008 // Drive it. A suspended run has no thread to notice anything, so an
2009 // operator's stop would sit unobserved until the deadline swept it —
2010 // which for a run waiting on a six-week obligation is not a stop.
2011 //
2012 // Resuming a run that has already concluded is a no-op inside `replay`,
2013 // which reads the recorded status back rather than re-executing — so
2014 // there is no "is it finished?" check to race with here.
2015 //
2016 // Driven **only when the run is genuinely idle**: `replay` claims the
2017 // lease before it reads, and a `LeaseHeld` refusal means somebody is
2018 // executing the run right now — here or on another instance. That is
2019 // not a failure of this call. The request is already durable, and the
2020 // live owner checks for it at every step boundary (the cancellation
2021 // read at the top of the executor's ready-set loop), so the stop
2022 // arrives without a second executor ever touching the run. Resuming
2023 // anyway would, on the owner's own instance, hand the resume the *same
2024 // epoch* the live execution is writing under — two executors on one
2025 // chain that fencing, by construction, cannot tell apart.
2026 //
2027 // A plane holding no provider, case store or key ring for the run — an
2028 // operator's terminal, a library plane — records the request and
2029 // leaves the driving to the plane that does: that plane's next resume
2030 // finds the request standing.
2031 if fresh {
2032 match self.replay(run, Mode::Resume).await {
2033 Ok(_)
2034 | Err(
2035 RuntimeError::LeaseHeld { .. }
2036 | RuntimeError::NoProvider { .. }
2037 | RuntimeError::NoCaseStore { .. }
2038 | RuntimeError::PayloadsSealed { .. },
2039 ) => {}
2040 Err(e) => return Err(e),
2041 }
2042 }
2043 Ok(fresh)
2044 }
2045
2046 /// Answer a quarantine, and let the runtime decide the run again.
2047 ///
2048 /// A quarantine is this runtime saying it does not know, and the one
2049 /// conclusion it cannot reach past on its own. Everything else that stops a
2050 /// run has a party who can move it — a resume for a crash, a raised ceiling
2051 /// for an exhaustion, a reviewer for a suspension. This is the party for
2052 /// this one.
2053 ///
2054 /// # What this does and does not decide
2055 ///
2056 /// The person decides *whether the run may be judged again*, or that it
2057 /// never will be. They do not decide the outcome. A
2058 /// [`Reopen`](crate::core::QuarantineDecision::Reopen) hands the run back
2059 /// to the executor, which re-derives its verdict from a history that now
2060 /// holds whatever the person established with
2061 /// [`reconcile_effect`](Self::reconcile_effect) — and quarantines it again,
2062 /// on the record, if the answer was wrong or incomplete. An
2063 /// [`Abandon`](crate::core::QuarantineDecision::Abandon) closes it where it
2064 /// stands, unwinding nothing.
2065 ///
2066 /// The decision is journaled **before** it is acted on, so a crash between
2067 /// the two leaves the instruction standing rather than lost: the next
2068 /// resume finds it and finishes the job.
2069 ///
2070 /// Returns the outcome of the pass this drove — so an operator who reopens
2071 /// a run learns immediately whether it got any further, rather than having
2072 /// to go and look.
2073 ///
2074 /// # Errors
2075 ///
2076 /// [`RuntimeError::NotQuarantined`] if the run's last conclusion is
2077 /// anything else, [`RuntimeError::LeaseHeld`] if somebody is executing it,
2078 /// and [`RuntimeError`] for a store failure or a refused resume.
2079 ///
2080 /// # Panics
2081 ///
2082 /// Never: the `expect` covers a lease this function has just acquired.
2083 pub async fn decide_quarantine(
2084 &self,
2085 run: RunId,
2086 decider: &crate::core::Operator,
2087 reason: &str,
2088 decision: crate::core::QuarantineDecision,
2089 ) -> Result<RunOutcome, RuntimeError> {
2090 // The whole weight of this record is that a named person took
2091 // responsibility for a fact the runtime could not establish, and
2092 // "unknown decided this for no stated reason" documents nothing while
2093 // looking like process. Only the reason is checked here: `Operator` has
2094 // no empty value, so the decider is guaranteed by the type on every
2095 // path that builds one rather than by a check on this one.
2096 if reason.trim().is_empty() {
2097 return Err(RuntimeError::PlanContract(
2098 "answering a quarantine needs a decider and a reason — who looked, and what they \
2099 found"
2100 .to_owned(),
2101 ));
2102 }
2103
2104 let lease = self
2105 .store
2106 .acquire(run, &self.owner, self.lease_ttl)
2107 .await
2108 .map_err(RuntimeError::from_store)?;
2109 let appended = self
2110 .record_decision(run, &lease, decider, reason, decision)
2111 .await;
2112 if let Err(e) = appended {
2113 // The lease was claimed by this call and never handed to a resume,
2114 // so it is this call's to give back. Leaving it would strand the
2115 // run for a TTL and hand it to the recovery sweep, which would
2116 // "recover" a run nobody was running.
2117 let _ = self.store.release_lease(run, lease.epoch).await;
2118 return Err(e);
2119 }
2120 // Handed over rather than released and re-taken: the same window a wake
2121 // path closes. Both decisions travel this path — an abandonment is
2122 // concluded inside the resume, from the record just written. The run
2123 // still stands quarantined on the record, so it keeps its listing
2124 // whatever this pass does.
2125 self.replay_releasing(run, Pass::Resume, Some(lease), Claim::Standing)
2126 .await
2127 }
2128
2129 /// Answer a quarantine **without driving the run**.
2130 ///
2131 /// [`decide_quarantine`](Self::decide_quarantine) records the decision and
2132 /// then hands the run to the executor, which is what an operator on a
2133 /// running plane wants: they learn immediately whether a reopened run got
2134 /// any further.
2135 ///
2136 /// A terminal holding a store cannot do the second half. It has the
2137 /// journal and not the agent — a declarative run's program is a manifest
2138 /// this process was not given — so driving the run would mean refusing the
2139 /// verb for want of an argument the operator may not have during an
2140 /// incident.
2141 ///
2142 /// **That is a supported state rather than a lesser one.** The decision is
2143 /// journaled before it is acted on precisely so a crash between the two
2144 /// leaves the instruction standing, and this is the same position reached
2145 /// deliberately: the next resume finds the record and finishes the job —
2146 /// so what the caller must not do is report the run as moved.
2147 ///
2148 /// # Errors
2149 ///
2150 /// As [`decide_quarantine`](Self::decide_quarantine), minus everything the
2151 /// resume could have raised.
2152 pub async fn record_quarantine_decision(
2153 &self,
2154 run: RunId,
2155 decider: &crate::core::Operator,
2156 reason: &str,
2157 decision: crate::core::QuarantineDecision,
2158 ) -> Result<(), RuntimeError> {
2159 if reason.trim().is_empty() {
2160 return Err(RuntimeError::PlanContract(
2161 "answering a quarantine needs a decider and a reason — who looked, and what they \
2162 found"
2163 .to_owned(),
2164 ));
2165 }
2166 let lease = self
2167 .store
2168 .acquire(run, &self.owner, self.lease_ttl)
2169 .await
2170 .map_err(RuntimeError::from_store)?;
2171 let appended = self
2172 .record_decision(run, &lease, decider, reason, decision)
2173 .await;
2174 // Always given back: this verb never executes the run, so the lease is
2175 // held only for the width of one append — the rule
2176 // `reconcile_effect` follows.
2177 let _ = self.store.release_lease(run, lease.epoch).await;
2178 appended
2179 }
2180
2181 /// Append the decision, once the run is known to be asking for one.
2182 async fn record_decision(
2183 &self,
2184 run: RunId,
2185 lease: &crate::journal::Lease,
2186 decider: &crate::core::Operator,
2187 reason: &str,
2188 decision: crate::core::QuarantineDecision,
2189 ) -> Result<(), RuntimeError> {
2190 let records = self
2191 .store
2192 .read(run, 1)
2193 .await
2194 .map_err(RuntimeError::from_store)?;
2195 if records.is_empty() {
2196 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2197 run.to_string(),
2198 )));
2199 }
2200 let status = recorded_conclusion(&records);
2201 // A run whose payloads were erased is asking the same question a
2202 // quarantine asks — *the runtime cannot go on, and a person must
2203 // decide* — so it is answered by the same verb whatever outcome it was
2204 // left under. Without this it has no verb at all: it cannot be
2205 // resumed, and a cancellation promises an unwind that erased arguments
2206 // make impossible. Abandonment is the honest ending and this is the
2207 // door to it.
2208 if status.as_deref() != Some("quarantined") && !payloads_erased(&records) {
2209 return Err(RuntimeError::NotQuarantined {
2210 run: run.to_string(),
2211 status: status.unwrap_or_else(|| "still running".to_owned()),
2212 });
2213 }
2214 let mut append = Append::new(
2215 run,
2216 RecordKind::QuarantineDecided {
2217 decider: decider.clone(),
2218 reason: reason.to_owned(),
2219 decision,
2220 },
2221 );
2222 if let Some(case) = records.iter().find_map(|r| r.body.case) {
2223 append = append.case(case);
2224 }
2225 self.store
2226 .append(lease.epoch, vec![append])
2227 .await
2228 .map_err(RuntimeError::from_store)?;
2229 Ok(())
2230 }
2231
2232 /// Supply, out of band, the outcome of an effect the journal cannot decide.
2233 ///
2234 /// The manual twin of [`Effect::reconcile`](crate::core::Effect::reconcile),
2235 /// and it produces the same record for the same reason: an operator who
2236 /// looks the charge up in the provider's console is answering exactly the
2237 /// question a probe asks, so the answer belongs in the place a probe's
2238 /// answer goes, where replay reads it back and nothing re-derives it.
2239 ///
2240 /// What separates the two is on the record too. A probe's verdict is
2241 /// attributed to the run; this one names the person, because "the provider
2242 /// told us" and "somebody asserted it" are different evidence.
2243 ///
2244 /// **It supplies a missing fact and never replaces a recorded one.** Only
2245 /// an effect [`undecided_effects`](crate::journal::undecided_effects) names
2246 /// may be answered. Without that rule an operator could talk a run out of
2247 /// compensating work that is standing in the world, and the journal would
2248 /// show an orderly reconciliation while it happened.
2249 ///
2250 /// This changes no status by itself: the run stays quarantined until
2251 /// [`decide_quarantine`](Self::decide_quarantine) hands it back. Several
2252 /// doubts can be answered before the run is judged again, and answering one
2253 /// of three is not a decision.
2254 ///
2255 /// # Errors
2256 ///
2257 /// [`RuntimeError::NotUndecided`] if that effect's outcome is already on
2258 /// the record, [`RuntimeError::LeaseHeld`] if somebody is executing the
2259 /// run, and [`RuntimeError`] for a store failure.
2260 pub async fn reconcile_effect(
2261 &self,
2262 run: RunId,
2263 effect: crate::core::EffectKey,
2264 assertion: crate::core::Assertion,
2265 asserted_by: &crate::core::Operator,
2266 note: &str,
2267 ) -> Result<(), RuntimeError> {
2268 // `asserted_by` needs no emptiness check: `Operator` has none to fail.
2269 if note.trim().is_empty() {
2270 return Err(RuntimeError::PlanContract(
2271 "an assertion about an effect needs a name and a note — who established this, and \
2272 how"
2273 .to_owned(),
2274 ));
2275 }
2276 let lease = self
2277 .store
2278 .acquire(run, &self.owner, self.lease_ttl)
2279 .await
2280 .map_err(RuntimeError::from_store)?;
2281 let result = self
2282 .record_assertion(run, &lease, effect, assertion, asserted_by, note)
2283 .await;
2284 // Always given back: this verb never executes the run, so the lease is
2285 // held only for the width of one append.
2286 let _ = self.store.release_lease(run, lease.epoch).await;
2287 result
2288 }
2289
2290 async fn record_assertion(
2291 &self,
2292 run: RunId,
2293 lease: &crate::journal::Lease,
2294 effect: crate::core::EffectKey,
2295 assertion: crate::core::Assertion,
2296 asserted_by: &crate::core::Operator,
2297 note: &str,
2298 ) -> Result<(), RuntimeError> {
2299 let records = self
2300 .store
2301 .read(run, 1)
2302 .await
2303 .map_err(RuntimeError::from_store)?;
2304 if records.is_empty() {
2305 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2306 run.to_string(),
2307 )));
2308 }
2309 let Some(undecided) = crate::journal::undecided_effects(&records)
2310 .into_iter()
2311 .find(|u| u.effect == effect)
2312 else {
2313 return Err(RuntimeError::NotUndecided {
2314 run: run.to_string(),
2315 effect: effect.to_string(),
2316 });
2317 };
2318
2319 let output = match &assertion {
2320 crate::core::Assertion::Landed(value) => Some(value.clone()),
2321 crate::core::Assertion::DidNotHappen => None,
2322 };
2323 let mut append = Append::new(
2324 run,
2325 RecordKind::EffectReconciled {
2326 disposition: assertion.disposition(),
2327 // Stated on the record rather than left to a reader's default.
2328 // A durable format that does not say what it means is one a
2329 // later build guesses about, and the guess here is the label on
2330 // a value the run is about to act on.
2331 declared: output
2332 .is_some()
2333 .then(crate::core::DeclaredOutput::untrusted),
2334 output,
2335 // A person's assertion costs nothing. What the call itself
2336 // consumed, if it consumed anything, is already on the record
2337 // of the attempt this answers.
2338 spend: Spend::ZERO,
2339 // No probe ran: this is a person's assertion, so the
2340 // provider-facing field stays empty and their account goes in
2341 // the field that is read without a key.
2342 detail: None,
2343 note: Some(note.to_owned()),
2344 asserted_by: Some(asserted_by.clone()),
2345 },
2346 )
2347 .step(undecided.step)
2348 .phase(undecided.phase)
2349 .effect(effect);
2350 if let Some(case) = records.iter().find_map(|r| r.body.case) {
2351 append = append.case(case);
2352 }
2353 self.store
2354 .append(lease.epoch, vec![append])
2355 .await
2356 .map_err(RuntimeError::from_store)?;
2357 tracing::info!(
2358 target: telemetry::RECONCILED,
2359 %run,
2360 step = %undecided.step,
2361 verdict = ?assertion.disposition(),
2362 actor = %asserted_by,
2363 );
2364 self.meter
2365 .count(metrics::RECONCILIATIONS, assertion.disposition().as_str());
2366 Ok(())
2367 }
2368
2369 /// Every mutating effect this run's journal cannot decide the outcome of.
2370 ///
2371 /// What an operator holding a quarantine has to go and look up, and the
2372 /// keys they quote back to [`reconcile_effect`](Self::reconcile_effect).
2373 ///
2374 /// # Errors
2375 ///
2376 /// If the store is unreachable.
2377 pub async fn undecided(&self, run: RunId) -> Result<Vec<crate::core::Undecided>, RuntimeError> {
2378 let records = self
2379 .store
2380 .read(run, 1)
2381 .await
2382 .map_err(RuntimeError::from_store)?;
2383 Ok(crate::journal::undecided_effects(&records))
2384 }
2385
2386 /// Stop this tenant from starting new work, or let it start again.
2387 ///
2388 /// The emergency stop. `Some(reason)` halts, `None` lifts, and the reason is
2389 /// required because the next person to look will be somebody else, possibly
2390 /// at three in the morning, and *why* is the whole question.
2391 ///
2392 /// **What it stops, precisely.** New admissions, across every instance,
2393 /// because the flag is in the store rather than in this process — a switch
2394 /// that stops only the instance it was thrown on is the in-process-counter
2395 /// failure arriving during an incident. Refusals are their own error, not a
2396 /// ceiling: a ceiling means *not right now* and invites a retry, which is
2397 /// exactly what somebody pulling this switch is trying to stop.
2398 ///
2399 /// **What a workload-scoped halt does not stop, deliberately.** Runs
2400 /// already executing, and suspended runs resuming. Those are existing work,
2401 /// and refusing to let them continue would strand them mid-saga with
2402 /// reversals unrun — turning an incident into a second one. To stop work in
2403 /// flight, cancel it: that unwinds what it did and records who asked. This
2404 /// is the front door, not a power cut, and saying so is the difference
2405 /// between a control an operator can reason about and one they discover the
2406 /// shape of during an outage.
2407 ///
2408 /// **How wide it reaches** is the [`HaltScope`](crate::quota::HaltScope),
2409 /// and the scopes divide on *which axis they name* rather than on how
2410 /// broad they are. Three name a workload — the tenant, every revision of
2411 /// one declared agent, or one exact reviewed digest — and stop admission.
2412 /// The fourth names an **authority**, any principal on the delegation
2413 /// chain a run was admitted under, and is the one that reaches work
2414 /// already running:
2415 /// there the incident *is* the credential, so a run carrying on under it is
2416 /// the harm rather than a side effect, and it pauses at its next step
2417 /// boundary with its completed work standing
2418 /// ([`withdraws_from`](crate::quota::HaltScope::withdraws_from) is the
2419 /// predicate, so the distinction is checked rather than remembered).
2420 /// Scopes are independent rows — lifting a narrow halt leaves a broad one
2421 /// standing. An ungoverned run, with no manifest, is stopped only by a
2422 /// tenant halt; there is nothing narrower to key it on.
2423 ///
2424 /// `at` is the caller's, never a clock read here: every lifecycle instant in
2425 /// this crate is supplied for the reason a hold's is — a pass that reads its
2426 /// own clock cannot be tested against an ageing plane, and [`I1`] forbids the
2427 /// runtime an ambient clock outside the effect protocol.
2428 ///
2429 /// Requires a quota store; without one there is nowhere durable to keep the
2430 /// flag, and an emergency stop that a restart forgets is not one.
2431 ///
2432 /// [`I1`]: crate::journal
2433 ///
2434 /// # Errors
2435 ///
2436 /// If no quota store is wired, or the store is unreachable.
2437 pub async fn set_halt(
2438 &self,
2439 scope: &crate::quota::HaltScope,
2440 by: &crate::core::Operator,
2441 at: crate::core::Timestamp,
2442 reason: &str,
2443 ) -> Result<(), RuntimeError> {
2444 let quotas = self.quota_store()?;
2445 quotas
2446 .set_halt(scope, by, at, reason)
2447 .await
2448 .map_err(RuntimeError::Store)?;
2449 self.forget_credentials(scope);
2450 Ok(())
2451 }
2452
2453 /// Drop the peer credentials held for a subject a halt names, when it is
2454 /// thrown and when it is lifted — not only when a hop next meets it, which
2455 /// may never happen while the halt stands.
2456 fn forget_credentials(&self, scope: &crate::quota::HaltScope) {
2457 if let (crate::quota::HaltScope::Subject { id }, Some(wiring)) =
2458 (scope, self.peers.as_ref())
2459 {
2460 wiring.registry.forget(id);
2461 }
2462 }
2463
2464 /// Lift one, and let work at that scope start again — recording who did.
2465 ///
2466 /// Answers the run holding the lift record and whether this call removed
2467 /// the row, or `None` when nothing was standing at `scope`: an operator who
2468 /// lifts a scope nobody halted is told so rather than told *done*, and
2469 /// nothing is written.
2470 ///
2471 /// **The record comes first, then the row goes.** The lift is written as
2472 /// the sole [`HaltLifted`](crate::journal::RecordKind::HaltLifted) record
2473 /// of a sealed run of its own, carrying `by`, `at` and the whole stop it
2474 /// ended; only then is the register's row removed. The journal and the
2475 /// register share no transaction, so a lift that cannot be recorded does
2476 /// not happen, and a removal that fails after the record is written is an
2477 /// error naming the record's run — the halt still stands, the register is
2478 /// the answer to what is in force, and lifting again writes a second
2479 /// record. The record attests the instruction and the row as read, not
2480 /// that the removal succeeded.
2481 ///
2482 /// **Only the row the record names goes.** The removal is
2483 /// [`QuotaStore::lift_halt_if`](crate::quota::QuotaStore::lift_halt_if)
2484 /// with that row, so a halt re-thrown between the read and the removal is
2485 /// left standing and the call fails naming the record's run. A row another
2486 /// lift already removed answers `removed: false`.
2487 ///
2488 /// [`QuotaStore::lift_halt`](crate::quota::QuotaStore::lift_halt) is the
2489 /// raw register verb; a direct call to it is unrecorded.
2490 ///
2491 /// # Errors
2492 ///
2493 /// If no quota store is wired, the store is unreachable, or the record
2494 /// cannot be written; [`RuntimeError::ControlStands`] if the row cannot be
2495 /// removed after the record was written, or was re-thrown in between.
2496 pub async fn lift_halt(
2497 &self,
2498 scope: &crate::quota::HaltScope,
2499 by: &crate::core::Operator,
2500 at: crate::core::Timestamp,
2501 ) -> Result<Option<ControlLifted>, RuntimeError> {
2502 let quotas = self.quota_store()?;
2503 let standing = quotas.halts().await.map_err(RuntimeError::Store)?;
2504 let Some(halt) = standing.into_iter().find(|h| &h.scope == scope) else {
2505 return Ok(None);
2506 };
2507 let run = self
2508 .seal_operator_record(
2509 RecordKind::HaltLifted {
2510 scope: scope.key(),
2511 by: by.clone(),
2512 at,
2513 reason: halt.reason.clone(),
2514 thrown_by: halt.by.clone(),
2515 thrown_at: halt.at,
2516 },
2517 None,
2518 HALT_LIFTED_OUTCOME,
2519 None,
2520 )
2521 .await?;
2522 let removed = quotas
2523 .lift_halt_if(&halt)
2524 .await
2525 .map_err(|e| removal_failed(run, &e))?;
2526 self.forget_credentials(scope);
2527 if !removed {
2528 let now = quotas.halts().await.map_err(|e| removal_failed(run, &e))?;
2529 if now.iter().any(|h| &h.scope == scope) {
2530 return Err(RuntimeError::ControlStands {
2531 run: run.to_string(),
2532 detail: format!(
2533 "the halt on '{}' was re-thrown after this lift read it, and the new \
2534 halt still stands",
2535 scope.key()
2536 ),
2537 });
2538 }
2539 }
2540 Ok(Some(ControlLifted {
2541 record: run,
2542 removed,
2543 }))
2544 }
2545
2546 /// Release the legal hold on `case` — recording who did.
2547 ///
2548 /// The hold's counterpart of [`lift_halt`](Self::lift_halt), in the same
2549 /// order for the same reason: the
2550 /// [`HoldReleased`](crate::journal::RecordKind::HoldReleased) record is
2551 /// written, stamped with `case`, before the row goes. It names the releaser
2552 /// and the hold's placing operator and instant, and not the hold's reason,
2553 /// so the erasure the release permits leaves its authorization readable.
2554 ///
2555 /// `None` when the matter is not held, with nothing written; `removed:
2556 /// false` when another release removed the row first.
2557 ///
2558 /// **Only the hold the record names goes.** The removal is
2559 /// [`CaseStore::release_hold_if`](crate::case::CaseStore::release_hold_if)
2560 /// with that hold, so a hold placed again between the read and the removal
2561 /// is left standing and the call fails naming the record's run.
2562 /// [`CaseStore::release_hold`](crate::case::CaseStore::release_hold) is the
2563 /// raw register verb; a direct call to it is unrecorded.
2564 ///
2565 /// # Errors
2566 ///
2567 /// [`StoreError::NotFound`](crate::core::StoreError::NotFound) if the case
2568 /// does not exist; [`RuntimeError::ControlStands`] if the row cannot be
2569 /// removed after the record was written, or was placed again in between;
2570 /// otherwise if no case store is
2571 /// wired, the store is unreachable, or the record cannot be written.
2572 pub async fn release_hold(
2573 &self,
2574 case: crate::core::CaseId,
2575 by: &crate::core::Operator,
2576 at: crate::core::Timestamp,
2577 ) -> Result<Option<ControlLifted>, RuntimeError> {
2578 let cases = self.cases.as_ref().ok_or_else(|| {
2579 RuntimeError::PlanContract("releasing a hold needs a case store".to_owned())
2580 })?;
2581 let Some(standing) = cases.hold(case).await.map_err(RuntimeError::Store)? else {
2582 return match cases.case(case).await.map_err(RuntimeError::Store)? {
2583 Some(_) => Ok(None),
2584 None => Err(RuntimeError::Store(crate::core::StoreError::NotFound(
2585 format!("case {case}"),
2586 ))),
2587 };
2588 };
2589 let run = self
2590 .seal_operator_record(
2591 RecordKind::HoldReleased {
2592 by: by.clone(),
2593 at,
2594 placed_by: standing.by.clone(),
2595 placed_at: standing.placed_at,
2596 },
2597 Some(case),
2598 HOLD_RELEASED_OUTCOME,
2599 None,
2600 )
2601 .await?;
2602 let removed = cases
2603 .release_hold_if(case, &standing)
2604 .await
2605 .map_err(|e| removal_failed(run, &e))?;
2606 if !removed
2607 && cases
2608 .hold(case)
2609 .await
2610 .map_err(|e| removal_failed(run, &e))?
2611 .is_some()
2612 {
2613 return Err(RuntimeError::ControlStands {
2614 run: run.to_string(),
2615 detail: format!(
2616 "the hold on case {case} was placed again after this release read it, \
2617 and the new hold still stands"
2618 ),
2619 });
2620 }
2621 Ok(Some(ControlLifted {
2622 record: run,
2623 removed,
2624 }))
2625 }
2626
2627 /// Every recorded halt lift, newest first: the one record of each run.
2628 ///
2629 /// # Errors
2630 ///
2631 /// If the journal is unreachable, or a lift run does not hold a lift record.
2632 pub async fn lifted_halts(&self, limit: usize) -> Result<Vec<Record>, RuntimeError> {
2633 self.operator_records(HALT_LIFTED_OUTCOME, limit).await
2634 }
2635
2636 /// Every recorded hold release, newest first: the one record of each run.
2637 ///
2638 /// # Errors
2639 ///
2640 /// If the journal is unreachable, or a release run does not hold a release
2641 /// record.
2642 pub async fn released_holds(&self, limit: usize) -> Result<Vec<Record>, RuntimeError> {
2643 self.operator_records(HOLD_RELEASED_OUTCOME, limit).await
2644 }
2645
2646 /// The first record of each run sealed under `outcome`, in the index's
2647 /// newest-first order. A run whose first record is not the act its outcome
2648 /// names is a store error, never skipped: a history that silently thins is
2649 /// one that reads as complete.
2650 async fn operator_records(
2651 &self,
2652 outcome: &str,
2653 limit: usize,
2654 ) -> Result<Vec<Record>, RuntimeError> {
2655 let runs = self
2656 .store
2657 .runs_by_outcome(outcome, limit)
2658 .await
2659 .map_err(RuntimeError::from_store)?;
2660 let mut found = Vec::with_capacity(runs.len());
2661 for run in runs {
2662 let first = self
2663 .store
2664 .read_page(run, 1, 1)
2665 .await
2666 .map_err(RuntimeError::from_store)?
2667 .into_iter()
2668 .next();
2669 match first {
2670 Some(record)
2671 if matches!(
2672 (outcome, record.kind()),
2673 (HALT_LIFTED_OUTCOME, RecordKind::HaltLifted { .. })
2674 | (HOLD_RELEASED_OUTCOME, RecordKind::HoldReleased { .. })
2675 ) =>
2676 {
2677 found.push(record);
2678 }
2679 _ => {
2680 return Err(RuntimeError::Store(crate::core::StoreError::Corrupt {
2681 seq: 1,
2682 detail: format!(
2683 "run {run} is sealed as '{outcome}' but holds no such record"
2684 ),
2685 }));
2686 }
2687 }
2688 }
2689 Ok(found)
2690 }
2691
2692 /// The quota store, when one is wired.
2693 pub(crate) fn quota_store_if_wired(&self) -> Option<&Arc<dyn crate::quota::QuotaStore>> {
2694 self.quotas.as_ref()
2695 }
2696
2697 /// Every run holding spend against this tenant's periods, and how much.
2698 ///
2699 /// The runs a [`QuotaError::SpentOut`](crate::quota::QuotaError::SpentOut)
2700 /// refusal counts as *reserved*. A run holds its reservation until it
2701 /// concludes, so the ones that stopped waiting on a person — quarantined,
2702 /// exhausted, withheld, failed — hold it until that person acts;
2703 /// [`attention`](Self::attention) names those.
2704 ///
2705 /// # Errors
2706 ///
2707 /// If no quota store is wired, or it is unreachable.
2708 pub async fn reservations(
2709 &self,
2710 limit: usize,
2711 ) -> Result<Vec<crate::quota::Held>, RuntimeError> {
2712 let quotas = self.quotas.as_ref().ok_or_else(|| {
2713 RuntimeError::Store(crate::core::StoreError::Backend(
2714 "no quota store is wired, so no run holds spend against a period".to_owned(),
2715 ))
2716 })?;
2717 quotas
2718 .reservations(limit)
2719 .await
2720 .map_err(RuntimeError::Store)
2721 }
2722
2723 /// The quota store, or the refusal that says why an emergency stop needs one.
2724 fn quota_store(&self) -> Result<&std::sync::Arc<dyn crate::quota::QuotaStore>, RuntimeError> {
2725 self.quotas.as_ref().ok_or_else(|| {
2726 RuntimeError::Store(crate::core::StoreError::Backend(
2727 "an emergency stop needs a quota store to keep the flag in — an \
2728 in-process one is forgotten by a restart and never seen by a \
2729 second instance"
2730 .to_owned(),
2731 ))
2732 })
2733 }
2734
2735 /// Every standing emergency stop on this tenant, and why.
2736 ///
2737 /// The operator's question — *what is stopped right now?* — which a
2738 /// per-scope lookup cannot answer without already knowing what to ask
2739 /// about.
2740 ///
2741 /// # Errors
2742 ///
2743 /// If no quota store is wired, or the store is unreachable.
2744 pub async fn halts(&self) -> Result<Vec<crate::quota::Halt>, RuntimeError> {
2745 let quotas = self.quotas.as_ref().ok_or_else(|| {
2746 RuntimeError::Store(crate::core::StoreError::Backend(
2747 "no quota store is wired, so no emergency stop can be set or read".to_owned(),
2748 ))
2749 })?;
2750 quotas.halts().await.map_err(RuntimeError::Store)
2751 }
2752
2753 /// Which runs are executing **right now**, and under whose authority.
2754 ///
2755 /// [`runs_by_outcome`](crate::journal::JournalStore::runs_by_outcome)
2756 /// answers from the journal's index, which only *concluded* runs are in; the
2757 /// live set is the tenant's admission slots.
2758 ///
2759 /// **The join is the point, not the ids.** An incident is rarely *cancel
2760 /// this run* but a revoked credential or a bad revision, so each entry
2761 /// carries the declared agent and the delegation subject it was admitted
2762 /// under.
2763 ///
2764 /// A slot whose lease has lapsed is marked `stranded` rather than omitted:
2765 /// the recovery sweep resumes those, and cancelling one undoes work that was
2766 /// about to continue.
2767 ///
2768 /// Bounded by `limit`, and reads a page of each run's header records to
2769 /// attribute it — an operator question rather than a hot path.
2770 ///
2771 /// # Errors
2772 ///
2773 /// If no quota store is wired, or either store is unreachable.
2774 pub async fn running_runs(&self, limit: usize) -> Result<Vec<LiveRun>, RuntimeError> {
2775 let quotas = self.quotas.as_ref().ok_or_else(|| {
2776 RuntimeError::Store(crate::core::StoreError::Backend(
2777 "no quota store is wired, so nothing tracks which runs hold this \
2778 tenant's admission slots"
2779 .to_owned(),
2780 ))
2781 })?;
2782 let live = quotas
2783 .running_runs(limit)
2784 .await
2785 .map_err(RuntimeError::Store)?;
2786 // One read, not one per run: the stranded set is small and the whole of
2787 // it is needed to answer any entry.
2788 let stranded: std::collections::BTreeSet<RunId> = self
2789 .store
2790 .abandoned_runs(limit)
2791 .await
2792 .map_err(RuntimeError::Store)?
2793 .into_iter()
2794 .collect();
2795
2796 let mut out = Vec::with_capacity(live.len());
2797 for run in live {
2798 // The header records only. `RunAdmitted` is the first and
2799 // `IdentityBound` follows it when the caller presented a chain, so a
2800 // short page carries both without reading a run's whole history —
2801 // which on a long-running agent is the difference between an
2802 // operator listing and a scan.
2803 let head = self
2804 .store
2805 .read_page(run, 1, ATTRIBUTION_RECORDS)
2806 .await
2807 .map_err(RuntimeError::Store)?;
2808 let governed_by = head.iter().find_map(|r| match r.kind() {
2809 crate::journal::RecordKind::RunAdmitted { governed_by, .. } => {
2810 governed_by.as_ref().map(|id| (**id).clone())
2811 }
2812 _ => None,
2813 });
2814 let subject = head.iter().find_map(|r| match r.kind() {
2815 crate::journal::RecordKind::IdentityBound { chain } => {
2816 chain.first().map(|p| p.id.clone())
2817 }
2818 _ => None,
2819 });
2820 out.push(LiveRun {
2821 stranded: stranded.contains(&run),
2822 run,
2823 governed_by,
2824 subject,
2825 });
2826 }
2827 Ok(out)
2828 }
2829
2830 /// The runs that are waiting, and what each one waits for.
2831 ///
2832 /// The third listing, beside concluded runs and running ones: a suspended
2833 /// run has no outcome and holds no admission slot, so neither of the others
2834 /// names it. Re-arming these is the recovery runbook's last step, and until
2835 /// this existed that step named a verb with no argument an operator could
2836 /// obtain.
2837 ///
2838 /// Soonest due first, bounded by `limit`. Straight through to the store's
2839 /// maintained index — no page of records is read, because what a run waits
2840 /// for is on the suspension itself.
2841 ///
2842 /// # Errors
2843 ///
2844 /// If the store is unreachable.
2845 pub async fn waiting_runs(
2846 &self,
2847 limit: usize,
2848 ) -> Result<Vec<crate::journal::WaitingRun>, RuntimeError> {
2849 self.store
2850 .waiting_runs(limit)
2851 .await
2852 .map_err(RuntimeError::from_store)
2853 }
2854
2855 /// Stop this instance taking on work, then wait for what it is already
2856 /// doing.
2857 ///
2858 /// A process killed mid-step leaves an announced effect with no terminal
2859 /// record, so its declared [`Recovery`](crate::core::Recovery) has to
2860 /// decide whether it happened — the right answer to an accident, and an
2861 /// expensive one to schedule on every deploy.
2862 ///
2863 /// Admission closes **first**: every later [`start`](Self::start) is
2864 /// refused
2865 /// [`RuntimeError::Draining`](crate::core::RuntimeError::Draining), leaving
2866 /// no lease, no slot and no journal. Then this waits up to `grace` for the
2867 /// runs it put on a background task to reach a journaled resting point.
2868 /// Closing first is what makes the answer worth having — a wait alone
2869 /// reports about an instant — and it is also the only signal a long-lived
2870 /// read gets, so an open A2A subscription ends rather than holding a
2871 /// graceful shutdown for as long as the run it watches.
2872 ///
2873 /// **Run it beside a server's own graceful shutdown, not after it** —
2874 /// `tokio::join!` of the two. Serialising deadlocks one on the other in
2875 /// whichever order you pick: a run awaited inside a request handler is
2876 /// finished by the server, and an open subscription is ended by this.
2877 ///
2878 /// Three things it deliberately does not do. It does not wait for runs on
2879 /// somebody else's task; those belong to that caller's own shutdown. It
2880 /// does not gate resumes or an agent commissioning another one — both
2881 /// continue work this plane already owns, and a commission is a step of a
2882 /// run this is waiting for. And it does not hand back the leases of runs it
2883 /// did not finish: a release says takeover is safe, and for a run still
2884 /// inside `perform` the next owner would re-perform the call beside the one
2885 /// in flight. Left to expire, the lease is the signal the recovery sweep
2886 /// reads as *an instance died holding this run* — which is what a
2887 /// drained-out run is.
2888 ///
2889 /// Idempotent: draining an instance that is already draining waits again.
2890 pub async fn drain(&self, grace: std::time::Duration) -> super::DrainReport {
2891 self.inflight.drain(grace).await
2892 }
2893
2894 /// Whether this instance has stopped admitting runs.
2895 ///
2896 /// For an embedder whose server keeps its listener open while it drains and
2897 /// therefore needs something to answer a readiness probe with. The shipped
2898 /// server needs none: it closes its listeners first, so a probe that
2899 /// connects already fails — which is why this crate's HTTP surface has no
2900 /// `/health` route to flip. See [`crate::api`] on why it has none at all.
2901 #[must_use]
2902 pub fn is_draining(&self) -> bool {
2903 self.inflight.is_draining()
2904 }
2905
2906 /// The cancellation recorded against a run, if one was.
2907 ///
2908 /// Reads the journal rather than a flag: a cancellation is a record, so
2909 /// this answers the same thing after a restart and on another instance.
2910 ///
2911 /// # Errors
2912 ///
2913 /// If the store is unreachable.
2914 pub async fn cancellation(
2915 &self,
2916 run: RunId,
2917 ) -> Result<Option<crate::journal::Cancellation>, RuntimeError> {
2918 self.store
2919 .cancellation(run)
2920 .await
2921 .map_err(RuntimeError::from_store)
2922 }
2923
2924 /// The policy engine, if this runtime has one.
2925 ///
2926 /// Exposed because a surface that faces strangers has to be able to ask
2927 /// whether one exists at all. Inside the process the caller is the
2928 /// embedder's own code and an absent engine is a choice; on a socket it is
2929 /// a hole, and the HTTP surface refuses to start without one.
2930 #[must_use]
2931 pub fn policy(&self) -> Option<&Arc<dyn crate::core::PolicyEngine>> {
2932 self.policy.as_ref()
2933 }
2934
2935 /// Which tenant this plane runs as.
2936 #[must_use]
2937 pub fn tenant(&self) -> &crate::core::TenantId {
2938 &self.tenant
2939 }
2940
2941 /// What this plane's policy engine cannot evaluate for a served caller
2942 /// that presented no chain.
2943 ///
2944 /// The build probes the chain shapes this plane's own runs take: its own
2945 /// chain where it has one, and both shapes where it has none. A served
2946 /// surface adds the one the build cannot know about — a caller acting
2947 /// under no chain on a plane that has one — so every surface that admits
2948 /// work for a stranger asks this before it starts.
2949 #[cfg(any(feature = "a2a-server", feature = "mcp-server"))]
2950 pub(crate) fn served_policy_problems(&self) -> Vec<String> {
2951 match (&self.policy, &self.identity) {
2952 (Some(engine), Some(_)) => runtime_policy_problems(engine.as_ref(), None),
2953 _ => Vec::new(),
2954 }
2955 }
2956
2957 /// How far from its owner the plane's own chain is, or zero without one.
2958 ///
2959 /// The depth a run bound by the plane's chain but acting under none starts
2960 /// from: acting under no chain does not put a served caller closer to the
2961 /// owner than the plane that serves it.
2962 pub(crate) fn own_depth(&self) -> usize {
2963 self.identity
2964 .as_ref()
2965 .map_or(0, crate::core::Delegation::depth)
2966 }
2967
2968 /// The journal, under the name the batch driver reads it by.
2969 #[must_use]
2970 pub(crate) fn meter(&self) -> &super::metrics::Meter {
2971 &self.meter
2972 }
2973
2974 /// Write an operator act as the sole record of a sealed run of its own.
2975 ///
2976 /// The act, then its conclusion, then the seal — so it carries the chain,
2977 /// the signature, Merkle inclusion and the witness, and an offline reader
2978 /// checks it without being taught what the act was. `case` stamps the
2979 /// record onto a matter's history; `reason` is the conclusion's, for the
2980 /// one act whose reason lives there.
2981 async fn seal_operator_record(
2982 &self,
2983 kind: RecordKind,
2984 case: Option<crate::core::CaseId>,
2985 outcome: &str,
2986 reason: Option<&str>,
2987 ) -> Result<RunId, RuntimeError> {
2988 let run = RunId::generate();
2989 let epoch = BREAK_GLASS_EPOCH;
2990 let mut act = Append::new(run, kind);
2991 if let Some(case) = case {
2992 act = act.case(case);
2993 }
2994 self.store
2995 .append(epoch, vec![act])
2996 .await
2997 .map_err(RuntimeError::from_store)?;
2998 let head = self
2999 .store
3000 .head(run)
3001 .await
3002 .map_err(RuntimeError::from_store)?;
3003 self.store
3004 .append(
3005 epoch,
3006 vec![Append::new(
3007 run,
3008 RecordKind::RunConcluded {
3009 outcome: outcome.to_owned(),
3010 reason: reason.map(str::to_owned),
3011 exhaustion: None,
3012 live_spend: Spend::default(),
3013 chain_head: head.hash,
3014 },
3015 )],
3016 )
3017 .await
3018 .map_err(RuntimeError::from_store)?;
3019 self.store
3020 .seal(run, epoch, outcome)
3021 .await
3022 .map_err(RuntimeError::from_store)?;
3023 Ok(run)
3024 }
3025
3026 /// Record an operator deliberately crossing into this tenant, and return
3027 /// the run that holds the record.
3028 ///
3029 /// Every other tenancy control keeps a cross-tenant read from being reached
3030 /// by accident. This is the designed exception, and the rule for it is that an
3031 /// exception without a record is indistinguishable from the breach it is
3032 /// meant to be. So the access is written into **this** tenant's journal —
3033 /// the one whose data is about to be reached — in a sealed run of its own,
3034 /// exactly as a sweep writes its decisions. It therefore inherits the hash
3035 /// chain, the per-record signature and the Merkle inclusion, and the
3036 /// offline audit tool reports it without being taught that break-glass
3037 /// exists.
3038 ///
3039 /// **The record is written before any data is served, and a failure to
3040 /// write it is a failure to access.** That direction is the whole control:
3041 /// the alternative — serve first, record best-effort — is a break-glass
3042 /// that works exactly as well when its own evidence is lost.
3043 ///
3044 /// # Errors
3045 ///
3046 /// If `reason` is blank, or if the record cannot be written. An
3047 /// unexplained exception is the thing this record exists to prevent, so it
3048 /// is refused rather than stored empty.
3049 pub async fn record_break_glass(
3050 &self,
3051 actor: &crate::core::Operator,
3052 roles: &[String],
3053 reason: &str,
3054 ) -> Result<RunId, RuntimeError> {
3055 if reason.trim().is_empty() {
3056 return Err(RuntimeError::PlanContract(
3057 "break-glass needs a reason: an unexplained crossing of the tenant \
3058 boundary is what this record exists to prevent"
3059 .to_owned(),
3060 ));
3061 }
3062 let run = self
3063 .seal_operator_record(
3064 RecordKind::BreakGlass {
3065 actor: actor.clone(),
3066 roles: roles.to_vec(),
3067 reason: reason.to_owned(),
3068 },
3069 None,
3070 BREAK_GLASS_OUTCOME,
3071 Some(reason),
3072 )
3073 .await?;
3074 tracing::warn!(
3075 tenant = %self.tenant(),
3076 %actor,
3077 %run,
3078 reason,
3079 "break-glass: an operator crossed the tenant boundary"
3080 );
3081 Ok(run)
3082 }
3083
3084 /// The journal, for reads the runtime does not mediate.
3085 ///
3086 /// This is also the embedder's **resumable output stream**:
3087 /// [`JournalStore::read`] is a seq-cursored read over a run's records, so
3088 /// polling from the last sequence seen is a durable, reconnect-safe
3089 /// progress feed that any instance can serve — the exact mechanism the
3090 /// A2A streaming surface is built on. There is deliberately no curated
3091 /// event type between an embedder and the records: a third vocabulary
3092 /// beside the journal's and the wire's would be a second truth wearing
3093 /// ergonomics, and the records are already the history the stream must
3094 /// agree with.
3095 pub fn journal(&self) -> &Arc<dyn JournalStore> {
3096 &self.store
3097 }
3098
3099 /// Which case a run belongs to, or `None` if it belongs to none.
3100 ///
3101 /// Read from the **journal**, not from a column beside it. The binding is
3102 /// stamped on the run's own records at admission, so answering from there
3103 /// is answering from the plan of record — a case-store column saying the
3104 /// same thing would be a second copy of one fact, and the two could
3105 /// disagree about a run the case layer never saw.
3106 ///
3107 /// It is the first question an operator surface asks, which is why it is a
3108 /// method rather than a documented one-liner: every caller was otherwise
3109 /// going to reach for the first record and read `body.case` off it, and a
3110 /// caller who reached for the *last* one instead would still be right today
3111 /// and wrong the moment a run is admitted before its case is known.
3112 ///
3113 /// # Errors
3114 ///
3115 /// If the journal cannot be read. An unknown run is `Ok(None)` rather than
3116 /// an error: *no such run* and *a run in no case* are both honest answers
3117 /// to this question, and neither is a fault.
3118 pub async fn case_of(&self, run: RunId) -> Result<Option<crate::core::CaseId>, RuntimeError> {
3119 Ok(self
3120 .store
3121 .read(run, 1)
3122 .await
3123 .map_err(RuntimeError::from_store)?
3124 .first()
3125 .and_then(|record| record.body.case))
3126 }
3127
3128 #[must_use]
3129 /// The witness set and quorum this plane was built with, if any.
3130 pub(crate) fn witnessing(&self) -> Option<&Arc<Witnessing>> {
3131 self.witnesses.as_ref()
3132 }
3133
3134 pub fn store(&self) -> &Arc<dyn JournalStore> {
3135 &self.store
3136 }
3137
3138 /// This **process instance's** identity, as it appears in run leases.
3139 ///
3140 /// Not the agent's name: several instances of one agent are normal. See
3141 /// [`RuntimeBuilder::owner`].
3142 ///
3143 /// Public because "which instance holds this run" is a question an operator
3144 /// asks of a stuck system, and the answer is otherwise only in a store row.
3145 #[must_use]
3146 pub fn owner_id(&self) -> &str {
3147 &self.owner
3148 }
3149
3150 /// The input as the declared admission rules leave it: refused with no
3151 /// record, as an admission policy denial is, or with its sensitivity
3152 /// raised to every matched classification — which `RunAdmitted` then
3153 /// records as the input's label.
3154 #[cfg(feature = "manifest")]
3155 fn admit_content(
3156 &self,
3157 agent: &str,
3158 input: Tainted<Value>,
3159 ) -> Result<Tainted<Value>, RuntimeError> {
3160 let Some(manifest) = self
3161 .resolve(agent)
3162 .ok()
3163 .and_then(|skill| self.governing(skill.as_ref()))
3164 else {
3165 return Ok(input);
3166 };
3167 let outcome = super::ctx::judged(
3168 manifest.spec.security.content.as_ref(),
3169 crate::content::At::Admission,
3170 input.peek(),
3171 )
3172 .map_err(RuntimeError::PolicyDenied)?;
3173 if let Some(hit) = outcome.refused.first() {
3174 return Err(RuntimeError::PolicyDenied(
3175 crate::core::PolicyError::Content {
3176 rule: hit.rule.clone(),
3177 pointer: hit.pointer.clone(),
3178 },
3179 ));
3180 }
3181 let raised = outcome.raise(input.label().sensitivity);
3182 if raised == input.label().sensitivity {
3183 return Ok(input);
3184 }
3185 Ok(input.raised_to(raised))
3186 }
3187
3188 #[cfg(not(feature = "manifest"))]
3189 #[allow(clippy::unused_self, clippy::unnecessary_wraps)]
3190 fn admit_content(
3191 &self,
3192 _agent: &str,
3193 input: Tainted<Value>,
3194 ) -> Result<Tainted<Value>, RuntimeError> {
3195 Ok(input)
3196 }
3197
3198 /// The content rules' verdict on an event delivered to a suspended run,
3199 /// under the declaration governing the run's skill; `None` where no rule
3200 /// matched or none is declared.
3201 #[cfg(feature = "manifest")]
3202 fn delivered_verdict(
3203 &self,
3204 history: &[crate::journal::Record],
3205 payload: &serde_json::Value,
3206 ) -> Option<crate::core::ContentVerdict> {
3207 let capability = history.iter().find_map(|r| match r.kind() {
3208 RecordKind::RunAdmitted { capability, .. } => Some(capability.clone()),
3209 _ => None,
3210 })?;
3211 let manifest = self.governing(self.resolve(&capability).ok()?.as_ref())?;
3212 super::ctx::verdict(super::ctx::judged(
3213 manifest.spec.security.content.as_ref(),
3214 crate::content::At::Source(super::ctx::AWAIT_KIND),
3215 payload,
3216 ))
3217 }
3218
3219 #[cfg(not(feature = "manifest"))]
3220 #[allow(clippy::unused_self)]
3221 fn delivered_verdict(
3222 &self,
3223 _history: &[crate::journal::Record],
3224 _payload: &serde_json::Value,
3225 ) -> Option<crate::core::ContentVerdict> {
3226 None
3227 }
3228
3229 /// The declaration governing a skill, if its agent has one.
3230 ///
3231 /// Per skill, not per plane: a runtime runs several agents, and a step must
3232 /// be judged against the manifest of the agent whose skill it is. Looking up
3233 /// one plane-wide manifest would apply another agent's ceilings.
3234 #[cfg(feature = "manifest")]
3235 fn governing(&self, skill: &dyn Skill) -> Option<Arc<crate::manifest::Manifest>> {
3236 self.governed_by.get(&skill.descriptor().name).cloned()
3237 }
3238
3239 /// Renew this run's lease until the returned guard is dropped.
3240 ///
3241 /// Requires a Tokio runtime, as the rest of this crate's timing does.
3242 fn heartbeat(&self, run: RunId, epoch: crate::core::Epoch) -> Heartbeat {
3243 let store = Arc::clone(&self.store);
3244 let owner = self.owner.clone();
3245 let ttl = self.lease_ttl;
3246 // A third of the TTL, so two renewals can be lost to a slow store before
3247 // the lease lapses. Renewing *at* the TTL would mean any hesitation
3248 // leaves it expired, and an expired lease is one anybody may take —
3249 // including, per `acquire`, this caller, which would fence the run with
3250 // its own heartbeat.
3251 let period = ttl / 3;
3252 Heartbeat(tokio::spawn(async move {
3253 loop {
3254 tokio::time::sleep(period).await;
3255 // `renew`, never `acquire`. A renewal extends only a lease
3256 // still held by exactly this `(owner, epoch)`; it cannot
3257 // claim. That distinction is load-bearing here: this task
3258 // races the run's own conclusion, and an acquire arriving
3259 // just after `conclude` released the lease would re-take it —
3260 // a live, never-released lease over a concluded run, which
3261 // the recovery sweep then "recovers" forever.
3262 //
3263 // Stop only when the store says the lease is gone — released
3264 // by the conclusion, lapsed and reclaimed, or taken over: the
3265 // run's next append will be fenced, which is the correct
3266 // outcome, and claiming now would only resurrect ownership
3267 // this instance no longer has. Any other failure is the store
3268 // not answering, and giving up then would let a run that is
3269 // still executing lapse into somebody else's recovery.
3270 match store.renew(run, &owner, epoch, ttl).await {
3271 Ok(_) => {}
3272 Err(crate::core::StoreError::LeaseNotHeld { .. }) => return,
3273 Err(error) => tracing::warn!(
3274 target: "agentplane::lease",
3275 run = %run,
3276 error = %error,
3277 "a lease renewal failed; the next one retries"
3278 ),
3279 }
3280 }
3281 }))
3282 }
3283
3284 /// The standing withdrawal that reaches `identity`, if one does.
3285 ///
3286 /// The one predicate the step boundary and a subject-bound hop both ask,
3287 /// so the two cannot disagree about whose authority is withdrawn. Read on
3288 /// a live pass only — a halt thrown today must not decide what a run did
3289 /// last year — and an unreachable store propagates, because carrying on
3290 /// would leave a withdrawal unenforced exactly while accounting is down.
3291 pub(crate) async fn withdrawal_against(
3292 &self,
3293 identity: Option<&crate::core::Delegation>,
3294 ) -> Result<Option<Withdrawal>, crate::core::StoreError> {
3295 let (Some(quotas), Some(chain)) = (self.quotas.as_ref(), identity) else {
3296 return Ok(None);
3297 };
3298 let halts = quotas.halts().await?;
3299 Ok(halts.into_iter().find_map(|halt| {
3300 let subject = halt.scope.withdraws_from(chain)?.to_owned();
3301 Some(Withdrawal {
3302 subject,
3303 reason: halt.reason,
3304 by: halt.by,
3305 })
3306 }))
3307 }
3308
3309 /// Stop a run because the authority it acts under was withdrawn.
3310 ///
3311 /// A **pause**: `stop` is reached with a status `maybe_unwind` passes
3312 /// straight through, so the run's mutations stand and it stays resumable.
3313 /// That is the treatment exhaustion gets and for the same reason — the run
3314 /// did nothing wrong, and reversing correct work because a credential lapsed
3315 /// would be a second incident.
3316 ///
3317 /// `record` is false on a resume that finds the withdrawal still standing:
3318 /// the withholding already on the chain says so, and a second one per resume
3319 /// attempt would turn a queue of retries into a queue of records.
3320 #[allow(clippy::too_many_arguments)]
3321 async fn withhold(
3322 &self,
3323 cx: Unwind<'_>,
3324 withdrawal: Withdrawal,
3325 record: bool,
3326 completed: &[(StepId, Capability)],
3327 outputs: &BTreeMap<StepId, Tainted<Value>>,
3328 cursor: &mut ReplayCursor,
3329 case_id: Option<crate::core::CaseId>,
3330 quota: &QuotaPass,
3331 ) -> Result<RunOutcome, RuntimeError> {
3332 let Withdrawal {
3333 subject,
3334 reason,
3335 by,
3336 } = withdrawal;
3337 if record {
3338 let stamped = (cx.stamp)(Append::new(
3339 cx.run,
3340 RecordKind::AuthorityWithheld {
3341 subject: subject.clone(),
3342 reason: reason.clone(),
3343 by: by.clone(),
3344 },
3345 ));
3346 self.store
3347 .append(cx.epoch, vec![stamped])
3348 .await
3349 .map_err(RuntimeError::from_store)?;
3350 }
3351 self.stop(
3352 cx,
3353 RunStatus::Withheld { subject, reason },
3354 completed,
3355 outputs,
3356 cursor,
3357 case_id,
3358 quota,
3359 )
3360 .await
3361 }
3362
3363 /// Refuse a run whose tenant is at a ceiling.
3364 ///
3365 /// Fails **closed**: an unreachable quota store refuses rather than admits,
3366 /// because a ceiling that yields when its accounting is down is a ceiling an
3367 /// attacker removes by taking the accounting down.
3368 ///
3369 /// Live admission only, and replay and resume are skipped for **different**
3370 /// reasons. Re-checking during replay would let a run that happened produce
3371 /// a different history when it is re-read: a ceiling crossed since admission
3372 /// would rewrite the past into a refusal. Resume is not that — it decides
3373 /// only what happens next — and it is ungated because refusing it strands a
3374 /// run mid-saga with its reversals unrun. What that costs the ceiling is
3375 /// stated where the ceiling is defined ([`crate::quota`]); it is a real cost
3376 /// rather than an absent one.
3377 async fn check_quota(
3378 &self,
3379 run: RunId,
3380 governed_by: Option<&crate::journal::AgentIdentity>,
3381 chain: Option<&crate::core::Delegation>,
3382 at: crate::core::Timestamp,
3383 (budget, width): (&Budget, usize),
3384 ) -> Result<QuotaPass, RuntimeError> {
3385 let Some(quotas) = self.quotas.as_ref() else {
3386 return Ok(QuotaPass::disabled());
3387 };
3388 let pass = QuotaPass::at(&self.quota, at, true).sized_for(width);
3389
3390 // The halt is checked **before** the unlimited shortcut, because an
3391 // emergency stop is not a ceiling and a tenant with no ceilings is
3392 // exactly the one an operator is most likely to need to stop. Reading it
3393 // fails closed for the same reason the ceilings do: a switch that yields
3394 // when its store is unreachable is a switch an attacker throws by taking
3395 // the store down.
3396 //
3397 // Every standing halt in one read, and the **narrowest match wins the
3398 // message**: an operator who stopped one agent and whoever is refused
3399 // are looking for different sentences, and "the tenant is halted" told
3400 // to the caller of agent 12 sends them to the wrong incident. Ordering
3401 // the scopes puts `Subject` before `Revision` before `Agent` before
3402 // `Tenant`, so the reason reported is the most specific one that
3403 // actually covers this run — and a withdrawn authority outranks every
3404 // workload scope, because it is the answer that tells the caller their
3405 // credential is the problem rather than the agent.
3406 match quotas.halts().await {
3407 Ok(halts) => {
3408 if let Some(halt) = halts
3409 .into_iter()
3410 .filter(|halt| halt.scope.covers(governed_by, chain))
3411 .max_by(|a, b| a.scope.cmp(&b.scope))
3412 {
3413 return Err(RuntimeError::QuotaExceeded(
3414 crate::quota::QuotaError::Halted {
3415 tenant: self.tenant.as_str().to_owned(),
3416 scope: halt.scope,
3417 reason: halt.reason,
3418 },
3419 ));
3420 }
3421 }
3422 Err(e) => {
3423 return Err(RuntimeError::QuotaExceeded(
3424 crate::quota::QuotaError::Unavailable(e.to_string()),
3425 ));
3426 }
3427 }
3428
3429 // What the run can cost at most, held against the period in the same
3430 // store transaction that checks the ceiling and takes the slot. A run
3431 // that leaves a bounded unit unbounded is refused here: admitted, it
3432 // would hold nothing against a period it can spend without limit.
3433 let hold = match crate::quota::reservation(&self.quota, budget, width as u64) {
3434 Ok(amount) => {
3435 amount
3436 .zip(pass.period())
3437 .map(|(amount, period)| crate::quota::SpendHold {
3438 period: period.to_owned(),
3439 amount,
3440 })
3441 }
3442 Err(field) => {
3443 return Err(RuntimeError::QuotaExceeded(
3444 crate::quota::QuotaError::Unbounded {
3445 tenant: self.tenant.as_str().to_owned(),
3446 unit: if field.contains("minor") {
3447 "money"
3448 } else {
3449 "tokens"
3450 },
3451 field,
3452 },
3453 ));
3454 }
3455 };
3456
3457 quotas
3458 .reserve(run, &self.quota, hold.as_ref(), at)
3459 .await
3460 .map_err(RuntimeError::QuotaExceeded)?;
3461 Ok(pass)
3462 }
3463
3464 /// Atomically settle this pass's spend and admission slot.
3465 ///
3466 /// Required before physical sealing and lease release. A failure keeps the
3467 /// lease owned so the abandonment sweep retries from journal evidence; a
3468 /// best-effort write here would turn a transient store outage into permanent
3469 /// under-accounting. The period is the one captured when this pass started,
3470 /// never recomputed here.
3471 async fn settle_quota(
3472 &self,
3473 run: RunId,
3474 epoch: crate::core::Epoch,
3475 spend: Spend,
3476 pass: &QuotaPass,
3477 concludes: bool,
3478 ) -> Result<(), RuntimeError> {
3479 let Some(quotas) = self.quotas.as_ref() else {
3480 return Ok(());
3481 };
3482 let pending = |error: crate::core::StoreError| RuntimeError::QuotaSettlementPending {
3483 run: run.to_string(),
3484 epoch,
3485 detail: error.to_string(),
3486 };
3487 if !pass.enabled {
3488 // A conclusion no pass opened — an abandonment decided while the
3489 // run was stopped — still ends the run, and a run that has ended
3490 // holds nothing against the period.
3491 if concludes {
3492 quotas.release(run).await.map_err(pending)?;
3493 }
3494 return Ok(());
3495 }
3496 let settlement = pass
3497 .settlement(run, epoch, spend, concludes)
3498 .expect("an enabled quota pass has a settlement");
3499 quotas.settle(&settlement).await.map_err(pending)
3500 }
3501
3502 /// Release a reservation for an admission whose journal never existed.
3503 pub(crate) async fn release_empty_quota_reservation(
3504 &self,
3505 run: RunId,
3506 ) -> Result<(), crate::core::StoreError> {
3507 match self.quotas.as_ref() {
3508 Some(quotas) => quotas.release(run).await,
3509 None => Ok(()),
3510 }
3511 }
3512
3513 /// Settle and release the admission slots sealed runs still hold.
3514 ///
3515 /// A plane with no quota store may seal a run whose admission holds a
3516 /// slot in this ledger, and no resume or recovery selects a sealed run
3517 /// again. Settlement is idempotent, so a slot another path already settled
3518 /// is not billed twice. Returns how many slots were given back.
3519 pub(crate) async fn release_sealed_slots(&self, limit: usize) -> Result<usize, RuntimeError> {
3520 let Some(quotas) = self.quotas.as_ref() else {
3521 return Ok(0);
3522 };
3523 let held = quotas
3524 .running_runs(limit)
3525 .await
3526 .map_err(RuntimeError::Store)?;
3527 let positions = self
3528 .store
3529 .log_positions(&held)
3530 .await
3531 .map_err(RuntimeError::from_store)?;
3532 let mut released = 0;
3533 for (run, sealed) in held.into_iter().zip(positions) {
3534 if sealed.is_none() {
3535 continue;
3536 }
3537 let records = self
3538 .store
3539 .read(run, 1)
3540 .await
3541 .map_err(RuntimeError::from_store)?;
3542 match self.settle_recorded_quota_passes(run, &records).await {
3543 Ok(()) => released += 1,
3544 Err(error) => tracing::warn!(
3545 %run,
3546 %error,
3547 "a sealed run's admission slot could not be released; the next sweep retries"
3548 ),
3549 }
3550 }
3551 Ok(released)
3552 }
3553
3554 /// Seal the lease-free runs whose sealing conclusion is written and whose
3555 /// seal is not.
3556 ///
3557 /// Each [`LEASE_FREE_OUTCOMES`] writer appends its conclusion and then
3558 /// seals in a second call, and holds no lease, so recovery never selects
3559 /// a run whose writer died between the two. The conclusion alone already
3560 /// lists the run under its outcome, so a history reads it as done while
3561 /// the audit finds it in no log. The conclusion is what was decided; the
3562 /// seal adds nothing to it but the log position, so finishing it here
3563 /// writes no decision the writer did not make.
3564 ///
3565 /// Looks at the newest `limit` runs of each outcome. Returns how many it
3566 /// sealed.
3567 pub(crate) async fn seal_unsealed_conclusions(
3568 &self,
3569 limit: usize,
3570 ) -> Result<usize, RuntimeError> {
3571 let mut sealed = 0;
3572 for &outcome in LEASE_FREE_OUTCOMES {
3573 let runs = self
3574 .store
3575 .runs_by_outcome(outcome, limit)
3576 .await
3577 .map_err(RuntimeError::from_store)?;
3578 let positions = self
3579 .store
3580 .log_positions(&runs)
3581 .await
3582 .map_err(RuntimeError::from_store)?;
3583 for (run, position) in runs.into_iter().zip(positions) {
3584 if position.is_some() {
3585 continue;
3586 }
3587 match self.store.seal(run, LEASE_FREE_EPOCH, outcome).await {
3588 Ok(_) => sealed += 1,
3589 Err(error) => tracing::warn!(
3590 %run,
3591 %error,
3592 "a concluded run's seal could not be finished; the next sweep retries"
3593 ),
3594 }
3595 }
3596 }
3597 Ok(sealed)
3598 }
3599
3600 /// Settle every quota pass present in history, idempotently.
3601 ///
3602 /// Reached before a resume can dispatch. A normally settled pass is a cheap
3603 /// receipt lookup; a pass whose process died is accrued here. Conclusions
3604 /// carry the exact in-memory live total, including a known outcome whose
3605 /// terminal effect record failed before a later append recovered. An
3606 /// unconcluded crash has only durable terminal effect records to sum.
3607 async fn settle_recorded_quota_passes(
3608 &self,
3609 run: RunId,
3610 records: &[Record],
3611 ) -> Result<(), RuntimeError> {
3612 // A plane with no ledger resumes a run only when no pass of it accrued
3613 // spend to a period, which would otherwise go unbilled. It cannot give
3614 // back the concurrency slot of an admission whose process died
3615 // mid-pass; the ledger's own plane does, on its next sweep once the
3616 // run is sealed.
3617 let Some(quotas) = self.quotas.as_ref() else {
3618 if records.iter().any(|record| {
3619 matches!(
3620 record.kind(),
3621 RecordKind::QuotaPassStarted {
3622 period: Some(_),
3623 ..
3624 }
3625 )
3626 }) {
3627 return Err(RuntimeError::PlanContract(
3628 "this run has passes that accrued spend to a quota period, and the resuming \
3629 plane has no quota store to settle them"
3630 .to_owned(),
3631 ));
3632 }
3633 return Ok(());
3634 };
3635
3636 for record in records {
3637 let RecordKind::QuotaPassStarted {
3638 period,
3639 release_slot,
3640 } = record.kind()
3641 else {
3642 continue;
3643 };
3644 let epoch = record.body.epoch;
3645 let spend = records
3646 .iter()
3647 .rev()
3648 .find_map(|candidate| (candidate.body.epoch == epoch).then_some(candidate.kind()))
3649 .and_then(|kind| match kind {
3650 RecordKind::RunConcluded { live_spend, .. } => Some(*live_spend),
3651 _ => None,
3652 })
3653 .unwrap_or_else(|| spend_recorded_in_epoch(records, epoch));
3654 quotas
3655 .settle(&crate::quota::QuotaSettlement {
3656 run,
3657 epoch,
3658 period: period.clone(),
3659 spend,
3660 release_slot: *release_slot,
3661 concludes: concluded_in(records, epoch),
3662 })
3663 .await
3664 .map_err(|error| RuntimeError::QuotaSettlementPending {
3665 run: run.to_string(),
3666 epoch,
3667 detail: error.to_string(),
3668 })?;
3669 }
3670 // A run whose history has ended holds nothing against the period,
3671 // whichever path ended it — including one no pass marker opened.
3672 if records.iter().any(is_sealing_conclusion) {
3673 quotas
3674 .release(run)
3675 .await
3676 .map_err(|error| RuntimeError::QuotaSettlementPending {
3677 run: run.to_string(),
3678 epoch: records.last().map_or(0, |r| r.body.epoch),
3679 detail: error.to_string(),
3680 })?;
3681 }
3682 Ok(())
3683 }
3684
3685 /// Repair quota receipts, and finish a conclusion an owner died after.
3686 ///
3687 /// Under recovery, a run whose last record is a conclusion lost its owner
3688 /// between concluding and handing the lease back. What remains is the seal and the release, never the business
3689 /// work: recovery is not permission to retry a failure, with or without a
3690 /// quota store. A run that crashed before concluding continues through
3691 /// replay.
3692 async fn recover_quota_before_resume(
3693 &self,
3694 run: RunId,
3695 records: &[Record],
3696 lease: &crate::journal::Lease,
3697 recovering: bool,
3698 ) -> Result<Option<RunOutcome>, RuntimeError> {
3699 self.settle_recorded_quota_passes(run, records).await?;
3700 if !recovering {
3701 return Ok(None);
3702 }
3703 let Some(last) = records.last() else {
3704 return Ok(None);
3705 };
3706 let RecordKind::RunConcluded {
3707 outcome,
3708 reason,
3709 exhaustion,
3710 ..
3711 } = last.kind()
3712 else {
3713 return Ok(None);
3714 };
3715
3716 let status = recorded_status(outcome, reason.as_deref(), exhaustion.as_ref(), records);
3717 let chain_head = if SEALED_OUTCOMES.contains(&outcome.as_str()) {
3718 let head = self
3719 .store
3720 .seal(run, lease.epoch, outcome)
3721 .await
3722 .map_err(RuntimeError::from_store)?;
3723 // A repaired seal is a conclusion like any other.
3724 self.retire_waits(run).await;
3725 head
3726 } else {
3727 self.store
3728 .head(run)
3729 .await
3730 .map_err(RuntimeError::from_store)?
3731 .hash
3732 };
3733 Ok(Some(RunOutcome {
3734 run_id: run,
3735 status,
3736 chain_head,
3737 consumed: Consumed::default(),
3738 output: None,
3739 }))
3740 }
3741
3742 /// Return a closed run without continuing it, repairing a missing physical
3743 /// seal when a crash landed after the conclusion append.
3744 async fn closed_resume_outcome(
3745 &self,
3746 run: RunId,
3747 records: &[Record],
3748 lease: &crate::journal::Lease,
3749 ) -> Result<Option<RunOutcome>, RuntimeError> {
3750 let Some(status) = resume_is_closed(records) else {
3751 return Ok(None);
3752 };
3753 let outcome = recorded_conclusion(records).unwrap_or_else(|| status.as_str().to_owned());
3754 let chain_head = if SEALED_OUTCOMES.contains(&outcome.as_str()) {
3755 let head = self
3756 .store
3757 .seal(run, lease.epoch, &outcome)
3758 .await
3759 .map_err(RuntimeError::from_store)?;
3760 // The crash this repairs may have landed before the conclusion
3761 // retired the run's waits; retiring twice is harmless.
3762 self.retire_waits(run).await;
3763 head
3764 } else {
3765 self.store
3766 .head(run)
3767 .await
3768 .map_err(RuntimeError::from_store)?
3769 .hash
3770 };
3771 Ok(Some(RunOutcome {
3772 run_id: run,
3773 status,
3774 chain_head,
3775 output: None,
3776 consumed: Consumed::default(),
3777 }))
3778 }
3779
3780 /// Finish an abandonment a decision asked for and a crash interrupted.
3781 ///
3782 /// The instruction is journaled before it is acted on, so this is what
3783 /// makes the ending arrive whatever happened in between — and it is the
3784 /// same path the verb itself takes, rather than a repair beside it.
3785 ///
3786 /// Nothing is unwound and no step runs: the run seals holding whatever it
3787 /// left in the world, which is the whole content of the decision.
3788 async fn abandon_if_decided(
3789 &self,
3790 run: RunId,
3791 records: &[Record],
3792 lease: &crate::journal::Lease,
3793 ) -> Result<Option<RunOutcome>, RuntimeError> {
3794 let Some(status) = pending_abandonment(records) else {
3795 return Ok(None);
3796 };
3797 self.conclude(
3798 run,
3799 lease.epoch,
3800 status,
3801 None,
3802 true,
3803 recorded_case_id(records),
3804 Consumed::default(),
3805 Spend::ZERO,
3806 &QuotaPass::disabled(),
3807 )
3808 .await
3809 .map(Some)
3810 }
3811
3812 /// This plane, writing through a journal that puts `quota`'s marker on
3813 /// the pass's first append — so a pass that writes nothing leaves no
3814 /// marker. `None` for a pass with no marker.
3815 fn marking_pass(&self, run: RunId, quota: &QuotaPass) -> Option<Self> {
3816 let marker = quota.started()?;
3817 Some(Self {
3818 store: Arc::new(super::pass_journal::PassJournal::new(
3819 Arc::clone(&self.store),
3820 run,
3821 marker,
3822 )),
3823 ..self.clone()
3824 })
3825 }
3826
3827 /// Open the quota identity of the live tail a resume is about to execute.
3828 ///
3829 /// Its marker is not written here: it rides the pass's first append (see
3830 /// [`super::pass_journal`]), so a resume that writes nothing leaves none.
3831 async fn start_replay_quota_pass(
3832 &self,
3833 run: RunId,
3834 mode: Pass,
3835 width: usize,
3836 ) -> Result<QuotaPass, RuntimeError> {
3837 let (Pass::Resume, Some(quotas)) = (mode, self.quotas.as_ref()) else {
3838 return Ok(QuotaPass::disabled());
3839 };
3840 let pass = QuotaPass::at(&self.quota, now_for_admission(), false).sized_for(width);
3841 // The run spends in the period it resumes in, so what it still holds
3842 // moves there — never refused, because a resume is not.
3843 if let Some(period) = pass.period() {
3844 quotas
3845 .carry(run, period)
3846 .await
3847 .map_err(RuntimeError::Store)?;
3848 }
3849 Ok(pass)
3850 }
3851
3852 /// The ceilings a run gets: its agent's, or the plane's if it has no agent.
3853 ///
3854 /// Per agent, because that is who declared them. A plane-wide budget would
3855 /// let one agent's generosity bound another's runs, which is the whole
3856 /// reason a declaration belongs to an identity rather than to a process.
3857 fn budget_for(&self, target: &str) -> Budget {
3858 #[cfg(feature = "manifest")]
3859 if let Ok(skill) = self.resolve(target)
3860 && let Some(m) = self.governing(skill.as_ref())
3861 {
3862 return m.budget();
3863 }
3864 let _ = target;
3865 self.budget
3866 }
3867
3868 /// Which declaration governs runs of this capability, if a declared agent
3869 /// does.
3870 ///
3871 /// Resolved the same way the budget is, and for the same reason: a run is
3872 /// governed by the agent that answers its entry capability. A commissioned
3873 /// sub-run opens its own run and records its own governor, so every run in a
3874 /// room has exactly one — there is no case where this has to pick.
3875 ///
3876 /// A manifest that cannot produce a digest is one that could not have been
3877 /// published; recording `None` would claim the run was ungoverned, so the
3878 /// failure is surfaced as an absent identity rather than a false one.
3879 #[cfg(feature = "manifest")]
3880 fn identity_for(&self, target: &str) -> Option<crate::journal::AgentIdentity> {
3881 let skill = self.resolve(target).ok()?;
3882 self.identity_of(self.governing(skill.as_ref())?.as_ref())
3883 }
3884
3885 /// What the agent answering `capability` may do, as its declaration says.
3886 ///
3887 /// `None` when nothing on this plane answers it; a reach with no
3888 /// declaration when something does and no declaration governs it.
3889 #[cfg(feature = "manifest")]
3890 pub(crate) fn reach_of(&self, capability: &str) -> Option<crate::core::Reach> {
3891 let skill = self.resolve(capability).ok()?;
3892 let declaration = self.governing(skill.as_ref()).and_then(|manifest| {
3893 let digest = manifest.digest().ok()?;
3894 Some(super::reach::declared(&manifest, digest, |server| {
3895 self.peers.as_ref().is_some_and(|wiring| {
3896 wiring
3897 .registry
3898 .grant(&crate::peers::PeerId::new(server))
3899 .is_some()
3900 })
3901 }))
3902 });
3903 Some(crate::core::Reach {
3904 capability: capability.to_owned(),
3905 declaration,
3906 })
3907 }
3908
3909 /// The declaration's identity, as every gate reports it.
3910 ///
3911 /// One construction, because admission and each effect gate must name the
3912 /// same revision the same way: two spellings of *which agent is acting*
3913 /// would let a rule permitted at the door mean something else at the sink.
3914 #[cfg(feature = "manifest")]
3915 fn identity_of(&self, m: &crate::manifest::Manifest) -> Option<crate::journal::AgentIdentity> {
3916 Some(crate::journal::AgentIdentity {
3917 name: m.metadata.name.clone(),
3918 version: m.metadata.version.clone(),
3919 digest: m.digest().ok()?,
3920 publisher: self.published_by.get(&m.metadata.name).cloned(),
3921 })
3922 }
3923
3924 /// Whether anything on this plane answers `target`.
3925 ///
3926 /// For a catalogue built before the first call: a surface that advertises a
3927 /// capability nothing provides offers a caller — often a model — a verb
3928 /// that fails at admission every time, and the failure names the plane
3929 /// rather than the catalogue that lied about it.
3930 ///
3931 /// Gated on the feature that builds a catalogue, so a build without one
3932 /// answers *this question is not asked here* rather than carrying a method
3933 /// nothing reaches.
3934 #[cfg(feature = "mcp-server")]
3935 pub(crate) fn provides(&self, target: &str) -> bool {
3936 self.resolve(target).is_ok()
3937 }
3938
3939 /// Whether a grant's server component names a registered A2A peer — a
3940 /// call the run may wait on.
3941 #[cfg(feature = "mcp-server")]
3942 pub(crate) fn wires_peer(&self, server: &str) -> bool {
3943 self.peers.as_ref().is_some_and(|wiring| {
3944 wiring
3945 .registry
3946 .grant(&crate::peers::PeerId::new(server))
3947 .is_some()
3948 })
3949 }
3950
3951 fn resolve(&self, target: &str) -> Result<Arc<dyn Skill>, RuntimeError> {
3952 if let Some(s) = self.skills.get(target) {
3953 return Ok(Arc::clone(s));
3954 }
3955 let cap = Capability::new(target);
3956 if let Some(name) = self.by_capability.get(&cap)
3957 && let Some(s) = self.skills.get(name)
3958 {
3959 return Ok(Arc::clone(s));
3960 }
3961 // Capabilities rather than skill names: `run` is documented to take the
3962 // capability, and listing the names would answer a question nobody
3963 // asked with vocabulary that does not work in the call that failed.
3964 let mut available: Vec<String> = self
3965 .by_capability
3966 .keys()
3967 .map(std::string::ToString::to_string)
3968 .collect();
3969 available.sort();
3970 Err(RuntimeError::NoProvider {
3971 target: target.to_owned(),
3972 available,
3973 })
3974 }
3975
3976 /// Start a run: a capability a skill answers, or a plan.
3977 ///
3978 /// The one door. What the run is bound to — a case, a caller's chain, a
3979 /// pinned declaration — goes in [`RunRequest::terms`]; whether it runs at
3980 /// most once goes in [`RunRequest::once`], which changes what the answer
3981 /// is; and the terminal says whether to wait for it.
3982 ///
3983 /// ```no_run
3984 /// # use std::sync::Arc;
3985 /// # use agentplane::core::Tainted;
3986 /// # use agentplane::runtime::{Runtime, RunTerms};
3987 /// # async fn f(rt: Arc<Runtime>, input: Tainted<serde_json::Value>) -> Result<(), agentplane::RuntimeError> {
3988 /// let outcome = rt.start("order.settle", input.clone()).run().await?;
3989 /// let admission = rt
3990 /// .start("order.settle", input)
3991 /// .terms(RunTerms::default().correlated("order", &[]))
3992 /// .once("billing:msg-7")
3993 /// .run()
3994 /// .await?;
3995 /// # Ok(()) }
3996 /// ```
3997 ///
3998 /// The input arrives **labelled**, and that is the whole of the decision.
3999 /// [`Tainted::trusted`](crate::core::Tainted::trusted) is what an operator
4000 /// writes for a literal, a constant or a configuration value they vouch for;
4001 /// anything that came from outside — an inbound event, a queue message, a
4002 /// counterparty's payload — keeps the label it arrived with. A bare value
4003 /// admitted as trusted would satisfy `require_trusted` fields with
4004 /// attacker-chosen data and give the egress ceiling nothing to join with.
4005 pub fn start(
4006 &self,
4007 target: impl Into<Target>,
4008 input: Tainted<Value>,
4009 ) -> RunRequest<'_, Unkeyed> {
4010 RunRequest {
4011 runtime: self,
4012 target: target.into(),
4013 input,
4014 terms: RunTerms::default(),
4015 key: Unkeyed,
4016 }
4017 }
4018
4019 /// This plane as the handle a background run keeps.
4020 ///
4021 /// Every `Runtime` is built inside an `Arc` that holds `self_ref`, and a
4022 /// borrow of it is a borrow of that `Arc`, so the upgrade cannot fail
4023 /// while `&self` exists.
4024 fn owned(&self) -> Arc<Self> {
4025 self.self_ref
4026 .upgrade()
4027 .expect("a runtime is only reachable through the Arc its builder made")
4028 }
4029
4030 /// Admit and execute, blocking until the run rests.
4031 async fn admit_target(
4032 &self,
4033 target: Target,
4034 input: Tainted<Value>,
4035 terms: RunTerms,
4036 ) -> Result<RunOutcome, RuntimeError> {
4037 match target {
4038 Target::Capability(capability) => self.admit(&capability, input, terms).await,
4039 Target::Plan(plan) => self.admit_plan(plan, input, terms).await,
4040 }
4041 }
4042
4043 /// Admit, and let the run proceed in the background.
4044 async fn spawn_target(
4045 self: &Arc<Self>,
4046 target: Target,
4047 input: Tainted<Value>,
4048 terms: RunTerms,
4049 ) -> Result<RunId, RuntimeError> {
4050 // Resolved before the id is minted: an unknown capability is the
4051 // caller's mistake and must be an error, not a run that exists and
4052 // immediately fails. A plan is validated inside admission, which also
4053 // precedes the id becoming visible.
4054 let plan = match target {
4055 Target::Capability(capability) => {
4056 let skill = self.resolve(&capability)?;
4057 PlanIR::single(first_capability(&skill.descriptor()))
4058 }
4059 Target::Plan(plan) => plan,
4060 };
4061
4062 let run = RunId::generate();
4063 // Registered before admission, not after. Admission is several store
4064 // writes, and a drain beginning inside them would take its snapshot
4065 // without this run, report a complete stop, and let the process exit
4066 // while the run was still being admitted. Registered first, both orders
4067 // work out: a drain that starts after this waits for the run, and one
4068 // that starts before it is seen by the gate inside `admit_only`, which
4069 // refuses and drops the ticket on the way out.
4070 let ticket = self.inflight.enter(run);
4071 let admitted = self
4072 .admit_only(run, plan, input, terms, Entry::Outside)
4073 .await?;
4074
4075 let plane = Arc::clone(self);
4076 // In the caller's span, as a blocking run is: a background run's
4077 // trace is still the trace of whatever started it.
4078 tokio::spawn(
4079 async move {
4080 let _ticket = ticket;
4081 plane.execute_admitted(admitted).await
4082 }
4083 .in_current_span(),
4084 );
4085 Ok(run)
4086 }
4087
4088 /// Admission under a key: the one copy, for blocking and background,
4089 /// capability and plan.
4090 ///
4091 /// Two gates, and only the second arbitrates. The read below is an
4092 /// optimisation — it saves a redelivery the cost of a policy evaluation, a
4093 /// quota reservation, a lease and a correlation lookup — and settles
4094 /// nothing, since two instances racing both see nothing here. The
4095 /// constraint inside `append` decides, on the same division
4096 /// `CaseStore::correlate_or_open` runs on.
4097 async fn claimed<T, F>(&self, key: &str, admit: F) -> Result<Claimed<T>, RuntimeError>
4098 where
4099 F: Future<Output = Result<T, RuntimeError>>,
4100 {
4101 validate_admission_key(key)?;
4102 if let Some(held) = self.holder_of(key).await? {
4103 return Ok(Claimed::Held(held));
4104 }
4105 match admit.await {
4106 Ok(fresh) => Ok(Claimed::Fresh(fresh)),
4107 Err(RuntimeError::Store(crate::core::StoreError::DuplicateAdmission {
4108 run, ..
4109 })) => Ok(Claimed::Held(parse_holder(&run)?)),
4110 Err(e) => Err(e),
4111 }
4112 }
4113
4114 /// Wait until `run` rests and answer with its outcome — the value
4115 /// re-derived by strict replay when it concluded.
4116 ///
4117 /// A run whose lease is free is resumed here: its holder has gone. One
4118 /// whose lease is held is followed on the journal until it rests or the
4119 /// lease lapses, so a holder that dies mid-wait is taken over rather than
4120 /// waited for forever.
4121 async fn settled(&self, run: RunId) -> Result<RunOutcome, RuntimeError> {
4122 loop {
4123 if let Some(rested) = self.recorded_outcome(run).await? {
4124 return self.with_output(rested).await;
4125 }
4126 // Subscribed before the attempt, so an append between the two
4127 // still wakes the wait below.
4128 let mut follower = self.store.watch(run).await;
4129 match self.replay(run, Mode::Resume).await {
4130 Err(RuntimeError::LeaseHeld { remaining_secs, .. }) => {
4131 let lapse = Duration::from_secs(remaining_secs.max(1));
4132 match follower.as_mut() {
4133 Some(follower) => {
4134 let _ = tokio::time::timeout(lapse, follower.changed()).await;
4135 }
4136 None => tokio::time::sleep(lapse).await,
4137 }
4138 }
4139 resumed => return resumed,
4140 }
4141 }
4142 }
4143
4144 /// A rested run's outcome with its value: strict replay re-derives what a
4145 /// concluded run answered, performing nothing. A suspension has no answer
4146 /// yet, and is returned as recorded.
4147 async fn with_output(&self, rested: RunOutcome) -> Result<RunOutcome, RuntimeError> {
4148 if rested.status == RunStatus::Succeeded {
4149 self.replay(rested.run_id, Mode::Strict).await
4150 } else {
4151 Ok(rested)
4152 }
4153 }
4154
4155 /// Which run holds this admission key, if any.
4156 async fn holder_of(&self, key: &str) -> Result<Option<RunId>, RuntimeError> {
4157 self.store
4158 .admitted_as(key)
4159 .await
4160 .map_err(RuntimeError::from_store)
4161 }
4162
4163 /// Answer a duplicate with what the original run has on the record.
4164 async fn answer_with(&self, held: RunId) -> Result<Admission, RuntimeError> {
4165 Ok(match self.recorded_outcome(held).await? {
4166 Some(outcome) => Admission::Replayed(outcome),
4167 None => Admission::InFlight(held),
4168 })
4169 }
4170
4171 /// What a run has recorded, **without resuming it**.
4172 ///
4173 /// `None` while the run is still working — it has records but has reached
4174 /// neither a conclusion nor a suspension.
4175 ///
4176 /// No lease is taken and nothing is appended. The original is frequently
4177 /// live when its redelivery lands, and a read that acquired a lease would
4178 /// fence a healthy run mid-step in order to report on it.
4179 ///
4180 /// The **last** record decides, never any record: a run that suspended, was
4181 /// resumed and carried on has a suspension in its history and is not
4182 /// suspended now. Same rule as the A2A status projection, so the two
4183 /// surfaces cannot disagree about one run.
4184 ///
4185 /// # Errors
4186 ///
4187 /// If the journal cannot be read.
4188 pub async fn recorded_outcome(&self, run: RunId) -> Result<Option<RunOutcome>, RuntimeError> {
4189 let records = self
4190 .store
4191 .read(run, 1)
4192 .await
4193 .map_err(RuntimeError::from_store)?;
4194 let Some(last) = records.last() else {
4195 return Ok(None);
4196 };
4197 let Some(status) = observed_status(&records) else {
4198 return Ok(None);
4199 };
4200 Ok(Some(RunOutcome {
4201 run_id: run,
4202 status,
4203 chain_head: last.hash,
4204 // Reading a record performs nothing, so it consumes nothing — the
4205 // same answer the closed-run path gives, and for the same reason:
4206 // the spend belongs to the pass that did the work.
4207 consumed: Consumed::default(),
4208 // A step's result is reconstructed by replay rather than stored, so
4209 // there is none to hand back here. See `Admission::Replayed`.
4210 output: None,
4211 }))
4212 }
4213
4214 /// Rebuild a run's case binding **from its history**, never by re-deriving.
4215 ///
4216 /// Two facts are read back rather than recomputed, and each has its own
4217 /// failure if it is not.
4218 ///
4219 /// The **case id**, because re-correlating could land on a different case if
4220 /// the keys were since released — silently rewriting which business fact the
4221 /// run belongs to.
4222 ///
4223 /// The **business keys**, because a case accumulates them over months. A run
4224 /// that resolved `memory.formation.subject: $correlation/meter` would
4225 /// otherwise resolve it against a set the live run never saw, write a second
4226 /// memory under a second subject, and produce a history that disagrees with
4227 /// itself with nothing on the record explaining why.
4228 fn recorded_case(&self, records: &[Record]) -> Option<CaseContext> {
4229 let (case_id, correlation) = records.iter().find_map(|r| match r.kind() {
4230 RecordKind::CaseBound { correlation, .. } => {
4231 r.body.case.map(|case_id| (case_id, correlation.clone()))
4232 }
4233 _ => None,
4234 })?;
4235 self.cases.as_ref().map(|cases| CaseContext {
4236 cases: Arc::clone(cases),
4237 tasks: self.tasks.clone(),
4238 events: self.events.clone(),
4239 calendar: Arc::clone(&self.calendar),
4240 case_id,
4241 correlation,
4242 })
4243 }
4244
4245 /// The contract this runtime enforces on every plan.
4246 pub(crate) fn contract(&self) -> crate::plan::Contract {
4247 let contract = crate::plan::Contract::new(self.by_capability.keys().cloned());
4248 if self.require_verifier {
4249 contract.require_verifier()
4250 } else {
4251 contract
4252 }
4253 }
4254
4255 async fn admit(
4256 &self,
4257 target: &str,
4258 input: Tainted<Value>,
4259 terms: RunTerms,
4260 ) -> Result<RunOutcome, RuntimeError> {
4261 // A bare target is the degenerate plan: one node, terminal.
4262 let skill = self.resolve(target)?;
4263 let capability = first_capability(&skill.descriptor());
4264 self.admit_plan(PlanIR::single(capability), input, terms)
4265 .await
4266 }
4267
4268 async fn admit_plan(
4269 &self,
4270 plan: PlanIR,
4271 input: Tainted<Value>,
4272 terms: RunTerms,
4273 ) -> Result<RunOutcome, RuntimeError> {
4274 self.admit_plan_as(RunId::generate(), plan, input, terms, Entry::Outside)
4275 .await
4276 }
4277
4278 /// Admit the run one already-executing run ordered from another agent.
4279 ///
4280 /// Apart from every other entry point for one reason: a commission is a
4281 /// *step* of a run this process is executing, so the drain gate must not
4282 /// see it. A drain that refused here would fail the commissioning step of
4283 /// exactly the runs it is waiting for — and that step's failure is
4284 /// `Interrupted`, the in-doubt classification, because a commissioning
4285 /// caller cannot know what the agent it ordered from managed to do. The
4286 /// drain would manufacture the state it exists to prevent.
4287 pub(crate) async fn commission_run(
4288 &self,
4289 target: &str,
4290 input: Tainted<Value>,
4291 terms: RunTerms,
4292 ) -> Result<RunOutcome, RuntimeError> {
4293 let skill = self.resolve(target)?;
4294 let capability = first_capability(&skill.descriptor());
4295 self.admit_plan_as(
4296 RunId::generate(),
4297 PlanIR::single(capability),
4298 input,
4299 terms,
4300 Entry::Nested,
4301 )
4302 .await
4303 }
4304
4305 /// Record the chain this run acts under, beside the plan it authorizes.
4306 ///
4307 /// The two are read back together: a chain without its plan says nothing
4308 /// about what it was allowed to do, and a plan without its chain says
4309 /// nothing about who was allowed to run it.
4310 fn bind_identity(
4311 run: RunId,
4312 chain: Option<&crate::core::Delegation>,
4313 records: &mut Vec<Append>,
4314 ) {
4315 if let Some(chain) = chain {
4316 records.push(Append::new(
4317 run,
4318 RecordKind::IdentityBound {
4319 chain: chain.links().cloned().collect(),
4320 },
4321 ));
4322 }
4323 }
4324
4325 /// Check the plan against the delegation chain's authority.
4326 ///
4327 /// The plan is the authorization graph, so this is where authority belongs:
4328 /// a plan that names a capability outside the chain's scope must never
4329 /// start, rather than failing at whichever step happens to reach it first.
4330 /// Checking here also makes the refusal deterministic — it depends only on
4331 /// the frozen plan and the chain, both of which are recorded.
4332 ///
4333 /// The chain's other two bounds are checked beside it, and only here: a
4334 /// chain issued for another plane or past its validity is refused before
4335 /// the run exists, and a run admitted under a live chain is never re-judged
4336 /// — replay reads the recorded chain back.
4337 ///
4338 /// `acts_under` is whether the run acts under `ceiling` or is only bounded
4339 /// by it. A refusal names the chain's subject only in the first case: a
4340 /// caller bounded by the plane's chain is not its subject, and a denial
4341 /// naming the operator would say the operator was refused.
4342 fn authorize_scope(
4343 &self,
4344 plan: &PlanIR,
4345 ceiling: Option<&crate::core::Delegation>,
4346 acts_under: bool,
4347 ) -> Result<(), RuntimeError> {
4348 let Some(chain) = ceiling else {
4349 return Ok(());
4350 };
4351 chain.admissible(self.tenant.as_str(), now_for_admission())?;
4352 let scope = chain.effective_scope();
4353 for node in &plan.nodes {
4354 if !scope.permits(&node.capability) {
4355 return Err(RuntimeError::PolicyDenied(
4356 crate::core::PolicyError::Denied {
4357 principal: if acts_under {
4358 chain.subject().id.clone()
4359 } else {
4360 node.capability.to_string()
4361 },
4362 action: crate::core::ACTION_ADMIT.to_owned(),
4363 resource: node.capability.to_string(),
4364 },
4365 ));
4366 }
4367 }
4368 Ok(())
4369 }
4370
4371 /// Authorize starting a run, before the run exists.
4372 ///
4373 /// A denial here leaves no journal at all, which is correct: nothing
4374 /// happened, and a run record for something that was never allowed to start
4375 /// would be a run nobody can explain.
4376 fn authorize_admission(
4377 &self,
4378 capability: &str,
4379 governed_by: Option<&crate::journal::AgentIdentity>,
4380 chain: Option<&crate::core::Delegation>,
4381 input: &Value,
4382 ) -> Result<(), RuntimeError> {
4383 let Some(engine) = self.policy.as_ref() else {
4384 return Ok(());
4385 };
4386 // Who is acting, and what is being asked for. Passing one string as both
4387 // made every admission rule a tautology — `principal == resource` cannot
4388 // express "this agent may not run that capability", which is the whole
4389 // question at admission on a plane hosting several agents.
4390 //
4391 // The principal is an **authenticated** identity or it is nothing. The
4392 // scope check a few lines above already denies under the delegation
4393 // subject, so anything else here would give one refused run two answers
4394 // to "who was refused" in the same error type.
4395 //
4396 // The agent's `metadata.name` is deliberately *not* used, tempting as it
4397 // is. A name is self-asserted — a manifest is a file, and its name is
4398 // whatever the author typed — so a rule granting authority to a name
4399 // grants it to any file claiming that name. Worse, a fallback would be
4400 // *silent*: `principal == X` would mean an authenticated identity in a
4401 // deployment with a delegation chain and a self-asserted label in one
4402 // without, so the same rule would change meaning with the wiring.
4403 //
4404 // Rules that need to bind to the agent bind to `context.agent.digest`,
4405 // which is content-addressed and pins what the declaration actually
4406 // said — so "this exact declaration may admit, and an edited one may
4407 // not" is expressible. The capability is the fallback principal because
4408 // it claims nothing: it is what was asked for, not who asked.
4409 //
4410 // Built by the function `policy check` also calls, from the values
4411 // `RunAdmitted` and `IdentityBound` record.
4412 let request = crate::policy::requests::admission(
4413 &crate::policy::requests::Acting {
4414 tenant: self.tenant.as_str(),
4415 capability,
4416 agent: governed_by,
4417 chain,
4418 },
4419 input,
4420 );
4421 // Every refusal, not only a rule's. `let … else` over `Deny` alone
4422 // admits the run for every other decision, so a policy set that cannot
4423 // be evaluated would open the door it exists to hold — the fail-open a
4424 // widened vocabulary creates for free unless the permit is the arm that
4425 // has to be named.
4426 let decision = engine.authorize(&request.as_request());
4427 let malformed = decision.is_malformed();
4428 let Some(reason) = decision.reason().map(ToOwned::to_owned) else {
4429 return Ok(());
4430 };
4431
4432 tracing::error!(
4433 target: telemetry::POLICY_DENIED,
4434 action = crate::core::ACTION_ADMIT,
4435 resource = %capability,
4436 policy_error = malformed,
4437 %reason,
4438 );
4439 self.meter
4440 .count(metrics::POLICY_DENIALS, crate::core::ACTION_ADMIT);
4441 Err(RuntimeError::PolicyDenied(
4442 crate::core::PolicyError::Denied {
4443 principal: request.principal,
4444 action: crate::core::ACTION_ADMIT.to_owned(),
4445 resource: capability.to_owned(),
4446 },
4447 ))
4448 }
4449
4450 /// The record that opens a run.
4451 ///
4452 /// Its own function because the label matters: journaled rather than
4453 /// recomputed, so a replay reaches the same verdict at every taint gate as
4454 /// the run it reproduces.
4455 fn admission(
4456 &self,
4457 capability: &str,
4458 governed_by: Option<crate::journal::AgentIdentity>,
4459 input: &Tainted<Value>,
4460 idempotency_key: Option<String>,
4461 admitted_by: Option<String>,
4462 acting_as: &Acting,
4463 ) -> RecordKind {
4464 RecordKind::RunAdmitted {
4465 capability: capability.to_owned(),
4466 governed_by: governed_by.map(Box::new),
4467 input: input.peek().clone(),
4468 input_label: input.label().clone(),
4469 input_field_labels: input
4470 .field_labels()
4471 .map(|(path, label)| (path.to_owned(), label.clone()))
4472 .collect(),
4473 policy_bundle: self.policy.as_ref().map(|p| Box::new(p.bundle())),
4474 canon: crate::core::canon::VERSION,
4475 idempotency_key,
4476 admitted_by,
4477 served_unchained: matches!(acting_as, Acting::Nobody),
4478 plane_chain: match acting_as {
4479 Acting::Plane => self.identity.is_some(),
4480 Acting::PlaneDelegate(_) => true,
4481 Acting::Caller(_) | Acting::Nobody => false,
4482 },
4483 }
4484 }
4485
4486 /// Everything up to and including the admission records, and no work.
4487 ///
4488 /// The seam a non-blocking submit needs. Admission is what makes a run
4489 /// *exist*: the policy gate, the lease, and the records that say what is
4490 /// about to happen. Splitting there means a caller can be told the run was
4491 /// accepted — or refused — before any of it runs, and a refusal stays an
4492 /// immediate answer rather than becoming a task that silently never appears.
4493 #[allow(clippy::too_many_lines)]
4494 async fn admit_only(
4495 &self,
4496 run: RunId,
4497 plan: PlanIR,
4498 input: Tainted<Value>,
4499 terms: RunTerms,
4500 entry: Entry,
4501 ) -> Result<Admitted, RuntimeError> {
4502 // First, and before the plan is even read: a draining instance is going
4503 // away, and everything below this line takes on work it may not be able
4504 // to finish. Refusing here leaves no lease, no quota slot and no
4505 // journal — the same nothing a policy denial leaves.
4506 if entry == Entry::Outside && self.inflight.is_draining() {
4507 return Err(RuntimeError::Draining);
4508 }
4509 crate::plan::validate(&plan, &self.contract())
4510 .map_err(|e| RuntimeError::PlanContract(e.to_string()))?;
4511
4512 let agent = plan
4513 .nodes
4514 .first()
4515 .map_or_else(|| "plan".to_owned(), |n| n.capability.to_string());
4516
4517 // Resolved once: the gate and the record must agree about which
4518 // declaration governs this run, and computing it twice is how they
4519 // start disagreeing.
4520 #[cfg(feature = "manifest")]
4521 let governed_by = self.identity_for(&agent);
4522 #[cfg(not(feature = "manifest"))]
4523 let governed_by: Option<crate::journal::AgentIdentity> = None;
4524
4525 if let Some(expected) = terms.expected_declaration {
4526 let found = governed_by.as_ref().map(|id| id.digest);
4527 if found != Some(expected) {
4528 return Err(RuntimeError::DeclarationPinMismatch {
4529 agent,
4530 expected,
4531 found,
4532 });
4533 }
4534 }
4535
4536 // The caller's chain where one was presented, the plane's otherwise —
4537 // one choice, made once, so the gate, the policy context and the
4538 // record cannot disagree about whom this run acts for.
4539 let chain = terms.acting_as.resolve(self.identity.as_ref());
4540 self.authorize_scope(
4541 &plan,
4542 terms.acting_as.ceiling(self.identity.as_ref()),
4543 chain.is_some(),
4544 )?;
4545 self.authorize_admission(&agent, governed_by.as_ref(), chain, input.peek())?;
4546 let input = self.admit_content(&agent, input)?;
4547
4548 self.admit_reserved(run, plan, input, terms, agent, governed_by)
4549 .await
4550 }
4551
4552 /// Admission past the quota reservation: the lease, the records, the case
4553 /// binding, the outbox.
4554 ///
4555 /// Every error path hands the lease back. A lease left standing over a
4556 /// failed admission is not idle bookkeeping: it expires unreleased, which
4557 /// is the exact signature the recovery sweep reads as "an instance died
4558 /// holding this run" — so a half-admitted run (records appended, outbox
4559 /// registration failed) would be *executed* by the sweeper a TTL later,
4560 /// having never been admitted. The one error that concludes rather than
4561 /// merely unwinding is the post-append one, where records already exist:
4562 /// the run is concluded `failed` in its own chain first, so nothing later
4563 /// reads an open journal as work in progress.
4564 async fn admit_reserved(
4565 &self,
4566 run: RunId,
4567 plan: PlanIR,
4568 input: Tainted<Value>,
4569 terms: RunTerms,
4570 agent: String,
4571 governed_by: Option<crate::journal::AgentIdentity>,
4572 ) -> Result<Admitted, RuntimeError> {
4573 // Ownership precedes reservation. A process can die between any two
4574 // store calls; with this order, a slot reserved before the admission
4575 // batch always has an owned lease beside it, and the abandonment sweep
4576 // can release both. Reserving first could strand a slot over no run and
4577 // no lease — invisible to every recovery query.
4578 let lease = self
4579 .store
4580 .acquire(run, &self.owner, self.lease_ttl)
4581 .await
4582 .map_err(RuntimeError::from_store)?;
4583
4584 let budget = self.budget_for(&agent);
4585 // Boxed: the quota decision holds its store round trips across awaits,
4586 // and inlined it pushes every admission future past the size clippy
4587 // allows on the served surfaces that await one.
4588 let quota = match Box::pin(self.check_quota(
4589 run,
4590 governed_by.as_ref(),
4591 // **The chain this run acts under, not the plane's.** A served
4592 // surface admits each run under its caller's chain, so reading
4593 // the plane's here would check a withdrawal against the
4594 // operator's own subject and never match the caller's — a
4595 // refusal that does not happen, which leaves no trace anywhere.
4596 terms.acting_as.resolve(self.identity.as_ref()),
4597 now_for_admission(),
4598 (&budget, reserved_width(&budget, &plan)),
4599 ))
4600 .await
4601 {
4602 Ok(pass) => pass,
4603 Err(error) => {
4604 let _ = self.store.release_lease(run, lease.epoch).await;
4605 return Err(error);
4606 }
4607 };
4608
4609 let out = self
4610 .admit_under_lease(run, &lease, plan, input, terms, &agent, governed_by, quota)
4611 .await;
4612 if out.is_err() {
4613 if let Some(quotas) = self.quotas.as_ref()
4614 && let Err(error) = quotas.release(run).await
4615 {
4616 tracing::debug!(%run, %error, "could not release the quota slot of a failed admission");
4617 }
4618 if let Err(error) = self.store.release_lease(run, lease.epoch).await {
4619 tracing::debug!(
4620 %run,
4621 %error,
4622 "could not release the lease of a failed admission; it will expire on its own"
4623 );
4624 }
4625 }
4626 out
4627 }
4628
4629 #[allow(clippy::too_many_arguments, clippy::too_many_lines)]
4630 async fn admit_under_lease(
4631 &self,
4632 run: RunId,
4633 lease: &crate::journal::Lease,
4634 plan: PlanIR,
4635 input: Tainted<Value>,
4636 terms: RunTerms,
4637 agent: &str,
4638 governed_by: Option<crate::journal::AgentIdentity>,
4639 quota: QuotaPass,
4640 ) -> Result<Admitted, RuntimeError> {
4641 let RunTerms {
4642 case,
4643 idempotency_key,
4644 acting_as,
4645 admitted_by,
4646 expected_declaration: _,
4647 subjects,
4648 } = terms;
4649 let chain = acting_as.resolve(self.identity.as_ref());
4650 let mut records = vec![
4651 Append::new(
4652 run,
4653 self.admission(
4654 agent,
4655 governed_by,
4656 &input,
4657 idempotency_key,
4658 admitted_by,
4659 &acting_as,
4660 ),
4661 ),
4662 // From here the plan is an authorization graph: compiled from
4663 // trusted input, frozen before anything untrusted was read, and
4664 // recorded so the journal that follows can be checked against it.
4665 Append::new(
4666 run,
4667 RecordKind::PlanFrozen {
4668 steps: plan
4669 .nodes
4670 .iter()
4671 .map(|n| n.capability.to_string())
4672 .collect(),
4673 plan: serde_json::to_value(&plan)?,
4674 },
4675 ),
4676 ];
4677
4678 Self::bind_identity(run, chain, &mut records);
4679 if let Some(started) = quota.started() {
4680 records.push(Append::new(run, started));
4681 }
4682
4683 #[cfg(feature = "manifest")]
4684 let declared: Vec<crate::journal::SubjectBinding> = self
4685 .resolve(agent)
4686 .ok()
4687 .and_then(|skill| self.governing(skill.as_ref()))
4688 .map(|m| {
4689 m.spec
4690 .data_subjects
4691 .iter()
4692 .map(crate::manifest::DataSubject::binding)
4693 .collect()
4694 })
4695 .unwrap_or_default();
4696 #[cfg(not(feature = "manifest"))]
4697 let declared: Vec<crate::journal::SubjectBinding> = Vec::new();
4698 // Before correlation, because `correlate_or_open` commits a case: a
4699 // binding that refuses the run must refuse it before anything outside
4700 // the journal names it. `$case` alone needs the case, and the case is
4701 // bound whenever the run asked for one, so it cannot refuse later.
4702 let bound = bind_subjects(&declared, subjects, &input, case.is_some())?;
4703
4704 // Correlation is deterministic and runs before planning: which case a
4705 // message belongs to is a matter of fact, settled by a lookup.
4706 let case_ctx = match (case, self.cases.as_ref()) {
4707 (Some(CaseBinding::Correlate { kind, keys }), Some(cases)) => {
4708 let correlation = cases
4709 .correlate_or_open(&kind, &keys, now_for_admission())
4710 .await
4711 .map_err(RuntimeError::from_store)?;
4712 let case_id = correlation.case_id();
4713 // The case's own keys, not the ones this message carried. A run
4714 // that *joined* an existing case may have been admitted with one
4715 // key while the case holds three, and a binding naming any of
4716 // them must resolve — which is the difference between "the keys
4717 // in this request" and "the business facts this matter is
4718 // identified by".
4719 let bound_keys = case_correlation(cases.as_ref(), case_id, &keys).await?;
4720 // Stamp the case on the records already queued as well: every
4721 // record of a case-bound run carries its case, which is what
4722 // makes "show me everything about this matter" one range scan.
4723 for r in &mut records {
4724 r.case = Some(case_id);
4725 }
4726 records.push(
4727 Append::new(
4728 run,
4729 RecordKind::CaseBound {
4730 case_kind: kind,
4731 opened: correlation.is_new(),
4732 correlation: bound_keys.clone(),
4733 },
4734 )
4735 .case(case_id),
4736 );
4737 Some(CaseContext {
4738 cases: Arc::clone(cases),
4739 tasks: self.tasks.clone(),
4740 events: self.events.clone(),
4741 calendar: Arc::clone(&self.calendar),
4742 case_id,
4743 correlation: bound_keys,
4744 })
4745 }
4746 (Some(CaseBinding::Existing(case_id)), Some(cases)) => {
4747 let existing = cases
4748 .case(case_id)
4749 .await
4750 .map_err(RuntimeError::from_store)?
4751 .ok_or_else(|| {
4752 RuntimeError::PlanContract(format!("no such case: {case_id}"))
4753 })?;
4754 if existing.status.is_closed() {
4755 return Err(RuntimeError::PlanContract(format!(
4756 "case '{case_id}' is closed and cannot accept another run"
4757 )));
4758 }
4759 for record in &mut records {
4760 record.case = Some(case_id);
4761 }
4762 records.push(
4763 Append::new(
4764 run,
4765 RecordKind::CaseBound {
4766 case_kind: existing.kind,
4767 opened: false,
4768 correlation: existing.correlation.clone(),
4769 },
4770 )
4771 .case(case_id),
4772 );
4773 Some(CaseContext {
4774 cases: Arc::clone(cases),
4775 tasks: self.tasks.clone(),
4776 events: self.events.clone(),
4777 calendar: Arc::clone(&self.calendar),
4778 case_id,
4779 correlation: existing.correlation,
4780 })
4781 }
4782 (Some(_), None) => {
4783 return Err(RuntimeError::PlanContract(
4784 "this run was admitted with correlation keys but the runtime has no case \
4785 store — build it with `.cases(store)`"
4786 .into(),
4787 ));
4788 }
4789 (None, _) => None,
4790 };
4791
4792 let bound = fill_case_subjects(bound, case_ctx.as_ref());
4793 let subject_refs = subject_refs(run, &bound);
4794 if !bound.is_empty() {
4795 let mut append = Append::new(run, RecordKind::DataSubjectBound { bindings: bound });
4796 append.case = case_ctx.as_ref().map(CaseContext::id);
4797 records.push(append);
4798 }
4799
4800 // The append claims the admission key, so a key already held is refused
4801 // here, before anything outside the journal names the run.
4802 self.store
4803 .append(lease.epoch, records)
4804 .await
4805 .map_err(RuntimeError::from_store)?;
4806
4807 // Registered before the run is handed back, and its failure fails
4808 // admission: a case that does not list the run, or destinations that
4809 // never see its history.
4810 //
4811 // After the append rather than before: a registration written for a
4812 // run whose admission then failed — or whose process died — would name
4813 // a run no journal holds, and nothing could tell which case to take it
4814 // off. The price of that order is paid here instead: the admission
4815 // records above are already durable, so a registration failure must
4816 // **conclude** the run rather than merely return — an open journal
4817 // under a lease that then lapses is exactly what the recovery sweep
4818 // reads as a crashed run, and it would execute a run whose admission
4819 // failed. Concluded `failed`, in the chain, with the reason beside it;
4820 // the caller's cleanup then releases the lease, and a resume registers
4821 // again before it executes.
4822 if let Err(open_failed) = self
4823 .register(run, case_ctx.as_ref().map(CaseContext::id))
4824 .await
4825 {
4826 let head = self
4827 .store
4828 .head(run)
4829 .await
4830 .map_err(RuntimeError::from_store)?;
4831 self.store
4832 .append(
4833 lease.epoch,
4834 vec![Append::new(
4835 run,
4836 RecordKind::RunConcluded {
4837 outcome: RunStatus::Failed(String::new()).as_str().to_owned(),
4838 reason: Some(format!(
4839 "the run's registration was refused ({open_failed}), so this \
4840 run was concluded without ever executing"
4841 )),
4842 exhaustion: None,
4843 live_spend: Spend::default(),
4844 chain_head: head.hash,
4845 },
4846 )],
4847 )
4848 .await
4849 .map_err(RuntimeError::from_store)?;
4850 return Err(RuntimeError::from_store(open_failed));
4851 }
4852
4853 Ok(Admitted {
4854 run,
4855 epoch: lease.epoch,
4856 quota,
4857 budget: self.budget_for(agent),
4858 agent: agent.to_owned(),
4859 plan,
4860 input,
4861 case: case_ctx,
4862 identity: chain.cloned(),
4863 subjects: subject_refs,
4864 })
4865 }
4866
4867 /// Execute a run that has already been admitted.
4868 async fn execute_admitted(&self, a: Admitted) -> Result<RunOutcome, RuntimeError> {
4869 let mut cursor = ReplayCursor::default();
4870 // Named, not `_`: `let _ = ` drops immediately, which would renew
4871 // nothing at all while looking exactly like this.
4872 let _heartbeat = self.heartbeat(a.run, a.epoch);
4873 self.execute(
4874 Execution {
4875 run: a.run,
4876 epoch: a.epoch,
4877 quota: a.quota,
4878 plan: &a.plan,
4879 input: a.input,
4880 mode: Pass::Live,
4881 case: a.case,
4882 budget: a.budget,
4883 agent: a.agent,
4884 identity: a.identity,
4885 subjects: a.subjects,
4886 refusal: None,
4887 withheld_subject: None,
4888 successors: Vec::new(),
4889 started: BTreeSet::new(),
4890 finished: BTreeSet::new(),
4891 succeeded: BTreeSet::new(),
4892 recorded_groups: BTreeMap::new(),
4893 },
4894 &mut cursor,
4895 )
4896 .await
4897 }
4898
4899 /// Admit and execute, which is what every blocking entry point does.
4900 ///
4901 /// The run id is the caller's rather than minted here, for batches: an
4902 /// item's run id is written to the batch store *before* the run starts, so
4903 /// that a crash leaves a reservation pointing at a journal that can be
4904 /// replayed rather than an item that must be guessed about.
4905 pub(crate) async fn admit_plan_as(
4906 &self,
4907 run: RunId,
4908 plan: PlanIR,
4909 input: Tainted<Value>,
4910 terms: RunTerms,
4911 entry: Entry,
4912 ) -> Result<RunOutcome, RuntimeError> {
4913 let admitted = self.admit_only(run, plan, input, terms, entry).await?;
4914 self.execute_admitted(admitted).await
4915 }
4916
4917 /// Why an open run may not cross its history frontier on this plane, if it
4918 /// may not.
4919 ///
4920 /// Two facts recorded at admission are compared with what this plane holds:
4921 /// the policy bundle (who may authorize an effect) and, for a declarative
4922 /// agent, the declaration (what the agent *is*). A resume under either one
4923 /// changed would continue one run's journal under another's rules. The
4924 /// answer is a refusal to journal rather than an error to raise; the caller
4925 /// decides where it goes.
4926 fn resume_refusal(&self, records: &[Record]) -> Result<Option<RuntimeError>, RuntimeError> {
4927 if let Some(refusal) = self.policy_bundle_refusal(records)? {
4928 return Ok(Some(refusal));
4929 }
4930 #[cfg(feature = "manifest")]
4931 if let Some(refusal) = self.declaration_refusal(records)? {
4932 return Ok(Some(refusal));
4933 }
4934 Ok(None)
4935 }
4936
4937 /// [`RuntimeError::NoProvider`] when a capability any of the run's plans
4938 /// names resolves to nothing on this plane.
4939 ///
4940 /// Every plan the run froze is checked, not only the one in force: an
4941 /// unwind resolves the steps that ran, and those may belong to a plan a
4942 /// replan replaced.
4943 fn refuse_undrivable<'p>(
4944 &self,
4945 plans: impl IntoIterator<Item = &'p PlanIR>,
4946 ) -> Result<(), RuntimeError> {
4947 for node in plans.into_iter().flat_map(|p| &p.nodes) {
4948 self.resolve(&node.capability.0)?;
4949 }
4950 Ok(())
4951 }
4952
4953 /// Name the run where it is watched from outside its journal: on its case,
4954 /// and to every outbox destination.
4955 ///
4956 /// Only after the admission's append, and idempotent — an existing case
4957 /// row and an existing delivery cursor are kept — so every writing resume
4958 /// repeats it.
4959 async fn register(
4960 &self,
4961 run: RunId,
4962 case: Option<crate::core::CaseId>,
4963 ) -> Result<(), crate::core::StoreError> {
4964 if let (Some(case), Some(cases)) = (case, self.cases.as_ref()) {
4965 cases.attach_run(case, run).await?;
4966 }
4967 #[cfg(feature = "push")]
4968 if let Some(outbox) = &self.outbox {
4969 outbox.open(run).await?;
4970 }
4971 Ok(())
4972 }
4973
4974 /// [`RuntimeError::NoCaseStore`] when the run is bound to a case and this
4975 /// plane has no case store.
4976 ///
4977 /// Everything a resume writes belongs to the case, and without the store
4978 /// it would land under the run alone, where erasing the case misses it.
4979 /// About this plane rather than the run, so another plane may resume it.
4980 fn refuse_caseless(&self, run: RunId, records: &[Record]) -> Result<(), RuntimeError> {
4981 match recorded_case_id(records) {
4982 Some(case) if self.cases.is_none() => Err(RuntimeError::NoCaseStore {
4983 run: run.to_string(),
4984 case: case.to_string(),
4985 }),
4986 _ => Ok(()),
4987 }
4988 }
4989
4990 /// Journal a resume refusal as the run's quarantine, naming what moved.
4991 async fn quarantine_refused(
4992 &self,
4993 run: RunId,
4994 records: &[Record],
4995 lease: &crate::journal::Lease,
4996 refusal: &RuntimeError,
4997 ) -> Result<RunOutcome, RuntimeError> {
4998 self.conclude(
4999 run,
5000 lease.epoch,
5001 RunStatus::Quarantined(refusal.to_string()),
5002 None,
5003 true,
5004 recorded_case_id(records),
5005 Consumed::default(),
5006 Spend::ZERO,
5007 &QuotaPass::disabled(),
5008 )
5009 .await
5010 }
5011
5012 /// The bundle half of [`resume_refusal`](Self::resume_refusal).
5013 ///
5014 /// Resume can dispatch new effects after it reaches the end of history.
5015 /// They must be judged by the same complete bundle recorded at admission,
5016 /// or one run would claim one policy while later effects were authorized
5017 /// by another.
5018 fn policy_bundle_refusal(
5019 &self,
5020 records: &[Record],
5021 ) -> Result<Option<RuntimeError>, RuntimeError> {
5022 let recorded = records
5023 .iter()
5024 .find_map(|record| match record.kind() {
5025 RecordKind::RunAdmitted { policy_bundle, .. } => {
5026 Some(policy_bundle.as_deref().cloned())
5027 }
5028 _ => None,
5029 })
5030 .ok_or_else(|| {
5031 RuntimeError::PlanContract("journal has no RunAdmitted record".into())
5032 })?;
5033 let configured = self.policy.as_ref().map(|policy| policy.bundle());
5034 if recorded != configured {
5035 return Ok(Some(RuntimeError::PolicyBundleChanged {
5036 recorded: recorded.as_ref().map(PolicyBundleIdentity::digest),
5037 configured: configured.as_ref().map(PolicyBundleIdentity::digest),
5038 }));
5039 }
5040 Ok(None)
5041 }
5042
5043 /// The declaration half of [`resume_refusal`](Self::resume_refusal).
5044 ///
5045 /// For a declarative agent the manifest is the whole behaviour, and a resume
5046 /// under an edited one runs a different program over the first one's
5047 /// journal — which replay would eventually catch as a key mismatch, several
5048 /// effects in and without being able to say what changed. The digest is
5049 /// already on the record at admission, so refusing here costs a comparison
5050 /// and turns a late diagnosis into a named remedy.
5051 ///
5052 /// **Silent in two cases, and deliberately.** A run admitted with no
5053 /// declaration is a coded skill, whose behaviour is the embedder's binary:
5054 /// this crate cannot identify that and will not pretend to. And a plane that
5055 /// holds no manifest under the recorded name cannot run the agent
5056 /// declaratively at all, so the refusal it needs is the one plan resolution
5057 /// already gives.
5058 #[cfg(feature = "manifest")]
5059 fn declaration_refusal(
5060 &self,
5061 records: &[Record],
5062 ) -> Result<Option<RuntimeError>, RuntimeError> {
5063 let Some(recorded) = records.iter().find_map(|record| match record.kind() {
5064 RecordKind::RunAdmitted { governed_by, .. } => governed_by.as_deref().cloned(),
5065 _ => None,
5066 }) else {
5067 return Ok(None);
5068 };
5069 let Some(current) = self
5070 .governed_by
5071 .values()
5072 .find(|m| m.metadata.name == recorded.name)
5073 else {
5074 return Ok(None);
5075 };
5076 let configured = current
5077 .digest()
5078 .map_err(|e| RuntimeError::PlanContract(e.to_string()))?;
5079 if configured != recorded.digest {
5080 return Ok(Some(RuntimeError::DeclarationChanged {
5081 agent: recorded.name,
5082 recorded: recorded.digest,
5083 configured,
5084 }));
5085 }
5086 Ok(None)
5087 }
5088
5089 /// A plan left sealed is erased only where a key ring could have opened
5090 /// it.
5091 ///
5092 /// A wired ring opens every payload whose key still exists and fails the
5093 /// read when it cannot be reached, so a payload still sealed behind one was
5094 /// erased. Behind no ring it says nothing about the key at all: the data
5095 /// is intact, and telling a reader it is gone sends them to the wrong
5096 /// remedy.
5097 fn sealed_or_erased(&self, error: RuntimeError) -> RuntimeError {
5098 match error {
5099 RuntimeError::PayloadsErased { run } if !self.holds_key_ring() => {
5100 RuntimeError::PayloadsSealed { run }
5101 }
5102 other => other,
5103 }
5104 }
5105
5106 /// Whether this plane was handed a key ring to open sealed payloads with.
5107 #[cfg_attr(not(feature = "keyring"), allow(clippy::unused_self))]
5108 const fn holds_key_ring(&self) -> bool {
5109 #[cfg(feature = "keyring")]
5110 {
5111 self.keyring.is_some()
5112 }
5113 #[cfg(not(feature = "keyring"))]
5114 {
5115 false
5116 }
5117 }
5118
5119 /// Re-execute a recorded run from its journal.
5120 ///
5121 /// * [`Mode::Strict`] verifies determinism: every effect must match, and the
5122 /// run must not want any effect the journal lacks.
5123 /// * [`Mode::Resume`] recovers a crashed run: history is replayed, then
5124 /// execution continues live from wherever the record ends.
5125 ///
5126 /// Either way no external effect is performed for anything already in the
5127 /// journal. That is the whole point — a resumed run does not re-issue the
5128 /// invoice it already issued.
5129 pub async fn replay(&self, run: RunId, mode: Mode) -> Result<RunOutcome, RuntimeError> {
5130 let mode = Pass::from(mode);
5131 // ── Ownership before observation ───────────────────────────────────
5132 //
5133 // Resume writes, so it takes the lease *before* it reads. The old
5134 // order — snapshot the records, then acquire — left a window in which
5135 // a live owner could append past the snapshot; the resume then
5136 // replayed a prefix of the true history and continued live from a
5137 // frontier that was not the frontier, re-deriving keys the owner had
5138 // already written. Claiming first means the snapshot below is taken
5139 // under this instance's own fence. Strict verifies and never writes,
5140 // so it needs no lease — and must not take one, or every regression
5141 // check would fence a run somebody may be legitimately running.
5142 let lease = if mode == Pass::Resume {
5143 Some(
5144 self.store
5145 .acquire(run, &self.owner, self.lease_ttl)
5146 .await
5147 .map_err(RuntimeError::from_store)?,
5148 )
5149 } else {
5150 None
5151 };
5152
5153 self.replay_releasing(run, mode, lease, Claim::Standing)
5154 .await
5155 }
5156
5157 /// Replay a recorded run strictly and say what it found.
5158 ///
5159 /// [`replay`](Self::replay) with [`Mode::Strict`], answered as a
5160 /// [`Verdict`](super::Verdict) rather than a status: *verified* when every
5161 /// recorded effect was asked for again and the run reached the ending it
5162 /// recorded, whatever that ending was; *diverged* at the first effect this
5163 /// build asked for differently; *cannot replay* when the run cannot be
5164 /// re-derived here, for a reason that is not the declaration. Both
5165 /// revisions are named on every answer — the one the run was admitted
5166 /// under and the one this plane holds by that name.
5167 ///
5168 /// Writes nothing, takes no lease and dispatches nothing, as a strict
5169 /// replay never does. It is the question an author asks of an edited
5170 /// manifest: replayed under the edit, this run's history either reaches
5171 /// nothing it changed or names the first effect it did.
5172 ///
5173 /// # Errors
5174 ///
5175 /// If the run does not exist or its journal does not verify, or on a store
5176 /// failure — never for a divergence or a reason the verdict names.
5177 pub async fn verify(&self, run: RunId) -> Result<super::Verdict, RuntimeError> {
5178 use super::{CannotReplay, Finding};
5179
5180 let records = self
5181 .store
5182 .read(run, 1)
5183 .await
5184 .map_err(RuntimeError::from_store)?;
5185 if records.is_empty() {
5186 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
5187 run.to_string(),
5188 )));
5189 }
5190 let recorded = records.iter().find_map(|r| match r.kind() {
5191 RecordKind::RunAdmitted { governed_by, .. } => governed_by.as_deref().cloned(),
5192 _ => None,
5193 });
5194 let candidate = self.candidate_revision(recorded.as_ref());
5195 let verdict = |finding| super::Verdict {
5196 run,
5197 recorded: recorded.clone(),
5198 candidate: candidate.clone(),
5199 finding,
5200 };
5201
5202 // A property of the history rather than of how it ended, so it is
5203 // named whatever the run's state: every key moves under another rule.
5204 if let Err(RuntimeError::CanonicalizationChanged {
5205 recorded,
5206 implemented,
5207 }) = ensure_replayable_canon(&records)
5208 {
5209 return Ok(verdict(Finding::CannotReplay(
5210 CannotReplay::CanonicalizationChanged {
5211 recorded,
5212 implemented,
5213 },
5214 )));
5215 }
5216 let Some(ending) = recorded_ending(&records) else {
5217 return Ok(verdict(Finding::CannotReplay(CannotReplay::NotConcluded)));
5218 };
5219 if ended_by_an_act(&ending) {
5220 return Ok(verdict(Finding::CannotReplay(CannotReplay::EndedByAnAct {
5221 outcome: ending,
5222 })));
5223 }
5224
5225 let mut dispatched = false;
5226 let mut divergence = None;
5227 let replayed = self
5228 .replay_under(
5229 run,
5230 Pass::Strict,
5231 None,
5232 Claim::Standing,
5233 &mut dispatched,
5234 &mut divergence,
5235 )
5236 .await;
5237 let finding = match (replayed, divergence) {
5238 (Ok(_), Some(d)) => Finding::Diverged(d),
5239 (Ok(outcome), None) if outcome.status.as_str() == ending => {
5240 Finding::Verified { outcome: ending }
5241 }
5242 (Ok(outcome), None) => Finding::OutcomeDiffers {
5243 recorded: ending,
5244 replayed: outcome.status.as_str().to_owned(),
5245 },
5246 (Err(RuntimeError::PayloadsErased { .. }), _) => {
5247 Finding::CannotReplay(CannotReplay::Erased)
5248 }
5249 (Err(RuntimeError::PayloadsSealed { .. }), _) => {
5250 Finding::CannotReplay(CannotReplay::KeyAbsent)
5251 }
5252 (Err(RuntimeError::NoProvider { target, .. }), _) => {
5253 Finding::CannotReplay(CannotReplay::EntryPointRemoved { capability: target })
5254 }
5255 (Err(other), _) => return Err(other),
5256 };
5257 Ok(verdict(finding))
5258 }
5259
5260 /// The declaration this plane holds under the recorded agent's name.
5261 #[cfg(feature = "manifest")]
5262 fn candidate_revision(
5263 &self,
5264 recorded: Option<&crate::journal::AgentIdentity>,
5265 ) -> Option<crate::journal::AgentIdentity> {
5266 let name = &recorded?.name;
5267 let held = self
5268 .governed_by
5269 .values()
5270 .find(|m| &m.metadata.name == name)?;
5271 self.identity_of(held)
5272 }
5273
5274 /// Without the manifest there are no declarations to name.
5275 #[cfg(not(feature = "manifest"))]
5276 #[allow(clippy::unused_self)]
5277 const fn candidate_revision(
5278 &self,
5279 _recorded: Option<&crate::journal::AgentIdentity>,
5280 ) -> Option<crate::journal::AgentIdentity> {
5281 None
5282 }
5283
5284 /// Resume a run under a lease the caller already holds.
5285 ///
5286 /// The wake path's handover: a timer or event delivery records the wake
5287 /// under its own lease and then resumes the run. Releasing that lease and
5288 /// letting the resume acquire a fresh one opened a window — a crash
5289 /// between the two left a *released* lease over a run whose timer was
5290 /// disarmed and whose subscription was retired, which no queue in the
5291 /// system names: the abandonment sweep looks only for leases that expired
5292 /// while still holding an owner. Handing the lease over closes the window;
5293 /// the run is owned continuously from wake to conclusion, and a crash
5294 /// anywhere in between leaves an expired *owned* lease the sweep drains.
5295 ///
5296 /// A resume that fails does the same thing a crash does, and for the same
5297 /// reason: the recorded wake took the run off the waiting listing, so the
5298 /// lease is the only thing left that names it.
5299 pub(crate) async fn resume_holding(
5300 &self,
5301 run: RunId,
5302 lease: crate::journal::Lease,
5303 ) -> Result<RunOutcome, RuntimeError> {
5304 self.replay_releasing(run, Pass::Resume, Some(lease), Claim::Unlisted)
5305 .await
5306 }
5307
5308 /// Recover an owner-abandoned run without turning settlement repair into
5309 /// an operator-requested retry of a recorded conclusion.
5310 ///
5311 /// A recovery that fails for a reason no retry changes — the journal, not
5312 /// the moment, is what refuses — quarantines the run under the lease the
5313 /// failed pass kept, naming the reason. Left alone it would lapse owned
5314 /// and be recovered, and refused, every lease period for ever; a
5315 /// quarantine is listed, counted, and answered by a person.
5316 pub(crate) async fn recover_abandoned_run(
5317 &self,
5318 run: RunId,
5319 ) -> Result<RunOutcome, RuntimeError> {
5320 let lease = self
5321 .store
5322 .acquire(run, &self.owner, self.lease_ttl)
5323 .await
5324 .map_err(RuntimeError::from_store)?;
5325 match self
5326 .replay_releasing(run, Pass::Resume, Some(lease.clone()), Claim::Recovering)
5327 .await
5328 {
5329 Err(error) if recurs_on_every_resume(&error) => {
5330 let records = self
5331 .store
5332 .read(run, 1)
5333 .await
5334 .map_err(RuntimeError::from_store)?;
5335 let refusal = RuntimeError::PlanContract(format!(
5336 "recovery refused, and would be refused the same way on every attempt: \
5337 {error}"
5338 ));
5339 self.quarantine_refused(run, &records, &lease, &refusal)
5340 .await
5341 }
5342 other => other,
5343 }
5344 }
5345
5346 async fn replay_releasing(
5347 &self,
5348 run: RunId,
5349 mode: Pass,
5350 lease: Option<crate::journal::Lease>,
5351 claim: Claim,
5352 ) -> Result<RunOutcome, RuntimeError> {
5353 let mut dispatched = false;
5354 let outcome = self
5355 .replay_under(run, mode, lease.as_ref(), claim, &mut dispatched, &mut None)
5356 .await;
5357
5358 // Idempotent: a resume that reached a conclusion already handed its
5359 // lease back in `conclude`, and re-releasing the same epoch changes
5360 // nothing. Released on success, and on a failure only where the run
5361 // is still listed somewhere else: the caller claimed a run whose
5362 // journal already says where it stands, and the refusal came before
5363 // anything past the frontier was written. Every other failure keeps
5364 // the lease so it lapses owned, which is what puts the run on the
5365 // abandonment listing the recovery sweep drains. A wake or a delivery
5366 // has already recorded the awaited result, so the run is off the
5367 // waiting listing; a recovery holds a run nothing else lists; and a
5368 // pass that failed mid-flight wrote records past a conclusion that no
5369 // longer describes it. Releasing any of those strands the run with no
5370 // driver and on no listing.
5371 let release = match &outcome {
5372 Ok(_) => true,
5373 // Settlement is retried by recovery, which finds the run through
5374 // this lease.
5375 Err(RuntimeError::QuotaSettlementPending { .. }) => false,
5376 Err(_) => claim == Claim::Standing && !dispatched,
5377 };
5378 if release
5379 && let Some(lease) = &lease
5380 && let Err(e) = self.store.release_lease(run, lease.epoch).await
5381 {
5382 tracing::debug!(
5383 %run,
5384 error = %e,
5385 "could not hand back the resume lease; it will expire on its own"
5386 );
5387 }
5388 outcome
5389 }
5390
5391 #[allow(clippy::too_many_lines)]
5392 async fn replay_under(
5393 &self,
5394 run: RunId,
5395 mode: Pass,
5396 lease: Option<&crate::journal::Lease>,
5397 claim: Claim,
5398 dispatched: &mut bool,
5399 divergence: &mut Option<crate::journal::Divergence>,
5400 ) -> Result<RunOutcome, RuntimeError> {
5401 let records = self
5402 .store
5403 .read(run, 1)
5404 .await
5405 .map_err(RuntimeError::from_store)?;
5406 if records.is_empty() {
5407 return Err(RuntimeError::Store(crate::core::StoreError::NotFound(
5408 run.to_string(),
5409 )));
5410 }
5411
5412 // Never trust a journal that does not verify. A tampered or truncated
5413 // history would let replay "confirm" something that never happened.
5414 //
5415 // Per-record hashes are already checked on read, so a single altered
5416 // record fails before we get here; this catches the structural attacks
5417 // that survive individually-valid records — deletion, reordering, and
5418 // splicing history from another run.
5419 Record::verify_chain(&records, Digest::ZERO).map_err(RuntimeError::from_store)?;
5420
5421 ensure_replayable_canon(&records)?;
5422
5423 if mode == Pass::Resume
5424 && let Some(outcome) = self
5425 .recover_quota_before_resume(
5426 run,
5427 &records,
5428 lease.expect("resume holds a lease"),
5429 claim == Claim::Recovering,
5430 )
5431 .await?
5432 {
5433 return Ok(outcome);
5434 }
5435
5436 if mode == Pass::Resume
5437 && let Some(outcome) = self
5438 .closed_resume_outcome(run, &records, lease.expect("resume holds a lease"))
5439 .await?
5440 {
5441 return Ok(outcome);
5442 }
5443
5444 // Placed *before* the compensation guard below on purpose: a run that
5445 // quarantined partway through an unwind is exactly the one an operator
5446 // has no other way to close, and it must not be the one this refuses to
5447 // close.
5448 if mode == Pass::Resume
5449 && let Some(outcome) = self
5450 .abandon_if_decided(run, &records, lease.expect("resume holds a lease"))
5451 .await?
5452 {
5453 return Ok(outcome);
5454 }
5455
5456 if mode == Pass::Resume {
5457 refuse_resume_over_undone_work(run, &records)?;
5458 }
5459
5460 // **Read after the conclusions above, and that ordering is the rule.**
5461 // A plan is what a run needs to *execute*; concluding one needs no
5462 // plan, and a cancellation and an abandonment are recorded in the
5463 // clear. Reading here rather than at the top of this function is what
5464 // leaves an operator a verb when the plan can no longer be read at all
5465 // — which is an ordinary state, because erasing a case's key erases
5466 // the plan of every run in it.
5467 let input = records
5468 .iter()
5469 .find_map(Record::admitted_input)
5470 .ok_or_else(|| {
5471 RuntimeError::PlanContract("journal has no RunAdmitted record".into())
5472 })?;
5473 let (plan, successors) =
5474 frozen_plans(run, &records).map_err(|e| self.sealed_or_erased(e))?;
5475
5476 // Only a plane that can drive the run judges it. A plane holding no
5477 // provider for the run's steps — an operator's terminal, a fleet
5478 // member serving other agents — answers `NoProvider` before comparing
5479 // its policy or declarations with the run's: a mismatch there says
5480 // nothing about the run, and quarantining it would stop a run this
5481 // plane could never have continued.
5482 if mode == Pass::Resume {
5483 self.refuse_undrivable(std::iter::once(&plan).chain(&successors))?;
5484 self.refuse_caseless(run, &records)?;
5485 }
5486
5487 // Resume can dispatch new effects past the end of history, so it is
5488 // held to the bundle and the declaration recorded at admission. Strict
5489 // replay performs no effects and remains usable as an offline verifier
5490 // without loading the historical engine.
5491 //
5492 // A refusal here is journaled as a quarantine rather than returned. The
5493 // callers that reach this unattended — a wake, a delivery, the recovery
5494 // sweep — have already recorded the result that took the run off its
5495 // waiting listing, and an error there reaches nobody: the run would
5496 // sit on no listing, or be retried every tick into the same refusal.
5497 // A quarantine is the state that already means *the runtime cannot go
5498 // on and a person must decide*, it is listed and counted, and both of
5499 // its answers work here — reopen on a plane holding the recorded
5500 // revision, or abandon.
5501 if mode == Pass::Resume
5502 && let Some(refusal) = self.resume_refusal(&records)?
5503 {
5504 let lease = lease.expect("resume holds a lease");
5505 return self
5506 .quarantine_refused(run, &records, lease, &refusal)
5507 .await;
5508 }
5509
5510 // A resume executes the run, so it registers it first: what it adds is
5511 // whatever an admission that failed or died partway never wrote.
5512 if mode == Pass::Resume {
5513 self.register(run, recorded_case_id(&records))
5514 .await
5515 .map_err(RuntimeError::from_store)?;
5516 }
5517
5518 *dispatched = true;
5519 // The agent recorded at admission, so a replay is bounded by the
5520 // ceilings the run actually had — and dispatched no wider than the
5521 // width its reservation was sized for.
5522 let budget = self.budget_for(&recorded_agent(&records));
5523 let quota = self
5524 .start_replay_quota_pass(run, mode, reserved_width(&budget, &plan))
5525 .await?;
5526 let through = self.marking_pass(run, "a);
5527 let this = through.as_ref().unwrap_or(self);
5528
5529 let case_ctx = self.recorded_case(&records);
5530
5531 let mut cursor = ReplayCursor::from_records(&records);
5532
5533 // Strict verification must not write, and it does not: it holds no
5534 // lease, and appends happen only past the end of history, which
5535 // Strict mode refuses to reach. Resume writes under the lease it
5536 // claimed before reading.
5537 let epoch = lease.map_or_else(|| records.last().map_or(1, |r| r.body.epoch), |l| l.epoch);
5538
5539 // Strict verification never writes, so it holds no lease to renew.
5540 let _heartbeat = lease.map(|l| self.heartbeat(run, l.epoch));
5541 this.execute(
5542 Execution {
5543 run,
5544 epoch,
5545 quota,
5546 plan: &plan,
5547 input,
5548 mode,
5549 case: case_ctx,
5550 budget,
5551 agent: recorded_agent(&records),
5552 identity: recorded_chain(&records)?,
5553 subjects: recorded_subjects(run, &records),
5554 // A step-level refusal has no effect key, so it cannot ride the
5555 // replay cursor like an effect's does. It is lifted here from
5556 // the records `replay` has already read.
5557 refusal: recorded_step_refusal(&records),
5558 withheld_subject: standing_withholding(&records),
5559 successors,
5560 started: recorded_started_steps(&records),
5561 finished: recorded_finished_steps(&records),
5562 succeeded: recorded_succeeded_steps(&records),
5563 recorded_groups: recorded_groups(&records),
5564 },
5565 &mut cursor,
5566 )
5567 .await
5568 .inspect(|_| *divergence = cursor.divergence().cloned())
5569 }
5570
5571 /// The trace root. Every span below is a child, so "what did this run do"
5572 /// is one query rather than a correlation exercise across logs.
5573 ///
5574 /// Instrumented rather than entered: an `Entered` guard held across an
5575 /// `.await` belongs to the thread, and with concurrent dispatch that
5576 /// reparents whatever runs next onto this span.
5577 async fn execute(
5578 &self,
5579 plan: Execution<'_>,
5580 cursor: &mut ReplayCursor,
5581 ) -> Result<RunOutcome, RuntimeError> {
5582 let span = tracing::info_span!(
5583 telemetry::RUN_SPAN,
5584 { telemetry::GEN_AI_OPERATION } = telemetry::GEN_AI_INVOKE_AGENT,
5585 { telemetry::RUN_ID } = tracing::field::display(plan.run),
5586 // Which declaration this run is executing. A plane serving several
5587 // agents cannot otherwise separate them, and separating them is
5588 // the first thing an operator does.
5589 { telemetry::GEN_AI_AGENT_NAME } = plan.agent.as_str(),
5590 { telemetry::MODE } = telemetry::mode_str(plan.mode),
5591 { telemetry::GEN_AI_CONVERSATION_ID } = plan
5592 .case
5593 .as_ref()
5594 .map(|c| super::ctx::CaseContext::id(c).to_string()),
5595 // Empty under the default policy, which is what `Omitted` means.
5596 // Recorded rather than declared so the attribute is absent rather
5597 // than blank — a collector cannot read *not disclosed* as *no
5598 // tenant* if there is no field at all.
5599 { telemetry::TENANT } = tracing::field::Empty,
5600 { telemetry::OUTCOME } = tracing::field::Empty,
5601 { telemetry::SEMCONV } = telemetry::SEMCONV_VERSION,
5602 { telemetry::OTEL_NAME } = format!("{} {}", telemetry::GEN_AI_INVOKE_AGENT, plan.agent),
5603 { telemetry::OTEL_KIND } = "internal",
5604 );
5605 if !self.meter.tenant().is_empty() {
5606 span.record(telemetry::TENANT, self.meter.tenant());
5607 }
5608 self.execute_inner(plan, cursor).instrument(span).await
5609 }
5610
5611 /// The driver loop: ready set, admit, dispatch, apply, repeat.
5612 ///
5613 /// Long by line count and deliberately not split further. Every *step* is
5614 /// already its own method — `admit_ready`, `dispatch`, `collect`, `apply`,
5615 /// `adopt_successor`, `stop`. What is left is the order they happen in, and
5616 /// that order is the algorithm. Breaking it up again would scatter one
5617 /// readable sequence across functions that exist only to satisfy a line
5618 /// count, which is the opposite of the thing the lint is protecting.
5619 #[allow(clippy::too_many_lines)]
5620 async fn execute_inner(
5621 &self,
5622 plan: Execution<'_>,
5623 cursor: &mut ReplayCursor,
5624 ) -> Result<RunOutcome, RuntimeError> {
5625 let Execution {
5626 run,
5627 epoch,
5628 quota,
5629 plan: ir,
5630 input,
5631 mode,
5632 case,
5633 budget,
5634 agent,
5635 identity,
5636 subjects,
5637 refusal: recorded_refusal,
5638 withheld_subject,
5639 successors,
5640 started,
5641 finished,
5642 succeeded,
5643 recorded_groups,
5644 } = plan;
5645 // Every value derived from what the run took in carries its subjects.
5646 let input = input.attributed(&subjects);
5647 let writing = !matches!(mode, Pass::Strict);
5648 let case_id = case.as_ref().map(super::ctx::CaseContext::id);
5649 let stamp = |a: Append| match case_id {
5650 Some(c) => a.case(c),
5651 None => a,
5652 };
5653
5654 // The plan in force. Owned rather than borrowed, because a replan
5655 // replaces it and a reference into the version list could not
5656 // survive that. `recorded_successors` is empty on a live run and seeded
5657 // from the journal on a replay, so a successor is read back rather than
5658 // re-synthesised.
5659 let mut current: PlanIR = ir.clone();
5660 let mut replans: u32 = 0;
5661 // A withholding the recorded history left standing, if it did. Seeded
5662 // from the records for the same reason the step refusal is: it is a fact
5663 // about *this* run that a resume has to answer, and asking the journal
5664 // again at every boundary would be a read per ready set for a question
5665 // whose answer only this loop can change.
5666 let mut withheld: Option<String> = withheld_subject;
5667 let recorded_successors = successors;
5668
5669 // One ledger for the run. A step never gets its own allowance to blow.
5670 let ledger = Arc::new(std::sync::Mutex::new(Ledger::new(budget)));
5671
5672 // Steps already completed, and what they produced. Rebuilt from the
5673 // journal on replay so a resumed run knows where it got to.
5674 let mut done: BTreeSet<StepId> = BTreeSet::new();
5675
5676 // The same steps as `done`, in the order they finished, **with the
5677 // capability that actually ran**. Unwinding needs both: a set has no
5678 // order to reverse, and after a replan the current plan may have
5679 // different work — or nothing at all — at a completed step's id.
5680 // Resolving the compensation from the live plan then undoes something
5681 // that never ran, which is a refund for a charge nobody made.
5682 let mut completed: Vec<(StepId, Capability)> = Vec::new();
5683 // The node each completed step ran as. A successor may carry a
5684 // completed step over only as this exact node: `done` is by id, so a
5685 // redeclared node would count as done on the strength of other work.
5686 let mut ran: BTreeMap<StepId, PlanNode> = BTreeMap::new();
5687 let mut outputs: BTreeMap<StepId, Tainted<Value>> = BTreeMap::new();
5688
5689 // ── Ready-set scheduling ───────────────────────────────────────────
5690 //
5691 // Dispatch order is a deterministic total order (ascending step id), so
5692 // replay reproduces it exactly. A plan with parallelism that
5693 // dispatched in completion order would replay differently every time.
5694 // The new work of a ready set whose recorded successes replay first.
5695 let mut deferred: Option<Vec<StepId>> = None;
5696 loop {
5697 let mut ready = current.ready(&done);
5698 if let Some(rest) = deferred.take() {
5699 let held: Vec<StepId> =
5700 ready.iter().copied().filter(|s| rest.contains(s)).collect();
5701 if !held.is_empty() {
5702 ready = held;
5703 }
5704 }
5705 // A resume's ready set can hold a step the record shows succeeding
5706 // beside one it does not — parallel work cut off mid-flight. The
5707 // recorded steps replay as a set of their own first, so the checks
5708 // below meet the frontier with their outputs already restored; the
5709 // rest is the next set, with nothing their replay made ready
5710 // joining it.
5711 if writing
5712 && ready.iter().any(|s| succeeded.contains(s))
5713 && ready.iter().any(|s| !succeeded.contains(s))
5714 {
5715 let (recorded, rest) = ready.into_iter().partition(|s| succeeded.contains(s));
5716 ready = recorded;
5717 deferred = Some(rest);
5718 }
5719 // ── The stop check, at a step boundary and nowhere else ────────
5720 //
5721 // Between boundaries an effect may be announced and not yet
5722 // recorded, and interrupting there manufactures the in-doubt case
5723 // the whole protocol exists to avoid. Checking here costs one
5724 // store read per ready set and buys a cancellation that can never
5725 // strand an effect.
5726 //
5727 // Skipped while replaying: a recorded run's history already
5728 // contains whatever stop it received, and re-reading the live
5729 // request would let a cancellation arriving *today* rewrite what a
5730 // run did last year.
5731 //
5732 // A resume asks only at its frontier — once the ready set holds a
5733 // step the record does not show succeeding. Before that, the
5734 // replayed steps have not yet refilled `completed` and `outputs`,
5735 // and an unwind from there would hand each compensation nothing
5736 // and walk the steps in id order rather than in reverse of the
5737 // order they finished.
5738 let at_frontier = ready.is_empty() || ready.iter().any(|s| !succeeded.contains(s));
5739 if writing
5740 && at_frontier
5741 && let Some(c) = self
5742 .store
5743 .cancellation(run)
5744 .await
5745 .map_err(RuntimeError::from_store)?
5746 {
5747 let status = RunStatus::Cancelled {
5748 actor: c.actor.clone(),
5749 reason: c.reason.clone(),
5750 };
5751 // Journaled *before* unwinding, so the reason the run stopped is
5752 // in the chain even if compensation then fails and quarantines
5753 // it. An operator reading a half-unwound run must be able to see
5754 // that somebody asked for this.
5755 self.store
5756 .append(
5757 epoch,
5758 vec![stamp(Append::new(
5759 run,
5760 RecordKind::RunCancelled {
5761 actor: c.actor,
5762 reason: c.reason,
5763 },
5764 ))],
5765 )
5766 .await
5767 .map_err(RuntimeError::from_store)?;
5768 // A step that paused with its group open holds work only that
5769 // step can take back: its reversals are built from the members'
5770 // outputs as the step runs. Re-run it once to abandon the group
5771 // — members read back, the pause aborts instead of waiting —
5772 // and only then unwind the steps before it.
5773 let abandon: Vec<StepId> = ready
5774 .iter()
5775 .copied()
5776 .filter(|&s| {
5777 recorded_groups.iter().any(|((step, phase, _), g)| {
5778 *step == s && *phase == Phase::Forward && g.opened > g.settled
5779 })
5780 })
5781 .collect();
5782 if !abandon.is_empty() {
5783 let dispatched = self
5784 .dispatch(
5785 &abandon,
5786 cursor,
5787 Batch {
5788 agent: &agent,
5789 identity: identity.as_ref(),
5790 subjects: &subjects,
5791 run,
5792 epoch,
5793 ir: ¤t,
5794 mode,
5795 case: &case,
5796 ledger: &ledger,
5797 writing,
5798 stamp: &stamp,
5799 input: &input,
5800 outputs: &outputs,
5801 started: &started,
5802 finished: &finished,
5803 recorded_groups: &recorded_groups,
5804 parallelism: 1,
5805 abandoning: true,
5806 },
5807 )
5808 .await;
5809 collect(dispatched, &abandon, cursor)?;
5810 }
5811 return self
5812 .stop(
5813 Unwind {
5814 agent: &agent,
5815 identity: identity.as_ref(),
5816 subjects: &subjects,
5817 run,
5818 epoch,
5819 ir: ¤t,
5820 mode,
5821 case: case.clone(),
5822 ledger: &ledger,
5823 writing,
5824 stamp: &stamp,
5825 },
5826 status,
5827 &completed,
5828 &outputs,
5829 cursor,
5830 case_id,
5831 "a,
5832 )
5833 .await;
5834 }
5835
5836 // ── The withdrawal check, at the same boundary ─────────────────
5837 //
5838 // Beside the stop check and for its reasons. Unlike a cancellation
5839 // this is a **pause**: `stop` is reached with a status
5840 // `maybe_unwind` passes straight through, as exhaustion is.
5841 if writing && at_frontier {
5842 let standing = self
5843 .withdrawal_against(identity.as_ref())
5844 .await
5845 .map_err(RuntimeError::Store)?;
5846 match (standing, withheld.take()) {
5847 // Lifted since the run was withheld: that decision is a
5848 // fact about the run and goes on the record, beside the
5849 // withholding it supersedes rather than instead of it.
5850 // Without it a strict replay would stop at the old pause and
5851 // report `withheld` about a run whose own later records show
5852 // it finishing.
5853 (None, Some(prior)) => {
5854 self.store
5855 .append(
5856 epoch,
5857 vec![stamp(Append::new(
5858 run,
5859 RecordKind::AuthorityRestored { subject: prior },
5860 ))],
5861 )
5862 .await
5863 .map_err(RuntimeError::from_store)?;
5864 }
5865 // Still withdrawn. The standing record already says so — no
5866 // second one, for the reason a re-exhausted run writes no
5867 // second refusal.
5868 // The prior is dropped rather than restored: this pass
5869 // returns, and the *records* are what a later resume reads
5870 // its standing state from.
5871 (Some(withdrawal), Some(_prior)) => {
5872 return self
5873 .withhold(
5874 Unwind {
5875 agent: &agent,
5876 identity: identity.as_ref(),
5877 subjects: &subjects,
5878 run,
5879 epoch,
5880 ir: ¤t,
5881 mode,
5882 case: case.clone(),
5883 ledger: &ledger,
5884 writing,
5885 stamp: &stamp,
5886 },
5887 withdrawal,
5888 false,
5889 &completed,
5890 &outputs,
5891 cursor,
5892 case_id,
5893 "a,
5894 )
5895 .await;
5896 }
5897 (Some(withdrawal), None) => {
5898 return self
5899 .withhold(
5900 Unwind {
5901 agent: &agent,
5902 identity: identity.as_ref(),
5903 subjects: &subjects,
5904 run,
5905 epoch,
5906 ir: ¤t,
5907 mode,
5908 case: case.clone(),
5909 ledger: &ledger,
5910 writing,
5911 stamp: &stamp,
5912 },
5913 withdrawal,
5914 true,
5915 &completed,
5916 &outputs,
5917 cursor,
5918 case_id,
5919 "a,
5920 )
5921 .await;
5922 }
5923 (None, None) => {}
5924 }
5925 }
5926 if ready.is_empty() {
5927 break;
5928 }
5929
5930 // Admission first, and deliberately not concurrent: which step a
5931 // ceiling refuses must be a property of the plan, not of which
5932 // future happened to poll first.
5933 let (admitted, refused) = self
5934 .admit_ready(
5935 &ready,
5936 &ledger,
5937 mode,
5938 recorded_refusal.as_ref(),
5939 cursor,
5940 Journalling {
5941 run,
5942 epoch,
5943 writing,
5944 stamp: &stamp,
5945 },
5946 )
5947 .await?;
5948
5949 let dispatched = self
5950 .dispatch(
5951 &admitted,
5952 cursor,
5953 Batch {
5954 agent: &agent,
5955 identity: identity.as_ref(),
5956 subjects: &subjects,
5957 run,
5958 epoch,
5959 ir: ¤t,
5960 mode,
5961 case: &case,
5962 ledger: &ledger,
5963 writing,
5964 stamp: &stamp,
5965 input: &input,
5966 outputs: &outputs,
5967 started: &started,
5968 finished: &finished,
5969 recorded_groups: &recorded_groups,
5970 parallelism: quota.parallelism(&budget),
5971 abandoning: false,
5972 },
5973 )
5974 .await;
5975 let outcomes = collect(dispatched, &ready, cursor)?;
5976
5977 // ── Strict verification consumes everything, per step ──────────
5978 //
5979 // Divergence has two directions, and the ordered key comparison
5980 // only sees one of them. A build that asks for a *different*
5981 // effect fails at the mismatch; a build that asks for **fewer**
5982 // effects than the record sails past the remainder and would
5983 // report the step verified — a false confirmation about exactly
5984 // the history a verification pass exists to check. So a step that
5985 // finished with journaled effects still unread is the same
5986 // quarantine-grade finding as a key mismatch, named by the first
5987 // effect nothing requested.
5988 if mode == Pass::Strict {
5989 for &(step, ref status, _) in &outcomes {
5990 // A step that stopped — suspended, failed, refused — ended
5991 // where its history says it ended, and the stop itself is
5992 // the verified finding. Only a step that claims to have
5993 // *finished* owes a fully consumed slice.
5994 if !matches!(status, RunStatus::Succeeded) {
5995 continue;
5996 }
5997 if let Some(missing) = cursor.unconsumed_in(step, Phase::Forward) {
5998 cursor.note_divergence(crate::journal::Divergence::unconsumed(
5999 step,
6000 Phase::Forward,
6001 &missing,
6002 ));
6003 self.meter.count(metrics::DIVERGENCES, "");
6004 tracing::error!(
6005 target: telemetry::NONDETERMINISM,
6006 %step, key = %missing.key, unconsumed = true,
6007 );
6008 return self
6009 .conclude(
6010 run,
6011 epoch,
6012 RunStatus::Quarantined(format!(
6013 "strict replay verified less than the run recorded: \
6014 step {step} finished with journaled effect {missing} \
6015 never requested — this build performs fewer effects than the \
6016 recorded one"
6017 )),
6018 None,
6019 writing,
6020 case_id,
6021 Consumed::default(),
6022 Spend::default(),
6023 "a,
6024 )
6025 .await;
6026 }
6027 }
6028 }
6029
6030 // A step that stops for any reason stops the run: whatever remains
6031 // either depended on it, or will be dispatched when it resumes.
6032 // Anything already done may need undoing first.
6033 let mut stopped = apply(
6034 ¤t,
6035 outcomes,
6036 &mut done,
6037 &mut completed,
6038 &mut ran,
6039 &mut outputs,
6040 )
6041 .or_else(|| refused.map(RunStatus::Exhausted));
6042
6043 if let Some(RunStatus::Replanning(reason)) = &stopped {
6044 let recorded = recorded_successors.get(replans as usize);
6045 let journal = Journalling {
6046 run,
6047 epoch,
6048 writing: writing && recorded.is_none(),
6049 stamp: &stamp,
6050 };
6051 let cx = Replan {
6052 run,
6053 current: ¤t,
6054 ran: &ran,
6055 reason,
6056 already_replanned: replans,
6057 max_replans: budget.max_replans,
6058 recorded,
6059 };
6060 match self
6061 .adopt_successor(cx, journal, &outputs, &completed)
6062 .await?
6063 {
6064 Ok(next) => {
6065 replans += 1;
6066 current = next;
6067 continue;
6068 }
6069 Err(refusal) => stopped = Some(refusal),
6070 }
6071 }
6072
6073 if let Some(status) = stopped {
6074 return self
6075 .stop(
6076 Unwind {
6077 agent: &agent,
6078 identity: identity.as_ref(),
6079 subjects: &subjects,
6080 run,
6081 epoch,
6082 ir: ¤t,
6083 mode,
6084 case: case.clone(),
6085 ledger: &ledger,
6086 writing,
6087 stamp: &stamp,
6088 },
6089 status,
6090 &completed,
6091 &outputs,
6092 cursor,
6093 case_id,
6094 "a,
6095 )
6096 .await;
6097 }
6098 }
6099
6100 // ── Strict verification consumes everything, at the end ────────────
6101 //
6102 // The per-batch check above covers steps this build dispatched; this
6103 // covers the ones it never asked for at all — a step a changed plan no
6104 // longer reaches, a compensation slice nothing requested. Either way the
6105 // record holds work this build cannot account for, and "verified" must
6106 // not be the answer.
6107 if mode == Pass::Strict
6108 && let Some((step, phase, missing)) = cursor.first_unconsumed()
6109 {
6110 cursor.note_divergence(crate::journal::Divergence::unconsumed(
6111 step, phase, &missing,
6112 ));
6113 self.meter.count(metrics::DIVERGENCES, "");
6114 tracing::error!(
6115 target: telemetry::NONDETERMINISM,
6116 %step, key = %missing.key, unconsumed = true,
6117 );
6118 return self
6119 .conclude(
6120 run,
6121 epoch,
6122 RunStatus::Quarantined(format!(
6123 "strict replay verified less than the run recorded: journaled \
6124 effect {missing} at step {step}, {phase:?} phase, was never \
6125 requested — this build performs fewer effects than the recorded one"
6126 )),
6127 None,
6128 writing,
6129 case_id,
6130 Consumed::default(),
6131 Spend::default(),
6132 "a,
6133 )
6134 .await;
6135 }
6136
6137 // Read into a value first. A `MutexGuard` built inline as an argument
6138 // lives until the end of the full expression — which here is *after*
6139 // the await — so the lock would be held across a suspension. That is
6140 // the same shape as the `Span::enter()` bug: a guard whose
6141 // scope is wider than it looks, and invisible until something else
6142 // needs the lock.
6143 let (consumed, live_spend) = {
6144 let l = ledger.lock().expect("budget mutex");
6145 (l.consumed(), l.live_spend())
6146 };
6147 self.conclude(
6148 run,
6149 epoch,
6150 completion(¤t, &done),
6151 run_output(¤t, &outputs),
6152 writing,
6153 case_id,
6154 consumed,
6155 live_spend,
6156 "a,
6157 )
6158 .await
6159 }
6160
6161 /// Read the clock into the ledger, when a wall-clock ceiling is declared.
6162 ///
6163 /// Separate so the guard's scope is two statements that cannot span the
6164 /// await between them, and so the condition is asked once rather than
6165 /// spelled at the caller.
6166 async fn observe_clock(
6167 cx: &mut StepCtx<'_>,
6168 ledger: &Arc<std::sync::Mutex<Ledger>>,
6169 ) -> Result<(), crate::core::StepError> {
6170 if !ledger.lock().expect("budget mutex").tracks_wallclock() {
6171 return Ok(());
6172 }
6173 let at = cx.now().await?;
6174 ledger.lock().expect("budget mutex").observe_clock(at);
6175 Ok(())
6176 }
6177
6178 /// Run a ready set concurrently, at most `max_parallel_steps` at a time.
6179 ///
6180 /// The ready set is every node whose predecessors are done, so nothing in
6181 /// it depends on anything else in it — running them one at a time is a
6182 /// choice, and the wrong one when steps are waiting on models and networks.
6183 ///
6184 /// Each step takes its own slice of history, which is what makes this sound:
6185 /// a step touches only its own effects, so no shared mutable state is left
6186 /// between them, and the per-step replay cursor verifies each one's order
6187 /// independently of how the journal happened to interleave them.
6188 ///
6189 /// The width is bounded because it is not otherwise bounded by anything:
6190 /// a fan-out is as wide as its author wrote it, and every step in flight
6191 /// holds a connection, a provider's rate limit and one operation's worth
6192 /// of spend the metered ceilings have checked and not yet billed. Ordered
6193 /// rather than completion-ordered — `buffered`, not `buffer_unordered` —
6194 /// so the outcomes still apply in ready order and a plan's dispatch is a
6195 /// property of the plan rather than of which future finished first.
6196 async fn dispatch(
6197 &self,
6198 admitted: &[StepId],
6199 cursor: &mut ReplayCursor,
6200 batch: Batch<'_>,
6201 ) -> Vec<Dispatched> {
6202 use futures_util::StreamExt;
6203
6204 let slices: Vec<(StepId, StepCursor)> = admitted
6205 .iter()
6206 .map(|&s| (s, cursor.take(s, Phase::Forward)))
6207 .collect();
6208 // Floored at one because a width of zero would poll nothing forever.
6209 // A zero ceiling is refused at parse and at build, so this is the
6210 // second half of a rule rather than the only half — and the failure it
6211 // guards against is a hang, which is the one shape a later reader
6212 // cannot diagnose from the outside.
6213 let width = batch.parallelism.max(1);
6214
6215 futures_util::stream::iter(slices.into_iter().map(|(step, slice)| async move {
6216 let node = batch
6217 .ir
6218 .node(step)
6219 .ok_or_else(|| RuntimeError::PlanContract(format!("no node {step}")))?;
6220 let (status, out, slice) = self
6221 .run_step(
6222 StepRun {
6223 agent: batch.agent,
6224 identity: batch.identity,
6225 subjects: batch.subjects,
6226 run: batch.run,
6227 epoch: batch.epoch,
6228 node,
6229 phase: Phase::Forward,
6230 mode: batch.mode,
6231 case: batch.case.clone(),
6232 ledger: batch.ledger,
6233 writing: batch.writing,
6234 stamp: batch.stamp,
6235 already_started: batch.started.contains(&step),
6236 already_finished: batch.finished.contains(&step),
6237 recorded_groups: batch
6238 .recorded_groups
6239 .iter()
6240 .filter(|((s, p, _), _)| *s == step && *p == Phase::Forward)
6241 .map(|((_, _, name), n)| (name.clone(), *n))
6242 .collect(),
6243 abandoning: batch.abandoning,
6244 },
6245 batch.input,
6246 batch.outputs,
6247 slice,
6248 )
6249 .await?;
6250 Ok((step, status, out, slice))
6251 }))
6252 .buffered(width)
6253 .collect()
6254 .await
6255 }
6256
6257 /// Take the successor a step asked for: check it, record it, announce it.
6258 ///
6259 /// The nested result separates two failures. The outer `RuntimeError` is the
6260 /// runtime itself failing — the journal would not accept the plan — and is
6261 /// never recoverable. The inner `RunStatus` is the *request* being refused,
6262 /// which is an ordinary outcome the run reports.
6263 async fn adopt_successor(
6264 &self,
6265 cx: Replan<'_>,
6266 journal: Journalling<'_>,
6267 outputs: &BTreeMap<StepId, Tainted<Value>>,
6268 completed: &[(StepId, Capability)],
6269 ) -> Result<Result<PlanIR, RunStatus>, RuntimeError> {
6270 // What each step that started and journaled an effect ran as, read
6271 // from the journal only when a successor is about to be checked — a
6272 // replay reads the successor back instead.
6273 let started = if cx.recorded.is_none() {
6274 let records = self
6275 .store
6276 .read(cx.run, 1)
6277 .await
6278 .map_err(RuntimeError::from_store)?;
6279 let effectful: BTreeSet<StepId> = records
6280 .iter()
6281 .filter(|r| r.body.phase.is_forward())
6282 .filter_map(|r| match r.kind() {
6283 RecordKind::EffectStarted { .. } => r.body.step,
6284 _ => None,
6285 })
6286 .collect();
6287 let mut started = started_skills(&records);
6288 started.retain(|step, _| effectful.contains(step));
6289 started
6290 } else {
6291 BTreeMap::new()
6292 };
6293 let next = match self.successor(cx, outputs, completed, &started).await {
6294 Ok(next) => next,
6295 Err(refusal) => return Ok(Err(refusal)),
6296 };
6297 if journal.writing {
6298 self.freeze(journal.run, journal.epoch, &next, journal.stamp)
6299 .await?;
6300 }
6301 announce_replan(
6302 &self.meter,
6303 journal.run,
6304 &next,
6305 next.reason.as_deref().unwrap_or(""),
6306 );
6307 Ok(Ok(next))
6308 }
6309
6310 /// Record a plan version in the journal.
6311 ///
6312 /// A successor is frozen exactly as a first plan is, so replay reads it back
6313 /// rather than asking a planner that may since have changed its mind.
6314 async fn freeze(
6315 &self,
6316 run: RunId,
6317 epoch: u64,
6318 plan: &PlanIR,
6319 stamp: &(dyn Fn(Append) -> Append + Send + Sync),
6320 ) -> Result<(), RuntimeError> {
6321 self.store
6322 .append(
6323 epoch,
6324 vec![stamp(Append::new(
6325 run,
6326 RecordKind::PlanFrozen {
6327 steps: plan.nodes.iter().map(|n| n.capability.0.clone()).collect(),
6328 plan: serde_json::to_value(plan)?,
6329 },
6330 ))],
6331 )
6332 .await
6333 .map_err(RuntimeError::from_store)?;
6334 Ok(())
6335 }
6336
6337 /// Produce the successor plan a step asked for, or say why not.
6338 ///
6339 /// Three gates, and the first is not negotiable.
6340 ///
6341 /// **Provenance.** The frozen plan is an authorization graph compiled from
6342 /// trusted input only. A replan *changes that graph*, so once any
6343 /// untrusted value has reached working memory, anything shaping the new plan
6344 /// may be attacker-chosen — and choosing the authorization graph is the
6345 /// whole game. `plan-then-execute` is enforced here, structurally. A run
6346 /// that wants a different plan after reading untrusted input is describing
6347 /// exactly the attack.
6348 ///
6349 /// **Budget.** A run that replans without bound has stopped making progress
6350 /// and started thrashing.
6351 ///
6352 /// **On replay, the successor is read back, never re-synthesised.** A
6353 /// planner asked twice can answer differently — a changed router, a
6354 /// different model — and replay would then verify the run against a plan
6355 /// that never governed it. Same rule as the first plan, for the same
6356 /// reason.
6357 async fn successor(
6358 &self,
6359 cx: Replan<'_>,
6360 outputs: &BTreeMap<StepId, Tainted<Value>>,
6361 completed: &[(StepId, Capability)],
6362 started: &BTreeMap<StepId, String>,
6363 ) -> Result<PlanIR, RunStatus> {
6364 if let Some(source) = untrusted_in(outputs) {
6365 return Err(RunStatus::Failed(format!(
6366 "replanning refused: untrusted data from {source} is already in working memory, and the plan is an authorization graph — letting it change now would let that data choose what runs next ({})",
6367 cx.reason
6368 )));
6369 }
6370
6371 let spent = cx.already_replanned;
6372 if let Some(max) = cx.max_replans
6373 && spent >= max
6374 {
6375 return Err(RunStatus::Exhausted(crate::core::BudgetExceeded::Replans {
6376 allowed: max,
6377 }));
6378 }
6379
6380 // Replay: the successor is in the journal, at the position this replan
6381 // reached. Reading it back is what keeps a re-planned run replayable.
6382 if let Some(recorded) = cx.recorded {
6383 return Ok(recorded.clone());
6384 }
6385
6386 let replanner = self.replanner.as_ref().ok_or_else(|| {
6387 RunStatus::Failed(format!(
6388 "a step asked to replan and this runtime has no planner — build it with `.replanner(..)` ({})",
6389 cx.reason
6390 ))
6391 })?;
6392
6393 let next = replanner
6394 .replan(cx.current, cx.reason, completed)
6395 .await
6396 .map_err(|e| RunStatus::Failed(format!("replanning failed: {e}")))?;
6397
6398 // A successor faces the same contract a first plan does. One that fails
6399 // validation stops the run rather than half-applying.
6400 crate::plan::validate(&next, &self.contract())
6401 .map_err(|e| RunStatus::Failed(format!("the successor plan is invalid: {e}")))?;
6402
6403 // A completed step carries over only as the node it ran as, or not at
6404 // all. Other work at its id breaks replay — effect keys derive from the
6405 // step id — and has the saga undo something that never ran. The same
6406 // capability redeclared is refused too: `done` is by id, so a node with
6407 // new arguments, dependencies or flags would count as done on the
6408 // strength of work done before any of them held — a terminal verifier
6409 // satisfied without ever seeing what it verifies.
6410 for (step, node_ran) in cx.ran {
6411 let Some(node) = next.node(*step) else {
6412 continue;
6413 };
6414 if node.capability != node_ran.capability {
6415 return Err(RunStatus::Failed(format!(
6416 "the successor plan reuses step {step} — which already ran \
6417 as '{}' — for '{}'. Keep a completed step's node or leave \
6418 the step out; effect keys are derived from the step id, so \
6419 new work at a used id cannot be replayed",
6420 node_ran.capability.0, node.capability.0
6421 )));
6422 }
6423 if node != node_ran {
6424 return Err(RunStatus::Failed(format!(
6425 "the successor plan redeclares completed step {step} ('{}') \
6426 with different arguments, dependencies or flags. A \
6427 completed step counts as done by its id, so the new node \
6428 would be satisfied by work done under the old one — carry \
6429 it over unchanged, or give the new work a fresh id",
6430 node.capability.0
6431 )));
6432 }
6433 }
6434
6435 // A step that started, journaled an effect and did not complete holds
6436 // keys derived from its id just the same — it may have charged before
6437 // asking to replan — so its id may not be reused for work another
6438 // skill would do. One that journaled nothing, the step that asked for
6439 // a fallback before touching anything, leaves its id free.
6440 for (step, skill) in started {
6441 if cx.ran.contains_key(step) {
6442 continue;
6443 }
6444 let Some(node) = next.node(*step) else {
6445 continue;
6446 };
6447 let resolved = self
6448 .resolve(&node.capability.0)
6449 .map(|s| s.descriptor().name)
6450 .ok();
6451 if resolved.as_deref() != Some(skill.as_str()) {
6452 return Err(RunStatus::Failed(format!(
6453 "the successor plan reuses step {step} — which already \
6454 started as '{skill}' and journaled effects — for '{}'. Keep the step's \
6455 capability or leave the step out; effect keys are derived \
6456 from the step id, and the unwind undoes a started step as \
6457 the skill that ran",
6458 node.capability.0
6459 )));
6460 }
6461 }
6462
6463 if next.derived_from != Some(cx.current.digest()) {
6464 return Err(RunStatus::Failed(
6465 "the successor plan does not name its predecessor — use `PlanIR::succeed_with`, or the audit trail has a hole where the lineage should be"
6466 .into(),
6467 ));
6468 }
6469
6470 // The third leg of the lineage `succeed_with` sets. A successor with
6471 // no reason would freeze into the journal as a plan that replaced
6472 // another with nothing on the record saying why — the half of an
6473 // incident (*what changed the run's mind*) that versioned replanning
6474 // exists to keep.
6475 if next.reason.as_deref().is_none_or(str::is_empty) {
6476 return Err(RunStatus::Failed(
6477 "the successor plan carries no reason — use `PlanIR::succeed_with`, or the \
6478 journal freezes a plan that replaced another with nothing saying why"
6479 .into(),
6480 ));
6481 }
6482
6483 Ok(next)
6484 }
6485
6486 /// Decide which of a ready set may start, in ready order.
6487 ///
6488 /// Three regimes, one per mode, and the middle one is the subtle case:
6489 ///
6490 /// * **Strict** consumes recorded verdicts and recomputes none. A run
6491 /// replayed under a larger budget stopped where it stopped; recomputing
6492 /// made a step-limited run replay as *succeeded* — a false audit result.
6493 /// * **Resume** takes each verdict from wherever the truth lives. A step
6494 /// with unconsumed history was admitted by the run that recorded it,
6495 /// and the history *is* the verdict. A step at the frontier is about to
6496 /// run **live**, and every live step is metered — keying this on "is
6497 /// the mode a replay" instead of "will this step dispatch live" let a
6498 /// resumed run cross `max_steps` unmetered for as long as it kept
6499 /// running. A *recorded refusal* is re-evaluated against the ledger now
6500 /// in force: exhaustion is a pause, and raising the ceiling and
6501 /// resuming is how it un-pauses.
6502 /// * **Live** asks the ledger, which is the ordinary case.
6503 async fn admit_ready(
6504 &self,
6505 ready: &[StepId],
6506 ledger: &Arc<std::sync::Mutex<Ledger>>,
6507 mode: Pass,
6508 recorded: Option<&(StepId, String, String)>,
6509 cursor: &ReplayCursor,
6510 journal: Journalling<'_>,
6511 ) -> Result<(Vec<StepId>, Option<crate::core::BudgetExceeded>), RuntimeError> {
6512 let mut admitted = Vec::new();
6513
6514 for &step in ready {
6515 // ── A recorded step refusal at this step ───────────────────────
6516 if let Some((at, limit, used)) = recorded
6517 && *at == step
6518 {
6519 if mode == Pass::Strict {
6520 // Verbatim: the recorded run stopped here, and a
6521 // verification pass reaches the same conclusion without
6522 // consulting today's ledger.
6523 return Ok((
6524 admitted,
6525 Some(crate::core::BudgetExceeded::Recorded {
6526 limit: limit.clone(),
6527 used: used.clone(),
6528 }),
6529 ));
6530 }
6531 // Resume: the replayed prefix has already billed the journaled
6532 // historical usage into this ledger, so the question is "does
6533 // the budget now in force admit one more step over what the
6534 // run already spent". Still no → the run concludes exhausted
6535 // again, and the standing refusal already says so — no second
6536 // record. Yes → the re-admission is its own record, beside
6537 // the refusal it supersedes, so a strict replay of this
6538 // history reads refusal-then-continuation as a decision
6539 // somebody made rather than as divergence.
6540 if let Err(exceeded) = ledger
6541 .lock()
6542 .expect("budget mutex")
6543 .admit_step(admitted.len())
6544 {
6545 return Ok((admitted, Some(exceeded)));
6546 }
6547 if journal.writing {
6548 self.store
6549 .append(
6550 journal.epoch,
6551 vec![(journal.stamp)(
6552 Append::new(
6553 journal.run,
6554 RecordKind::BudgetReadmitted {
6555 limit: limit.clone(),
6556 },
6557 )
6558 .step(step),
6559 )],
6560 )
6561 .await
6562 .map_err(RuntimeError::from_store)?;
6563 }
6564 admitted.push(step);
6565 continue;
6566 }
6567
6568 let verdict = match mode {
6569 Pass::Strict => Ok(()),
6570 Pass::Resume if !cursor.exhausted(step, Phase::Forward) => Ok(()),
6571 Pass::Live | Pass::Resume => ledger
6572 .lock()
6573 .expect("budget mutex")
6574 .admit_step(admitted.len()),
6575 };
6576
6577 let Err(exceeded) = verdict else {
6578 admitted.push(step);
6579 continue;
6580 };
6581
6582 if journal.writing {
6583 // The refusal goes in the journal under the step it refused, so
6584 // replay reads the verdict rather than recomputing it.
6585 let used = format!("{:?}", ledger.lock().expect("budget mutex").consumed());
6586 self.store
6587 .append(
6588 journal.epoch,
6589 vec![(journal.stamp)(
6590 Append::new(
6591 journal.run,
6592 RecordKind::BudgetRefused {
6593 limit: exceeded.to_string(),
6594 used,
6595 },
6596 )
6597 .step(step),
6598 )],
6599 )
6600 .await
6601 .map_err(RuntimeError::from_store)?;
6602 }
6603 return Ok((admitted, Some(exceeded)));
6604 }
6605
6606 Ok((admitted, None))
6607 }
6608
6609 /// End a run that stopped early: undo what warrants undoing, then seal.
6610 ///
6611 /// Both places a run can stop short go through here, so the unwind can never
6612 /// be attached to one of them and forgotten on the other.
6613 #[allow(clippy::too_many_arguments)]
6614 async fn stop(
6615 &self,
6616 cx: Unwind<'_>,
6617 status: RunStatus,
6618 completed: &[(StepId, Capability)],
6619 outputs: &BTreeMap<StepId, Tainted<Value>>,
6620 cursor: &mut ReplayCursor,
6621 case_id: Option<crate::core::CaseId>,
6622 quota: &QuotaPass,
6623 ) -> Result<RunOutcome, RuntimeError> {
6624 let (run, epoch, writing, ir) = (cx.run, cx.epoch, cx.writing, cx.ir);
6625 let output = run_output(ir, outputs);
6626 // Cloned before `maybe_unwind` consumes `cx`; read *after* it, because an
6627 // item's cost is what the whole attempt consumed, compensation included.
6628 let ledger = cx.ledger.clone();
6629 let unwound = self
6630 .maybe_unwind(cx, status, completed, outputs, cursor)
6631 .await?;
6632 // A strict pass that stops early is still a verification, and its
6633 // verdict must not outrun what it consumed. The ready loop's
6634 // unconsumed-effects check runs only when the loop drains normally,
6635 // and every early conclusion — a failure, an unwind, a recorded
6636 // suspension — comes through here instead; without this check a build
6637 // that performs fewer effects than the record (a compensation that
6638 // shrank, a step that now suspends earlier) would return the recorded
6639 // status as "verified" over effects it never requested. Checked after
6640 // `maybe_unwind` because a strict pass consumes recorded compensation
6641 // slices there, and what remains after that is genuinely unclaimed.
6642 //
6643 // A run that already quarantined keeps its own reason. Divergence
6644 // leaves history unconsumed as a matter of course — the step stopped
6645 // where it disagreed — so reporting the leftovers here would replace
6646 // the precise finding ("expected this effect, recomputed that one")
6647 // with a vaguer restatement of its consequence, and an operator would
6648 // be told the build performs fewer effects when what it actually did
6649 // was perform a different one.
6650 if !writing
6651 && !matches!(unwound, RunStatus::Quarantined(_))
6652 && let Some((step, phase, missing)) = cursor.first_unconsumed()
6653 {
6654 cursor.note_divergence(crate::journal::Divergence::unconsumed(
6655 step, phase, &missing,
6656 ));
6657 self.meter.count(metrics::DIVERGENCES, "");
6658 tracing::error!(
6659 target: telemetry::NONDETERMINISM,
6660 %step, key = %missing.key, unconsumed = true,
6661 );
6662 return self
6663 .conclude(
6664 run,
6665 epoch,
6666 RunStatus::Quarantined(format!(
6667 "strict replay verified less than the run recorded: journaled \
6668 effect {missing} at step {step}, {phase:?} phase, was never \
6669 requested — this build performs fewer effects than the recorded one"
6670 )),
6671 None,
6672 writing,
6673 case_id,
6674 Consumed::default(),
6675 Spend::default(),
6676 quota,
6677 )
6678 .await;
6679 }
6680 // Scoped before the await — see `execute_inner` on why an inline guard
6681 // outlives the call it is passed to.
6682 let (consumed, live_spend) = {
6683 let l = ledger.lock().expect("budget mutex");
6684 (l.consumed(), l.live_spend())
6685 };
6686 self.conclude(
6687 run, epoch, unwound, output, writing, case_id, consumed, live_spend, quota,
6688 )
6689 .await
6690 }
6691
6692 /// Undo the completed steps, if the way the run stopped calls for it.
6693 ///
6694 /// # When a run does *not* unwind
6695 ///
6696 /// * **Quarantined.** The run holds an effect whose outcome is unknown, and
6697 /// you cannot safely undo around one: compensating a payment that may
6698 /// never have gone out creates a refund for money nobody took. Everything
6699 /// stays exactly where it is until a person supplies the fact the journal
6700 /// lacks — [`Runtime::reconcile_effect`](super::Runtime::reconcile_effect)
6701 /// and [`decide_quarantine`](super::Runtime::decide_quarantine) — and
6702 /// giving up
6703 /// ([`Abandon`](crate::core::QuarantineDecision::Abandon)) still unwinds
6704 /// nothing, because impatience is not evidence. This is the rule that
6705 /// separates a saga that is honest about distributed systems from one that
6706 /// tidies up and hopes.
6707 /// * **Suspended.** The run is healthy and waiting. Nothing has failed.
6708 ///
6709 /// A failure *after the pivot* also stops the unwind at the pivot: once the
6710 /// business has committed, reversing the decisions leading up to it would
6711 /// contradict something the outside world has already acted on.
6712 ///
6713 /// # Two rules every unwind takes
6714 ///
6715 /// **It must not unwind around an unknown outcome.** The same rule
6716 /// quarantine already enforces, applied at the unwind's own door. For a
6717 /// cancelled run the check is unconditional — cancellation gets no second
6718 /// pass, so an unresolved outcome stays unresolved forever. For a failed
6719 /// run it is conditional on the unwind list: an orphan under a failure is
6720 /// resolvable, because the announcement is journaled, the effect declared
6721 /// a `Recovery`, and resuming resolves it — but only while nothing was
6722 /// compensated, since compensation closes the run to resume. A failure
6723 /// that would compensate around doubt therefore quarantines instead; a
6724 /// failure with nothing to unwind stays a failure and keeps its resume.
6725 ///
6726 /// **It must undo the steps it interrupted.** Compensation walks
6727 /// *completed* steps, which covers the step that failed only if it never
6728 /// mutated. A stop arrives from outside while a step is typically
6729 /// suspended — holding effects it performed and never completing — and a
6730 /// failure in one concurrent sibling interrupts the others the same way:
6731 /// severity ordering can conclude the run while a suspended sibling holds
6732 /// a landed mutation. Unwinding only completed steps would leave exactly
6733 /// that work standing — a run that posted to a ledger and then suspended
6734 /// for approval would conclude with the posting still in the world and,
6735 /// once anything else was compensated, no resume left to finish it.
6736 async fn maybe_unwind(
6737 &self,
6738 cx: Unwind<'_>,
6739 status: RunStatus,
6740 completed: &[(StepId, Capability)],
6741 outputs: &BTreeMap<StepId, Tainted<Value>>,
6742 cursor: &mut ReplayCursor,
6743 ) -> Result<RunStatus, RuntimeError> {
6744 match status {
6745 // A cancelled run unwinds exactly as a failed one does. Stopping a
6746 // run that has already moved money and leaving the movement in
6747 // place is not stopping it — and the operator who asked is entitled
6748 // to assume "stop" means the world is put back, not that the
6749 // process merely exited.
6750 //
6751 // **Exhausted is deliberately not here.** Exhaustion is a pause,
6752 // not a fault: the run did what it was told, and what it was told
6753 // included a ceiling. Its mutations stand, because the operator's
6754 // two honest options both need them standing — raise the ceiling
6755 // and resume (which continues *over* the completed work), or
6756 // cancel (which unwinds through this same function under a status
6757 // that does). Unwinding on exhaustion would make the three ends of
6758 // an exhausted run contradict each other: the work reversed, the
6759 // run still resumable, and the resume reporting success over a
6760 // world where the work no longer stands.
6761 RunStatus::Failed(_) | RunStatus::Cancelled { .. } => {}
6762 other => return Ok(other),
6763 }
6764
6765 let (list, evidence) = match self.gated_unwind_list(&cx, &status, completed).await? {
6766 Ok(scope) => scope,
6767 Err(quarantine) => return Ok(quarantine),
6768 };
6769 let UnwindEvidence {
6770 mutated,
6771 started: _,
6772 undone: already_undone,
6773 recorded_groups,
6774 failed: _,
6775 } = evidence;
6776 let completed = &list[..];
6777
6778 for (step, capability) in completed.iter().rev().cloned() {
6779 // Resolved from what ran, not from the plan in force. After a replan
6780 // the two can differ, and undoing whatever now occupies that slot is
6781 // how a saga compensates work that never happened.
6782 let skill = self.resolve(&capability.0)?;
6783 let declared = skill.compensation();
6784
6785 match declared {
6786 // The point of no return. Everything from here back stays.
6787 crate::core::Compensation::Pivot => break,
6788 crate::core::Compensation::Unnecessary => continue,
6789 crate::core::Compensation::Undeclared => {
6790 if !mutated.contains(&step) {
6791 // Nothing to undo, and the journal proves it.
6792 continue;
6793 }
6794 return Ok(RunStatus::Quarantined(format!(
6795 "step {step} ('{}') changed external state and declares no \
6796 compensation, so the run cannot be safely unwound — \
6797 declare Compensation on it, or resolve this by hand",
6798 capability.0
6799 )));
6800 }
6801 crate::core::Compensation::Compensatable => {}
6802 }
6803
6804 let result = self
6805 .run_compensation(
6806 &cx,
6807 step,
6808 skill.as_ref(),
6809 outputs,
6810 cursor,
6811 recorded_groups
6812 .iter()
6813 .filter(|((s, p, _), _)| *s == step && *p == Phase::Compensating)
6814 .map(|((_, _, name), n)| (name.clone(), *n))
6815 .collect(),
6816 )
6817 .await;
6818
6819 // A compensation may legitimately need to wait — a refund that needs
6820 // four eyes is still a refund. Suspension is not failure: the run is
6821 // healthy, its frame is durable, and it will finish unwinding when
6822 // the answer arrives. Reporting it as a failure would quarantine a
6823 // run doing exactly the right thing and tell the operator the
6824 // compensation had broken.
6825 if let Err(crate::core::SkillError::Step(crate::core::StepError::Suspended(reason))) =
6826 &result
6827 {
6828 return Ok(RunStatus::Suspended(reason.clone()));
6829 }
6830
6831 let outcome = match &result {
6832 Ok(()) => "compensated".to_owned(),
6833 Err(e) => e.to_string(),
6834 };
6835
6836 // Re-run but do not re-record. On resume the compensation executes
6837 // again with every effect served from the journal, which is what
6838 // keeps strict verification meaningful — but a second
6839 // `StepCompensated` would report one compensation as two.
6840 if result.is_ok() {
6841 tracing::info!(target: telemetry::COMPENSATED, run = %cx.run, %step);
6842 self.meter.count(metrics::COMPENSATIONS, "done");
6843 }
6844 if cx.writing && !already_undone.contains(&step) {
6845 self.store
6846 .append(
6847 cx.epoch,
6848 vec![(cx.stamp)(
6849 Append::new(
6850 cx.run,
6851 RecordKind::StepCompensated {
6852 compensation: declared,
6853 outcome: outcome.clone(),
6854 },
6855 )
6856 .step(step)
6857 .phase(Phase::Compensating),
6858 )],
6859 )
6860 .await
6861 .map_err(RuntimeError::from_store)?;
6862 }
6863
6864 if let Err(fault) = &result {
6865 tracing::error!(
6866 target: telemetry::COMPENSATION_FAILED,
6867 run = %cx.run,
6868 %step,
6869 error_type = fault.class(),
6870 reason_digest = %telemetry::reason_digest(&outcome),
6871 );
6872 self.meter.count(metrics::COMPENSATIONS, "failed");
6873 // Not a problem more compensation solves. Unwinding further
6874 // would undo steps *before* one that is now in an unknown
6875 // state, which is strictly worse than stopping and saying so.
6876 return Ok(RunStatus::Quarantined(format!(
6877 "compensation failed for step {step} ('{}'): {outcome} — the run is \
6878 partially unwound and needs an operator",
6879 capability.0
6880 )));
6881 }
6882 }
6883
6884 Ok(status)
6885 }
6886
6887 /// Whether `run`'s journal holds a landed mutation no compensation undid.
6888 ///
6889 /// The attention roll-up's question about a failed run: `failed` alone may
6890 /// be cleared by a resume, but one standing on landed work leaves that
6891 /// work in the world until somebody finishes or unwinds it.
6892 pub(super) async fn holds_landed_work(&self, run: RunId) -> Result<bool, RuntimeError> {
6893 let evidence = self.unwind_evidence(run).await?;
6894 Ok(!evidence.mutated.is_subset(&evidence.undone))
6895 }
6896
6897 /// What the journal knows about an unwind before it starts:
6898 /// `(steps that changed something, steps already compensated)`.
6899 ///
6900 /// Evidence, not bookkeeping. The journal already knows both, it knows the
6901 /// same thing on replay, and nothing has to be threaded through the executor
6902 /// to keep a parallel copy honest.
6903 ///
6904 /// **A mutating effect that provably did not happen does not mark its
6905 /// step.** The announcement alone is not the evidence: an effect whose
6906 /// terminal record classifies it `DidNotHappen` — a peer that refused
6907 /// before acting, a call that never left — changed nothing, and
6908 /// compensating it is a refund for money nobody took. Everything else
6909 /// counts, and the defaults lean that way: an announcement with no
6910 /// terminal record is an orphan and therefore in doubt, and each new
6911 /// attempt starts from doubt again, so a retry that lands after a refusal
6912 /// is still evidence the step touched the world.
6913 ///
6914 /// **Nor does a member of a group that settled `Aborted`.** That settlement
6915 /// is written only once every landed member was reversed, so the group's
6916 /// effects — members and reversals alike — left nothing standing, and
6917 /// counting them marked a cleanly aborted step as mutated: a quarantine for
6918 /// a step declaring no compensation, a second undo for one that did.
6919 ///
6920 /// What this does **not** establish is that a `DidNotHappen` classification
6921 /// is true — that is the driver's claim, and a driver that mislabels a
6922 /// landed call leaves a mutation nothing here will undo.
6923 async fn unwind_evidence(&self, run: RunId) -> Result<UnwindEvidence, RuntimeError> {
6924 let records = self
6925 .store
6926 .read(run, 1)
6927 .await
6928 .map_err(RuntimeError::from_store)?;
6929
6930 // Per mutating forward effect: the step it belongs to, and whether the
6931 // record leaves it capable of having touched the world.
6932 let mut touching: BTreeMap<crate::core::EffectKey, (StepId, bool)> = BTreeMap::new();
6933 // The mutating effects of the group each step has open. A step holds
6934 // at most one open group, so the step is the key.
6935 let mut grouped: BTreeMap<StepId, Vec<crate::core::EffectKey>> = BTreeMap::new();
6936 let mut undone = BTreeSet::new();
6937 let mut failed = BTreeSet::new();
6938 for r in &records {
6939 let Some(step) = r.body.step else { continue };
6940 let key = r.effect_key();
6941 match r.kind() {
6942 RecordKind::StepFinished { outcome } if r.body.phase.is_forward() => {
6943 if outcome == RunStatus::Succeeded.as_str() {
6944 failed.remove(&step);
6945 } else {
6946 failed.insert(step);
6947 }
6948 }
6949 RecordKind::EffectStarted { mutates: true, .. } if r.body.phase.is_forward() => {
6950 if let Some(key) = key {
6951 // A fresh attempt is undecided until its own terminal
6952 // record says otherwise, so it supersedes whatever the
6953 // previous attempt concluded.
6954 touching.insert(key, (step, true));
6955 if let Some(members) = grouped.get_mut(&step) {
6956 members.push(key);
6957 }
6958 }
6959 }
6960 RecordKind::GroupOpened { .. } if r.body.phase.is_forward() => {
6961 grouped.insert(step, Vec::new());
6962 }
6963 RecordKind::GroupSettled { outcome, .. } if r.body.phase.is_forward() => {
6964 let members = grouped.remove(&step).unwrap_or_default();
6965 if *outcome == crate::core::GroupOutcome::Aborted {
6966 for member in members {
6967 touching.remove(&member);
6968 }
6969 }
6970 }
6971 RecordKind::EffectFailed { disposition, .. }
6972 | RecordKind::EffectReconciled { disposition, .. } => {
6973 if let Some(entry) = key.and_then(|k| touching.get_mut(&k)) {
6974 entry.1 = *disposition != crate::core::Disposition::DidNotHappen;
6975 }
6976 }
6977 RecordKind::StepCompensated { .. } => {
6978 undone.insert(step);
6979 }
6980 _ => {}
6981 }
6982 }
6983 let mutated = touching
6984 .into_values()
6985 .filter_map(|(step, touched)| touched.then_some(step))
6986 .collect();
6987 Ok(UnwindEvidence {
6988 mutated,
6989 started: started_skills(&records),
6990 undone,
6991 recorded_groups: recorded_groups(&records),
6992 failed,
6993 })
6994 }
6995
6996 /// The half of a step's frame that belongs to the **plane** rather than to
6997 /// the step.
6998 ///
6999 /// One place, for both a forward step and a compensation: a plane
7000 /// capability wired into one of the two and not the other is a control
7001 /// that applies going forward and not while unwinding.
7002 fn plane_frame(&self, step: StepFrame) -> super::ctx::Frame {
7003 let StepFrame {
7004 run,
7005 epoch,
7006 step,
7007 phase,
7008 mode,
7009 case,
7010 ledger,
7011 identity,
7012 subjects,
7013 agent,
7014 #[cfg(feature = "manifest")]
7015 manifest,
7016 recorded_groups,
7017 } = step;
7018 super::ctx::Frame {
7019 run,
7020 epoch,
7021 step,
7022 phase,
7023 mode,
7024 case,
7025 timers: self.timers.clone(),
7026 blobs: self.blobs.clone(),
7027 memories: self.memories.clone(),
7028 semantic: self.semantic.clone(),
7029 authorities: self.authorities.clone(),
7030 peers: self.peers.clone(),
7031 #[cfg(feature = "manifest")]
7032 tools: self.tools.clone(),
7033 journal_ceiling: self.journal_ceiling,
7034 #[cfg(feature = "manifest")]
7035 egress: self.egress.clone(),
7036 #[cfg(feature = "manifest")]
7037 checkers: self.checkers.clone(),
7038 streams: self.streams.clone(),
7039 #[cfg(feature = "manifest")]
7040 rates: self.rates.clone(),
7041 meter: self.meter.clone(),
7042 #[cfg(feature = "keyring")]
7043 keyring: self.keyring.clone(),
7044 tenant: self.tenant.clone(),
7045 ledger,
7046 policy: self.policy.clone(),
7047 identity,
7048 subjects,
7049 agent,
7050 plane: self.self_ref.clone(),
7051 #[cfg(feature = "manifest")]
7052 declaration: manifest.as_deref().and_then(|m| self.identity_of(m)),
7053 #[cfg(feature = "manifest")]
7054 manifest,
7055 signer: self.signer.clone(),
7056 recorded_groups,
7057 }
7058 }
7059
7060 /// Run one step's `compensate`, in its own phase and cursor slice.
7061 ///
7062 /// Split out so the unwind reads as the policy it is. The step gets a full
7063 /// `StepCtx` on purpose: compensating effects are journaled, retried,
7064 /// reconciled and replayed exactly like forward ones, and may suspend for a
7065 /// human — a refund that needs four eyes is still a refund.
7066 async fn run_compensation(
7067 &self,
7068 cx: &Unwind<'_>,
7069 step: StepId,
7070 skill: &dyn Skill,
7071 outputs: &BTreeMap<StepId, Tainted<Value>>,
7072 cursor: &mut ReplayCursor,
7073 recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
7074 ) -> Result<(), crate::core::SkillError> {
7075 // A step with no recorded output still gets compensated: the absence of
7076 // a result says nothing about whether it changed anything, and the
7077 // compensation is what knows.
7078 let output = outputs
7079 .get(&step)
7080 .cloned()
7081 .unwrap_or_else(|| Tainted::trusted(Value::Null));
7082
7083 let mut ctx = StepCtx::new(
7084 &self.store,
7085 cursor.take(step, Phase::Compensating),
7086 self.plane_frame(StepFrame {
7087 run: cx.run,
7088 epoch: cx.epoch,
7089 step,
7090 phase: Phase::Compensating,
7091 mode: cx.mode,
7092 case: cx.case.clone(),
7093 ledger: Arc::clone(cx.ledger),
7094 identity: cx.identity.cloned(),
7095 subjects: cx.subjects.clone(),
7096 agent: cx.agent.to_owned(),
7097 #[cfg(feature = "manifest")]
7098 manifest: self.governing(skill),
7099 recorded_groups,
7100 }),
7101 );
7102
7103 let result = skill.compensate(&mut ctx, &output).await;
7104 let result = moved_blob_wins(&ctx, result);
7105 let mut slice = ctx.into_cursor();
7106 if let Err(crate::core::SkillError::Step(error)) = &result
7107 && let Some(divergence) =
7108 crate::journal::Divergence::of(step, Phase::Compensating, error, &slice)
7109 {
7110 slice.note_divergence(divergence);
7111 }
7112 cursor.restore(step, Phase::Compensating, slice);
7113 result
7114 }
7115
7116 /// What the unwind walks, gated — or the quarantine that replaces it.
7117 ///
7118 /// Reads the journal's evidence and applies the stop gates. The gates run
7119 /// exactly when this conclusion closes the run to resume: always for a
7120 /// cancellation (it gets no second pass), and for a failure precisely
7121 /// when the unwind will compensate something — compensation is what
7122 /// closes resume, and a failure that compensates nothing stays open, so
7123 /// its orphans resolve and its suspended siblings re-register on the
7124 /// resume instead of being unwound around.
7125 async fn gated_unwind_list(
7126 &self,
7127 cx: &Unwind<'_>,
7128 status: &RunStatus,
7129 completed: &[(StepId, Capability)],
7130 ) -> Result<Result<(Vec<(StepId, Capability)>, UnwindEvidence), RunStatus>, RuntimeError> {
7131 // Which steps actually changed something outside. Read from the
7132 // journal rather than tracked in memory: it is the same evidence live
7133 // and on replay, and it is what lets an *undeclared* step be judged
7134 // on what it did instead of on what nobody said about it.
7135 let evidence = self.unwind_evidence(cx.run).await?;
7136 // The question is asked of the interrupted siblings too: a sibling
7137 // holding landed work the failure stranded is work the unwind must
7138 // reach, and asked of completed steps alone, a failure with nothing
7139 // completed to compensate left it standing. The failing step itself
7140 // is not asked — its own landed work leaves the run resumable.
7141 let candidates = Self::with_interrupted_siblings(completed, &evidence);
7142 let list = if status.is_cancelled() || self.will_compensate(&candidates, &evidence.mutated)
7143 {
7144 match self
7145 .stop_list(cx.run, completed, &evidence.started, &evidence.mutated)
7146 .await?
7147 {
7148 Ok(list) => list,
7149 Err(quarantine) => return Ok(Err(quarantine)),
7150 }
7151 } else {
7152 completed.to_vec()
7153 };
7154 Ok(Ok((list, evidence)))
7155 }
7156
7157 /// The unwind list for a stop, or the quarantine that replaces it.
7158 ///
7159 /// Both rules in `maybe_unwind`'s doc comment, applied in order: refuse to
7160 /// unwind while an outcome is unknown, and extend the list with the steps
7161 /// the stop interrupted. Called only for a conclusion that closes the run
7162 /// to resume — a cancellation, or a failure whose unwind will compensate
7163 /// — so doubt quarantines unconditionally here: there is no later pass on
7164 /// which an unresolved outcome could still resolve, and compensating
7165 /// around the one call nobody can account for would destroy the evidence
7166 /// an operator needs to resolve it by hand.
7167 async fn stop_list(
7168 &self,
7169 run: RunId,
7170 completed: &[(StepId, Capability)],
7171 started: &BTreeMap<StepId, String>,
7172 mutated: &BTreeSet<StepId>,
7173 ) -> Result<Result<Vec<(StepId, Capability)>, RunStatus>, RuntimeError> {
7174 if let Some(step) = self.undecided_effect(run).await? {
7175 return Ok(Err(RunStatus::Quarantined(format!(
7176 "step {step} holds a mutating effect whose outcome is unknown — it \
7177 was announced and never concluded, or concluded in doubt with no \
7178 reconciliation — so the run cannot be unwound: compensating around \
7179 it would undo everything except the one thing nobody can account for"
7180 ))));
7181 }
7182 Ok(Ok(Self::with_interrupted_steps(
7183 completed, started, mutated,
7184 )))
7185 }
7186
7187 /// Whether an unwind over these completed steps would compensate anything.
7188 ///
7189 /// The question that decides whether a failure closes its run: a declared
7190 /// `Compensatable` step will be compensated, and an `Undeclared` step
7191 /// that provably mutated quarantines inside the loop — either way the
7192 /// conclusion is not one a resume can reopen. `Pivot` and `Unnecessary`
7193 /// steps compensate nothing. Deliberately conservative on a capability
7194 /// that no longer resolves: an unresolvable skill is treated as
7195 /// compensating, so the unwind path — which will surface the resolution
7196 /// error loudly — is taken rather than silently skipped.
7197 fn will_compensate(
7198 &self,
7199 completed: &[(StepId, Capability)],
7200 mutated: &BTreeSet<StepId>,
7201 ) -> bool {
7202 completed.iter().any(|(step, capability)| {
7203 self.resolve(&capability.0)
7204 .map_or(true, |skill| match skill.compensation() {
7205 crate::core::Compensation::Compensatable => true,
7206 crate::core::Compensation::Undeclared => mutated.contains(step),
7207 crate::core::Compensation::Pivot | crate::core::Compensation::Unnecessary => {
7208 false
7209 }
7210 })
7211 })
7212 }
7213
7214 /// The completed steps, plus every sibling a failure interrupted holding
7215 /// landed work: a step that mutated, never completed, and did not itself
7216 /// fail. Undone as the skill its `StepStarted` names, as
7217 /// [`with_interrupted_steps`](Self::with_interrupted_steps) undoes it.
7218 fn with_interrupted_siblings(
7219 completed: &[(StepId, Capability)],
7220 evidence: &UnwindEvidence,
7221 ) -> Vec<(StepId, Capability)> {
7222 let mut out = completed.to_vec();
7223 let done: BTreeSet<StepId> = out.iter().map(|(s, _)| *s).collect();
7224 for step in evidence
7225 .mutated
7226 .iter()
7227 .filter(|s| !done.contains(s) && !evidence.failed.contains(s))
7228 {
7229 if let Some(skill) = evidence.started.get(step) {
7230 out.push((*step, Capability::new(skill.as_str())));
7231 }
7232 }
7233 out
7234 }
7235
7236 /// The unwind list for a stop: completed steps, plus any step that mutated
7237 /// without completing — the one a cancellation interrupted, or the sibling
7238 /// a failure stranded mid-suspension. The interrupted steps go last, so
7239 /// the caller's reverse walk undoes them first.
7240 ///
7241 /// An interrupted step is undone as the skill its `StepStarted` names —
7242 /// the capability that ran — never as whatever the plan in force has at
7243 /// its id. A step that charged and then asked to replan is interrupted
7244 /// under a successor that may drop it or put other work at its id, and
7245 /// either reading would leave the charge standing or refund it through a
7246 /// skill that never ran.
7247 ///
7248 /// A step whose compensation is already on the record stays in the list,
7249 /// exactly as a completed step does. The walk re-runs its compensation
7250 /// with every effect served from the journal and the `StepCompensated`
7251 /// dedup keeps the record single — while filtering it out here made a
7252 /// *replayed* unwind walk fewer steps than the recorded one, so a strict
7253 /// pass over an honestly-unwound run left the interrupted step's
7254 /// compensation slice unconsumed and quarantined a history that was
7255 /// telling the truth.
7256 fn with_interrupted_steps(
7257 completed: &[(StepId, Capability)],
7258 started: &BTreeMap<StepId, String>,
7259 mutated: &BTreeSet<StepId>,
7260 ) -> Vec<(StepId, Capability)> {
7261 let mut out = completed.to_vec();
7262 let done: BTreeSet<StepId> = out.iter().map(|(s, _)| *s).collect();
7263 for step in mutated.iter().filter(|s| !done.contains(s)) {
7264 if let Some(skill) = started.get(step) {
7265 out.push((*step, Capability::new(skill.as_str())));
7266 }
7267 }
7268 out
7269 }
7270
7271 /// The lowest step holding an effect this journal cannot decide, if any.
7272 ///
7273 /// Ordinarily the run is already `Quarantined` when this is true, and a
7274 /// quarantined run never unwinds. Cancellation opens a second door into the
7275 /// unwind, and it has to be shut the same way — otherwise an operator's
7276 /// stop compensates every step *around* the one nobody can account for,
7277 /// which is precisely the refund for money nobody took, arriving through a
7278 /// control meant to make things safer.
7279 ///
7280 /// The rule itself is [`undecided_effects`](crate::journal::undecided_effects),
7281 /// shared with the operator API and the offline audit. This adds only the
7282 /// question the unwind asks of it: *how far back does the doubt reach*.
7283 async fn undecided_effect(&self, run: RunId) -> Result<Option<StepId>, RuntimeError> {
7284 let records = self
7285 .store
7286 .read(run, 1)
7287 .await
7288 .map_err(RuntimeError::from_store)?;
7289 Ok(crate::journal::undecided_effects(&records)
7290 .into_iter()
7291 .map(|u| u.step)
7292 .min())
7293 }
7294
7295 /// Execute one plan node.
7296 async fn run_step(
7297 &self,
7298 ctx: StepRun<'_>,
7299 run_input: &Tainted<Value>,
7300 outputs: &BTreeMap<StepId, Tainted<Value>>,
7301 cursor: crate::journal::StepCursor,
7302 ) -> Result<
7303 (
7304 RunStatus,
7305 Option<Tainted<Value>>,
7306 crate::journal::StepCursor,
7307 ),
7308 RuntimeError,
7309 > {
7310 let span = tracing::info_span!(
7311 telemetry::STEP_SPAN,
7312 { telemetry::STEP } = tracing::field::display(ctx.node.id),
7313 { telemetry::CAPABILITY } = tracing::field::display(&ctx.node.capability.0),
7314 { telemetry::PHASE } = ctx.phase.as_str(),
7315 { telemetry::MODE } = telemetry::mode_str(ctx.mode),
7316 { telemetry::OUTCOME } = tracing::field::Empty,
7317 );
7318 self.run_step_inner(ctx, run_input, outputs, cursor)
7319 .instrument(span)
7320 .await
7321 }
7322
7323 #[allow(clippy::too_many_lines)]
7324 async fn run_step_inner(
7325 &self,
7326 ctx: StepRun<'_>,
7327 run_input: &Tainted<Value>,
7328 outputs: &BTreeMap<StepId, Tainted<Value>>,
7329 cursor: crate::journal::StepCursor,
7330 ) -> Result<
7331 (
7332 RunStatus,
7333 Option<Tainted<Value>>,
7334 crate::journal::StepCursor,
7335 ),
7336 RuntimeError,
7337 > {
7338 let StepRun {
7339 identity,
7340 subjects,
7341 run,
7342 epoch,
7343 node,
7344 phase,
7345 mode,
7346 case,
7347 ledger,
7348 writing,
7349 stamp,
7350 agent,
7351 already_started,
7352 already_finished,
7353 recorded_groups,
7354 abandoning,
7355 } = ctx;
7356 let step = node.id;
7357 let skill = self.resolve(&node.capability.0)?;
7358
7359 // Assemble this step's input from its declared sources. Labels join, so
7360 // provenance flows through the graph without anyone threading it by hand.
7361 let step_input = assemble(node, run_input, outputs)?;
7362
7363 // Announce a step doing new work, once: always in `Live`, and on a
7364 // resume only for a step the journal never announced — a frontier
7365 // step running for the first time. Keying this on the mode alone got
7366 // both directions wrong: resumed frontier steps went unannounced, and
7367 // the `!Strict` finish gate below duplicated `StepFinished` for every
7368 // fully replayed step on every resume.
7369 let announce = writes_step_record(mode, !already_started);
7370 if announce {
7371 self.store
7372 .append(
7373 epoch,
7374 vec![stamp(
7375 Append::new(
7376 run,
7377 RecordKind::StepStarted {
7378 skill: skill.descriptor().name,
7379 },
7380 )
7381 .step(step),
7382 )],
7383 )
7384 .await
7385 .map_err(RuntimeError::from_store)?;
7386 }
7387
7388 let mut cx = StepCtx::new(
7389 &self.store,
7390 cursor,
7391 self.plane_frame(StepFrame {
7392 run,
7393 epoch,
7394 step,
7395 phase,
7396 mode,
7397 case,
7398 ledger: Arc::clone(ledger),
7399 identity: identity.cloned(),
7400 subjects: subjects.clone(),
7401 agent: agent.to_owned(),
7402 #[cfg(feature = "manifest")]
7403 manifest: self.governing(skill.as_ref()),
7404 recorded_groups,
7405 }),
7406 );
7407 cx.abandoning = abandoning;
7408 // The one place elapsed time is measured, and it is measured with a
7409 // journaled clock read like every other observation — a wall-clock
7410 // ceiling decided from an ambient clock would give a replayed run a
7411 // different verdict every time it was looked at.
7412 //
7413 // Paid only by a run that asked for the ceiling, which is why the
7414 // step's first effect is this one when a wall-clock budget is declared
7415 // and the skill's own when it is not. The consequence is worth stating:
7416 // that history can be replayed under a *raised* wall-clock ceiling, and
7417 // not under a build that removed it, because the recorded run performs
7418 // a read this one would not request.
7419 let result = match Self::observe_clock(&mut cx, ledger).await {
7420 Ok(()) => skill.invoke(&mut cx, step_input).await,
7421 Err(e) => Err(crate::core::SkillError::Step(e)),
7422 };
7423 let result = settle_abandoned_group(&mut cx, result).await;
7424 let result = moved_blob_wins(&cx, result);
7425 let wrote = cx.wrote_records();
7426 let mut cursor = cx.into_cursor();
7427 if let Err(crate::core::SkillError::Step(error)) = &result
7428 && let Some(divergence) = crate::journal::Divergence::of(step, phase, error, &cursor)
7429 {
7430 cursor.note_divergence(divergence);
7431 }
7432 ledger.lock().expect("budget mutex").record_step();
7433
7434 let fault = result.as_ref().err().map(crate::core::SkillError::class);
7435 let (status, output) = classify(&self.meter, result);
7436 tracing::Span::current().record(telemetry::OUTCOME, status.as_str());
7437 if let RunStatus::Quarantined(why) = &status {
7438 tracing::error!(
7439 target: telemetry::QUARANTINED,
7440 %run,
7441 %step,
7442 error_type = fault.unwrap_or("quarantined"),
7443 reason_digest = %telemetry::reason_digest(why),
7444 );
7445 }
7446
7447 // Record the ending only where the journal does not already hold it:
7448 // every live step; a resumed step that did new work; and a resumed
7449 // step reaching an ending the record never captured — how a suspended
7450 // step that finishes purely from replayed history (its awaited event
7451 // recorded by the delivery) still gets its one `StepFinished`. What
7452 // this rules out is a duplicate ending for every fully replayed
7453 // completed step, appended on every resume of a run doing nothing new.
7454 let record_ending = writes_step_record(mode, announce || wrote || !already_finished);
7455 if writing && record_ending {
7456 // A suspended step has not finished, so it records why it stopped
7457 // rather than claiming an outcome.
7458 let record = match &status {
7459 RunStatus::Suspended(reason) => RecordKind::RunSuspended {
7460 reason: reason.clone(),
7461 },
7462 other => RecordKind::StepFinished {
7463 outcome: other.as_str().to_owned(),
7464 },
7465 };
7466 self.store
7467 .append(epoch, vec![stamp(Append::new(run, record).step(step))])
7468 .await
7469 .map_err(RuntimeError::from_store)?;
7470 }
7471
7472 Ok((status, output, cursor))
7473 }
7474
7475 /// Whether the chain's last record is already this conclusion. `head` is
7476 /// the chain's last sequence.
7477 async fn last_is_conclusion(
7478 &self,
7479 run: RunId,
7480 head: crate::core::Seq,
7481 status: &RunStatus,
7482 ) -> Result<bool, RuntimeError> {
7483 if head < 1 {
7484 return Ok(false);
7485 }
7486 let page = self
7487 .store
7488 .read_page(run, head, 1)
7489 .await
7490 .map_err(RuntimeError::from_store)?;
7491 Ok(matches!(
7492 page.first().map(Record::kind),
7493 Some(RecordKind::RunConcluded { outcome, .. }) if outcome == status.as_str()
7494 ))
7495 }
7496
7497 /// Seal the run and report.
7498 /// Eight arguments, and each is a distinct fact about how the run ended
7499 /// that the caller already holds. Bundling them into a struct would move the
7500 /// same fields one indirection away without removing a single one.
7501 #[allow(clippy::too_many_arguments)]
7502 async fn conclude(
7503 &self,
7504 run: RunId,
7505 epoch: u64,
7506 status: RunStatus,
7507 output: Option<Tainted<Value>>,
7508 writing: bool,
7509 case: Option<crate::core::CaseId>,
7510 consumed: Consumed,
7511 live_spend: Spend,
7512 quota: &QuotaPass,
7513 ) -> Result<RunOutcome, RuntimeError> {
7514 // Loud toward the operator, ordinary toward the caller. A failed run is
7515 // a conclusion a resume can honestly answer, so it is not an incident —
7516 // but without this event nothing says why one failed except the
7517 // journal and an index that needs the HTTP surface mounted.
7518 if let RunStatus::Failed(reason) = &status {
7519 tracing::warn!(
7520 target: telemetry::RUN_FAILED,
7521 %run,
7522 error_type = "failed",
7523 reason_digest = %telemetry::reason_digest(reason),
7524 );
7525 }
7526
7527 // A suspended run is not sealed: its chain is going to be extended the
7528 // moment whatever it waits for arrives.
7529 let chain_head = if writing && !status.is_suspended() {
7530 // The conclusion goes *in* the chain — before the chain is closed
7531 // over it, where the conclusion is one that closes it. Two things
7532 // follow, and both were missing while the outcome lived only in a
7533 // side table: tamper detection covers how the run ended, and a
7534 // resumed run can read that fact from the same history it verifies
7535 // rather than inferring it from the last step that happened to
7536 // finish. The stores also derive the outcome index from this
7537 // record, last conclusion wins — which is what keeps a
7538 // failed-then-resumed-then-succeeded run from being listed as
7539 // failed forever.
7540 let before = self
7541 .store
7542 .head(run)
7543 .await
7544 .map_err(RuntimeError::from_store)?;
7545 // A resume that did nothing new reaches the same open conclusion
7546 // the record already carries, and appending it again would grow
7547 // the journal by one identical conclusion per resume. The chain's
7548 // own last record is the authority: only when it already *is*
7549 // this conclusion is the append skipped. A resume that worked and then concluded the same way
7550 // has its work between the two conclusions, so both are kept.
7551 // Scoped to open conclusions, because a sealing one is
7552 // unreachable twice: the seal refuses further appends and a
7553 // sealed run's replay is a no-op long before here.
7554 let repeated =
7555 !status.seals() && self.last_is_conclusion(run, before.seq, &status).await?;
7556 if repeated {
7557 before.hash
7558 } else {
7559 let mut sealed = Append::new(
7560 run,
7561 RecordKind::RunConcluded {
7562 outcome: status.as_str().to_owned(),
7563 reason: status.own_account().map(std::borrow::Cow::into_owned),
7564 exhaustion: match &status {
7565 RunStatus::Exhausted(limit) => Some(limit.clone()),
7566 _ => None,
7567 },
7568 live_spend,
7569 chain_head: before.hash,
7570 },
7571 );
7572 if let Some(c) = case {
7573 sealed = sealed.case(c);
7574 }
7575 let concluded = self
7576 .store
7577 .append(epoch, vec![sealed])
7578 .await
7579 .map_err(RuntimeError::from_store)?;
7580
7581 concluded.last().map_or(before.hash, |r| r.hash)
7582 }
7583 } else {
7584 self.store
7585 .head(run)
7586 .await
7587 .map_err(RuntimeError::from_store)?
7588 .hash
7589 };
7590
7591 // Hand the lease back rather than letting it time out.
7592 //
7593 // Whatever the outcome — sealed, suspended, exhausted — this instance is
7594 // finished with the run. Holding the lease until expiry would make every
7595 // failover wait out the TTL for nothing.
7596 //
7597 // Best-effort on purpose. A release that fails costs a TTL of patience;
7598 // turning it into a run failure would convert a tidiness problem into a
7599 // correctness one, after the work is already done and journaled.
7600 //
7601 // Only when this pass held a lease at all. A strict pass acquires
7602 // nothing — it is a read — and both stores release on epoch match
7603 // alone, so a strict verification of a run whose owner is live under
7604 // that same epoch would mark the *owner's* lease released: its
7605 // heartbeat stops renewing and any delivery or sweep may fence a
7606 // healthy run mid-step. The same guard keeps a read-only pass out of
7607 // the quota ledger and the metrics: a verification is not work the
7608 // tenant did today, and accruing a historical run's spend into the
7609 // current period on every audit would bill the past once per look.
7610 if writing {
7611 // Settlement precedes both physical sealing and lease release. If
7612 // it fails, the conclusion remains open under an owned lease; once
7613 // that lease expires, recovery derives this pass from the journal
7614 // and retries the idempotent receipt.
7615 //
7616 // Whatever the outcome this instance is finished with the run, so
7617 // a **suspended** run gives its slot back too — it costs a row,
7618 // not a thread, and holding the slot would mean a tenant waiting
7619 // on a hundred approvals could start nothing. The figure accrued
7620 // is what *this pass* dispatched, not the run's cumulative spend,
7621 // which would bill the replayed prefix once per resume.
7622 self.settle_quota(run, epoch, live_spend, quota, status.seals())
7623 .await?;
7624
7625 let chain_head = if status.seals() {
7626 let head = self
7627 .store
7628 .seal(run, epoch, status.as_str())
7629 .await
7630 .map_err(RuntimeError::from_store)?;
7631 self.retire_waits(run).await;
7632 head
7633 } else {
7634 chain_head
7635 };
7636
7637 if let Err(e) = self.store.release_lease(run, epoch).await {
7638 tracing::debug!(
7639 %run,
7640 error = %e,
7641 "could not hand back the lease; it will expire on its own"
7642 );
7643 }
7644
7645 announce(&self.meter, run, &status);
7646
7647 return Ok(RunOutcome {
7648 run_id: run,
7649 status,
7650 chain_head,
7651 consumed,
7652 output,
7653 });
7654 }
7655
7656 Ok(RunOutcome {
7657 run_id: run,
7658 status,
7659 chain_head,
7660 consumed,
7661 // The caller is outside the lattice, so the label is dropped at the
7662 // boundary rather than inside the graph.
7663 output,
7664 })
7665 }
7666}
7667
7668/// The run's result: the terminal step's output.
7669///
7670/// Not "whichever step finished last". That coincides with the terminal step
7671/// only while dispatch is sequential, and stops being well-defined the moment
7672/// two steps run at once. Lowest id wins when a plan has several terminals, so
7673/// the answer is a property of the plan rather than of the schedule.
7674fn run_output(ir: &PlanIR, outputs: &BTreeMap<StepId, Tainted<Value>>) -> Option<Tainted<Value>> {
7675 ir.nodes
7676 .iter()
7677 .filter(|n| n.terminal)
7678 .map(|n| n.id)
7679 .min()
7680 .and_then(|id| outputs.get(&id).cloned())
7681 // A run that stopped before any terminal step still has something to
7682 // report: the furthest output it did produce.
7683 .or_else(|| outputs.iter().next_back().map(|(_, v)| v.clone()))
7684}
7685
7686/// Effects a run announced and never recorded an outcome for — the waits a
7687/// closed run leaves behind.
7688fn unanswered_waits(records: &[Record]) -> Vec<crate::core::EffectKey> {
7689 let settled: BTreeSet<_> = records
7690 .iter()
7691 .filter(|r| {
7692 matches!(
7693 r.kind(),
7694 RecordKind::EffectDone { .. }
7695 | RecordKind::EffectFailed { .. }
7696 | RecordKind::EffectReconciled { .. }
7697 )
7698 })
7699 .filter_map(Record::effect_key)
7700 .collect();
7701 records
7702 .iter()
7703 .filter(|r| matches!(r.kind(), RecordKind::EffectStarted { .. }))
7704 .filter_map(Record::effect_key)
7705 .filter(|k| !settled.contains(k))
7706 .collect()
7707}
7708
7709/// Whether `epoch` holds the conclusion that sealed the run — the pass whose
7710/// settlement releases what the run still held.
7711fn concluded_in(records: &[Record], epoch: crate::core::Epoch) -> bool {
7712 records
7713 .iter()
7714 .any(|r| r.body.epoch == epoch && is_sealing_conclusion(r))
7715}
7716
7717fn is_sealing_conclusion(record: &Record) -> bool {
7718 matches!(
7719 record.kind(),
7720 RecordKind::RunConcluded { outcome, .. } if SEALED_OUTCOMES.contains(&outcome.as_str())
7721 )
7722}
7723
7724/// Spend durably recorded by effects written under one live pass epoch.
7725fn spend_recorded_in_epoch(records: &[Record], epoch: crate::core::Epoch) -> Spend {
7726 records
7727 .iter()
7728 .filter(|record| record.body.epoch == epoch)
7729 .fold(Spend::default(), |total, record| {
7730 let spend = match record.kind() {
7731 RecordKind::EffectDone { spend, .. }
7732 | RecordKind::EffectFailed { spend, .. }
7733 | RecordKind::EffectReconciled { spend, .. } => *spend,
7734 _ => Spend::default(),
7735 };
7736 total.plus(spend)
7737 })
7738}
7739
7740/// Whether the plan actually finished.
7741///
7742/// Structural, never self-reported: a workload asserting it is done is not
7743/// evidence, so the runtime checks that every terminal node ran.
7744fn completion(ir: &PlanIR, done: &BTreeSet<StepId>) -> RunStatus {
7745 if ir.is_complete(done) {
7746 return RunStatus::Succeeded;
7747 }
7748 let missing: Vec<String> = ir
7749 .nodes
7750 .iter()
7751 .filter(|n| n.terminal && !done.contains(&n.id))
7752 .map(|n| n.id.to_string())
7753 .collect();
7754 RunStatus::Failed(format!(
7755 "plan did not complete: terminal step(s) {} never ran",
7756 missing.join(", ")
7757 ))
7758}
7759
7760/// Say that a run changed its plan, and what it changed from.
7761fn announce_replan(meter: &super::metrics::Meter, run: RunId, next: &PlanIR, reason: &str) {
7762 tracing::info!(
7763 target: telemetry::REPLANNED,
7764 %run,
7765 from = next.derived_from.map(Digest::to_hex),
7766 version = next.version,
7767 reason_digest = %telemetry::reason_digest(reason),
7768 );
7769 meter.count(metrics::REPLANS, "");
7770}
7771
7772/// Say how a run ended, on the run span and — for the loud ones — as an event.
7773fn announce(meter: &super::metrics::Meter, run: RunId, status: &RunStatus) {
7774 // Counted here and nowhere else. A step that quarantines also fails its run,
7775 // so counting at both levels would report one incident as two — and the
7776 // terminal status is the fact an operator is counting.
7777 meter.count(metrics::RUNS, status.as_str());
7778 match status {
7779 RunStatus::Quarantined(why) => {
7780 tracing::error!(
7781 target: telemetry::QUARANTINED,
7782 %run,
7783 error_type = "quarantined",
7784 reason_digest = %telemetry::reason_digest(why),
7785 );
7786 meter.count(metrics::QUARANTINES, "");
7787 }
7788 // Loud, though a person asked for it. The operator who decided already
7789 // knows; this reaches whoever answers for what the run left standing.
7790 RunStatus::Abandoned { actor, reason } => {
7791 tracing::error!(
7792 target: telemetry::ABANDONED,
7793 %run,
7794 %actor,
7795 reason_digest = %telemetry::reason_digest(reason),
7796 );
7797 }
7798 RunStatus::Exhausted(limit) => {
7799 tracing::warn!(target: telemetry::BUDGET_REFUSED, %run, %limit);
7800 }
7801 _ => {}
7802 }
7803 tracing::Span::current().record(telemetry::OUTCOME, status.as_str());
7804}
7805
7806/// Record a batch's successes, then report the first step that stopped.
7807///
7808/// **Every** success is recorded, including those of siblings dispatched
7809/// alongside the one that stopped. Returning early on the first failure loses
7810/// them — and a sibling that already performed a mutating effect would then
7811/// never be compensated, because `completed` is what the unwind reverses. The
7812/// work happened; the saga has to know about it.
7813///
7814/// When siblings stop for different reasons, **severity wins over ready order**.
7815/// A suspension is the run working; a failure is the run over. Letting one
7816/// sibling's wait mask another's failure would defer the unwind until an event
7817/// that may never arrive — leaving the failed sibling's mutations in place
7818/// indefinitely. Within one severity, ready order decides, so the choice stays a
7819/// property of the plan rather than of the schedule.
7820fn apply(
7821 plan: &PlanIR,
7822 outcomes: Vec<StepOutcome>,
7823 done: &mut BTreeSet<StepId>,
7824 completed: &mut Vec<(StepId, Capability)>,
7825 ran: &mut BTreeMap<StepId, PlanNode>,
7826 outputs: &mut BTreeMap<StepId, Tainted<Value>>,
7827) -> Option<RunStatus> {
7828 let mut stopped: Option<RunStatus> = None;
7829 for (step, status, output) in outcomes {
7830 let RunStatus::Succeeded = status else {
7831 if stopped
7832 .as_ref()
7833 .is_none_or(|held| severity(&status) > severity(held))
7834 {
7835 stopped = Some(status);
7836 }
7837 continue;
7838 };
7839 if let Some(v) = output {
7840 outputs.insert(step, v);
7841 }
7842 done.insert(step);
7843 if let Some(node) = plan.node(step) {
7844 completed.push((step, node.capability.clone()));
7845 ran.insert(step, node.clone());
7846 }
7847 }
7848 stopped
7849}
7850
7851/// How much a stop reason dominates a competing one.
7852///
7853/// `Quarantined` is highest because it is the only one that must *not* unwind:
7854/// something is undecidable, and compensating around it can make the damage
7855/// worse. `Suspended` is lowest because it is not a stop at all — the run is
7856/// healthy and waiting.
7857fn severity(status: &RunStatus) -> u8 {
7858 match status {
7859 RunStatus::Quarantined(_) => 3,
7860 // A stop ranks with a failure, not above it: both end the run, both
7861 // unwind, and when they arrive together the run is over either way.
7862 // Abandonment is not produced by a step and never competes here; it is
7863 // named so a new stop reason cannot be given a rank by omission.
7864 RunStatus::Cancelled { .. } | RunStatus::Abandoned { .. } | RunStatus::Failed(_) => 2,
7865 // A withdrawal ranks with exhaustion for the same reason: both pause a
7866 // healthy run, neither unwinds, and a sibling that actually failed has
7867 // decided more about the run than either.
7868 RunStatus::Exhausted(_) | RunStatus::Withheld { .. } => 1,
7869 // A replan request is the weakest signal in a batch: a sibling that
7870 // failed outright has already decided the run, and re-planning around a
7871 // failure is not what the requesting step was asking for.
7872 // Neither is reachable here at all — a sweep, a crossing and a lift are
7873 // sealed at birth with no steps to compete — and both are named for the reason
7874 // abandonment is: a rank given by omission is a decision nobody made.
7875 RunStatus::Replanning(_)
7876 | RunStatus::Suspended(_)
7877 | RunStatus::Succeeded
7878 | RunStatus::Swept
7879 // Unreachable for the same reason, and named rather than left to a
7880 // wildcard: an observed session has no steps of this plane's to
7881 // compete, because this plane ran none of it.
7882 | RunStatus::Observed
7883 | RunStatus::BrokeGlass { .. }
7884 | RunStatus::HaltLifted { .. }
7885 | RunStatus::HoldReleased { .. } => 0,
7886 }
7887}
7888
7889/// Gather a dispatched batch back into ready order.
7890///
7891/// Not completion order. `completed` is what the unwind reverses, and a saga
7892/// whose compensation order depended on which future finished first would undo
7893/// a plan differently on every run.
7894type Dispatched = Result<(StepId, RunStatus, Option<Tainted<Value>>, StepCursor), RuntimeError>;
7895
7896/// One step's result, once its history has been handed back.
7897type StepOutcome = (StepId, RunStatus, Option<Tainted<Value>>);
7898
7899fn collect(
7900 dispatched: Vec<Dispatched>,
7901 ready: &[StepId],
7902 cursor: &mut ReplayCursor,
7903) -> Result<Vec<StepOutcome>, RuntimeError> {
7904 let mut outcomes = Vec::with_capacity(dispatched.len());
7905 for result in dispatched {
7906 let (step, status, out, slice) = result?;
7907 cursor.restore(step, Phase::Forward, slice);
7908 outcomes.push((step, status, out));
7909 }
7910 // The key is an `Option`, and `None` would sort *first* — so the invariant
7911 // that makes this total is worth naming rather than trusting: every
7912 // dispatched step came from `admit_ready(&ready, ..)`, which returns a
7913 // subset of `ready`. A future that dispatched a step outside the ready set
7914 // would silently re-order this batch rather than fail, and the order is what
7915 // an unwind reverses.
7916 outcomes.sort_by_key(|(step, _, _)| ready.iter().position(|r| r == step));
7917 Ok(outcomes)
7918}
7919
7920/// What one ready set's dispatch needs.
7921struct Batch<'a> {
7922 agent: &'a str,
7923 identity: Option<&'a crate::core::Delegation>,
7924 subjects: &'a BTreeSet<crate::core::SubjectRef>,
7925 run: RunId,
7926 epoch: u64,
7927 ir: &'a PlanIR,
7928 mode: Pass,
7929 case: &'a Option<CaseContext>,
7930 ledger: &'a Arc<std::sync::Mutex<Ledger>>,
7931 writing: bool,
7932 stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
7933 input: &'a Tainted<Value>,
7934 outputs: &'a BTreeMap<StepId, Tainted<Value>>,
7935 /// Steps whose `StepStarted` the journal already holds. Empty on a live
7936 /// run.
7937 started: &'a BTreeSet<StepId>,
7938 /// Steps whose `StepFinished` the journal already holds. Empty on a live
7939 /// run.
7940 finished: &'a BTreeSet<StepId>,
7941 /// Group records the journal already holds, by writing step and phase.
7942 /// Empty on a live run.
7943 recorded_groups: &'a BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
7944 /// How many of this ready set may be in flight at once.
7945 parallelism: usize,
7946 /// Re-running steps only to abandon groups a pause left open.
7947 abandoning: bool,
7948}
7949
7950/// A standing withdrawal, as the step boundary meets it.
7951///
7952/// Carries the operator from the halt rather than only its reason: the
7953/// withholding record is the one durable trace of this act on the run it
7954/// stopped, and a pause nobody is named on cannot be asked about afterwards.
7955pub(crate) struct Withdrawal {
7956 pub(crate) subject: String,
7957 pub(crate) reason: String,
7958 pub(crate) by: crate::core::Operator,
7959}
7960
7961/// What the journal proves about a run an unwind is deciding over.
7962struct UnwindEvidence {
7963 /// Steps that announced a mutating forward effect.
7964 mutated: BTreeSet<StepId>,
7965 /// The skill each step that started ran as, from its `StepStarted`.
7966 started: BTreeMap<StepId, String>,
7967 /// Steps whose compensation is already on the record.
7968 undone: BTreeSet<StepId>,
7969 /// Group records already on the journal, by writing step and phase.
7970 recorded_groups: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
7971 /// Steps whose last forward ending is not a success: the failing step,
7972 /// which stays resumable, as opposed to a sibling the failure interrupted.
7973 failed: BTreeSet<StepId>,
7974}
7975
7976/// What producing a successor plan needs.
7977struct Replan<'a> {
7978 run: RunId,
7979 current: &'a PlanIR,
7980 /// The node each completed step ran as.
7981 ran: &'a BTreeMap<StepId, PlanNode>,
7982 reason: &'a str,
7983 already_replanned: u32,
7984 max_replans: Option<u32>,
7985 /// The successor this run produced when it first ran, if this is a replay.
7986 recorded: Option<&'a PlanIR>,
7987}
7988
7989/// The first untrusted value in working memory, if any.
7990///
7991/// Returns the source so the refusal can name it: "replanning refused" without
7992/// saying *what* made it unsafe sends an operator looking through the whole run.
7993fn untrusted_in(outputs: &BTreeMap<StepId, Tainted<Value>>) -> Option<String> {
7994 outputs.values().find_map(|v| {
7995 let label = v.label();
7996 label.is_untrusted().then(|| {
7997 label
7998 .provenance
7999 .first()
8000 .map_or_else(|| "an untrusted source".to_owned(), ToString::to_string)
8001 })
8002 })
8003}
8004
8005/// Where a refusal is recorded, when one is.
8006struct Journalling<'a> {
8007 run: RunId,
8008 epoch: u64,
8009 writing: bool,
8010 stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8011}
8012
8013/// What unwinding a run needs.
8014struct Unwind<'a> {
8015 identity: Option<&'a crate::core::Delegation>,
8016 subjects: &'a BTreeSet<crate::core::SubjectRef>,
8017 run: RunId,
8018 epoch: u64,
8019 ir: &'a PlanIR,
8020 mode: Pass,
8021 case: Option<CaseContext>,
8022 ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8023 writing: bool,
8024 stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8025 agent: &'a str,
8026}
8027
8028/// What one step's execution needs.
8029// Each flag is an independent fact about this dispatch; no combination is
8030// invalid, so an enum over them would satisfy the lint at the reader's expense.
8031#[allow(clippy::struct_excessive_bools)]
8032struct StepRun<'a> {
8033 identity: Option<&'a crate::core::Delegation>,
8034 subjects: &'a BTreeSet<crate::core::SubjectRef>,
8035 run: RunId,
8036 epoch: u64,
8037 node: &'a PlanNode,
8038 phase: Phase,
8039 mode: Pass,
8040 case: Option<CaseContext>,
8041 ledger: &'a Arc<std::sync::Mutex<Ledger>>,
8042 writing: bool,
8043 stamp: &'a (dyn Fn(Append) -> Append + Send + Sync),
8044 agent: &'a str,
8045 /// Whether the journal already announced this step. Always `false` live.
8046 already_started: bool,
8047 /// Whether the journal already records this step finishing. Always
8048 /// `false` live.
8049 already_finished: bool,
8050 /// Group records this step and phase already wrote. Always empty live.
8051 recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
8052 /// Whether this step is re-run only to abandon a group a pause left
8053 /// open, because the run was cancelled while paused.
8054 abandoning: bool,
8055}
8056
8057/// The half of a `StepCtx`'s frame that belongs to the **step**.
8058///
8059/// A struct rather than a dozen positional arguments, for the reason
8060/// [`StepRun`] is one: the other half comes from the plane, and a caller who
8061/// transposed `phase` and `mode` positionally would compile.
8062struct StepFrame {
8063 run: RunId,
8064 epoch: crate::core::Epoch,
8065 step: StepId,
8066 phase: Phase,
8067 mode: Pass,
8068 case: Option<CaseContext>,
8069 ledger: Arc<std::sync::Mutex<Ledger>>,
8070 identity: Option<crate::core::Delegation>,
8071 subjects: BTreeSet<crate::core::SubjectRef>,
8072 agent: String,
8073 #[cfg(feature = "manifest")]
8074 manifest: Option<Arc<crate::manifest::Manifest>>,
8075 recorded_groups: BTreeMap<String, super::ctx::RecordedGroup>,
8076}
8077
8078/// Where the recorded run was refused by a *step* limit, if the refusal still
8079/// stands.
8080///
8081/// A step-level refusal has no effect key, so it cannot ride the replay cursor
8082/// the way an effect's does; it is lifted from the records instead.
8083///
8084/// **The last word wins.** An exhausted run may be resumed under a raised
8085/// ceiling; when the resume re-admits the refused step it journals a
8086/// `BudgetReadmitted` beside the old refusal, and from then on the refusal is
8087/// history that was *superseded*, not a verdict to re-serve. Without this, a
8088/// strict replay of the resumed history would stop at the old refusal and
8089/// report `Exhausted` about a run whose own later records show it finishing —
8090/// the refusal followed by the continuation would read as divergence.
8091fn recorded_step_refusal(records: &[Record]) -> Option<(StepId, String, String)> {
8092 let mut standing: Option<(StepId, String, String)> = None;
8093 for r in records {
8094 match r.kind() {
8095 RecordKind::BudgetRefused { limit, used } if r.effect_key().is_none() => {
8096 if let Some(step) = r.body.step {
8097 standing = Some((step, limit.clone(), used.clone()));
8098 }
8099 }
8100 RecordKind::BudgetReadmitted { .. }
8101 if standing
8102 .as_ref()
8103 .is_some_and(|(s, _, _)| Some(*s) == r.body.step) =>
8104 {
8105 standing = None;
8106 }
8107 _ => {}
8108 }
8109 }
8110 standing
8111}
8112
8113/// The subject of a withholding the records left standing, if one stands.
8114///
8115/// **The last word wins**, exactly as it does for a step refusal: a withheld run
8116/// resumes once somebody lifts the halt, and the resume journals an
8117/// `AuthorityRestored` beside the withholding. From then on the withholding is
8118/// history that was *superseded* rather than a verdict to re-serve — without
8119/// this, a resumed run would be treated as still withheld and would conclude
8120/// that way forever, with each resume adding nothing.
8121fn standing_withholding(records: &[Record]) -> Option<String> {
8122 let mut standing: Option<String> = None;
8123 for r in records {
8124 match r.kind() {
8125 RecordKind::AuthorityWithheld { subject, .. } => standing = Some(subject.clone()),
8126 RecordKind::AuthorityRestored { .. } => standing = None,
8127 _ => {}
8128 }
8129 }
8130 standing
8131}
8132
8133/// The run's most recent recorded conclusion, if it has one.
8134///
8135/// `None` for a run that has only ever suspended — a suspension is not a
8136/// conclusion, and `conclude` writes no `RunSealed` for it.
8137fn recorded_conclusion(records: &[Record]) -> Option<String> {
8138 records.iter().rev().find_map(|r| match r.kind() {
8139 RecordKind::RunConcluded { outcome, .. } => Some(outcome.clone()),
8140 _ => None,
8141 })
8142}
8143
8144/// Whether this pass writes a step-level record: always live, never under
8145/// strict verification, and on a resume only when `new_fact` — the record is
8146/// not already in the journal, or the step did new work.
8147const fn writes_step_record(mode: Pass, new_fact: bool) -> bool {
8148 match mode {
8149 Pass::Live => true,
8150 Pass::Resume => new_fact,
8151 Pass::Strict => false,
8152 }
8153}
8154
8155/// The steps whose start is already on the record.
8156///
8157/// Read back so a resume neither re-announces a step history already announced
8158/// nor forgets to announce one it is starting for the first time.
8159fn recorded_started_steps(records: &[Record]) -> BTreeSet<StepId> {
8160 records
8161 .iter()
8162 .filter_map(|r| match r.kind() {
8163 RecordKind::StepStarted { .. } => r.body.step,
8164 _ => None,
8165 })
8166 .collect()
8167}
8168
8169/// The skill each step started as, from its forward `StepStarted`.
8170///
8171/// The capability that actually ran, which after a replan the plan in force
8172/// may no longer name at that id — or may name as other work.
8173fn started_skills(records: &[Record]) -> BTreeMap<StepId, String> {
8174 records
8175 .iter()
8176 .filter(|r| r.body.phase.is_forward())
8177 .filter_map(|r| match (r.kind(), r.body.step) {
8178 (RecordKind::StepStarted { skill }, Some(step)) => Some((step, skill.clone())),
8179 _ => None,
8180 })
8181 .collect()
8182}
8183
8184/// The steps whose ending is already on the record.
8185///
8186/// Read back so a resume records a `StepFinished` exactly for the steps that
8187/// reach an ending the journal does not yet hold: a fully replayed completed
8188/// step keeps its one record, and a suspended step that finally finishes —
8189/// even purely from replayed history, its awaited event having been recorded
8190/// by the delivery — gets the one it never had.
8191fn recorded_finished_steps(records: &[Record]) -> BTreeSet<StepId> {
8192 records
8193 .iter()
8194 .filter_map(|r| match r.kind() {
8195 RecordKind::StepFinished { .. } => r.body.step,
8196 _ => None,
8197 })
8198 .collect()
8199}
8200
8201/// The case the run is bound to, read from its `CaseBound` record.
8202fn recorded_case_id(records: &[Record]) -> Option<crate::core::CaseId> {
8203 records.iter().find_map(|r| match r.kind() {
8204 RecordKind::CaseBound { .. } => r.body.case,
8205 _ => None,
8206 })
8207}
8208
8209/// The steps whose latest forward ending on the record is a success.
8210fn recorded_succeeded_steps(records: &[Record]) -> BTreeSet<StepId> {
8211 let mut succeeded = BTreeSet::new();
8212 for r in records.iter().filter(|r| r.body.phase.is_forward()) {
8213 if let (RecordKind::StepFinished { outcome }, Some(step)) = (r.kind(), r.body.step) {
8214 if outcome == RunStatus::Succeeded.as_str() {
8215 succeeded.insert(step);
8216 } else {
8217 succeeded.remove(&step);
8218 }
8219 }
8220 }
8221 succeeded
8222}
8223
8224/// The group records already on the journal, by writing step and phase.
8225///
8226/// Read back for the same reason the step sets are: group records are not
8227/// effects, so the cursor cannot dedup them, and a resumed step that re-opens
8228/// its group or reaches its end at the frontier — cursor exhausted, writes
8229/// enabled — would otherwise write the same record a second time.
8230fn recorded_groups(
8231 records: &[Record],
8232) -> BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup> {
8233 let mut recorded: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup> =
8234 BTreeMap::new();
8235 for r in records {
8236 let (group, opened) = match r.kind() {
8237 RecordKind::GroupOpened { group, .. } => (group, true),
8238 RecordKind::GroupSettled { group, .. } => (group, false),
8239 _ => continue,
8240 };
8241 let Some(step) = r.body.step else { continue };
8242 let entry = recorded
8243 .entry((step, r.body.phase, group.clone()))
8244 .or_default();
8245 if opened {
8246 entry.opened += 1;
8247 } else {
8248 entry.settled += 1;
8249 }
8250 }
8251 recorded
8252}
8253
8254/// Every capability an agent advertises is provided by one of **its own**
8255/// skills.
8256///
8257/// An agent advertising a capability none of its skills provide is a card that
8258/// lies, and the caller who believed it finds out at dispatch — in production —
8259/// rather than here at startup.
8260///
8261/// This checked a plane-wide map, and the difference is not pedantry. A skill
8262/// registered on the *builder* rather than on the agent — `.agent(Agent::new(&m))`
8263/// followed by `.skill(s)` — satisfied a plane-wide check while being
8264/// **ungoverned**: `governed_by` is keyed from the agent's own skills, so that
8265/// skill gets no manifest. It runs under the plane's default budget instead of
8266/// the declared one, and `StepCtx::gate` never refuses a model or tool the file
8267/// did not list, because there is no file.
8268///
8269/// The plane built cleanly and the assertion's own message said "its skills",
8270/// so the only signal was a `None` from `cx.manifest()` that a skill has no
8271/// reason to check. That is a declaration reading as a control while governing
8272/// nothing, which is the one shape this codebase refuses everywhere.
8273#[cfg(feature = "manifest")]
8274fn check_declaration_matches_skills(
8275 m: &crate::manifest::Manifest,
8276 mine: &HashSet<Capability>,
8277) -> Result<(), BuildError> {
8278 let missing: Vec<String> = m
8279 .spec
8280 .capabilities
8281 .provides
8282 .iter()
8283 .filter(|c| !mine.contains(&Capability::new(c.as_str())))
8284 .cloned()
8285 .collect();
8286 if !missing.is_empty() {
8287 return Err(BuildError::AdvertisesWhatItCannotProvide {
8288 agent: m.metadata.name.clone(),
8289 missing,
8290 });
8291 }
8292
8293 // And the other direction, which is the one that leaves no trace. A skill
8294 // registered under a manifest is *governed* by it — the manifest's budget,
8295 // model grants, egress ceiling and policy identity all apply — so a
8296 // capability the skill answers and the declaration never names is a
8297 // reviewed surface with an unreviewed door in it. The declaration is the
8298 // artifact that gets read, digested and pinned, and the A2A card is built
8299 // from it, so the extra capability is served and advertised nowhere.
8300 //
8301 // It is the same argument the manifest already makes about prompts: a
8302 // system prompt composed in the deployer's code has no version, changes in
8303 // a deploy, and nothing connects the change to the runs it affected. A
8304 // capability added in code does the same to the agent's surface.
8305 let undeclared: Vec<String> = {
8306 let declared: HashSet<Capability> = m
8307 .spec
8308 .capabilities
8309 .provides
8310 .iter()
8311 .map(|c| Capability::new(c.as_str()))
8312 .collect();
8313 let mut extra: Vec<String> = mine
8314 .iter()
8315 .filter(|c| !declared.contains(c))
8316 .map(|c| c.0.clone())
8317 .collect();
8318 // Deterministic, so the message does not depend on hash order.
8319 extra.sort();
8320 extra
8321 };
8322 if !undeclared.is_empty() {
8323 return Err(BuildError::ProvidesWhatItDoesNotAdvertise {
8324 agent: m.metadata.name.clone(),
8325 undeclared,
8326 });
8327 }
8328 Ok(())
8329}
8330
8331/// Add one skill to the plane's two lookup tables, refusing a collision.
8332///
8333/// Both maps are plane-wide, and a bare `insert` would take a second
8334/// registration silently. That was tolerable when a plane was one agent and is
8335/// not now: dispatch resolves a capability to a skill *and to the manifest
8336/// governing it*, so a silent overwrite does not merely shadow the loser — it
8337/// moves work the loser still advertises out from under the loser's budget,
8338/// model grants and egress ceiling. Nothing in the journal would show it,
8339/// because the winner looks like the only claimant that ever existed.
8340///
8341/// Returns the skill's name, which is the key governance is recorded under.
8342///
8343/// # Errors
8344///
8345/// If another skill already holds this name, or another skill already claims one
8346/// of its capabilities.
8347fn register_skill(
8348 skill: Arc<dyn Skill>,
8349 caps: &mut HashMap<Capability, String>,
8350 skills: &mut HashMap<String, Arc<dyn Skill>>,
8351) -> Result<String, BuildError> {
8352 let d = skill.descriptor();
8353 if let Some(existing) = skills.get(&d.name)
8354 // Registering the *same* `Arc` twice is idempotent rather than a
8355 // mistake; two distinct skills under one name is the collision.
8356 && !Arc::ptr_eq(existing, &skill)
8357 {
8358 return Err(BuildError::DuplicateSkillName { name: d.name });
8359 }
8360 // `capabilities()`, not the raw field: a descriptor that declared nothing
8361 // answers its own name, and registering that here is what makes
8362 // `run("greet")` and an `agent/greet` grant resolve without a hello-world
8363 // program inventing a second name for its one skill.
8364 for cap in d.capabilities() {
8365 if let Some(first) = caps.get(&cap)
8366 && first != &d.name
8367 {
8368 return Err(BuildError::CapabilityClaimedTwice {
8369 capability: cap.0,
8370 first: first.clone(),
8371 second: d.name,
8372 });
8373 }
8374 caps.insert(cap, d.name.clone());
8375 }
8376 skills.insert(d.name.clone(), skill);
8377 Ok(d.name)
8378}
8379
8380/// The capability a bare `run(target, ..)` admits: the first declared, or the
8381/// skill's own name — [`SkillDescriptor::capabilities`] is never empty.
8382fn first_capability(descriptor: &SkillDescriptor) -> Capability {
8383 descriptor
8384 .capabilities()
8385 .into_iter()
8386 .next()
8387 .unwrap_or_else(|| Capability::new(descriptor.name.clone()))
8388}
8389
8390/// The business keys a case is identified by, for the binding record.
8391///
8392/// Read from the case rather than taken from the request, because a run joining
8393/// an existing case is admitted with whichever key its message carried while the
8394/// case may be identified by several — and a manifest binding naming any of them
8395/// must resolve.
8396///
8397/// Falls back to the admitted keys if the case cannot be read back. That is not
8398/// defensive padding: `correlate_or_open` has already committed the binding, so
8399/// a failure here is a read that lost a race with nothing, and the keys the run
8400/// was admitted with are a true subset of the case's. Recording them is strictly
8401/// better than recording none, and *nothing* is what would silently make a
8402/// binding unresolvable for a run whose case is perfectly well identified.
8403async fn case_correlation(
8404 cases: &dyn CaseStore,
8405 case_id: crate::core::CaseId,
8406 admitted: &[CorrelationKey],
8407) -> Result<Vec<CorrelationKey>, RuntimeError> {
8408 let stored = cases
8409 .case(case_id)
8410 .await
8411 .map_err(RuntimeError::from_store)?
8412 .map(|case| case.correlation)
8413 .unwrap_or_default();
8414 let mut keys = if stored.is_empty() {
8415 admitted.to_vec()
8416 } else {
8417 stored
8418 };
8419 // Canonical order, so the record is byte-stable for a given set and two
8420 // stores that answer in different orders produce the same journal.
8421 keys.sort();
8422 keys.dedup();
8423 Ok(keys)
8424}
8425
8426/// A lease owner that no other process will accidentally share.
8427///
8428/// The previous default was the constant `"agentplane"`, which every replica and
8429/// every restart used. Two consequences, both silent:
8430///
8431/// * Two replicas each saw the other's lease as their own and renewed it
8432/// without bumping the epoch — two writers on one run, which is the exact
8433/// situation fencing exists to make impossible.
8434/// * A process restarting after a crash "renewed" the dead process's lease
8435/// instead of waiting for expiry and fencing it, so a zombie still holding a
8436/// socket could keep writing under the same epoch as its replacement.
8437///
8438/// A per-process random identity turns both into the correct behaviour: a
8439/// different owner cannot renew, so it waits for expiry and takes over with
8440/// `epoch + 1`.
8441///
8442/// Not derived from a hostname or PID: containers reuse both. Randomness is the
8443/// property that matters; readability is what the `owner` override is for, and a
8444/// deployment with a real instance identity — a pod name — should pass it.
8445fn default_owner() -> String {
8446 use std::collections::hash_map::RandomState;
8447 use std::hash::{BuildHasher, Hasher};
8448 use std::sync::atomic::{AtomicU64, Ordering};
8449
8450 // OS entropy, and deliberately *not* the clock: this crate forbids reading
8451 // the wall clock outside a journaled effect, and rightly — a lease owner is
8452 // a poor reason to make an exception to a rule that keeps replay honest.
8453 // `RandomState` is seeded by the operating system, so two processes differ
8454 // even where a container has reused a PID.
8455 static SEED: std::sync::OnceLock<u64> = std::sync::OnceLock::new();
8456 // A counter beside it, so two runtimes built in one process — which tests do
8457 // constantly — never alias each other either.
8458 static SEQ: AtomicU64 = AtomicU64::new(0);
8459
8460 let seed = *SEED.get_or_init(|| RandomState::new().build_hasher().finish());
8461 let n = SEQ.fetch_add(1, Ordering::Relaxed);
8462 format!("agentplane-{seed:016x}-{n}")
8463}
8464
8465/// Resolve a run's data-subject bindings against its input, before the run
8466/// is bound to a case.
8467///
8468/// Declared bindings first, in declaration order, then the subjects the
8469/// embedder named; each entry's position is the index every reference to it
8470/// names. A binding that resolves to nothing refuses the run rather than
8471/// attributing its data to nobody. A `$case` binding is refused when the run
8472/// asked for no case, and otherwise left for [`fill_case_subjects`].
8473fn bind_subjects(
8474 declared: &[crate::journal::SubjectBinding],
8475 named: Vec<String>,
8476 input: &Tainted<Value>,
8477 in_case: bool,
8478) -> Result<Vec<crate::journal::BoundSubject>, RuntimeError> {
8479 use crate::journal::SubjectBinding;
8480 let refuse = |binding: &SubjectBinding, reason: &str| RuntimeError::SubjectUnbound {
8481 binding: binding.to_string(),
8482 reason: reason.to_owned(),
8483 };
8484 let mut resolved = Vec::new();
8485 let named = named
8486 .into_iter()
8487 .map(|subject| (SubjectBinding::Named, Some(subject)));
8488 for (position, (binding, given)) in declared
8489 .iter()
8490 .cloned()
8491 .map(|binding| (binding, None))
8492 .chain(named)
8493 .enumerate()
8494 {
8495 let (subject, trusted) = match (&binding, given) {
8496 (_, Some(subject)) => (subject, true),
8497 (SubjectBinding::Input { pointer }, None) => {
8498 let Some(selected) = input.project_pointer(pointer) else {
8499 return Err(refuse(&binding, "it selects nothing in the run's input"));
8500 };
8501 let subject = match selected.peek() {
8502 Value::String(text) => text.clone(),
8503 Value::Number(number) => number.to_string(),
8504 _ => {
8505 return Err(refuse(&binding, "it selects neither a string nor a number"));
8506 }
8507 };
8508 (
8509 subject,
8510 selected.label().trust == crate::core::Trust::Trusted,
8511 )
8512 }
8513 (SubjectBinding::Case, None) => {
8514 if !in_case {
8515 return Err(refuse(&binding, "the run belongs to no case"));
8516 }
8517 (CASE_PENDING.to_owned(), true)
8518 }
8519 (SubjectBinding::Named, None) => {
8520 return Err(refuse(&binding, "no subject was named"));
8521 }
8522 };
8523 if subject.trim().is_empty() {
8524 return Err(refuse(&binding, "it resolves to an empty subject"));
8525 }
8526 let index = u16::try_from(position)
8527 .map_err(|_| refuse(&binding, "a run binds at most 65536 subjects"))?;
8528 resolved.push(crate::journal::BoundSubject {
8529 index,
8530 binding,
8531 trusted,
8532 subject,
8533 });
8534 }
8535 Ok(resolved)
8536}
8537
8538/// What a `$case` binding holds between [`bind_subjects`] and
8539/// [`fill_case_subjects`]: never journaled.
8540const CASE_PENDING: &str = "$case";
8541
8542/// Give each `$case` binding the id of the case the run was bound to.
8543fn fill_case_subjects(
8544 mut bound: Vec<crate::journal::BoundSubject>,
8545 case: Option<&CaseContext>,
8546) -> Vec<crate::journal::BoundSubject> {
8547 if let Some(case) = case {
8548 for entry in &mut bound {
8549 if entry.binding == crate::journal::SubjectBinding::Case {
8550 entry.subject = case.case_id.to_string();
8551 }
8552 }
8553 }
8554 bound
8555}
8556
8557/// The references a run's `DataSubjectBound` record makes available to every
8558/// value it reads.
8559fn subject_refs(
8560 run: RunId,
8561 bound: &[crate::journal::BoundSubject],
8562) -> BTreeSet<crate::core::SubjectRef> {
8563 bound
8564 .iter()
8565 .map(|b| crate::core::SubjectRef {
8566 run,
8567 index: b.index,
8568 })
8569 .collect()
8570}
8571
8572/// The references a recorded run's `DataSubjectBound` names, read back so a
8573/// resume attributes exactly as the live run did.
8574fn recorded_subjects(run: RunId, records: &[Record]) -> BTreeSet<crate::core::SubjectRef> {
8575 records
8576 .iter()
8577 .find_map(|r| match r.kind() {
8578 RecordKind::DataSubjectBound { bindings } => Some(subject_refs(run, bindings)),
8579 _ => None,
8580 })
8581 .unwrap_or_default()
8582}
8583
8584/// Refuse a history this build cannot re-derive.
8585///
8586/// Before recomputing anything, because every effect key a replay derives comes
8587/// out of the canonicalizer: a run written under another rule recomputes
8588/// different keys and would be quarantined as *non-determinism* — the most
8589/// serious conclusion this runtime reaches, reported for a healthy run because
8590/// the rule moved underneath it.
8591///
8592/// The chain itself is fine and always was: it hashes the bytes it stored rather
8593/// than re-canonicalizing them. What moved is everything *derived*, which is
8594/// exactly the class replay compares.
8595fn ensure_replayable_canon(records: &[Record]) -> Result<(), RuntimeError> {
8596 if let Some(recorded) = records.iter().find_map(recorded_canon)
8597 && recorded != crate::core::canon::VERSION
8598 {
8599 return Err(RuntimeError::CanonicalizationChanged {
8600 recorded,
8601 implemented: crate::core::canon::VERSION,
8602 });
8603 }
8604 Ok(())
8605}
8606
8607/// Which canonicalization rule wrote this run's derived digests.
8608fn recorded_canon(r: &Record) -> Option<u16> {
8609 match r.kind() {
8610 RecordKind::RunAdmitted { canon, .. } => Some(*canon),
8611 _ => None,
8612 }
8613}
8614
8615/// The principal a run was admitted as.
8616///
8617/// Read back rather than recomputed, for the same reason the plan is: the
8618/// principal a run was authorized as is a fact *about that run*, and deriving it
8619/// again from a plan that may since have been edited would silently re-attribute
8620/// history.
8621/// The chain a recorded run acted under, re-checked structurally on the way
8622/// back — never re-verified as a credential, for the reason `core::identity`
8623/// gives. `None` for a run admitted without one.
8624fn recorded_chain(records: &[Record]) -> Result<Option<crate::core::Delegation>, RuntimeError> {
8625 records
8626 .iter()
8627 .find_map(|r| match r.kind() {
8628 RecordKind::IdentityBound { chain } => Some(chain.clone()),
8629 _ => None,
8630 })
8631 .map(crate::core::Delegation::rehydrate)
8632 .transpose()
8633 .map_err(RuntimeError::Delegation)
8634}
8635
8636/// Whether a resume refused with `error` would be refused the same way on
8637/// every attempt: the journal it read is what refuses, not the store, a lease,
8638/// a quota or this plane's configuration.
8639///
8640/// Named rather than negated: a failure not listed here is retried, so a
8641/// transient one added later costs a retry loop rather than a quarantine.
8642/// `PlanContract`, `NoProvider` and `NoCaseStore` are not listed because they
8643/// are about this plane — a quota store, an agent or a case store it lacks —
8644/// and another plane may settle what this one cannot.
8645const fn recurs_on_every_resume(error: &RuntimeError) -> bool {
8646 matches!(
8647 error,
8648 RuntimeError::Encoding(_)
8649 | RuntimeError::ChainBroken { .. }
8650 | RuntimeError::CanonicalizationChanged { .. }
8651 | RuntimeError::Delegation(_)
8652 )
8653}
8654
8655fn recorded_agent(records: &[Record]) -> String {
8656 records
8657 .iter()
8658 .find_map(|r| match r.kind() {
8659 RecordKind::RunAdmitted { capability, .. } => Some(capability.clone()),
8660 _ => None,
8661 })
8662 .unwrap_or_default()
8663}
8664
8665/// Build a step's input from its declared argument sources.
8666///
8667/// Labels join across sources, so a step reading anything untrusted produces an
8668/// untrusted input without the plan author having to say so.
8669fn assemble(
8670 node: &PlanNode,
8671 run_input: &Tainted<Value>,
8672 outputs: &BTreeMap<StepId, Tainted<Value>>,
8673) -> Result<Tainted<Value>, RuntimeError> {
8674 // The common case — a single argument — passes the value through rather than
8675 // wrapping it in a one-key object, so simple plans stay legible.
8676 if node.args.len() == 1
8677 && let Some((_, only)) = node.args.iter().next()
8678 {
8679 return resolve_arg(node, only, run_input, outputs);
8680 }
8681
8682 let mut fields = Vec::with_capacity(node.args.len());
8683 for (name, source) in &node.args {
8684 let v = resolve_arg(node, source, run_input, outputs)?;
8685 fields.push((name.clone(), v));
8686 }
8687 Ok(Tainted::object(fields))
8688}
8689
8690fn resolve_arg(
8691 node: &PlanNode,
8692 source: &ArgSource,
8693 run_input: &Tainted<Value>,
8694 outputs: &BTreeMap<StepId, Tainted<Value>>,
8695) -> Result<Tainted<Value>, RuntimeError> {
8696 Ok(match source {
8697 // Projected as a step's output is: a field keeps the label it was
8698 // admitted with, and one admitted without its own inherits the whole
8699 // value's — the parts of an untrusted document are untrusted.
8700 ArgSource::RunInput { field } => project(run_input, field.as_deref()),
8701 ArgSource::Const { value } => Tainted::trusted(value.clone()),
8702 ArgSource::Node { step, field } => {
8703 // The contract already proved this is upstream, so a miss here means
8704 // the scheduler dispatched out of order — a bug worth naming rather
8705 // than papering over with a null.
8706 let upstream = outputs.get(step).ok_or_else(|| {
8707 RuntimeError::PlanContract(format!(
8708 "step {} read step {step}, which has not produced a value",
8709 node.id
8710 ))
8711 })?;
8712 project(upstream, field.as_deref())
8713 }
8714 })
8715}
8716
8717/// A value, or one of its top-level fields with its descendants' labels; an
8718/// absent field is a null under the whole value's label.
8719fn project(value: &Tainted<Value>, field: Option<&str>) -> Tainted<Value> {
8720 match field {
8721 Some(field) => value
8722 .project_field(field)
8723 .unwrap_or_else(|| Tainted::with_label(Value::Null, value.label().clone())),
8724 None => value.clone(),
8725 }
8726}
8727
8728/// A step that read a blob rewritten under its address ends unreproducible,
8729/// whatever the skill returned: its history cannot be reproduced, whether the
8730/// skill propagated the error, retried it or swallowed it.
8731fn moved_blob_wins<T>(
8732 cx: &StepCtx<'_>,
8733 result: Result<T, crate::core::SkillError>,
8734) -> Result<T, crate::core::SkillError> {
8735 match cx.moved_blob() {
8736 Some(what) => Err(crate::core::SkillError::Step(
8737 crate::core::StepError::Unreproducible {
8738 what: what.to_owned(),
8739 detail: "a pinned read returned other bytes under its address".to_owned(),
8740 },
8741 )),
8742 None => result,
8743 }
8744}
8745
8746/// Settle a group the skill left open, because `Drop` cannot.
8747///
8748/// A skill that fails with `?` never reaches `commit` or `abort`, so the handle
8749/// is dropped with members standing. Reversing them is async and `Drop` is not,
8750/// which is why the group lives on the context and the executor finishes what
8751/// the handle abandoned — the same relationship the executor already has with a
8752/// step's compensation.
8753///
8754/// Three situations, and they are not the same:
8755///
8756/// * **suspended** — the step has not ended. Its frame is persisted and it will
8757/// re-run from the top, rebuilding the group from the journal as it replays
8758/// the members. Reversing here would undo a run that is merely waiting.
8759/// * **failed** — abort, unless the failure leaves the world in doubt. Doubt is
8760/// the one condition under which nothing may be reversed.
8761/// * **succeeded with a group still open** — an author bug, and the safe
8762/// reading is that the group was never meant to take. It is reversed and the
8763/// step fails loudly, because a group that commits by being forgotten is
8764/// worse than one that does not commit at all.
8765async fn settle_abandoned_group(
8766 cx: &mut StepCtx<'_>,
8767 result: Result<Outcome, crate::core::SkillError>,
8768) -> Result<Outcome, crate::core::SkillError> {
8769 use crate::core::{SkillError, StepError};
8770
8771 let Some(name) = cx.open_group().map(|g| g.name.clone()) else {
8772 return result;
8773 };
8774 // A pause leaves the group open for the resumed step to settle — unless
8775 // the run was cancelled while paused and this pass exists to abandon it.
8776 if !cx.abandoning
8777 && matches!(&result, Err(SkillError::Step(e)) if crate::runtime::group::is_pause(e))
8778 {
8779 return result;
8780 }
8781
8782 // A member whose failure may have reached the world travels no further.
8783 // Reversing around a call that may have — or did — happen leaves the world
8784 // holding a write no `Aborted` settlement can honestly claim to have undone.
8785 let doubt = match &result {
8786 Err(SkillError::Step(e)) => crate::runtime::group::may_have_externalised(e),
8787 _ => false,
8788 };
8789 if doubt {
8790 let detail = match &result {
8791 Err(e) => e.to_string(),
8792 Ok(_) => String::new(),
8793 };
8794 let settled = cx
8795 .settle_open_group(crate::core::GroupOutcome::Quarantined, Some(&detail))
8796 .await;
8797 return match settled {
8798 Ok(()) => result,
8799 Err(e) => Err(SkillError::Step(e)),
8800 };
8801 }
8802
8803 match cx
8804 .abort_open_group("the step ended without settling the group")
8805 .await
8806 {
8807 // The abort itself could not be completed. That outranks whatever the
8808 // step was reporting: a partly unwound group is the more dangerous fact.
8809 Err(e) => Err(SkillError::Step(e)),
8810 Ok(()) => match result {
8811 // A reported failure keeps its own reason. `Outcome::Fail` is an
8812 // `Ok` at the type level and a failure in fact: leaving the group
8813 // to the runtime is the ordinary path there, not an author bug, and
8814 // overwriting the reason would tell an operator the step "returned
8815 // successfully" while hiding why it actually stopped.
8816 Err(e) => Err(e),
8817 failed @ Ok(Outcome::Fail { .. }) => failed,
8818 // Anything else claimed to make progress while leaving a group
8819 // unsettled.
8820 Ok(_) => Err(SkillError::Step(StepError::GroupAborted {
8821 what: format!(
8822 "step made progress with group '{name}' still open — it was \
8823 reversed, because a group that commits by being forgotten is worse \
8824 than one that does not commit at all"
8825 ),
8826 })),
8827 },
8828 }
8829}
8830
8831/// Turn a step's result into a run status.
8832///
8833/// The distinction that matters is between an ordinary failure and a run whose
8834/// *history can no longer be trusted*. Divergence and orphaned effects are the
8835/// latter: they mean the journal no longer describes what this code does, so a
8836/// human has to look before anything else happens. Folding them into `Failed`
8837/// would put them in the same bucket as "the invoice was rejected", and they
8838/// would be retried like one.
8839fn classify(
8840 meter: &super::metrics::Meter,
8841 result: Result<Outcome, crate::core::SkillError>,
8842) -> (RunStatus, Option<Tainted<Value>>) {
8843 use crate::core::{SkillError, StepError};
8844
8845 match result {
8846 // The label travels with the value. Stripping it here would silently
8847 // launder provenance at every step boundary: a downstream step reading
8848 // an untrusted upstream output would receive it marked trusted, and the
8849 // taint gates further on would have nothing to act on.
8850 Ok(Outcome::Done(v)) => (RunStatus::Succeeded, Some(v)),
8851 Ok(Outcome::Fail { reason }) => (RunStatus::Failed(reason), None),
8852 // Not a failure: the executor decides whether a new plan is allowed,
8853 // because the answer depends on the run's provenance and budget, which
8854 // a step cannot see.
8855 Ok(Outcome::Replan { reason }) => (RunStatus::Replanning(reason), None),
8856 // Suspension is not a failure: the run is healthy and waiting. It
8857 // reaches here as an error only because that is how control leaves a
8858 // skill.
8859 Err(SkillError::Step(StepError::Suspended(reason))) => (RunStatus::Suspended(reason), None),
8860 // The ceiling did its job. Reporting this as a failure would have
8861 // operators debugging a system that behaved exactly as instructed.
8862 Err(SkillError::Step(StepError::Budget(exceeded))) => {
8863 (RunStatus::Exhausted(exceeded), None)
8864 }
8865 // A withdrawal found at a hop is the boundary's pause, reached
8866 // mid-step: the hop recorded the withholding before refusing.
8867 Err(SkillError::Step(StepError::Withheld { subject, reason })) => {
8868 (RunStatus::Withheld { subject, reason }, None)
8869 }
8870 Err(e) => {
8871 let msg = e.to_string();
8872 // Matched structurally, never on the message: a match on the word
8873 // "quarantined" would let a reworded error silently downgrade a run
8874 // to `Failed` — history kept, the flag that said not to trust it
8875 // lost.
8876 // Each of these is a failure P7 exists to make loud, and each gets
8877 // its own event so "did this happen" is a query rather than a grep.
8878 match &e {
8879 SkillError::Step(StepError::NonDeterminism {
8880 seq,
8881 expected,
8882 actual,
8883 detail,
8884 }) => {
8885 tracing::error!(
8886 target: telemetry::NONDETERMINISM,
8887 %seq, %expected, %actual,
8888 error_type = "nondeterminism",
8889 reason_digest = %telemetry::reason_digest(detail),
8890 );
8891 meter.count(metrics::DIVERGENCES, "");
8892 }
8893 SkillError::Step(StepError::ReplayOverrun { actual, kind }) => {
8894 tracing::error!(
8895 target: telemetry::NONDETERMINISM,
8896 %actual, %kind, overrun = true,
8897 );
8898 meter.count(metrics::DIVERGENCES, "");
8899 }
8900 SkillError::Step(StepError::Undecidable { key, detail, .. }) => {
8901 tracing::error!(
8902 target: telemetry::UNDECIDABLE,
8903 %key,
8904 error_type = "undecidable",
8905 reason_digest = %telemetry::reason_digest(detail),
8906 );
8907 meter.count(metrics::UNDECIDABLE, "");
8908 }
8909 SkillError::Step(StepError::Unreproducible { what, detail }) => {
8910 tracing::error!(
8911 target: telemetry::UNREPRODUCIBLE,
8912 error_type = "unreproducible",
8913 reason_digest = %telemetry::reason_digest(&format!("{what}: {detail}")),
8914 );
8915 meter.count(metrics::UNREPRODUCIBLE, "");
8916 }
8917 _ => {}
8918 }
8919
8920 let untrustworthy = matches!(
8921 e,
8922 SkillError::Step(
8923 StepError::NonDeterminism { .. }
8924 | StepError::ReplayOverrun { .. }
8925 | StepError::Undecidable { .. }
8926 | StepError::GroupUnsettled { .. }
8927 | StepError::Unreproducible { .. }
8928 )
8929 );
8930 if untrustworthy {
8931 (RunStatus::Quarantined(msg), None)
8932 } else {
8933 (RunStatus::Failed(msg), None)
8934 }
8935 }
8936 }
8937}
8938
8939/// Everything one execution needs, gathered so the executor is not called with
8940/// eight positional arguments — two of which are `u64`-shaped and would swap
8941/// silently.
8942struct Execution<'a> {
8943 /// Where the recorded run was refused by a step limit, if it was. `None`
8944 /// for a live run, which has no history to consult.
8945 refusal: Option<(StepId, String, String)>,
8946 /// The subject of a withholding the recorded history left standing. `None`
8947 /// for a live run, and for a resumed one whose withdrawal was already
8948 /// lifted and recorded.
8949 withheld_subject: Option<String>,
8950 /// Successor plans the recorded run produced, oldest first. Empty on a live
8951 /// run. Read back rather than re-synthesised, because a planner asked twice
8952 /// can answer differently.
8953 successors: Vec<PlanIR>,
8954 /// Steps whose `StepStarted` is already on the record. Empty on a live
8955 /// run. Read back so a resume neither re-announces a step nor forgets to
8956 /// announce one it starts for the first time.
8957 started: BTreeSet<StepId>,
8958 /// Steps whose `StepFinished` is already on the record. Empty on a live
8959 /// run.
8960 finished: BTreeSet<StepId>,
8961 /// Steps whose latest forward ending on the record is a success. Empty on
8962 /// a live run. A resume replays these before it asks whether to stop, so
8963 /// an unwind sees their outputs and the order they finished in.
8964 succeeded: BTreeSet<StepId>,
8965 /// Group records already on the journal, keyed by the step and phase
8966 /// that wrote them. Empty on a live run. Read back for the same reason
8967 /// `finished` is: group records are not effects, so the cursor cannot
8968 /// dedup them, and a resumed step that re-opens or re-settles its group
8969 /// at the frontier — where its cursor is exhausted and writes are
8970 /// enabled — would report one group as two.
8971 recorded_groups: BTreeMap<(StepId, Phase, String), super::ctx::RecordedGroup>,
8972 run: RunId,
8973 epoch: u64,
8974 quota: QuotaPass,
8975 plan: &'a PlanIR,
8976 input: Tainted<Value>,
8977 mode: Pass,
8978 case: Option<CaseContext>,
8979 budget: Budget,
8980 /// The run's data-subject references: from the `DataSubjectBound` just
8981 /// written on a live run, read back from it on a replay.
8982 subjects: BTreeSet<crate::core::SubjectRef>,
8983 /// Who is acting, for the policy principal. Read back from `RunAdmitted` on
8984 /// a replay rather than recomputed, for the same reason the plan is: the
8985 /// principal a run was authorized as is a fact about that run.
8986 agent: String,
8987 /// On whose behalf, for the policy context and for what this run
8988 /// commissions. Read back from `IdentityBound` on a replay for the same
8989 /// reason: a resumed run acting under the plane's chain instead of the one
8990 /// recorded for it would reach different verdicts than the live run did.
8991 identity: Option<crate::core::Delegation>,
8992}
8993
8994/// What a conclusion says when nothing on the chain says who ended the run.
8995///
8996/// Quarantine rather than a placeholder name, for the reason the exhausted arm
8997/// beside it quarantines when the typed ceiling verdict is missing: the record
8998/// is not trustworthy, and *not trustworthy* is a different bucket from *try
8999/// again*. A name invented here would be served through the operator API as the
9000/// person who did it.
9001const UNATTRIBUTED: &str = "the chain records this ending and not who asked for it — an \
9002 operator act with nobody on it cannot be answered by reading further";
9003
9004/// The recorded status of a run that must not be resumed.
9005///
9006/// The outcomes that close a run to recovery:
9007///
9008/// * **Succeeded** — there is nothing outstanding. Re-executing would repeat
9009/// work that is not an effect (a case-state write, say), which is the same
9010/// class of bug the effect protocol prevents, arriving through a side door.
9011/// * **Cancelled** — a stopped run stays stopped, or the next inbound event
9012/// resumes it and it carries on doing the thing somebody intervened to
9013/// prevent.
9014/// * **Abandoned** — a person already decided there will never be an answer.
9015/// * **Quarantined, until a person answers it.** Resuming an unanswered
9016/// quarantine would re-hit whatever could not be decided, and burying that in
9017/// a retry loop is exactly how an undecidable situation becomes an unnoticed
9018/// one. A recorded [`QuarantineDecided`](RecordKind::QuarantineDecided)
9019/// lifts it — the *precondition* is that a named person looked, and the
9020/// verdict is still the runtime's: a reopened run re-decides from a history
9021/// that now holds whatever they established, and reaches the same quarantine
9022/// again if they were wrong or incomplete.
9023///
9024/// A **failed** run is deliberately *not* terminal here: a process that died
9025/// mid-flight records a failure, and recovering it is the entire point.
9026///
9027/// # Why the seal, and not the last step
9028///
9029/// Scanning backwards for a `StepFinished` and reading its outcome is wrong
9030/// twice, and the second is severe:
9031///
9032/// * A step's outcome is not the run's. They coincide only in a one-step plan.
9033/// * `find_map` **skips** a record it does not recognise and keeps looking. A
9034/// run whose last step failed after earlier steps succeeded would match an
9035/// *earlier* `StepFinished { outcome: "succeeded" }` and be reported
9036/// closed-and-succeeded: every multi-step run suspended after a failure —
9037/// every saga waiting on an approval to finish unwinding — could never be
9038/// resumed, and would report success while stuck.
9039///
9040/// `RunSealed` is written by `conclude` for exactly the runs that reached a
9041/// conclusion, and never for a suspended one. That is the fact this needs, so
9042/// it is the fact it reads.
9043pub(crate) fn resume_is_closed(records: &[Record]) -> Option<RunStatus> {
9044 let outcome = records.iter().rev().find_map(|r| match r.kind() {
9045 RecordKind::RunConcluded { outcome, .. } => Some(outcome.as_str()),
9046 _ => None,
9047 })?;
9048
9049 match outcome {
9050 "succeeded" => Some(RunStatus::Succeeded),
9051 "quarantined" => quarantine_decision(records).is_none().then(|| {
9052 RunStatus::Quarantined(
9053 "recorded as quarantined; a named person has to answer it before it can run \
9054 again — reopen it once the doubt is resolved, or abandon it"
9055 .into(),
9056 )
9057 }),
9058 // A conclusion whose attribution record is missing is **quarantined**,
9059 // not attributed to a person called "unknown". The arm for an exhausted
9060 // run with no typed ceiling verdict already reads this way; these three
9061 // fabricated a name instead, and a fabricated one is worse than a
9062 // missing one because it is indistinguishable from a real operator with
9063 // that name and it is served through the operator API as fact.
9064 //
9065 // The reason is the one the person gave, from the record that holds
9066 // it, rather than a sentence written here: an operator refused a
9067 // resume is asking *why this run is closed*, and a canned restatement
9068 // of the outcome answers a question they did not ask.
9069 "abandoned" => Some(recorded_abandonment(records).map_or_else(
9070 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9071 |(actor, reason)| RunStatus::Abandoned { actor, reason },
9072 )),
9073 // A stopped run stays stopped. Otherwise the next inbound event resumes
9074 // it and it carries on doing the thing somebody intervened to prevent —
9075 // and the intervention would look, from the journal, like it worked.
9076 "cancelled" => Some(recorded_cancellation(records).map_or_else(
9077 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9078 |(actor, reason)| RunStatus::Cancelled { actor, reason },
9079 )),
9080 // The two conclusions that deliberately do not close a run: a failed
9081 // run resumes with its completed effects read back from history, and an
9082 // exhausted one continues once somebody raises the ceiling.
9083 //
9084 // Their relationship to [`RunStatus::seals`] is the load-bearing part
9085 // and is pinned by `a_sealing_conclusion_is_never_resumable`. The
9086 // direction that matters is *no sealing status may be resumable*: a
9087 // status that seals froze the journal and published a Merkle leaf, so
9088 // resuming it would grow the history past the leaf every later
9089 // checkpoint attests. The reverse direction is deliberately **not** an
9090 // equality — `Suspended` and `Replanning` also do not seal, and never
9091 // reach here at all, because neither is ever a recorded conclusion.
9092 // Stating that as "the two that do not seal" was wrong; there are four,
9093 // and only two of them can be a `RunSealed` outcome.
9094 // A withdrawal joins them, and is the clearest case of the three: the
9095 // run did nothing wrong, its mutations stand, and lifting the halt is
9096 // what continues it. Closing it would turn *stop acting under this
9097 // credential* into *destroy this work*, which is not what an operator
9098 // withdrawing a credential asked for — the operator who wants that
9099 // cancels, and cancelling unwinds.
9100 "failed" | "exhausted" | WITHHELD_OUTCOME => None,
9101 // Fail closed. An outcome this build does not recognise — a sweep's
9102 // `swept`, a future variant, a corrupted string — is not permission to
9103 // resume; it is a run whose recorded ending this code cannot interpret,
9104 // and continuing it would graft new behaviour onto a history that says
9105 // it ended.
9106 other => Some(RunStatus::Quarantined(format!(
9107 "recorded as '{other}', which this build does not recognise as resumable"
9108 ))),
9109 }
9110}
9111
9112/// Refuse a resume that would report success over work its own history reversed.
9113///
9114/// A run whose journal holds compensation-phase records is not resumable once it
9115/// stands concluded. The unwind *reversed* work — a hold released, a refund
9116/// issued — and a resume that replays the forward history and continues would
9117/// conclude success over a world where the work was undone: the journal would
9118/// say "done" about effects its own later records say were taken back.
9119///
9120/// Scoped to runs with a standing conclusion, deliberately: a run that is
9121/// mid-unwind (a compensation suspended for four eyes, a crash between
9122/// reversals) has concluded nothing, and resuming it is how the unwind finishes.
9123fn refuse_resume_over_undone_work(run: RunId, records: &[Record]) -> Result<(), RuntimeError> {
9124 let Some(outcome) = recorded_conclusion(records) else {
9125 return Ok(());
9126 };
9127 if !records
9128 .iter()
9129 .any(|r| matches!(r.kind(), RecordKind::StepCompensated { .. }))
9130 {
9131 return Ok(());
9132 }
9133 Err(RuntimeError::PlanContract(format!(
9134 "run {run} concluded '{outcome}' after compensating completed steps — the work was \
9135 reversed, and resuming over undone work would report success about a world where it no \
9136 longer stands. Start a fresh run"
9137 )))
9138}
9139
9140/// The person's answer standing against this run's quarantine, if there is one.
9141///
9142/// Scoped to what follows the **last** conclusion, so an answer is spent by the
9143/// pass it authorized. A reopened run that quarantines again is quarantined
9144/// again: the earlier decision was made about a history that has since moved,
9145/// and reading it as standing would turn one person's judgement into a
9146/// permanent licence to retry.
9147fn quarantine_decision(records: &[Record]) -> Option<(crate::core::QuarantineDecision, &Record)> {
9148 records
9149 .iter()
9150 .rev()
9151 .take_while(|r| !matches!(r.kind(), RecordKind::RunConcluded { .. }))
9152 .find_map(|r| match r.kind() {
9153 RecordKind::QuarantineDecided { decision, .. } => Some((*decision, r)),
9154 _ => None,
9155 })
9156}
9157
9158/// The abandonment this run's history asks for but has not yet recorded.
9159///
9160/// A decision is written before it is acted on, so a crash in between leaves
9161/// the instruction standing rather than lost. This is what the next resume
9162/// reads to finish the job — and it is why abandonment is not simply performed
9163/// inline by the verb that requests it.
9164fn pending_abandonment(records: &[Record]) -> Option<RunStatus> {
9165 let (decision, record) = quarantine_decision(records)?;
9166 if decision != crate::core::QuarantineDecision::Abandon {
9167 return None;
9168 }
9169 match record.kind() {
9170 RecordKind::QuarantineDecided {
9171 decider, reason, ..
9172 } => Some(RunStatus::Abandoned {
9173 actor: decider.clone(),
9174 reason: reason.clone(),
9175 }),
9176 _ => None,
9177 }
9178}
9179
9180/// The longest admission key this runtime accepts.
9181///
9182/// Bounded because the key is a **counterparty-controlled** string that becomes
9183/// part of a storage key: unbounded, it is an index entry an emitter chooses the
9184/// size of. 512 bytes is far above what an identity needs — a `CloudEvents`
9185/// `source` and `id` together are tens of bytes — and far below what a btree
9186/// index refuses, so the refusal is this crate's with a reason rather than the
9187/// backend's with an error code.
9188pub const MAX_ADMISSION_KEY_BYTES: usize = 512;
9189
9190/// Refuse a key that cannot mean what the caller thinks it means.
9191///
9192/// An **empty** key is the important one, and it is refused rather than
9193/// accepted as a value. A missing header or an unset variable arrives here as
9194/// `""`, and `""` is a perfectly good key: the first message claims it and every
9195/// later message — a *different* message — is answered with the first one's run.
9196/// That is silent, total message loss produced by a configuration mistake, and
9197/// it is exactly what an at-most-once control must not do quietly.
9198fn validate_admission_key(key: &str) -> Result<(), RuntimeError> {
9199 if key.trim().is_empty() {
9200 return Err(RuntimeError::PlanContract(
9201 "an admission key is empty — an unset header or variable arrives here as \
9202 an empty string, and accepting it would answer every later message with \
9203 the first one's run. Use the message's own identity, such as \
9204 `InboundEvent::dedup_key()`"
9205 .into(),
9206 ));
9207 }
9208 if key.len() > MAX_ADMISSION_KEY_BYTES {
9209 return Err(RuntimeError::PlanContract(format!(
9210 "an admission key of {} bytes exceeds the {MAX_ADMISSION_KEY_BYTES}-byte \
9211 limit — the key is chosen by the emitter and becomes part of a storage \
9212 key, so its size is not theirs to decide",
9213 key.len()
9214 )));
9215 }
9216 Ok(())
9217}
9218
9219/// The run id a store named as an admission key's holder.
9220///
9221/// An unparseable holder is corruption in a derived index. Refused rather than
9222/// admitted around: "the index is unreadable" is not evidence the key is free.
9223fn parse_holder(run: &str) -> Result<RunId, RuntimeError> {
9224 RunId::parse(run).map_err(|e| {
9225 RuntimeError::PlanContract(format!(
9226 "an admission key is held by run '{run}', which does not parse ({e}) — refusing rather than admitting a second run under a key whose holder this build cannot read"
9227 ))
9228 })
9229}
9230
9231/// What a run's history says its state is, if it says anything.
9232///
9233/// **One reader, for every surface that answers the question.** Idempotent
9234/// admission, the operator API and the A2A projection are three views of one
9235/// fact, and three copies of this match agree right up until somebody adds a
9236/// record kind or a conclusion arrives that this build cannot read — after
9237/// which a caller polling and the same caller streaming are told different
9238/// things about one run, which is worse than either answer being wrong. The
9239/// laxest copy is the one that decides what an operator sees, so there is one.
9240///
9241/// `None` means the run is still going: the last record is neither a suspension
9242/// nor a conclusion. It is not this function's business whether that means
9243/// *running* or *never existed* — an empty history answers `None` too, and the
9244/// caller knows which it asked for.
9245///
9246/// **The last record, not any record.** A run that waited, was resumed and
9247/// carried on has a suspension in its history and is not suspended now.
9248pub(crate) fn observed_status(records: &[Record]) -> Option<RunStatus> {
9249 Some(match records.last()?.kind() {
9250 RecordKind::RunSuspended { reason } => RunStatus::Suspended(reason.clone()),
9251 RecordKind::RunConcluded {
9252 outcome,
9253 reason,
9254 exhaustion,
9255 ..
9256 } => recorded_status(outcome, reason.as_deref(), exhaustion.as_ref(), records),
9257 _ => return None,
9258 })
9259}
9260
9261/// A sealed run's status, rebuilt from its recorded conclusion.
9262///
9263/// Distinct from `resume_is_closed`, which asks the narrower *may this run be
9264/// resumed* and returns `None` for the conclusions that stay resumable. Every
9265/// conclusion has an answer here: a failed run reported as "still working"
9266/// would send a redelivery into a second admission.
9267fn recorded_status(
9268 outcome: &str,
9269 reason: Option<&str>,
9270 exhaustion: Option<&crate::core::BudgetExceeded>,
9271 records: &[Record],
9272) -> RunStatus {
9273 // A conclusion that carries no reason gets a statement of that fact rather
9274 // than an empty string: the empty summary is what `RunStatus::reason`
9275 // exists to prevent, and this is the surface a retried delivery reads.
9276 let said = || {
9277 reason.map_or_else(
9278 || "concluded without a recorded reason".to_owned(),
9279 ToOwned::to_owned,
9280 )
9281 };
9282 match outcome {
9283 "succeeded" => RunStatus::Succeeded,
9284 "failed" => RunStatus::Failed(said()),
9285 "replanning" => RunStatus::Replanning(said()),
9286 "exhausted" => exhaustion.cloned().map_or_else(
9287 || {
9288 RunStatus::Quarantined(
9289 "recorded as exhausted without the typed ceiling verdict".to_owned(),
9290 )
9291 },
9292 RunStatus::Exhausted,
9293 ),
9294 "quarantined" => RunStatus::Quarantined(said()),
9295 "cancelled" => recorded_cancellation(records).map_or_else(
9296 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9297 |(actor, reason)| RunStatus::Cancelled { actor, reason },
9298 ),
9299 "abandoned" => recorded_abandonment(records).map_or_else(
9300 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9301 |(actor, reason)| RunStatus::Abandoned { actor, reason },
9302 ),
9303 // A withdrawal, read from the record that holds both halves. Without
9304 // this arm the catch-all answered *quarantined, and this build does
9305 // not recognise 'withheld'* — the plane calling its own record
9306 // foreign, on the surface an operator uses to decide whether to lift a
9307 // halt or to cancel.
9308 WITHHELD_OUTCOME => recorded_withholding(records).map_or_else(
9309 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9310 |(subject, reason)| RunStatus::Withheld { subject, reason },
9311 ),
9312 super::sweeper::SWEEP_OUTCOME => RunStatus::Swept,
9313 // Read back as itself, like every other ending this build writes. The
9314 // catch-all below would answer `Quarantined` — this plane calling its
9315 // own record foreign, on a run whose whole point is that somebody else
9316 // ran it.
9317 OBSERVED_OUTCOME => RunStatus::Observed,
9318 BREAK_GLASS_OUTCOME => recorded_crossing(records).map_or_else(
9319 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9320 |(actor, reason)| RunStatus::BrokeGlass { actor, reason },
9321 ),
9322 HALT_LIFTED_OUTCOME => recorded_halt_lift(records).map_or_else(
9323 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9324 |actor| RunStatus::HaltLifted { actor },
9325 ),
9326 HOLD_RELEASED_OUTCOME => recorded_hold_release(records).map_or_else(
9327 || RunStatus::Quarantined(UNATTRIBUTED.to_owned()),
9328 |actor| RunStatus::HoldReleased { actor },
9329 ),
9330 // Fail closed, as the resume path does: a conclusion this build cannot
9331 // interpret is not permission to treat the run as ordinary. Every
9332 // outcome this build *writes* has an arm above, and a guard holds the
9333 // two lists together — an ending answered here as unrecognised would be
9334 // this plane calling its own record foreign, and the surface it reaches
9335 // is the one an operator uses to read a break-glass crossing.
9336 other => RunStatus::Quarantined(format!(
9337 "recorded as '{other}', which this build does not recognise"
9338 )),
9339 }
9340}
9341
9342/// Whether this value is a sealed payload rather than a readable one.
9343///
9344/// Always false without `keyring`: nothing seals, so nothing can be sealed, and
9345/// a build with no key ring must not carry a dependency on the module that
9346/// defines the wrapper.
9347fn sealed_payload(value: &serde_json::Value) -> bool {
9348 #[cfg(feature = "keyring")]
9349 {
9350 crate::journal::payload::is_sealed(value)
9351 }
9352 #[cfg(not(feature = "keyring"))]
9353 {
9354 let _ = value;
9355 false
9356 }
9357}
9358
9359/// How the run's history says it last ended, or that it is waiting.
9360///
9361/// The last conclusion or suspension on the record, as the outcome word a
9362/// replay's own status reports, so the two compare directly. `None` for a run
9363/// that has neither: it is still in flight, and a strict replay of it would
9364/// stop at its frontier and call that divergence.
9365fn recorded_ending(records: &[Record]) -> Option<String> {
9366 records.iter().rev().find_map(|r| match r.kind() {
9367 RecordKind::RunConcluded { outcome, .. } => Some(outcome.clone()),
9368 RecordKind::RunSuspended { .. } => Some("suspended".to_owned()),
9369 _ => None,
9370 })
9371}
9372
9373/// Whether an ending was an act rather than something the run's steps reached.
9374///
9375/// A cancellation, an abandonment, a sweep, a break-glass crossing, a lift, a
9376/// withdrawal and an observation are each recorded by whoever took them; a
9377/// strict replay re-derives steps, and no step decided these.
9378fn ended_by_an_act(outcome: &str) -> bool {
9379 matches!(outcome, "cancelled" | "abandoned")
9380 || [
9381 super::sweeper::SWEEP_OUTCOME,
9382 BREAK_GLASS_OUTCOME,
9383 HALT_LIFTED_OUTCOME,
9384 HOLD_RELEASED_OUTCOME,
9385 WITHHELD_OUTCOME,
9386 OBSERVED_OUTCOME,
9387 ]
9388 .contains(&outcome)
9389}
9390
9391/// The plan the run was admitted under, and each successor a replan produced,
9392/// oldest first. Read back rather than recompiled, which could produce a graph
9393/// that never governed the run.
9394///
9395/// All or nothing. Successors are walked by position, so skipping one this
9396/// build cannot read would put every later replan on the wrong plan.
9397///
9398/// # Errors
9399///
9400/// [`RuntimeError::PayloadsErased`] when a plan is still sealed, named before
9401/// it is parsed: a sealed payload deserialises into a *missing field*, and that
9402/// sentence sends an operator looking for a corrupt journal rather than telling
9403/// them the data was erased on request.
9404fn frozen_plans(run: RunId, records: &[Record]) -> Result<(PlanIR, Vec<PlanIR>), RuntimeError> {
9405 let mut plans = records
9406 .iter()
9407 .filter_map(|r| match r.kind() {
9408 RecordKind::PlanFrozen { plan, .. } => Some(plan),
9409 _ => None,
9410 })
9411 .map(|plan| {
9412 if sealed_payload(plan) {
9413 return Err(RuntimeError::PayloadsErased {
9414 run: run.to_string(),
9415 });
9416 }
9417 serde_json::from_value(plan.clone()).map_err(RuntimeError::Encoding)
9418 })
9419 .collect::<Result<Vec<PlanIR>, _>>()?
9420 .into_iter();
9421 let admitted = plans
9422 .next()
9423 .ok_or_else(|| RuntimeError::PlanContract("journal has no PlanFrozen record".into()))?;
9424 Ok((admitted, plans.collect()))
9425}
9426
9427/// Whether this run's history is sealed to a key that no longer exists.
9428///
9429/// Read from the frozen plan, which every admitted run has and which is sealed
9430/// under the run's own erasure scope — so it is gone exactly when the run's
9431/// data is gone. Checking a payload rather than asking the key ring keeps this
9432/// a property of the records in hand: the same answer offline, on a restored
9433/// copy, and in a build with no ring wired.
9434fn payloads_erased(records: &[Record]) -> bool {
9435 records.iter().any(|r| match r.kind() {
9436 RecordKind::PlanFrozen { plan, .. } => sealed_payload(plan),
9437 _ => false,
9438 })
9439}
9440
9441/// Who asked for the stop, read back from the chain.
9442///
9443/// Read rather than remembered, for the same reason every other fact about a run
9444/// is: the journal gives the same answer on every subsequent read, and an
9445/// operator asking "who stopped this?" six weeks later is asking history.
9446///
9447/// **Both halves, from the one record that holds them.** The conclusion does
9448/// not repeat the reason — see [`RecordKind::RunConcluded`]'s `reason` for why
9449/// a second copy of it would be the worse copy.
9450fn recorded_cancellation(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9451 records.iter().rev().find_map(|r| match r.kind() {
9452 RecordKind::RunCancelled { actor, reason } => Some((actor.clone(), reason.clone())),
9453 _ => None,
9454 })
9455}
9456
9457/// Who answered the quarantine and what they said, read back from the chain.
9458fn recorded_abandonment(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9459 records.iter().rev().find_map(|r| match r.kind() {
9460 RecordKind::QuarantineDecided {
9461 decider, reason, ..
9462 } => Some((decider.clone(), reason.clone())),
9463 _ => None,
9464 })
9465}
9466
9467/// Who crossed into this tenant and why, read back from the chain.
9468fn recorded_crossing(records: &[Record]) -> Option<(crate::core::Operator, String)> {
9469 records.iter().rev().find_map(|r| match r.kind() {
9470 RecordKind::BreakGlass { actor, reason, .. } => Some((actor.clone(), reason.clone())),
9471 _ => None,
9472 })
9473}
9474
9475/// Who lifted the halt this run records, read back from its lift record.
9476fn recorded_halt_lift(records: &[Record]) -> Option<crate::core::Operator> {
9477 records.iter().rev().find_map(|r| match r.kind() {
9478 RecordKind::HaltLifted { by, .. } => Some(by.clone()),
9479 _ => None,
9480 })
9481}
9482
9483/// Who released the hold this run records, read back from its release record.
9484fn recorded_hold_release(records: &[Record]) -> Option<crate::core::Operator> {
9485 records.iter().rev().find_map(|r| match r.kind() {
9486 RecordKind::HoldReleased { by, .. } => Some(by.clone()),
9487 _ => None,
9488 })
9489}
9490
9491/// Whose authority was withdrawn, and what the operator said when withdrawing
9492/// it — read back from the withholding record.
9493///
9494/// The last one standing: a withdrawal may be lifted and thrown again within a
9495/// run's history, and the conclusion is about the one that stopped it.
9496fn recorded_withholding(records: &[Record]) -> Option<(String, String)> {
9497 records.iter().rev().find_map(|r| match r.kind() {
9498 RecordKind::AuthorityWithheld {
9499 subject, reason, ..
9500 } => Some((subject.clone(), reason.clone())),
9501 _ => None,
9502 })
9503}
9504
9505/// Wall-clock read for the case's `opened_at` stamp, and for a rate
9506/// reservation's instant.
9507///
9508/// Admission happens before any step exists, so there is no `StepCtx` to
9509/// journal through. Both values are store metadata and never participate in
9510/// replay: a rate refusal is journaled on the run and read back, never
9511/// re-counted — run-visible time still goes through `StepCtx::now`, which
9512/// journals it.
9513#[allow(clippy::disallowed_methods)]
9514pub(super) fn now_for_admission() -> crate::core::Timestamp {
9515 crate::core::Timestamp::now_utc()
9516}
9517
9518/// A lift was recorded in `run` and the register's row was not removed.
9519///
9520/// The control still stands; the operator is told the lift failed and which
9521/// run recorded the attempt.
9522fn removal_failed(run: RunId, e: &crate::core::StoreError) -> RuntimeError {
9523 RuntimeError::ControlStands {
9524 run: run.to_string(),
9525 detail: e.to_string(),
9526 }
9527}
9528
9529/// An operator's lift or release that found a control standing and recorded
9530/// ending it.
9531#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9532pub struct ControlLifted {
9533 /// The run holding the record.
9534 pub record: RunId,
9535 /// Whether this call removed the register row. `false` when another lift
9536 /// or release removed it between this call's read and its removal: the
9537 /// control is gone either way, and the record still stands.
9538 pub removed: bool,
9539}
9540
9541/// The epoch an operator act's own run is written under.
9542///
9543/// Such a run has no competing writer to fence against — it is created,
9544/// written and sealed in one call — so a constant is honest here for the same
9545/// reason it is in the sweeper.
9546const BREAK_GLASS_EPOCH: crate::core::Epoch = LEASE_FREE_EPOCH;
9547
9548/// How a break-glass run ends. Not a run status: it neither succeeded nor
9549/// failed at a goal, which is why a sweep seals as `swept` rather than
9550/// borrowing one.
9551const BREAK_GLASS_OUTCOME: &str = "broke-glass";
9552
9553/// How a run recording an operator lifting a halt ends.
9554pub const HALT_LIFTED_OUTCOME: &str = "halt-lifted";
9555
9556/// How a run recording an operator releasing a legal hold ends.
9557pub const HOLD_RELEASED_OUTCOME: &str = "hold-released";
9558
9559/// How a run ends when the authority it acts under is withdrawn under it.
9560///
9561/// Resumable, so it is deliberately absent from both
9562/// [`SEALED_OUTCOMES`] and [`OUTCOMES_OF_RECORD`]: the run has not ended, it is
9563/// waiting for somebody to lift a halt or to cancel it properly.
9564pub const WITHHELD_OUTCOME: &str = "withheld";
9565
9566/// The outcome a session this plane only **observed** is sealed under.
9567///
9568/// Its own word, and not `succeeded`: a session this runtime did not execute
9569/// has no success to report, and answering with one would put somebody else's
9570/// agent's work in the column an operator reads as *this plane did it*. The
9571/// distinction is the whole reason the status exists.
9572pub const OBSERVED_OUTCOME: &str = "observed";
9573
9574/// The epoch every run written without a lease is written under: a sweep's
9575/// record, an operator act's, an observed session's. Nothing takes such a run
9576/// over, so there is no ownership for the epoch to arbitrate, and one constant
9577/// lets the pass that finishes their seals use the epoch they were written at.
9578pub(crate) const LEASE_FREE_EPOCH: crate::core::Epoch = 1;
9579
9580/// The sealing outcomes whose runs are written without a lease.
9581///
9582/// Recovery selects a run by its lapsed lease, so it never finds one of these
9583/// whose writer died between conclusion and seal;
9584/// [`Runtime::seal_unsealed_conclusions`] is what finishes them.
9585pub(crate) const LEASE_FREE_OUTCOMES: &[&str] = &[
9586 super::sweeper::SWEEP_OUTCOME,
9587 BREAK_GLASS_OUTCOME,
9588 HALT_LIFTED_OUTCOME,
9589 HOLD_RELEASED_OUTCOME,
9590 OBSERVED_OUTCOME,
9591];
9592
9593/// One governed identity: a declaration and the skills that serve it.
9594///
9595/// A runtime **runs** agents; it is not one. It owns the journal, the stores,
9596/// the model drivers and the policy engine — infrastructure, shared. An agent
9597/// owns a manifest and its skills — governance, per-identity. Several agents on
9598/// one plane share a journal and are still separately declared, separately
9599/// bounded, and separately answerable.
9600///
9601/// Conflating the two forced a runtime per agent, which meant a lease owner per
9602/// agent for what is one process, a model driver registered once per agent, and
9603/// nowhere in the journal to record *which* agent governed a run.
9604#[cfg(feature = "manifest")]
9605#[derive(Debug, Default)]
9606pub struct Agent {
9607 manifest: Option<Arc<crate::manifest::Manifest>>,
9608 /// Who vouched for the declaration, when it came from a verified resolution.
9609 publisher: Option<crate::core::KeyId>,
9610 skills: Vec<Arc<dyn Skill>>,
9611}
9612
9613#[cfg(feature = "manifest")]
9614impl Agent {
9615 /// An agent governed by this declaration.
9616 ///
9617 /// Nobody has vouched for it. Prefer [`Agent::published_by`] where the
9618 /// manifest came from a verified registry resolution.
9619 #[must_use]
9620 pub fn new(manifest: &crate::manifest::Manifest) -> Self {
9621 Self {
9622 manifest: Some(Arc::new(manifest.clone())),
9623 publisher: None,
9624 skills: Vec::new(),
9625 }
9626 }
9627
9628 /// Record who vouched for this declaration.
9629 ///
9630 /// Takes the [`KeyId`](crate::core::KeyId) that
9631 /// [`Registry::resolve_verified`](crate::manifest::Registry::resolve_verified)
9632 /// returned beside the manifest — which is otherwise dropped on the floor,
9633 /// so a verified resolution and a parsed file become indistinguishable the
9634 /// moment they reach the runtime.
9635 ///
9636 /// # Why this is the grouping a policy wants
9637 ///
9638 /// A rule has to name *a set of agents*, and the obvious candidates do not
9639 /// survive contact with a deployment:
9640 ///
9641 /// * the **workload identity** is per-instance, so a rule naming one is a
9642 /// rule rewritten on every deploy;
9643 /// * the agent **name**, its **role**, or any group label in the manifest is
9644 /// self-asserted — a file claims it, so a rule granting authority to one
9645 /// grants it to any file that types the same string;
9646 /// * the **digest** is unforgeable but names exactly one revision, so every
9647 /// edit is a policy change.
9648 ///
9649 /// A publisher key is the only one that is both a group — many agents, many
9650 /// versions — and impossible to claim without holding the key. Bind the rule
9651 /// to the publisher, keep the digest for "this exact revision", and leave
9652 /// the name for humans reading logs.
9653 #[must_use]
9654 pub fn published_by(mut self, key_id: impl Into<crate::core::KeyId>) -> Self {
9655 self.publisher = Some(key_id.into());
9656 self
9657 }
9658
9659 /// Give it a skill.
9660 #[must_use]
9661 pub fn skill(mut self, skill: impl Skill + 'static) -> Self {
9662 self.skills.push(Arc::new(skill));
9663 self
9664 }
9665}
9666
9667/// Receives every model call's live output, with the run it belongs to.
9668///
9669/// What [`ModelStreamObserver`](crate::model::ModelStreamObserver) receives for
9670/// one call, for every call a plane makes — so a surface can show a run's model
9671/// writing while the journal holds only the finished answer.
9672pub trait RunStreamObserver: Send + Sync + std::fmt::Debug {
9673 /// Delivery is advisory and must not block the call.
9674 fn event(&self, run: RunId, event: Tainted<crate::model::ModelStreamEvent>);
9675}
9676
9677/// Assembles a [`Runtime`].
9678#[derive(Debug)]
9679pub struct RuntimeBuilder {
9680 store: Arc<dyn JournalStore>,
9681 /// The witnesses this plane submits checkpoints to, and the bar a round
9682 /// is held to. Both or neither: a witness list with no declared quorum is
9683 /// witnessing whose evidence is whatever happened to arrive.
9684 witnesses: Vec<Arc<dyn crate::journal::Witness>>,
9685 quorum: Option<crate::journal::WitnessQuorum>,
9686 witness_interval: Option<std::time::Duration>,
9687 signer: Option<Arc<dyn crate::core::Signer>>,
9688 skills: Vec<Arc<dyn Skill>>,
9689 #[cfg(feature = "manifest")]
9690 tools: Option<(
9691 Arc<crate::tools::ToolCatalog>,
9692 Arc<dyn crate::tools::ToolClient>,
9693 )>,
9694 #[cfg(feature = "manifest")]
9695 egress: Option<Arc<crate::core::Egress>>,
9696 #[cfg(feature = "manifest")]
9697 checkers: crate::content::Checkers,
9698 streams: Option<Arc<dyn RunStreamObserver>>,
9699 journal_ceiling: Option<crate::core::Sensitivity>,
9700 tenant: crate::core::TenantId,
9701 owner: Option<String>,
9702 lease_ttl: Duration,
9703 memories: Option<Arc<dyn crate::memory::MemoryStore>>,
9704 semantic: Option<Arc<super::SemanticMemory>>,
9705 authorities: Option<Arc<dyn crate::authority::AuthorityStore>>,
9706 peers: Option<Arc<PeerWiring>>,
9707 tenant_label: super::telemetry::TenantLabel,
9708 quotas: Option<Arc<dyn crate::quota::QuotaStore>>,
9709 quota: crate::quota::TenantQuota,
9710 budget: Budget,
9711 require_verifier: bool,
9712 /// Typed tools whose coherence with every agent is checked at `build`.
9713 #[cfg(feature = "manifest")]
9714 toolbox: Option<crate::tools::ToolBox>,
9715 /// Tool servers reached by some transport other than the box, by name.
9716 #[cfg(feature = "manifest")]
9717 tool_servers: Vec<(String, Arc<dyn crate::tools::ToolClient>)>,
9718 /// Agents registered on this plane, each with its own declaration.
9719 #[cfg(feature = "manifest")]
9720 agents: Vec<Agent>,
9721 /// Drivers by the name a manifest calls them.
9722 #[cfg(feature = "manifest")]
9723 providers: HashMap<String, Arc<dyn crate::model::ModelProvider>>,
9724 cases: Option<Arc<dyn CaseStore>>,
9725 events: Option<Arc<dyn EventStore>>,
9726 tasks: Option<Arc<dyn TaskStore>>,
9727 timers: Option<Arc<dyn TimerStore>>,
9728 blobs: Option<Arc<dyn crate::blob::BlobStore>>,
9729 disclosures: Option<Arc<dyn crate::disclosure::DisclosureRegister>>,
9730 #[cfg(feature = "push")]
9731 push: Option<Arc<dyn crate::push::PushStore>>,
9732 #[cfg(feature = "keyring")]
9733 keyring: Option<Arc<dyn crate::keyring::KeyRing>>,
9734 batches: Option<Arc<dyn crate::batch::BatchStore>>,
9735 policy: Option<Arc<dyn crate::core::PolicyEngine>>,
9736 identity: Option<crate::core::Delegation>,
9737 replanner: Option<Arc<dyn crate::plan::Replanner>>,
9738 calendar: Option<Arc<dyn Calendar>>,
9739 /// Destinations this deployment sends its own run events to.
9740 #[cfg(feature = "push")]
9741 outbox: Option<Arc<crate::push::Outbox>>,
9742}
9743
9744impl RuntimeBuilder {
9745 #[must_use]
9746 pub fn skill(mut self, s: impl Skill) -> Self {
9747 self.skills.push(Arc::new(s));
9748 self
9749 }
9750
9751 /// The workload identity this plane signs its outward claims with.
9752 ///
9753 /// What it buys is that a tool or peer can *check* who called it. Without a
9754 /// signer the provenance block still travels — a server can correlate on it
9755 /// — but it is an assertion any intermediary could have written, and a
9756 /// callee must not authorize on it.
9757 ///
9758 /// Give the store the same signer ([`signing_as`] there) so records and
9759 /// outward claims carry one identity. They are separate settings because a
9760 /// plane can legitimately have one without the other.
9761 ///
9762 /// [`signing_as`]: crate::store::RedbStore::signing_as
9763 #[must_use]
9764 pub fn signing_as(mut self, signer: Arc<dyn crate::core::Signer>) -> Self {
9765 self.signer = Some(signer);
9766 self
9767 }
9768
9769 /// How long this plane's run leases last.
9770 ///
9771 /// The trade is recovery speed against tolerance for a slow instance: a
9772 /// crashed owner's runs stay unclaimable for this long, and a live owner
9773 /// must renew within it. The runtime heartbeats while a run executes, so
9774 /// this bounds *crash* detection rather than how long a run may take.
9775 ///
9776 /// A TTL below [`MIN_LEASE_TTL`] is refused at
9777 /// [`build`](Self::build)/[`try_build`](Self::try_build) as
9778 /// [`BuildError::LeaseUnrenewable`](crate::runtime::BuildError::LeaseUnrenewable).
9779 /// Both stores keep lease expiry in **whole seconds** and treat
9780 /// `expires_at <= now` as lapsed, so a one-second lease expires the moment
9781 /// the clock ticks past the second it was written in — no matter how often
9782 /// it is renewed. Such a lease cannot be held by a live run, and a run that
9783 /// cannot hold its lease is one any instance may take away mid-flight.
9784 /// Refused where a manifest-serving plane can report it as a diagnostic
9785 /// rather than left as a panic in a setter, which `try_build`'s callers
9786 /// could never see as a value.
9787 #[must_use]
9788 pub fn lease_ttl(mut self, ttl: Duration) -> Self {
9789 self.lease_ttl = ttl;
9790 self
9791 }
9792
9793 /// Put this plane's tenant on what it reports about itself.
9794 ///
9795 /// Off by default, and one policy for both signals: the `tenant` field of
9796 /// every metric event, and `agentplane.tenant` on the run span. Read
9797 /// [`telemetry::TenantLabel`](super::telemetry::TenantLabel) before turning
9798 /// it on — a tenant name is often a customer name, and an observability
9799 /// backend is usually the least protected system in a deployment.
9800 ///
9801 /// Cardinality is bounded by construction — the label is *this plane's*
9802 /// tenant, so the number of streams is the number of planes configured, and
9803 /// no request can grow it.
9804 #[must_use]
9805 pub const fn tenant_label(mut self, label: super::telemetry::TenantLabel) -> Self {
9806 self.tenant_label = label;
9807 self
9808 }
9809
9810 /// Give this plane's agents a memory.
9811 ///
9812 /// Optional, and absent by default: an agent with no memory is a normal
9813 /// agent, and one that quietly gained persistent state because a store was
9814 /// wired for something else would be a surprise.
9815 ///
9816 /// Read [`crate::memory`] before wiring one. Writable memory is delayed
9817 /// code: what is written today is read into a context window tomorrow, where
9818 /// a model treats it as established fact.
9819 #[must_use]
9820 pub fn memory(mut self, memories: Arc<dyn crate::memory::MemoryStore>) -> Self {
9821 self.memories = Some(memories);
9822 self
9823 }
9824
9825 /// Give this plane a semantic index, and the embedder that speaks its
9826 /// language.
9827 ///
9828 /// Both together, never one at a time: `build` holds
9829 /// [`Embedder::revision`] to the [`IndexIdentity::query_revision`] the
9830 /// index declares it accepts, and refuses the pair otherwise
9831 /// ([`BuildError::EmbeddingSpaceMismatch`] says what a mismatch costs).
9832 ///
9833 /// Also needs [`memory`](Self::memory): the index holds only commitments,
9834 /// and every hit is materialised from the authoritative store before its
9835 /// content is exposed.
9836 ///
9837 /// [`Embedder::revision`]: crate::memory::Embedder::revision
9838 /// [`IndexIdentity::query_revision`]: crate::memory::IndexIdentity::query_revision
9839 /// [`BuildError::EmbeddingSpaceMismatch`]: crate::runtime::BuildError::EmbeddingSpaceMismatch
9840 #[must_use]
9841 pub fn semantic_memory(
9842 mut self,
9843 embedder: Arc<dyn crate::memory::Embedder>,
9844 retriever: Arc<dyn crate::memory::SemanticRetriever>,
9845 ) -> Self {
9846 self.semantic = Some(Arc::new(super::SemanticMemory {
9847 embedder,
9848 retriever,
9849 }));
9850 self
9851 }
9852
9853 /// Whether the plane's budget, or any registered agent's, caps money.
9854 fn states_a_money_ceiling(&self) -> bool {
9855 #[cfg(feature = "manifest")]
9856 if self
9857 .agents
9858 .iter()
9859 .filter_map(|agent| agent.manifest.as_ref())
9860 .any(|manifest| manifest.budget().max_minor_units.is_some())
9861 {
9862 return true;
9863 }
9864 self.budget.max_minor_units.is_some()
9865 }
9866
9867 /// Attach durable standing-authority accounting.
9868 ///
9869 /// The ceiling neither of the other two can express. A budget bounds one
9870 /// run; a quota bounds a tenant over a billing period. A standing authority
9871 /// bounds *an authorization* — what one customer approved, spanning as many
9872 /// runs as it takes, revocable when they change their mind.
9873 ///
9874 /// Without one, [`StepCtx::draw`](crate::runtime::StepCtx::draw) refuses
9875 /// rather than falling back to an in-process counter. That fallback would
9876 /// fail **open** the moment a second instance started, which is exactly when
9877 /// a shared ceiling was needed.
9878 #[must_use]
9879 pub fn authorities(mut self, authorities: Arc<dyn crate::authority::AuthorityStore>) -> Self {
9880 self.authorities = Some(authorities);
9881 self
9882 }
9883
9884 /// Bound what this tenant may consume, durably.
9885 ///
9886 /// Budgets bound one run; this bounds the tenant. Both are needed: a caller
9887 /// that can start runs can start a thousand, each within its own ceiling.
9888 ///
9889 /// The accounting lives in the store, so the ceiling survives a second
9890 /// instance — an in-process counter would silently double the moment
9891 /// somebody scales out, which is exactly when it was needed.
9892 ///
9893 /// Read [`crate::quota`] for what each ceiling does and does not bound; a
9894 /// limit believed to bound something it does not is worse than none.
9895 #[must_use]
9896 pub fn quota(
9897 mut self,
9898 quotas: Arc<dyn crate::quota::QuotaStore>,
9899 quota: crate::quota::TenantQuota,
9900 ) -> Self {
9901 self.quotas = Some(quotas);
9902 self.quota = quota;
9903 self
9904 }
9905
9906 /// This **process instance's** identity, as it appears in run leases.
9907 ///
9908 /// Not the agent's name. Fencing rests on the epoch — a claim never renews,
9909 /// and a renewal names the exact epoch it holds — so a shared string costs
9910 /// no safety; it costs the answer to *which process holds this run*, which
9911 /// is what an operator asks of a stuck one and what `LeaseHeld` reports.
9912 ///
9913 /// So keep it unique per running process, which is what the default is —
9914 /// override it only with a better instance identity than a random one,
9915 /// such as a pod name. An agent's *name* is
9916 /// [`Manifest::metadata`](crate::manifest::Metadata::name).
9917 ///
9918 /// The owner lives in the lease table and never in the chain, so it has no
9919 /// bearing on replay.
9920 #[must_use]
9921 pub fn owner(mut self, o: impl Into<String>) -> Self {
9922 self.owner = Some(o.into());
9923 self
9924 }
9925
9926 /// Cap what a run may consume.
9927 ///
9928 /// Defaults to [`Budget::unlimited`], which is right for a runtime whose
9929 /// effects are all free and local, and wrong the moment one of them calls a
9930 /// metered API.
9931 #[must_use]
9932 pub fn budget(mut self, budget: Budget) -> Self {
9933 self.budget = budget;
9934 self
9935 }
9936
9937 /// Refuse a plan in which nothing checks the work.
9938 ///
9939 /// A node that declares [`verifies`](crate::core::PlanNode::verifies) must
9940 /// already depend on what it checks; this makes carrying one a condition of
9941 /// admission. It binds every plan this runtime validates, and the one that
9942 /// matters is the one the embedder did not write: a
9943 /// [`Replanner`](crate::plan::Replanner)'s successor is a graph proposed
9944 /// mid-run, and a control that held for the first plan and not for its
9945 /// replacement is a control a replan removes.
9946 #[must_use]
9947 pub fn require_verifier(mut self) -> Self {
9948 self.require_verifier = true;
9949 self
9950 }
9951
9952 /// Attach long-lived case storage, enabling correlation and deadlines.
9953 #[must_use]
9954 pub fn cases(mut self, cases: Arc<dyn CaseStore>) -> Self {
9955 self.cases = Some(cases);
9956 self
9957 }
9958
9959 /// Send this plane's own events to destinations the deployment configured.
9960 ///
9961 /// Every run registers each destination **at admission**, so there is no
9962 /// window in which a run exists and nothing is watching it. Delivery is a
9963 /// separate, operator-scheduled sweep — see
9964 /// [`DeliveryWorker`](crate::push::DeliveryWorker) — reading each run's
9965 /// journal past a cursor that advances only on 2xx.
9966 ///
9967 /// This is the mirror image of A2A push, and the mirror is the point: there
9968 /// the *caller* names a URL and three controls exist because of it. Here the
9969 /// deployment names it, and the run's own history is the outbox rather than
9970 /// a queue that can fall out of sync with it.
9971 #[cfg(feature = "push")]
9972 #[must_use]
9973 pub fn outbox(mut self, outbox: Arc<crate::push::Outbox>) -> Self {
9974 self.outbox = Some(outbox);
9975 self
9976 }
9977
9978 /// Attach the webhook registration store.
9979 ///
9980 /// The same handle the A2A server and the [`Outbox`](crate::push::Outbox)
9981 /// are given: this is the *durable* half of push, and it belongs to the
9982 /// plane because its failures do. A receiver that answers permanently, or
9983 /// stays silent past the retry ceiling, has its registration
9984 /// [parked](crate::push::PushStore::park) rather than deleted — the cursor
9985 /// is the only record of how far that receiver got — and a parked row is a
9986 /// backlog an operator reads and re-arms on the operator API. Without this
9987 /// the plane serves the delivery and cannot answer for it.
9988 #[cfg(feature = "push")]
9989 #[must_use]
9990 pub fn push(mut self, push: Arc<dyn crate::push::PushStore>) -> Self {
9991 self.push = Some(push);
9992 self
9993 }
9994
9995 /// Attach inbound-event storage, enabling durable waits.
9996 #[must_use]
9997 pub fn events(mut self, events: Arc<dyn EventStore>) -> Self {
9998 self.events = Some(events);
9999 self
10000 }
10001
10002 /// Attach a worklist, enabling human tasks.
10003 #[must_use]
10004 pub fn tasks(mut self, tasks: Arc<dyn TaskStore>) -> Self {
10005 self.tasks = Some(tasks);
10006 self
10007 }
10008
10009 /// Supply the planner that produces successor plans.
10010 ///
10011 /// Without one, a step asking to replan fails with that as the reason —
10012 /// which is the honest outcome, not a silent no-op.
10013 #[must_use]
10014 pub fn replanner(mut self, r: Arc<dyn crate::plan::Replanner>) -> Self {
10015 self.replanner = Some(r);
10016 self
10017 }
10018
10019 /// Supply the durable-timer store.
10020 ///
10021 /// Needed by `StepCtx::sleep` and `sleep_until`, and by the sweep that wakes
10022 /// them. A runtime without one refuses to sleep rather than falling back to
10023 /// an in-process wait that a restart would forget.
10024 #[must_use]
10025 pub fn timers(mut self, timers: Arc<dyn TimerStore>) -> Self {
10026 self.timers = Some(timers);
10027 self
10028 }
10029
10030 /// Register an agent on this plane.
10031 ///
10032 /// A runtime runs agents; it is not one. This is where a declaration and
10033 /// its skills arrive together, so several agents can share one journal, one
10034 /// set of drivers and one process identity while each stays separately
10035 /// governed.
10036 ///
10037 /// The declaration **binds** for that agent's steps: an effect naming a
10038 /// model or tool its manifest never listed is refused before dispatch and
10039 /// journaled, and the egress and delegation ceilings combine with the sink's
10040 /// own — the stricter wins. Its budget bounds its runs. Architectural
10041 /// injection patterns are deliberately absent from the schema, because this
10042 /// runtime cannot prove that arbitrary skill code follows one.
10043 ///
10044 /// An agent declaring `spec.execution` needs no skill: the runtime supplies
10045 /// the behaviour. See [`provider`](Self::provider) for the driver mapping it
10046 /// needs.
10047 ///
10048 /// It does **not** set the lease owner. That identifies a *process*, and one
10049 /// plane running four agents is still one process — see
10050 /// [`owner`](Self::owner).
10051 ///
10052 /// # This call never fails
10053 ///
10054 /// It records the agent and returns. Every refusal an agent can cause — a
10055 /// capability none of its skills provide, a provider no driver is registered
10056 /// for, oversight with nowhere to put a decision — is raised at
10057 /// [`build`](Self::build), which panics, or returned by
10058 /// [`try_build`](Self::try_build), which does not.
10059 ///
10060 /// That split matters for a daemon assembling a plane from files it did not
10061 /// write: a bad manifest is an *input* there, and a panic takes down every
10062 /// other tenant in the process to report it. A `# Panics` section here,
10063 /// describing `build`'s refusals, would send readers looking for a
10064 /// fallible variant of the wrong call — the refusals are documented where
10065 /// they fire.
10066 #[cfg(feature = "manifest")]
10067 #[must_use]
10068 pub fn agent(mut self, agent: Agent) -> Self {
10069 self.agents.push(agent);
10070 self
10071 }
10072
10073 /// Register a model driver under the name a manifest uses for it.
10074 ///
10075 /// The seam a declarative agent needs. A manifest says `provider: anthropic`
10076 /// — a string a reviewer can read — and something has to map that to a
10077 /// driver holding a credential. That mapping is deployment wiring, not a
10078 /// property of the agent, which is exactly why it lives here and not in the
10079 /// file: an agent's declaration should not change when its API key does.
10080 ///
10081 /// Required only for [`ExecutionKind::Completion`] and the other declarative
10082 /// kinds. A hand-written skill constructs its own `ModelCall` and never
10083 /// consults this.
10084 ///
10085 /// [`ExecutionKind::Completion`]: crate::manifest::ExecutionKind::Completion
10086 #[cfg(feature = "manifest")]
10087 #[must_use]
10088 pub fn provider(
10089 mut self,
10090 name: impl Into<String>,
10091 provider: Arc<dyn crate::model::ModelProvider>,
10092 ) -> Self {
10093 self.providers.insert(name.into(), provider);
10094 self
10095 }
10096
10097 /// Supply content-addressed blob storage.
10098 ///
10099 /// Needed by `StepCtx::store_blob`, which is how bytes too large for a
10100 /// journal record get somewhere durable while the chain keeps only their
10101 /// digest. A runtime without one refuses rather than silently inlining
10102 /// megabytes into an append-only chain that can never take them back.
10103 #[must_use]
10104 pub fn blobs(mut self, blobs: Arc<dyn crate::blob::BlobStore>) -> Self {
10105 self.blobs = Some(blobs);
10106 self
10107 }
10108
10109 /// Where disclosures of a matter are recorded, so a retention pass names
10110 /// the copies of what it erased.
10111 #[must_use]
10112 pub fn disclosures(mut self, register: Arc<dyn crate::disclosure::DisclosureRegister>) -> Self {
10113 self.disclosures = Some(register);
10114 self
10115 }
10116
10117 /// The operator's tool catalogue, and the client that reaches those tools.
10118 ///
10119 /// Required by a `tool-calling` agent and by nothing else: a skill that
10120 /// calls tools builds its own [`ToolCall`](crate::tools::ToolCall), because
10121 /// it knows which client it means. A declarative agent has no code to make
10122 /// that choice, so the plane makes it once.
10123 ///
10124 /// The catalogue is the authority. A manifest grants a subset of it, the
10125 /// model is offered exactly that subset, and a name the model returns is
10126 /// matched against it byte for byte.
10127 #[cfg(feature = "manifest")]
10128 #[must_use]
10129 pub fn tools(
10130 mut self,
10131 catalog: Arc<crate::tools::ToolCatalog>,
10132 client: Arc<dyn crate::tools::ToolClient>,
10133 ) -> Self {
10134 self.tools = Some((catalog, client));
10135 self
10136 }
10137
10138 /// Typed tools, with their catalogue derived and their coherence enforced.
10139 ///
10140 /// The one-call form, and the reason it exists is not brevity. Deriving the
10141 /// catalogue and checking it against every agent's manifest were both
10142 /// possible before and both **optional**, and a control a caller may forget
10143 /// is not a control — it is advice that reads like one.
10144 ///
10145 /// So this does three things that were three things:
10146 ///
10147 /// * derives the catalogue from each agent's declaration, so a grant, its
10148 /// ceiling and its protected fields are stated once;
10149 /// * refuses to build if the tools this binary implements and the manifests
10150 /// a reviewer approved have drifted apart;
10151 /// * wires the box as the client.
10152 ///
10153 /// The work happens in [`build`](Self::build) rather than here, and that is
10154 /// the whole reason it is trustworthy: checking on this call would check
10155 /// against the agents registered *so far*, so `.toolbox(..).agent(..)` would
10156 /// pass by having nothing to disagree with. An enforcement that depends on
10157 /// the order a builder was written is not one.
10158 #[cfg(feature = "manifest")]
10159 #[must_use]
10160 pub fn toolbox(mut self, tools: crate::tools::ToolBox) -> Self {
10161 self.toolbox = Some(tools);
10162 self
10163 }
10164
10165 /// A tool server this plane reaches over some transport of its own.
10166 ///
10167 /// An MCP connection is the usual one. Registering it does three things:
10168 ///
10169 /// * a plane may reach **several** servers, because the router resolves the
10170 /// `tool://server/name` a grant carries rather than handing every id to one
10171 /// client;
10172 /// * typed in-process tools and remote servers can be used by the *same*
10173 /// agent, which is the ordinary shape;
10174 /// * a grant naming a server nobody wired is refused at build, in the same
10175 /// breath as a grant nothing implements — both mean the model would be
10176 /// offered a tool that fails when chosen.
10177 ///
10178 /// Composes with [`toolbox`](Self::toolbox); the box answers for the servers
10179 /// its own tools name and these answer for theirs. A server claimed twice is
10180 /// a panic, because registration order deciding which transport carries a
10181 /// call is the defect [`ToolRouter`](crate::tools::ToolRouter) exists to
10182 /// remove.
10183 #[cfg(feature = "manifest")]
10184 #[must_use]
10185 pub fn tool_server(
10186 mut self,
10187 name: impl Into<String>,
10188 client: Arc<dyn crate::tools::ToolClient>,
10189 ) -> Self {
10190 self.tool_servers.push((name.into(), client));
10191 self
10192 }
10193
10194 /// Where this plane's **tool transports** may connect.
10195 ///
10196 /// A `ToolClient` is the embedder's code, and the split is the one every
10197 /// other outbound path here makes: **the client owns the connection, the
10198 /// plane owns the destination.** A transport says where it goes with
10199 /// [`ToolClient::destination`](crate::tools::ToolClient::destination), and
10200 /// this decides whether that is somewhere it may go — before the effect
10201 /// exists, so nothing leaves, nothing is journaled and nothing is metered.
10202 ///
10203 /// A [`Destination::Local`](crate::tools::Destination::Local) transport is
10204 /// not judged: tools compiled into the binary and a stdio child process
10205 /// contact no host of this plane's choosing, so there is nothing for an
10206 /// allowlist to decide. What that does not claim is that the far side
10207 /// reaches nothing — the same residual a compromised allowlisted endpoint
10208 /// carries.
10209 ///
10210 /// Unset means no egress control on the tool path, spelled the way an
10211 /// absent policy engine is: by configuring nothing, rather than by
10212 /// configuring something that looks like a control and is not.
10213 ///
10214 /// ```ignore
10215 /// let plane = Runtime::builder(store)
10216 /// .tool_server("tickets", mcp_over_https)
10217 /// .egress(Egress::new().allow("mcp.tickets.example"))
10218 /// .build();
10219 /// ```
10220 #[cfg(feature = "manifest")]
10221 #[must_use]
10222 pub fn egress(mut self, egress: crate::core::Egress) -> Self {
10223 self.egress = Some(Arc::new(egress));
10224 self
10225 }
10226
10227 /// Forward every model call's live output, with the run it belongs to.
10228 ///
10229 /// Advisory and live-only: the journal's completion stays the canonical
10230 /// answer, a replay performs no call and so streams nothing, and an
10231 /// observer that drops events changes nothing a run does.
10232 #[must_use]
10233 pub fn observe_model_streams(mut self, observer: Arc<dyn RunStreamObserver>) -> Self {
10234 self.streams = Some(observer);
10235 self
10236 }
10237
10238 /// Register a content checker, by its [`name`](crate::content::ContentChecker::name).
10239 ///
10240 /// A manifest check naming a checker no builder registered, or mapping a
10241 /// category the checker does not declare, refuses the build: a declared
10242 /// check nothing could run is a control a reviewer approved that does not
10243 /// exist.
10244 #[cfg(feature = "manifest")]
10245 #[must_use]
10246 pub fn content_checker(mut self, checker: Arc<dyn crate::content::ContentChecker>) -> Self {
10247 Arc::make_mut(&mut self.checkers).insert(checker.name().to_owned(), checker);
10248 self
10249 }
10250
10251 /// The highest sensitivity this plane will write into an append-only record.
10252 ///
10253 /// # What this is, and what the key ring is
10254 ///
10255 /// Two different answers to *this data must be erasable*, and they compose.
10256 /// [`keyring`](Self::keyring) is the **seal it** half: payloads become
10257 /// ciphertext whose key an erasure destroys, which reaches every replica
10258 /// and every backup at once. This is the **refuse it** half: an argument
10259 /// more sensitive than the deployment is willing to make permanent is
10260 /// refused at dispatch, before it reaches the chain.
10261 ///
10262 /// The plane-level twin of `spec.security.max_sensitivity_journaled`, for
10263 /// a plane of hand-written skills that runs under no manifest. The default
10264 /// is the unsafe one — a journal keeps everything, indefinitely — so a
10265 /// plane must be able to state the ceiling whether or not it is
10266 /// declarative. It is an enforcement point at the same gate the manifest's
10267 /// field reaches, not a warning; where both are present **the stricter
10268 /// wins**, as a reviewed tool grant may only tighten what the operator's
10269 /// catalogue allows.
10270 ///
10271 /// Absent means no ceiling, spelled the way an absent policy engine is.
10272 ///
10273 /// ```ignore
10274 /// let plane = Runtime::builder(store)
10275 /// // Personal data goes in a blob and the chain gets the digest.
10276 /// .max_sensitivity_journaled(Sensitivity::Internal)
10277 /// .skill(Triage)
10278 /// .build();
10279 /// ```
10280 #[must_use]
10281 pub const fn max_sensitivity_journaled(mut self, ceiling: crate::core::Sensitivity) -> Self {
10282 self.journal_ceiling = Some(ceiling);
10283 self
10284 }
10285
10286 /// Which tenant this plane runs as.
10287 ///
10288 /// **One plane, one tenant — but one process, many planes.** A plane is the
10289 /// unit that is bound to a tenant; serving several is
10290 /// [`Planes`](crate::api::Planes)' job, and it resolves the plane from the
10291 /// authenticated caller's tenant rather than from the request, so a handler
10292 /// cannot reach a store it did not resolve. An unregistered tenant is
10293 /// refused rather than defaulted.
10294 ///
10295 /// The name scopes **data keys**, so one tenant's cryptographic erasure
10296 /// cannot reach another's bytes, and it reaches the **policy request**, so a
10297 /// rule can be written per tenant.
10298 ///
10299 /// It does **not** scope the store — that is a separate handle, scoped by
10300 /// `RedbStore::for_tenant` or `PostgresStore::for_tenant`. Two tenants may
10301 /// share one store, because the tenant is a key component of every row on
10302 /// both backends rather than a filter. Setting one and not the other is
10303 /// refused at [`build`](Self::build) rather than discovered later: a plane
10304 /// whose store is scoped elsewhere works perfectly and writes its runs into
10305 /// somebody else's keyspace.
10306 ///
10307 /// Defaults to `default`, which is a real tenant rather than an absence: the
10308 /// single-tenant path is then the same code as the multi-tenant one, and a
10309 /// special "no tenant" case is a second path that would not get tested.
10310 #[must_use]
10311 pub fn tenant(mut self, tenant: crate::core::TenantId) -> Self {
10312 self.tenant = tenant;
10313 self
10314 }
10315
10316 /// Seal payload bytes, and make erasure reach copies deletion cannot.
10317 ///
10318 /// With a key ring configured, everything written through
10319 /// [`StepCtx::blobs`](crate::runtime::StepCtx::blobs) — including
10320 /// [`store_blob`](crate::runtime::StepCtx::store_blob) and governed media —
10321 /// is encrypted under a data key belonging to the run's **case**. Erasing
10322 /// that case destroys the key, so every copy of those bytes becomes
10323 /// unreadable at once: the live store, the replicas, and every backup ever
10324 /// taken. Expiring blobs only reaches the first of those.
10325 ///
10326 /// The case is the erasure unit because it is already the retention unit —
10327 /// bytes are linked to their case at write time, and a second, differently
10328 /// shaped unit for keys would let the two disagree about what an erasure
10329 /// covered.
10330 ///
10331 /// It also seals the stores the plane holds — the journal's payloads, case
10332 /// state, task proposals and buffered event payloads — at
10333 /// [`build`](Self::build), so the order they were registered in cannot lose
10334 /// the guarantee — a store registered after this call is sealed just the
10335 /// same.
10336 ///
10337 /// **Governed memory is the one store this does not reach.**
10338 /// [`EncryptedMemoryStore`](crate::keyring::EncryptedMemoryStore)
10339 /// serialises subject erasure against writes and legal-hold changes with a
10340 /// process-local mutex, so it holds its contract on a single-writer
10341 /// deployment and nowhere else; wrapping it here would hand that adapter to
10342 /// an active-active `PostgreSQL` plane, where the mutex coordinates nothing
10343 /// and the hold race it exists to prevent is the result. Its erasure unit
10344 /// is one item version (`memory-item/<id>@<version>`) and outlives every
10345 /// case, so `erase_case` is not the act that reaches it either. Wrap it
10346 /// yourself, where the deployment's topology is visible:
10347 ///
10348 /// ```ignore
10349 /// let memories = EncryptedMemoryStore::new(inner, keys.clone(), tenant.clone());
10350 /// Runtime::builder(store).memory(Arc::new(memories)).keyring(keys).build()
10351 /// ```
10352 ///
10353 /// Without one, bytes are stored as given and erasure remains deletion.
10354 #[cfg(feature = "keyring")]
10355 #[must_use]
10356 pub fn keyring(mut self, keyring: Arc<dyn crate::keyring::KeyRing>) -> Self {
10357 self.keyring = Some(keyring);
10358 self
10359 }
10360
10361 /// Wrap every store a key ring can seal, at **build** time.
10362 ///
10363 /// One call, one guarantee. Before this, `keyring` sealed blob payloads
10364 /// and nothing else — which was honest when blobs were the only sealable
10365 /// surface and became a trap the moment they were not: a deployer who
10366 /// configured a key ring would reasonably read it as *this plane is
10367 /// encrypted* while the journal, the case store, the worklist and the
10368 /// event buffer stayed in the clear. Five independent wrapping calls is a
10369 /// control that can be forgotten four times, and forgetting looks exactly
10370 /// like remembering.
10371 ///
10372 /// Wrapped here rather than inside `keyring()` for the reason the tool
10373 /// catalogue is checked here: a wrap applied when the ring is supplied
10374 /// would cover only the stores registered so far, so `keyring(..)` before
10375 /// `cases(..)` would silently seal less than `keyring(..)` after it. An
10376 /// enforcement a reordering can lose is one a reformatter can delete.
10377 ///
10378 /// Every decorator is handed `self.tenant` rather than reading a name back
10379 /// out of the store it wraps, so all four derive the erasure scope from one
10380 /// value. `keyring()`'s own documentation names the one store deliberately
10381 /// left out, and why.
10382 #[cfg(feature = "keyring")]
10383 fn seal_stores(&mut self) {
10384 let Some(keys) = self.keyring.clone() else {
10385 return;
10386 };
10387 let tenant = self.tenant.clone();
10388 self.store = crate::keyring::SealedJournal::wrap(
10389 Arc::clone(&self.store),
10390 Arc::clone(&keys),
10391 tenant.clone(),
10392 );
10393 if let Some(cases) = self.cases.take() {
10394 self.cases = Some(crate::keyring::SealedCases::wrap(
10395 cases,
10396 Arc::clone(&keys),
10397 tenant.clone(),
10398 ));
10399 }
10400 if let Some(events) = self.events.take() {
10401 self.events = Some(crate::keyring::SealedEvents::wrap(
10402 events,
10403 Arc::clone(&keys),
10404 tenant.clone(),
10405 ));
10406 }
10407 // The outbox's store keeps the operator destinations' bearer tokens —
10408 // credentials like any caller's, and the one store here reached through
10409 // a handle the embedder built rather than registered. Sealing it in the
10410 // same sweep is what keeps `keyring()` one decision rather than five
10411 // and a sixth.
10412 #[cfg(feature = "push")]
10413 if let Some(outbox) = self.outbox.take() {
10414 self.outbox = Some(Arc::new(
10415 outbox
10416 .as_ref()
10417 .clone()
10418 .sealed(Arc::clone(&keys), tenant.clone()),
10419 ));
10420 }
10421 if let Some(tasks) = self.tasks.take() {
10422 self.tasks = Some(crate::keyring::SealedTasks::wrap(tasks, keys, tenant));
10423 }
10424 }
10425
10426 /// Without the `keyring` feature there is nothing to seal.
10427 ///
10428 /// `&mut self` matches the sealing arm above, which does mutate. Taking
10429 /// `&self` here would make the signature depend on a feature flag, and
10430 /// every caller with it.
10431 #[cfg(not(feature = "keyring"))]
10432 #[allow(clippy::unused_self, clippy::needless_pass_by_ref_mut)]
10433 fn seal_stores(&mut self) {}
10434
10435 /// Supply the store that tracks batch items.
10436 ///
10437 /// Only needed for [`Runtime::run_batch`]; a plane that runs no batches does
10438 /// not need one, and asking for it unconditionally would make the common
10439 /// case carry the uncommon one's setup.
10440 #[must_use]
10441 pub fn batches(mut self, batches: Arc<dyn crate::batch::BatchStore>) -> Self {
10442 self.batches = Some(batches);
10443 self
10444 }
10445
10446 /// Supply the authorization engine.
10447 ///
10448 /// Without one there is no policy layer — the information-flow gates still
10449 /// apply, but nothing asks whether the principal was allowed. That is a
10450 /// deliberate absence rather than a permissive default: see `core::policy`
10451 /// on why there is no `AllowAll` to configure by mistake.
10452 ///
10453 /// The engine's complete immutable bundle identity is recorded at admission,
10454 /// so both whether policy was on and exactly which executable semantics
10455 /// governed the run are answerable from the journal. An open run may resume
10456 /// only under that same identity.
10457 #[must_use]
10458 pub fn policy(mut self, policy: Arc<dyn crate::core::PolicyEngine>) -> Self {
10459 self.policy = Some(policy);
10460 self
10461 }
10462
10463 /// The chain this plane's own runs act under.
10464 ///
10465 /// The identity of a run the embedder starts in-process — checked against
10466 /// the plan at admission (the plan is the authorization graph, so one that
10467 /// exceeds the chain's authority never starts), held to the chain's
10468 /// audience and validity, and journaled, so "on whose behalf" is
10469 /// answerable from history rather than reconstructed from timestamps.
10470 ///
10471 /// It is **not** the identity of a run a served surface admits for a
10472 /// caller: those act under the chain the caller presented, or under none
10473 /// when it presented none ([`RunTerms::served`]). A plane that bound its own
10474 /// chain to every peer's run would be an ambient credential.
10475 ///
10476 /// Verification of the *credential* belongs to whatever authenticates the
10477 /// caller; what arrives here is already a chain, and its attenuation is
10478 /// guaranteed by its own constructors however it was obtained.
10479 #[must_use]
10480 pub fn acting_as(mut self, chain: crate::core::Delegation) -> Self {
10481 self.identity = Some(chain);
10482 self
10483 }
10484
10485 /// The peers this plane may call, and the transport that reaches them.
10486 ///
10487 /// The registry is wiring — what each peer is granted, which credential
10488 /// reaches it, how a failed call recovers. What a run may *ask* a peer
10489 /// for is a manifest grant, `tool://<peer>/<capability>`, dispatched
10490 /// through [`StepCtx::call_peer`](crate::runtime::StepCtx::call_peer).
10491 /// Refused at build: a peer named `agent` or like a wired tool server,
10492 /// and a grant naming a capability outside the peer's registry scope.
10493 #[must_use]
10494 pub fn peers(
10495 mut self,
10496 registry: crate::peers::PeerRegistry,
10497 client: Arc<dyn crate::peers::PeerClient>,
10498 ) -> Self {
10499 self.peers = Some(Arc::new(PeerWiring { registry, client }));
10500 self
10501 }
10502
10503 /// Submit this plane's checkpoints to witnesses, and hold each round to a
10504 /// quorum.
10505 ///
10506 /// This is the only control here whose value comes from *not* being this
10507 /// plane. The hash chain proves no record was edited, the signatures say
10508 /// which workload wrote them, and the Merkle log proves no sealed run was
10509 /// removed — but every input to all three comes from the party an auditor
10510 /// is being asked to trust. None of them detects a whole run deleted
10511 /// before anyone took a checkpoint, or two internally perfect histories
10512 /// shown to two auditors. A witness keeps the last checkpoint it saw and
10513 /// cosigns only a checkpoint that provably extends it, so those two stop
10514 /// being invisible.
10515 ///
10516 /// Availability never waits on it: submission runs on
10517 /// [`sweep`](Runtime::sweep), after sealing and off the run path, and a
10518 /// round that falls short is a report rather than a refusal. A run whose
10519 /// witnesses are unreachable finished long ago, and making the plane's
10520 /// availability depend on a third party is the wrong trade for evidence
10521 /// that is read after the fact.
10522 ///
10523 /// Point it at witnesses somebody else runs. A witness hosted beside the
10524 /// history it vouches for is an anchor one compromise removes, and
10525 /// [`MemoryWitness`](crate::journal::MemoryWitness) says so at its own
10526 /// definition.
10527 ///
10528 /// # Errors
10529 ///
10530 /// [`build`](RuntimeBuilder::build) refuses a quorum larger than the
10531 /// number of witnesses configured, and a witness list with no quorum.
10532 #[must_use]
10533 pub fn witnesses(
10534 mut self,
10535 witnesses: Vec<Arc<dyn crate::journal::Witness>>,
10536 quorum: crate::journal::WitnessQuorum,
10537 ) -> Self {
10538 self.witnesses = witnesses;
10539 self.quorum = Some(quorum);
10540 self
10541 }
10542
10543 /// Declare the maximum time between checkpoint submissions to each
10544 /// witness: [`sweep`](Runtime::sweep) re-submits an unchanged checkpoint
10545 /// once it has passed since the last round that met quorum, so an idle
10546 /// plane still carries a fresh witness timestamp. Without it an unchanged
10547 /// log is never re-submitted.
10548 ///
10549 /// The interval is kept only as often as `sweep` runs; a deployment
10550 /// sweeping on its own scheduler owns that cadence.
10551 ///
10552 /// # Errors
10553 ///
10554 /// [`build`](RuntimeBuilder::build) refuses an interval with no witnesses.
10555 #[must_use]
10556 pub fn witness_interval(mut self, interval: std::time::Duration) -> Self {
10557 self.witness_interval = Some(interval);
10558 self
10559 }
10560
10561 /// Supply the calendar that resolves deadline descriptions to instants.
10562 ///
10563 /// Defaults to [`WallClock`], which understands plain offsets and refuses
10564 /// anything it does not know rather than approximating it. Domain calendars
10565 /// — working days, holidays, cut-off hours — are the adapter's job.
10566 #[must_use]
10567 pub fn calendar(mut self, calendar: Arc<dyn Calendar>) -> Self {
10568 self.calendar = Some(calendar);
10569 self
10570 }
10571
10572 /// Derive the tool catalogue from the agents and refuse a disagreement.
10573 ///
10574 /// Every agent, not the first: a plane may host several, and a tool granted
10575 /// to none of them is still a tool this binary can be asked for.
10576 ///
10577 /// # Errors
10578 ///
10579 /// If the box and any agent's manifest disagree. A build-time
10580 /// misconfiguration has a fix and no recovery, and it is refused the way the
10581 /// tenant mismatch beside it is: before anything runs.
10582 #[cfg(feature = "manifest")]
10583 fn settle_toolbox(&mut self) -> Result<(), BuildError> {
10584 let servers = std::mem::take(&mut self.tool_servers);
10585 let tools = self.toolbox.take();
10586 // A peer's name is a server name a grant can carry, so it must mean
10587 // exactly one thing on this plane: not `agent`, and not a name some
10588 // tool transport also answers for.
10589 let peer_names: std::collections::BTreeSet<String> = self
10590 .peers
10591 .as_ref()
10592 .map(|wiring| wiring.registry.peers().map(ToString::to_string).collect())
10593 .unwrap_or_default();
10594 if peer_names.contains(crate::tools::AGENT_SERVER) {
10595 return Err(BuildError::ReservedToolServer);
10596 }
10597 for name in &peer_names {
10598 if servers.iter().any(|(server, _)| server == name)
10599 || tools
10600 .as_ref()
10601 .is_some_and(|t| t.servers().any(|s| s == name))
10602 {
10603 return Err(BuildError::PeerIsAlsoAToolServer {
10604 server: name.clone(),
10605 });
10606 }
10607 }
10608 if tools.is_none() && servers.is_empty() {
10609 return Ok(());
10610 }
10611 let tools = tools.unwrap_or_default();
10612 let remote_servers: std::collections::BTreeSet<String> =
10613 servers.iter().map(|(name, _)| name.clone()).collect();
10614 // What a grant may name without this box implementing it: a wired
10615 // transport, or a registered peer.
10616 let reachable_elsewhere: std::collections::BTreeSet<String> =
10617 remote_servers.union(&peer_names).cloned().collect();
10618 // `agent` names agents on this plane, and only them. A transport or a
10619 // typed tool under that name would let a deployment decide whether a
10620 // reviewed grant means "an agent here" or "somebody's server".
10621 if remote_servers.contains(crate::tools::AGENT_SERVER)
10622 || tools
10623 .servers()
10624 .any(|server| server == crate::tools::AGENT_SERVER)
10625 {
10626 return Err(BuildError::ReservedToolServer);
10627 }
10628 if remote_servers.len() != servers.len() {
10629 // The set lost an entry, so some name appears twice. Naming it beats
10630 // reporting a count a reader then has to go and diff by hand.
10631 let mut seen = std::collections::BTreeSet::new();
10632 let duplicate = servers
10633 .iter()
10634 .map(|(name, _)| name)
10635 .find(|name| !seen.insert((*name).clone()))
10636 .cloned()
10637 .unwrap_or_default();
10638 return Err(BuildError::DuplicateToolServer { server: duplicate });
10639 }
10640 // Both forms wired is not a merge and must not silently be one. The
10641 // hand-built catalogue is the operator saying something deliberate; the
10642 // derived one is the agent's declaration. Overwriting either with the
10643 // other would run a plane under grants nobody chose.
10644 if self.tools.is_some() {
10645 return Err(BuildError::ToolsWiredTwice);
10646 }
10647 let mut catalog = crate::tools::ToolCatalog::new();
10648 let mut declared = 0usize;
10649 // Which agent's declaration a tool's catalogue entry came from, so a
10650 // second agent declaring the same tool *differently* is a build-time
10651 // refusal rather than a silent overwrite.
10652 //
10653 // A plane has one catalogue and its agents have one manifest each, so
10654 // two agents granting `tool://ledger/read` with different protected
10655 // fields cannot both be satisfied. Merging by last-writer would resolve
10656 // it by **registration order** — the one thing this builder already
10657 // says an enforcement must never depend on — and it fails at a
10658 // distance: `declared` compares each agent's manifest against the
10659 // catalogue-derived descriptor exactly, so the agent that lost the race
10660 // is refused *every* call to that tool, in production, with a message
10661 // blaming a code-versus-manifest drift that neither file exhibits.
10662 let mut source: BTreeMap<crate::tools::ToolId, (String, crate::tools::ToolSafety)> =
10663 BTreeMap::new();
10664 for agent in &self.agents {
10665 let Some(manifest) = agent.manifest.as_ref() else {
10666 continue;
10667 };
10668 declared += 1;
10669 tools
10670 .check_against(manifest, &reachable_elsewhere)
10671 .map_err(|problems| BuildError::ToolDrift {
10672 agent: manifest.metadata.name.clone(),
10673 problems,
10674 })?;
10675 for (id, safety) in crate::tools::ToolCatalog::from_manifest(manifest).entries() {
10676 if let Some((first, existing)) = source.get(&id) {
10677 if existing != &safety {
10678 return Err(BuildError::ToolDeclaredTwoWays {
10679 tool: id.reference(),
10680 first: first.clone(),
10681 second: manifest.metadata.name.clone(),
10682 });
10683 }
10684 continue;
10685 }
10686 source.insert(id.clone(), (manifest.metadata.name.clone(), safety.clone()));
10687 catalog = catalog.allow(id, safety);
10688 }
10689 }
10690 // A box with nothing to be coherent *with* is the same defect one step
10691 // earlier: tools wired to a plane where no declaration admits them, so
10692 // nothing a reviewer reads describes what this binary can reach.
10693 if declared == 0 {
10694 return Err(BuildError::ToolsWithoutDeclaration);
10695 }
10696 // The typed argument type is the schema source. Overlay its
10697 // presentation only after every manifest has been checked, so the
10698 // model sees exactly what the body will deserialize rather than the
10699 // old permissive `{ type: object }` fallback.
10700 for id in tools.ids() {
10701 let (description, schema, _) = tools
10702 .declared(id)
10703 .expect("every registered typed tool has a declaration");
10704 let reviewed_description = catalog
10705 .declaration(id)
10706 .map_or_else(|| description.to_owned(), |(text, _)| text.to_owned());
10707 catalog = catalog.declare(id.clone(), reviewed_description, schema.clone());
10708 }
10709 // One client per server, resolved by the name a grant carries. A single
10710 // client handed every id could not tell `tool://ledger/read` from
10711 // `tool://tickets/read`, and a transport that never reads the server
10712 // component answers both — from whichever server it happens to hold.
10713 let router = servers.into_iter().fold(
10714 crate::tools::ToolRouter::new().toolbox(&Arc::new(tools)),
10715 |router, (name, client)| router.server(name, client),
10716 );
10717 self.tools = Some((
10718 Arc::new(catalog),
10719 Arc::new(router) as Arc<dyn crate::tools::ToolClient>,
10720 ));
10721 Ok(())
10722 }
10723
10724 /// Settle the tool catalogue: derive it if asked, then hold it to the
10725 /// declarations.
10726 ///
10727 /// One call because the two halves are one decision seen from either side.
10728 /// [`settle_toolbox`](Self::settle_toolbox) covers the derived catalogue,
10729 /// where code and manifest could disagree; the check after it covers the
10730 /// stated one, where operator and manifest could. Whichever way the
10731 /// catalogue arrived, it is checked before anything runs.
10732 #[cfg(feature = "manifest")]
10733 fn settle_tools(&mut self) -> Result<(), BuildError> {
10734 self.settle_toolbox()?;
10735 self.check_catalogue_not_laxer_than_grants()
10736 }
10737
10738 /// Refuse a stated catalogue that is **laxer** than a reviewed grant.
10739 ///
10740 /// `toolbox(..)` derives the catalogue from the manifests, so the two
10741 /// cannot drift. `tools(..)` states it by hand, and there the operator's
10742 /// entry and the agent's declaration are two copies of one decision — with
10743 /// nothing, until this, that noticed them disagreeing.
10744 ///
10745 /// Only one direction is a defect, the same one
10746 /// [`ToolBox::check_against`](crate::tools::ToolBox::check_against) refuses.
10747 /// An operator being **more** cautious than the declaration is fine and
10748 /// often right. An operator being **less** cautious is not, and it changes
10749 /// two things at once:
10750 ///
10751 /// * the whole-value taint gate stops firing, so model-chosen arguments
10752 /// reach something that changes the world;
10753 /// * `ToolSafety::read_only` carries `Recovery::Retry`, so a timed-out call
10754 /// to a money-moving tool is sent a second time.
10755 ///
10756 /// The dispatch gates already take the stricter `mutates` of the two, so
10757 /// the first is contained at runtime. The second is not — recovery is read
10758 /// from the catalogue alone — and neither should have to be, because this
10759 /// is a wiring mistake with a fix and no recovery. It is refused here,
10760 /// beside the rest of them.
10761 #[cfg(feature = "manifest")]
10762 fn check_catalogue_not_laxer_than_grants(&self) -> Result<(), BuildError> {
10763 let Some((catalog, _)) = self.tools.as_ref() else {
10764 return Ok(());
10765 };
10766 let mut problems = Vec::new();
10767 for agent in &self.agents {
10768 let Some(manifest) = agent.manifest.as_ref() else {
10769 continue;
10770 };
10771 for grant in &manifest.spec.tools {
10772 if !grant.mutates {
10773 continue;
10774 }
10775 let Some(id) = crate::tools::ToolId::parse(&grant.reference) else {
10776 continue;
10777 };
10778 if catalog.safety(&id).is_some_and(|s| !s.mutates) {
10779 problems.push(format!(
10780 "agent '{}' grants '{}' as mutating and the stated catalogue \
10781 calls it read-only",
10782 manifest.metadata.name, grant.reference
10783 ));
10784 }
10785 }
10786 }
10787 if problems.is_empty() {
10788 Ok(())
10789 } else {
10790 Err(BuildError::CatalogueLaxerThanGrant { problems })
10791 }
10792 }
10793
10794 /// Assemble the runtime, or panic naming the wiring mistake.
10795 ///
10796 /// The ordinary entry point. Every refusal below is a bug in code the author
10797 /// is looking at, so propagating it through `?` to a `main` that prints it
10798 /// is ceremony around an abort — and each is caught here at startup rather
10799 /// than at dispatch, in production, where the cost is a run that has already
10800 /// begun.
10801 ///
10802 /// Use [`try_build`](Self::try_build) where a manifest arrives at *runtime*
10803 /// — read from disk, pinned by a registry, or supplied per tenant. There a
10804 /// bad declaration is an input rather than a bug, and a panic would take
10805 /// every other tenant in the process down to report it.
10806 ///
10807 /// # Panics
10808 ///
10809 /// On any [`BuildError`]:
10810 ///
10811 /// * A manifest declares a capability in `spec.capabilities.provides` that
10812 /// no registered skill provides. **An agent has skills**, so a declaration
10813 /// advertising one it cannot perform is a card that lies.
10814 /// * Two agents claim the same capability. Dispatch resolves a capability to
10815 /// one skill *and to the manifest governing it*, so a second claim would
10816 /// silently take the first's work out from under the first's budget,
10817 /// model grants and egress ceiling.
10818 /// * Two skills share a name. A name is what a capability resolves to and
10819 /// what governance is keyed on; two of them make both lookups arbitrary.
10820 /// * A stated catalogue calls a tool read-only that a reviewed manifest
10821 /// grants as mutating. That exemption drops the whole-value taint gate
10822 /// and makes a timed-out money-moving call retryable — the one direction
10823 /// an operator cannot be right about.
10824 /// * A declarative agent names a provider no driver is registered for, or
10825 /// declares `spec.execution` without a privileged model to call.
10826 /// * A plane and its store — or its blob store — are scoped to different
10827 /// tenants. The two are set
10828 /// separately — this builder's tenant scopes data keys and the policy
10829 /// request, `for_tenant` scopes the store's keys — and the mismatch does
10830 /// not show up at runtime. It *works*, and writes this tenant's runs into
10831 /// another's keyspace while every erasure and every policy request names
10832 /// the right one.
10833 ///
10834 /// # Long-running services want [`try_build`](Self::try_build)
10835 ///
10836 /// This panics, which is the honest answer for a binary wiring its own
10837 /// skills: every variant above is a bug in code the author is looking at,
10838 /// and aborting reports it at the moment it can be fixed. A daemon is a
10839 /// different case — it wants to exit with a diagnostic, and a plane
10840 /// assembled from a manifest that arrived at runtime is handling an *input*,
10841 /// where a panic reports one tenant's typo by killing every other tenant's
10842 /// in-flight run. `try_build` returns the same `BuildError` instead. One
10843 /// implementation underneath both, so they cannot disagree about what is
10844 /// refused.
10845 #[must_use]
10846 pub fn build(self) -> Arc<Runtime> {
10847 // Not `expect`. That formats the error with `Debug`, which would print
10848 // `AdvertisesWhatItCannotProvide { agent: "…", missing: [...] }` — the
10849 // variant's *shape* — while the sentence explaining what to do about it
10850 // lives in `Display`. Panicking with `{error}` keeps the two entry
10851 // points telling one story, which is the whole point of `build` being
10852 // `try_build` underneath.
10853 match self.try_build() {
10854 Ok(runtime) => runtime,
10855 Err(error) => panic!("{error}"),
10856 }
10857 }
10858
10859 /// The driver for a declarative agent's privileged model — named rather
10860 /// than defaulted, because falling back to some other registered driver
10861 /// would run the agent on a model its own declaration does not name.
10862 /// Every declared content check names a registered checker, and maps only
10863 /// categories that checker declares.
10864 #[cfg(feature = "manifest")]
10865 fn check_content_checkers(
10866 checkers: &crate::content::Checkers,
10867 m: &crate::manifest::Manifest,
10868 ) -> Result<(), BuildError> {
10869 let Some(content) = m.spec.security.content.as_ref() else {
10870 return Ok(());
10871 };
10872 for check in &content.checks {
10873 let refused = |detail: String| BuildError::ContentCheck {
10874 agent: m.metadata.name.clone(),
10875 check: check.id.clone(),
10876 detail,
10877 };
10878 let Some(checker) = checkers.get(&check.checker) else {
10879 return Err(refused(format!(
10880 "no checker named '{}' is registered on this plane",
10881 check.checker
10882 )));
10883 };
10884 if let Some(unknown) = check
10885 .on
10886 .keys()
10887 .find(|category| !checker.categories().contains(*category))
10888 {
10889 return Err(refused(format!(
10890 "'{unknown}' is not a category checker '{}' declares",
10891 check.checker
10892 )));
10893 }
10894 }
10895 Ok(())
10896 }
10897
10898 #[cfg(feature = "manifest")]
10899 fn declared_provider(
10900 providers: &HashMap<String, Arc<dyn crate::model::ModelProvider>>,
10901 m: &crate::manifest::Manifest,
10902 ) -> Result<Arc<dyn crate::model::ModelProvider>, BuildError> {
10903 let model = m
10904 .spec
10905 .models
10906 .as_ref()
10907 .and_then(|x| x.privileged.as_ref())
10908 .ok_or_else(|| BuildError::DeclarativeWithoutModel {
10909 agent: m.metadata.name.clone(),
10910 })?;
10911 providers
10912 .get(&model.provider)
10913 .map(Arc::clone)
10914 .ok_or_else(|| BuildError::UnknownProvider {
10915 agent: m.metadata.name.clone(),
10916 provider: model.provider.clone(),
10917 })
10918 }
10919
10920 /// Assemble the runtime, or say why it cannot be.
10921 ///
10922 /// The same checks as [`build`](Self::build), returned rather than raised.
10923 /// One implementation behind both, so they cannot come to disagree about
10924 /// what is refused.
10925 ///
10926 /// # Errors
10927 ///
10928 /// Any [`BuildError`] — see [`build`](Self::build) for what each means.
10929 // `mut` is for `settle_toolbox`, which only exists when manifests do.
10930 #[cfg_attr(not(feature = "manifest"), allow(unused_mut))]
10931 #[allow(clippy::too_many_lines)]
10932 pub fn try_build(mut self) -> Result<Arc<Runtime>, BuildError> {
10933 if self.lease_ttl < MIN_LEASE_TTL {
10934 return Err(BuildError::LeaseUnrenewable {
10935 ttl: self.lease_ttl,
10936 minimum: MIN_LEASE_TTL,
10937 });
10938 }
10939 // A ceiling of zero is a budget already spent, and it refuses the
10940 // first effect of every run this plane will ever make. The manifest
10941 // refuses one at parse; a plane wired in Rust reaches the same `Budget`
10942 // without passing a parser, so the rule is applied at both doors from
10943 // the one definition in `Budget::bricked_ceiling`.
10944 if let Some(field) = self.budget.bricked_ceiling() {
10945 return Err(BuildError::BudgetPermitsNothing { field });
10946 }
10947 if let Some(engine) = self.policy.as_ref() {
10948 preflight_policy(engine.as_ref(), self.identity.as_ref())?;
10949 }
10950 // Before `seal_stores`, necessarily: sealing wraps each handle in one
10951 // scoped to the *plane's* tenant, after which every store agrees with
10952 // the plane and the question cannot be asked.
10953 // Both halves of push, because either can be wired to another
10954 // tenant's rows and the consequence is the same: registrations,
10955 // cursors and credentials filed under a scope this plane's erasure
10956 // never names.
10957 #[cfg(feature = "push")]
10958 let push = self
10959 .push
10960 .as_ref()
10961 .map(|s| s.tenant())
10962 .or_else(|| self.outbox.as_ref().map(|o| o.store_tenant()));
10963 #[cfg(not(feature = "push"))]
10964 let push: Option<&str> = None;
10965 check_same_tenant(
10966 self.store.as_ref(),
10967 self.blobs.as_ref(),
10968 self.memories.as_ref(),
10969 &[
10970 ("case", self.cases.as_ref().map(|s| s.tenant())),
10971 ("event", self.events.as_ref().map(|s| s.tenant())),
10972 ("task", self.tasks.as_ref().map(|s| s.tenant())),
10973 ("memory", self.memories.as_ref().map(|s| s.tenant())),
10974 ("quota", self.quotas.as_ref().map(|s| s.tenant())),
10975 ("timer", self.timers.as_ref().map(|s| s.tenant())),
10976 ("batch", self.batches.as_ref().map(|s| s.tenant())),
10977 ("authority", self.authorities.as_ref().map(|s| s.tenant())),
10978 ("push", push),
10979 ],
10980 &self.tenant,
10981 )?;
10982 // The one wiring mistake here that would otherwise never fail: a
10983 // cross-space query ranks rather than refusing.
10984 if let Some(semantic) = &self.semantic {
10985 let embedder = semantic.embedder.revision();
10986 let index = semantic.retriever.index().query_revision;
10987 if embedder != index {
10988 return Err(BuildError::EmbeddingSpaceMismatch { embedder, index });
10989 }
10990 if semantic.embedder.pricing().is_none() && self.states_a_money_ceiling() {
10991 return Err(BuildError::UnpricedEmbedder { embedder });
10992 }
10993 let Some(memories) = self.memories.take() else {
10994 return Err(BuildError::SemanticMemoryWithoutStore);
10995 };
10996 // Every erasure through the plane's store — and through a sealed
10997 // store's own subject erasure — reaches the index built from it.
10998 let reached = memories.erasure_index();
10999 let indexed: Arc<dyn crate::memory::MemoryStore> = match reached {
11000 Some(index) if Arc::ptr_eq(&index, &semantic.retriever) => memories,
11001 Some(_) => return Err(BuildError::MemoryIndexedElsewhere),
11002 None if memories.seals() => {
11003 return Err(BuildError::SealedMemoryMissesIndex);
11004 }
11005 None => Arc::new(crate::memory::IndexedMemoryStore::new(
11006 Arc::clone(&memories),
11007 Arc::clone(&semantic.retriever),
11008 )),
11009 };
11010 self.memories = Some(indexed);
11011 }
11012 self.seal_stores();
11013 #[cfg(feature = "manifest")]
11014 self.settle_tools()?;
11015
11016 let mut skills = HashMap::new();
11017 let mut by_capability = HashMap::new();
11018 #[cfg(feature = "manifest")]
11019 let mut governed_by: HashMap<String, Arc<crate::manifest::Manifest>> = HashMap::new();
11020 #[cfg(feature = "manifest")]
11021 let mut published_by: HashMap<String, crate::core::KeyId> = HashMap::new();
11022
11023 // Skills registered directly belong to the plane's anonymous agent: no
11024 // declaration, so nothing to enforce against them beyond the runtime's
11025 // own budget. That is a legitimate shape — not every agent needs a
11026 // manifest — and it is why `skill()` still exists beside `agent()`.
11027 for s in self.skills {
11028 register_skill(s, &mut by_capability, &mut skills)?;
11029 }
11030
11031 #[cfg(feature = "manifest")]
11032 for agent in self.agents {
11033 if let (Some(m), Some(key)) = (agent.manifest.as_ref(), agent.publisher.clone()) {
11034 published_by.insert(m.metadata.name.clone(), key);
11035 }
11036 let Some(m) = agent.manifest.clone() else {
11037 for s in agent.skills {
11038 register_skill(s, &mut by_capability, &mut skills)?;
11039 }
11040 continue;
11041 };
11042
11043 // The capabilities this agent's *own* skills provide, which is what
11044 // its declaration is checked against below.
11045 let mut mine: HashSet<Capability> = HashSet::new();
11046 for s in agent.skills {
11047 mine.extend(s.descriptor().capabilities());
11048 let name = register_skill(s, &mut by_capability, &mut skills)?;
11049 governed_by.insert(name, Arc::clone(&m));
11050 }
11051
11052 // A declarative agent needs no skill: the runtime supplies the
11053 // behaviour its manifest asked for.
11054 if let Some(execution) = &m.spec.execution {
11055 // A `call` calls no model, so it needs no provider.
11056 let provider = if execution.kind == crate::manifest::ExecutionKind::Call {
11057 None
11058 } else {
11059 Some(Self::declared_provider(&self.providers, &m)?)
11060 };
11061 if m.spec.capabilities.provides.is_empty() {
11062 return Err(BuildError::DeclarativeProvidesNothing {
11063 agent: m.metadata.name.clone(),
11064 });
11065 }
11066 // A plane whose only tools are agents needs no toolbox and no
11067 // transport: the catalogue is derived from the declaration and
11068 // dispatch is `commission`. The empty router is deliberate —
11069 // an agent-server call never reaches it, and anything else
11070 // arriving there is refused as unreachable rather than
11071 // silently absorbed.
11072 let tools = self.tools.clone().or_else(|| {
11073 // Agents on this plane and registered peers: both
11074 // dispatch through the runtime's own effects rather than
11075 // a tool transport, so a declaration made only of them
11076 // needs no box and no router.
11077 let all_agent = !m.spec.tools.is_empty()
11078 && m.spec.tools.iter().all(|g| {
11079 crate::tools::ToolId::parse(&g.reference).is_some_and(|id| {
11080 id.server == crate::tools::AGENT_SERVER
11081 || self.peers.as_ref().is_some_and(|wiring| {
11082 wiring
11083 .registry
11084 .grant(&crate::peers::PeerId::new(&id.server))
11085 .is_some()
11086 })
11087 })
11088 });
11089 all_agent.then(|| {
11090 (
11091 Arc::new(crate::tools::ToolCatalog::from_manifest(&m)),
11092 Arc::new(crate::tools::ToolRouter::new())
11093 as Arc<dyn crate::tools::ToolClient>,
11094 )
11095 })
11096 });
11097 // A tool loop with nothing to reach fails on every run with the
11098 // same sentence, and the manifest that says so was read at
11099 // build. `planned` only needs a catalogue when it has grants;
11100 // `tool-calling` needs one to exist at all, since offering a
11101 // model no tools is not a tool loop.
11102 if tools.is_none() {
11103 let needs = match execution.kind {
11104 crate::manifest::ExecutionKind::ToolCalling => {
11105 Some(if m.spec.tools.is_empty() {
11106 "no tool grants".to_owned()
11107 } else {
11108 format!("{} tool grant(s)", m.spec.tools.len())
11109 })
11110 }
11111 crate::manifest::ExecutionKind::Planned
11112 | crate::manifest::ExecutionKind::Call
11113 if !m.spec.tools.is_empty() =>
11114 {
11115 Some(format!("{} tool grant(s)", m.spec.tools.len()))
11116 }
11117 _ => None,
11118 };
11119 if let Some(grants) = needs {
11120 return Err(BuildError::DeclarativeToolsUnreachable {
11121 agent: m.metadata.name.clone(),
11122 kind: execution.kind.as_str(),
11123 grants,
11124 });
11125 }
11126 }
11127
11128 // Oversight needs somewhere to put the decision, and both
11129 // halves are known here. Left to run time it surfaces at the
11130 // first real approval — a person already waiting, on the code
11131 // path a test suite is least likely to reach.
11132 let declared = if m.spec.oversight.is_some() {
11133 Some("`spec.oversight`".to_owned())
11134 } else if m.spec.tools.iter().any(|g| g.requires_approval) {
11135 Some("a grant with `requires_approval: true`".to_owned())
11136 } else {
11137 None
11138 };
11139 if let Some(declared) = declared {
11140 // Ordered by what a reader fixes first: without a case
11141 // there is nothing for a task to hang off.
11142 //
11143 // Timers are deliberately **not** required, and the test
11144 // suite is why: four approval tests wire a case store and a
11145 // worklist, no timers, and run an approval end to end. A
11146 // task is opened, the run suspends, a person decides. What
11147 // a timer store adds is the sweeper firing `on_expiry`, and
11148 // refusing a configuration that works would be this check
11149 // asserting more than it knows — the failure mode it exists
11150 // to prevent, arriving from the other side.
11151 let missing = if self.cases.is_none() {
11152 Some(("case store", "cases"))
11153 } else if self.tasks.is_none() {
11154 Some(("worklist", "tasks"))
11155 } else {
11156 None
11157 };
11158 if let Some((missing, remedy)) = missing {
11159 return Err(BuildError::OversightUnreachable {
11160 agent: m.metadata.name.clone(),
11161 declared,
11162 missing,
11163 remedy,
11164 });
11165 }
11166 }
11167
11168 // A recall reads before the first model call and formation
11169 // writes after the answer, so a missing store costs a run its
11170 // first effect or its whole budget depending on which half was
11171 // declared. Both facts are here: the file says memories are
11172 // reached, the plane says there is nowhere to reach.
11173 if let Some(memory) = &m.spec.memory {
11174 let declared: [(&'static str, Option<&crate::manifest::MemorySubject>); 2] = [
11175 (
11176 "spec.memory.recall",
11177 memory.recall.as_ref().map(|r| &r.subject),
11178 ),
11179 (
11180 "spec.memory.formation",
11181 memory.formation.as_ref().map(|f| &f.subject),
11182 ),
11183 ];
11184 for (field, subject) in declared {
11185 let Some(subject) = subject else { continue };
11186 if self.memories.is_none() {
11187 return Err(BuildError::MemoryWithoutStore {
11188 agent: m.metadata.name.clone(),
11189 declared: field,
11190 });
11191 }
11192 if subject.needs_case() && self.cases.is_none() {
11193 return Err(BuildError::MemorySubjectUnbindable {
11194 agent: m.metadata.name.clone(),
11195 subject: subject.as_written(),
11196 });
11197 }
11198 }
11199 }
11200
11201 for cap in &m.spec.capabilities.provides {
11202 let skill: Arc<dyn Skill> = Arc::new(super::declarative::Declarative::new(
11203 execution.kind,
11204 cap.clone(),
11205 m.metadata.name.clone(),
11206 provider.clone(),
11207 tools.clone(),
11208 execution.max_turns,
11209 ));
11210 mine.insert(Capability::new(cap.as_str()));
11211 let name = register_skill(skill, &mut by_capability, &mut skills)?;
11212 governed_by.insert(name, Arc::clone(&m));
11213 }
11214 }
11215
11216 check_declaration_matches_skills(&m, &mine)?;
11217 }
11218
11219 // Agent grants are validated against the finished plane, because the
11220 // capability they name may belong to an agent registered *later* — a
11221 // check inside the loop would pass or fail on registration order.
11222 #[cfg(feature = "manifest")]
11223 {
11224 let mut checked = std::collections::BTreeSet::new();
11225 for m in governed_by.values() {
11226 if !checked.insert(m.metadata.name.clone()) {
11227 continue;
11228 }
11229 Self::check_content_checkers(&self.checkers, m)?;
11230 for grant in &m.spec.tools {
11231 let Some(id) = crate::tools::ToolId::parse(&grant.reference) else {
11232 continue;
11233 };
11234 // A grant naming a registered peer is a delegating hop,
11235 // and both things that would refuse it on every run are
11236 // knowable here: a capability the registry never gave the
11237 // peer, and a granting agent whose own ceiling is zero.
11238 if let Some(wiring) = self.peers.as_ref()
11239 && let Some(peer_grant) = wiring
11240 .registry
11241 .grant(&crate::peers::PeerId::new(&id.server))
11242 {
11243 if !peer_grant.scope.permits(&Capability::new(id.tool.as_str())) {
11244 return Err(BuildError::PeerGrantOutsideScope {
11245 agent: m.metadata.name.clone(),
11246 peer: id.server,
11247 capability: id.tool,
11248 });
11249 }
11250 let cannot_delegate = m
11251 .spec
11252 .topology
11253 .as_ref()
11254 .is_some_and(|t| t.role == crate::manifest::Role::Specialist)
11255 || m.spec.security.max_delegation_depth == Some(0);
11256 if cannot_delegate {
11257 return Err(BuildError::PeerGrantOnASpecialist {
11258 agent: m.metadata.name.clone(),
11259 peer: id.server,
11260 });
11261 }
11262 continue;
11263 }
11264 if id.server != crate::tools::AGENT_SERVER {
11265 continue;
11266 }
11267 if m.spec.capabilities.provides.contains(&id.tool) {
11268 return Err(BuildError::AgentToolSelfReference {
11269 agent: m.metadata.name.clone(),
11270 capability: id.tool,
11271 });
11272 }
11273 if !by_capability.contains_key(&Capability::new(id.tool.as_str())) {
11274 return Err(BuildError::AgentToolUnknownCapability {
11275 agent: m.metadata.name.clone(),
11276 capability: id.tool,
11277 });
11278 }
11279 }
11280 }
11281 }
11282
11283 // Every ceiling any declaration on this plane states, by tool: the
11284 // count is per tool, so the tightest binds every agent calling it.
11285 #[cfg(feature = "manifest")]
11286 let rates = {
11287 let mut declarations: Vec<&Arc<crate::manifest::Manifest>> =
11288 governed_by.values().collect();
11289 declarations.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
11290 let mut ceilings: BTreeMap<String, Vec<crate::quota::RateCeiling>> = BTreeMap::new();
11291 let mut first = None;
11292 for m in declarations {
11293 for grant in &m.spec.tools {
11294 let Some(rate) = grant.rate_limit else {
11295 continue;
11296 };
11297 first.get_or_insert_with(|| (m.metadata.name.clone(), grant.reference.clone()));
11298 let stated = ceilings.entry(grant.reference.clone()).or_default();
11299 let ceiling = crate::quota::RateCeiling::from(rate);
11300 if !stated.contains(&ceiling) {
11301 stated.push(ceiling);
11302 stated.sort();
11303 }
11304 }
11305 }
11306 if ceilings.is_empty() {
11307 None
11308 } else if let Some(store) = self.quotas.clone() {
11309 Some(Arc::new(super::ctx::Rates { store, ceilings }))
11310 } else {
11311 let (agent, grant) = first.unwrap_or_default();
11312 return Err(BuildError::RateLimitWithoutQuotaStore { agent, grant });
11313 }
11314 };
11315
11316 // Both or neither, checked here because a builder cannot see the other
11317 // half at the moment either is set.
11318 if self.witness_interval.is_some() && self.witnesses.is_empty() {
11319 return Err(BuildError::IntervalWithoutWitnesses);
11320 }
11321 let witnesses = match (self.witnesses.is_empty(), self.quorum) {
11322 (true, None) => None,
11323 (false, Some(quorum)) if quorum.required() <= self.witnesses.len() => {
11324 Some(Arc::new(Witnessing {
11325 witnesses: self.witnesses,
11326 quorum,
11327 submitted: std::sync::atomic::AtomicU64::new(0),
11328 interval: self.witness_interval,
11329 last_met: std::sync::Mutex::new(None),
11330 }))
11331 }
11332 (empty, quorum) => {
11333 return Err(BuildError::WitnessQuorumUnreachable {
11334 declared: quorum.map_or(0, |q| q.required()),
11335 configured: if empty { 0 } else { self.witnesses.len() },
11336 });
11337 }
11338 };
11339
11340 Ok(Arc::new_cyclic(|self_ref| Runtime {
11341 self_ref: self_ref.clone(),
11342 witnesses,
11343 inflight: Arc::default(),
11344 signer: self.signer,
11345 store: self.store,
11346 skills,
11347 by_capability,
11348 #[cfg(feature = "manifest")]
11349 published_by,
11350 #[cfg(feature = "manifest")]
11351 providers: self.providers,
11352 meter: super::metrics::Meter::new(self.tenant_label, &self.tenant),
11353 tenant: self.tenant,
11354 owner: self.owner.unwrap_or_else(default_owner),
11355 lease_ttl: self.lease_ttl,
11356 memories: self.memories,
11357 semantic: self.semantic,
11358 authorities: self.authorities,
11359 peers: self.peers,
11360 #[cfg(feature = "manifest")]
11361 tools: self.tools,
11362 journal_ceiling: self.journal_ceiling,
11363 #[cfg(feature = "manifest")]
11364 egress: self.egress,
11365 #[cfg(feature = "manifest")]
11366 checkers: self.checkers,
11367 streams: self.streams,
11368 quotas: self.quotas,
11369 #[cfg(feature = "manifest")]
11370 rates,
11371 quota: self.quota,
11372 budget: self.budget,
11373 require_verifier: self.require_verifier,
11374 cases: self.cases,
11375 events: self.events,
11376 tasks: self.tasks,
11377 timers: self.timers,
11378 blobs: self.blobs,
11379 disclosures: self.disclosures,
11380 #[cfg(feature = "keyring")]
11381 keyring: self.keyring,
11382 batches: self.batches,
11383 policy: self.policy,
11384 identity: self.identity,
11385 replanner: self.replanner,
11386 calendar: self.calendar.unwrap_or_else(|| Arc::new(WallClock)),
11387 #[cfg(feature = "push")]
11388 push: self.push,
11389 #[cfg(feature = "push")]
11390 outbox: self.outbox,
11391 #[cfg(feature = "manifest")]
11392 governed_by,
11393 }))
11394 }
11395}
11396
11397/// Ask the policy set the questions this plane will ask, before it asks them
11398/// for real.
11399///
11400/// Cedar evaluates **every** rule against **every** request, so a rule reading
11401/// an attribute a request does not carry does not quietly fail to match — it
11402/// errors, and an unevaluable rule may be the `forbid` that would have stopped
11403/// the call, so the gate refuses. One unguarded rule therefore denies every
11404/// effect of every run, from a policy set that compiled cleanly and validated
11405/// against its schema.
11406///
11407/// The probes cover every shape a runtime request takes: an effect that reads
11408/// and one that mutates, each with and without a label (only `sink` carries
11409/// one), a release, and an admission. `mutates` matters on its own: `&&`
11410/// short-circuits, so `context.mutates && context.label.trust == ..` evaluates
11411/// cleanly on a read and errors on every mutating call that carries no label.
11412///
11413/// The delegation attributes follow the plane. A plane with a chain carries
11414/// `owner`, `scope` and `delegation_depth` on every request, and a rule reading
11415/// them unguarded is correct there. A plane without one never carries them on
11416/// its own runs — and still admits runs under a caller's chain from a served
11417/// surface, so both shapes are asked.
11418///
11419/// What this does **not** establish: that the rules are *right*. A set that
11420/// permits everything passes here, as does one whose rules error only on a
11421/// shape carrying an attribute of an unexpected type. It answers one question
11422/// — can this plane's own requests be evaluated at all — which is the question
11423/// whose wrong answer looks like a working plane that refuses everything. The
11424/// served surfaces ask their own actions with their own contexts, and check
11425/// those where they are built.
11426#[cfg_attr(not(feature = "manifest"), allow(dead_code))]
11427fn preflight_policy(
11428 engine: &dyn crate::core::PolicyEngine,
11429 identity: Option<&crate::core::Delegation>,
11430) -> Result<(), BuildError> {
11431 let problems = runtime_policy_problems(engine, identity);
11432 if problems.is_empty() {
11433 return Ok(());
11434 }
11435 Err(BuildError::PolicyUnevaluable {
11436 problems: problems.join("; "),
11437 })
11438}
11439
11440/// The runtime tier's probes, answered by `engine`.
11441fn runtime_policy_problems(
11442 engine: &dyn crate::core::PolicyEngine,
11443 identity: Option<&crate::core::Delegation>,
11444) -> Vec<String> {
11445 use crate::policy::requests;
11446
11447 // Values are placeholders shaped like the real ones: a rule reading
11448 // `context.label.trust` on a release must find a label there, or the probe
11449 // reports a defect the real request would never hit. Nulls are absent for
11450 // the same reason — no gate sends one, and the adapter strips them.
11451 let run = RunId(ulid::Ulid::nil());
11452 let step = StepId(0);
11453 let label = crate::core::Label::untrusted(crate::core::SourceId::new("preflight"));
11454 let release = crate::core::Release::whole(
11455 crate::core::ReleaseScope::trust(),
11456 "preflight",
11457 "tool://preflight/probe",
11458 ["preflight"],
11459 );
11460 // A tool call's arguments are the one shape the runtime fixes for a kind,
11461 // so a rule scoped to `tool.call` may read `context.args.server` as-is.
11462 let tool_args = serde_json::json!({
11463 "server": "preflight",
11464 "tool": "preflight",
11465 "arguments": {},
11466 "protected_fields": [],
11467 });
11468
11469 // The chain shapes this plane's requests take. A chain of the plane's own
11470 // is on every request; without one, a run may still carry its caller's.
11471 let caller = crate::core::Delegation::root(crate::core::Principal::new(
11472 "preflight",
11473 crate::core::Scope::root(),
11474 ));
11475 let chains: Vec<Option<&crate::core::Delegation>> = match identity {
11476 Some(chain) => vec![Some(chain)],
11477 None => vec![None, Some(&caller)],
11478 };
11479
11480 // Built by the gates' own builders, so a probe cannot drift from the
11481 // shape it stands in for: an effect that reads and one that mutates, each
11482 // with and without a label (only `sink` carries one), a release, and an
11483 // admission.
11484 let mut built = Vec::new();
11485 for chain in chains {
11486 let acting = requests::Acting {
11487 tenant: "preflight",
11488 capability: "preflight.capability",
11489 agent: None,
11490 chain,
11491 };
11492 for (kind, args) in [
11493 ("preflight.effect", serde_json::json!({})),
11494 ("tool.call", tool_args.clone()),
11495 ] {
11496 for (mutates, labelled) in [(false, false), (true, false), (false, true), (true, true)]
11497 {
11498 built.push(requests::effect(
11499 &acting,
11500 run,
11501 step,
11502 kind,
11503 &args,
11504 mutates,
11505 labelled.then_some(&label),
11506 ));
11507 }
11508 }
11509 built.push(requests::release(&acting, run, step, &release, &label));
11510 built.push(requests::admission(&acting, &serde_json::json!({})));
11511 }
11512 let requests: Vec<crate::core::PolicyRequest<'_>> = built
11513 .iter()
11514 .map(requests::GatedRequest::as_request)
11515 .collect();
11516
11517 // Asked through the engine's own preflight rather than by calling
11518 // `authorize` here. An engine written as Rust code has no missing-attribute
11519 // trap and nothing to report; the engine that has the trap implements the
11520 // method, and knows which resources its own rules name.
11521 let mut problems = engine.preflight(&requests);
11522 problems.sort();
11523 problems.dedup();
11524 problems
11525}
11526
11527/// Refuse a plane whose stores answer for somebody else.
11528///
11529/// The plane's tenant scopes its data keys and names its policy requests; each
11530/// store handle is scoped separately. Nothing at runtime looks wrong when the
11531/// two disagree — the work is written into another tenant's keyspace while
11532/// every erasure and every policy request names the right one — so the mismatch
11533/// is easy to make and invisible once made. Asking each store who it serves is
11534/// what turns it into a startup refusal.
11535///
11536/// **A store that does not override its accessor answers `default`.** That is
11537/// the honest answer for a single-tenant deployment and the reason the check
11538/// can be unconditional, but it is also why this catches a wiring mistake
11539/// rather than proving isolation: an implementation that is scoped to `acme`
11540/// and never overrode the accessor reports `default` and is refused, which is
11541/// the safe direction — the unsafe one would be an implementation that claims a
11542/// tenant it does not enforce, and no build-time question can settle that.
11543fn check_same_tenant(
11544 store: &dyn JournalStore,
11545 blobs: Option<&Arc<dyn crate::blob::BlobStore>>,
11546 memories: Option<&Arc<dyn crate::memory::MemoryStore>>,
11547 state: &[(&'static str, Option<&str>)],
11548 tenant: &crate::core::TenantId,
11549) -> Result<(), BuildError> {
11550 if let Some(blobs) = blobs
11551 && blobs.tenant() != tenant.as_str()
11552 {
11553 return Err(BuildError::BlobStoreTenant {
11554 plane: tenant.to_string(),
11555 store: blobs.tenant().to_owned(),
11556 });
11557 }
11558 if store.tenant() != tenant.as_str() {
11559 return Err(BuildError::JournalStoreTenant {
11560 plane: tenant.to_string(),
11561 store: store.tenant().to_owned(),
11562 });
11563 }
11564
11565 // The stores whose state a key ring seals, in the caller's fixed order so a
11566 // plane with two mismatches reports the same one every build — an error
11567 // that varies between runs of the same wiring sends an operator looking for
11568 // a race that is not there.
11569 for &(store, serves) in state {
11570 if let Some(serves) = serves
11571 && serves != tenant.as_str()
11572 {
11573 return Err(BuildError::StateStoreTenant {
11574 store,
11575 plane: tenant.to_string(),
11576 tenant: serves.to_owned(),
11577 });
11578 }
11579 }
11580
11581 // After the tenant checks, deliberately: a store scoped to the wrong tenant
11582 // is the more basic fault, and reporting the erasure lock first would send
11583 // an operator to fix the second-most-wrong thing.
11584 //
11585 // A process-local erasure lock beside a store two instances can write is a
11586 // control that reads as present and is not. The window it fails to close is
11587 // between an erasure's legal-hold check and its key destruction: the other
11588 // instance writes an item, that item is sealed under a scope about to stop
11589 // existing, and the erasure reports success. Both facts are here, so the
11590 // refusal is here.
11591 if store.is_shared() && memories.is_some_and(|m| m.erasure_is_distributed() == Some(false)) {
11592 return Err(BuildError::ErasureCoordinatorNotShared);
11593 }
11594 Ok(())
11595}
11596
11597/// Refuse an event from outside whose kind this plane mints for itself.
11598fn refuse_reserved_kind(event: &InboundEvent) -> Result<(), RuntimeError> {
11599 if crate::core::is_reserved_kind(&event.kind) {
11600 return Err(RuntimeError::ReservedEventKind {
11601 kind: event.kind.clone(),
11602 });
11603 }
11604 Ok(())
11605}
11606
11607/// Inbound event delivery.
11608impl Runtime {
11609 /// Deliver an inbound event, resuming whichever run was waiting for it.
11610 ///
11611 /// # Ordering
11612 ///
11613 /// The event is **stored before** anyone looks for a waiter. That ordering
11614 /// is the whole reason this works: a message can arrive before its run
11615 /// reaches the wait, and one that is matched-then-discarded leaves that run
11616 /// waiting forever for something that already happened.
11617 ///
11618 /// A [`Delivery::Buffered`] result is therefore normal and not an error —
11619 /// it means "held until someone asks". Only the sweep
11620 /// ([`EventStore::sweep_unclaimed`](crate::case::EventStore::sweep_unclaimed))
11621 /// decides an event is genuinely unroutable, because that is a claim about
11622 /// the future rather than about this instant.
11623 ///
11624 /// # Errors
11625 ///
11626 /// [`RuntimeError::ReservedEventKind`] for a kind in the `agentplane.`
11627 /// namespace, which only this plane mints; [`RuntimeError`] for a store
11628 /// failure or a refused resume.
11629 pub async fn deliver(&self, event: &InboundEvent) -> Result<Delivery, RuntimeError> {
11630 refuse_reserved_kind(event)?;
11631 self.deliver_buffered(event, true).await
11632 }
11633
11634 /// Deliver an event this plane minted itself — a worklist decision.
11635 ///
11636 /// The one path into the reserved namespace [`deliver`](Self::deliver)
11637 /// refuses from outside.
11638 pub(crate) async fn deliver_minted(
11639 &self,
11640 event: &InboundEvent,
11641 ) -> Result<Delivery, RuntimeError> {
11642 self.deliver_buffered(event, false).await
11643 }
11644
11645 /// Store `event`, then offer it to the oldest eligible waiter.
11646 ///
11647 /// `rematch` offers a duplicate again: the buffer and the match are two
11648 /// writes, and a counterparty's retry is the one chance to repair a crash
11649 /// between them. A minted decision is never re-offered, because its
11650 /// duplicate is the answer *already on record*, and offering that stored
11651 /// answer under a second decider's call would report it as theirs.
11652 async fn deliver_buffered(
11653 &self,
11654 event: &InboundEvent,
11655 rematch: bool,
11656 ) -> Result<Delivery, RuntimeError> {
11657 let events = self.events.as_ref().ok_or_else(|| {
11658 RuntimeError::PlanContract(
11659 "this runtime has no event store — build it with `.events(store)`".into(),
11660 )
11661 })?;
11662
11663 let now = now_for_admission();
11664
11665 // Durable first. Deduplication by event id makes a counterparty's retry
11666 // — and they all retry — harmless.
11667 let fresh = events
11668 .buffer(event, now)
11669 .await
11670 .map_err(RuntimeError::from_store)?;
11671
11672 // The match claims only an unclaimed, live row, so a retry of a
11673 // message already consumed matches nothing.
11674 if !fresh && !rematch {
11675 return Ok(Delivery::Duplicate);
11676 }
11677 let Some(sub) = events
11678 .match_waiter(event, now)
11679 .await
11680 .map_err(RuntimeError::from_store)?
11681 else {
11682 return Ok(if fresh {
11683 Delivery::Buffered
11684 } else {
11685 Delivery::Duplicate
11686 });
11687 };
11688
11689 self.resume_subscription(events, sub, event).await
11690 }
11691
11692 /// Deliver an inbound event to exactly `run`.
11693 ///
11694 /// This is the task-addressed counterpart to [`Runtime::deliver`]. It is
11695 /// used by protocols such as A2A where a follow-up carries a concrete task
11696 /// id. Correlation alone is insufficient there: two tasks may wait on the
11697 /// same business key, and resuming the oldest would violate the request.
11698 ///
11699 /// The event store atomically inserts and claims the event for this run. A
11700 /// run that is not waiting leaves no buffered event behind for another run.
11701 ///
11702 /// # Errors
11703 ///
11704 /// As [`deliver`](Self::deliver), and [`RuntimeError::PlanContract`] when
11705 /// `run` is not waiting for this input. Addressing a run by id is no way
11706 /// around the reserved namespace: a run waiting on a human task is waiting
11707 /// for exactly the kind this refuses.
11708 pub async fn deliver_to(
11709 &self,
11710 run: RunId,
11711 event: &InboundEvent,
11712 ) -> Result<Delivery, RuntimeError> {
11713 refuse_reserved_kind(event)?;
11714 let events = self.events.as_ref().ok_or_else(|| {
11715 RuntimeError::PlanContract(
11716 "this runtime has no event store — build it with `.events(store)`".into(),
11717 )
11718 })?;
11719 match events
11720 .deliver_to(run, event, now_for_admission())
11721 .await
11722 .map_err(RuntimeError::from_store)?
11723 {
11724 crate::case::TargetedDelivery::Duplicate => Ok(Delivery::Duplicate),
11725 crate::case::TargetedDelivery::NotWaiting => Err(RuntimeError::PlanContract(format!(
11726 "run {run} is not waiting for this input"
11727 ))),
11728 crate::case::TargetedDelivery::Matched(sub) => {
11729 self.resume_subscription(events, sub, event).await
11730 }
11731 }
11732 }
11733
11734 async fn resume_subscription(
11735 &self,
11736 events: &Arc<dyn crate::case::EventStore>,
11737 sub: crate::core::Subscription,
11738 event: &InboundEvent,
11739 ) -> Result<Delivery, RuntimeError> {
11740 // Record the event as the awaited effect's result, then let replay do
11741 // the rest: the resumed run reads it back like any other completed
11742 // effect, and none of the suspension machinery exists twice.
11743 //
11744 // The lease may be briefly held: an event can arrive in the window
11745 // between the run registering its subscription and its owner finishing
11746 // the suspension bookkeeping — the owner is *concluding*, and will
11747 // release within milliseconds. Failing that race would be fatal in the
11748 // quietest possible way: the delivery fails `LeaseHeld` **after** the
11749 // event is durably claimed for this run, the counterparty's retry
11750 // deduplicates against the claimed event, and nothing ever resumes the
11751 // run — a message that arrived in time, parked forever. So the claim
11752 // is retried under a bounded backoff; if the owner is genuinely
11753 // stalled past the bound, the event is *left claimed* for this run
11754 // and reported as buffered — the sweep's redelivery pass finds a
11755 // waiting subscription with a claimed event and finishes the job on a
11756 // later tick, so the bound bounds latency, never delivery.
11757 let mut backoff = Duration::from_millis(25);
11758 let lease = loop {
11759 match self
11760 .store
11761 .acquire(sub.run, &self.owner, self.lease_ttl)
11762 .await
11763 {
11764 Ok(lease) => break lease,
11765 Err(crate::core::StoreError::LeaseHeld { .. })
11766 if backoff < Duration::from_secs(1) =>
11767 {
11768 tokio::time::sleep(backoff).await;
11769 backoff *= 2;
11770 }
11771 Err(crate::core::StoreError::LeaseHeld { .. }) => {
11772 // The claim retired the subscription, so nothing lists
11773 // this run as waiting any more — and the claimed event
11774 // blocks every dedup'd retry of itself. Parking the pair
11775 // is what keeps it findable: the sweep's redelivery pass
11776 // walks parked waits, re-claims (a claim by the same run
11777 // is returned, not filtered), and finishes the delivery
11778 // once the owner concludes.
11779 events
11780 .park_wait(&sub, now_for_admission())
11781 .await
11782 .map_err(RuntimeError::from_store)?;
11783 tracing::warn!(
11784 run = %sub.run,
11785 event = %event.id,
11786 "an event was claimed for a run whose owner did not conclude \
11787 within the retry window; the sweep will deliver it"
11788 );
11789 return Ok(Delivery::Buffered);
11790 }
11791 Err(e) => return Err(RuntimeError::from_store(e)),
11792 }
11793 };
11794
11795 let history = self
11796 .store
11797 .read(sub.run, 1)
11798 .await
11799 .map_err(RuntimeError::from_store)?;
11800 let already_recorded = history.iter().any(|record| {
11801 record.effect_key() == Some(sub.effect)
11802 && matches!(record.kind(), RecordKind::EffectDone { .. })
11803 });
11804 // A run whose conclusion is durable consumes nothing. An answer
11805 // appended after `RunConcluded` is one the run never acted on, and it
11806 // leaves a run its seal refuses forever. The resume below finishes it
11807 // instead — seals it and retires its waits, which hands this message,
11808 // still unconsumed, to the next waiter on its key.
11809 let closed = resume_is_closed(&history).is_some();
11810 if !already_recorded && !closed {
11811 // Judged here, where the output is recorded, under the run's own
11812 // declaration: the step that resumes reads the verdict back.
11813 let content = self.delivered_verdict(&history, &event.payload);
11814 self.store
11815 .append(
11816 lease.epoch,
11817 vec![{
11818 let mut a = Append::new(
11819 sub.run,
11820 RecordKind::EffectDone {
11821 output: event.payload.clone(),
11822 // The sender, so a replayed run rebuilds the same
11823 // provenance this delivery gave the value.
11824 source: Some(event.source.clone()),
11825 // And the operator who minted it, where one
11826 // did: the same record on the same path, so a
11827 // decision that resumes a suspended run is
11828 // attributed exactly as one consumed inline.
11829 by: event.by.clone(),
11830 spend: crate::core::Spend::default(),
11831 // An inbound payload is another party's data,
11832 // which is exactly the point this lattice value
11833 // names — the same one the run's own delivery
11834 // path records.
11835 declared: crate::core::DeclaredOutput::untrusted(),
11836 content,
11837 elapsed_ms: None,
11838 },
11839 )
11840 .effect(sub.effect)
11841 .step(sub.step)
11842 .phase(sub.phase);
11843 // Every record of a case-bound run carries its case, and a
11844 // record written from outside the run is no exception.
11845 if let Some(c) = sub.case {
11846 a = a.case(c);
11847 }
11848 a
11849 }],
11850 )
11851 .await
11852 .map_err(RuntimeError::from_store)?;
11853 }
11854
11855 if !closed {
11856 events
11857 .unsubscribe(sub.run, sub.effect)
11858 .await
11859 .map_err(RuntimeError::from_store)?;
11860 }
11861
11862 // The event is recorded as the awaited effect's result, and the resume
11863 // continues under the *same* lease. Releasing here and letting the
11864 // resume re-acquire opened the window this function's own doc names —
11865 // a crash between the release and the acquire left a released lease
11866 // over a run whose subscription was just retired: no driver, and
11867 // invisible to the abandonment queue, which lists only leases that
11868 // expired while still naming an owner. The handover keeps the run
11869 // owned continuously from delivery to conclusion.
11870 match self.resume_holding(sub.run, lease).await {
11871 // A `Fenced` means this lease lapsed mid-resume and somebody
11872 // claimed the run — a concurrent delivery, a recovery sweep. The
11873 // event is durably recorded, so whoever holds the lease replays
11874 // it; this delivery still delivered, and a sender told otherwise
11875 // retries into a duplicate.
11876 Ok(_) | Err(RuntimeError::Fenced { .. }) => {}
11877 // This plane holds no agent for the run — an operator's terminal
11878 // recording a decision. The answer is durable, and the lease is
11879 // kept so it lapses owned: that puts the run on the abandonment
11880 // listing, where the plane that runs the agent recovers it from
11881 // the recorded answer. Failing here would leave a recorded
11882 // decision reported as lost and the task it answers still claimed.
11883 // A plane holding no key ring for a sealed run, or no case store
11884 // for a case-bound one, is the same terminal: the plane that holds
11885 // it continues the run.
11886 Err(
11887 RuntimeError::NoProvider { .. }
11888 | RuntimeError::NoCaseStore { .. }
11889 | RuntimeError::PayloadsSealed { .. },
11890 ) => {
11891 return Ok(Delivery::Buffered);
11892 }
11893 Err(e) => return Err(e),
11894 }
11895 if closed {
11896 return Ok(Delivery::Buffered);
11897 }
11898 Ok(Delivery::Resumed { run: sub.run })
11899 }
11900
11901 /// Retire everything a run that concluded closed was still waiting on:
11902 /// its timers, its event subscriptions and its unanswered tasks.
11903 ///
11904 /// A sealed run can record nothing, so each of these left behind is work
11905 /// the plane keeps doing for nobody — a timer claimed and failed every
11906 /// lease period, the oldest waiter on a key taking the next event from a
11907 /// live run behind it, a task offered for a decision no answer reaches.
11908 ///
11909 /// Called after every seal of a concluded run — the conclusion's own and
11910 /// the one resume writes to repair a crash between conclusion and seal —
11911 /// and idempotent, so retiring twice is harmless.
11912 ///
11913 /// After the seal, and best-effort: the conclusion is already durable, and
11914 /// the stores pass over a sealed run's timers and subscriptions on their
11915 /// own, so a retirement that fails here costs clutter rather than a
11916 /// delivery. A task left pending by that failure is withdrawn by nothing
11917 /// else, which is why the failure is logged loudly.
11918 async fn retire_waits(&self, run: RunId) {
11919 if let Some(timers) = &self.timers
11920 && let Err(error) = timers.disarm_run(run).await
11921 {
11922 tracing::error!(%run, %error, "a closed run's timers could not be retired");
11923 }
11924 // The waits the run announced and never had answered: a message
11925 // claimed for one of them reached nobody and goes to the next waiter,
11926 // and a task opened for one is withdrawn. A task opened beside an
11927 // answer completes its own effect and is not among them: it is a
11928 // finding for a desk, and outlives the run by design.
11929 let unanswered = match self.store.read(run, 1).await {
11930 Ok(records) => unanswered_waits(&records),
11931 Err(error) => {
11932 tracing::error!(%run, %error, "a closed run's waits could not be read to retire them");
11933 return;
11934 }
11935 };
11936 if let Some(events) = &self.events {
11937 match events.unsubscribe_run(run, &unanswered).await {
11938 Ok(retired) => {
11939 for event in retired.released {
11940 self.offer_released(events, &event).await;
11941 }
11942 }
11943 Err(error) => {
11944 tracing::error!(%run, %error, "a closed run's subscriptions could not be retired");
11945 }
11946 }
11947 }
11948 if let Some(tasks) = &self.tasks {
11949 let awaited: Vec<_> = unanswered
11950 .iter()
11951 .map(|effect| crate::core::TaskId::derive(run, *effect))
11952 .collect();
11953 if let Err(error) = tasks.withdraw_run(run, &awaited).await {
11954 tracing::error!(%run, %error, "a closed run's open tasks could not be withdrawn");
11955 }
11956 }
11957 }
11958
11959 /// Offer a message a closed run held, and never consumed, to the next
11960 /// waiter on its key — the delivery it would have had if that run had not
11961 /// concluded first. Best effort, like the retirement that released it: a
11962 /// message no waiter takes now stays buffered for the next one.
11963 async fn offer_released(
11964 &self,
11965 events: &Arc<dyn crate::case::EventStore>,
11966 event: &InboundEvent,
11967 ) {
11968 let sub = match events.match_waiter(event, now_for_admission()).await {
11969 Ok(Some(sub)) => sub,
11970 Ok(None) => return,
11971 Err(error) => {
11972 tracing::error!(event = %event.id, %error, "a released message could not be offered");
11973 return;
11974 }
11975 };
11976 if let Err(error) = Box::pin(self.resume_subscription(events, sub, event)).await {
11977 tracing::error!(event = %event.id, %error, "a released message could not be delivered");
11978 }
11979 }
11980
11981 /// Finish deliveries that died between the claim and the resume.
11982 ///
11983 /// A delivery that raced a live owner leaves an event durably claimed for
11984 /// a run nothing resumed, and the counterparty's retries deduplicate
11985 /// against that claim — so no future delivery drives the run, and a
11986 /// message that arrived in time is stranded. The giving-up delivery parks
11987 /// the pair precisely so it stays findable; this pass walks the parked
11988 /// waits — never every registered wait, whose long legitimate members
11989 /// would fill the page ahead of a pair parked after them — re-claims
11990 /// (idempotently: a claim by the same run is returned, not filtered), and
11991 /// finishes the delivery. Runs whose owner is *still* live are skipped and
11992 /// found again next tick, so the delivery's retry bound bounds latency,
11993 /// never delivery.
11994 ///
11995 /// What this does **not** cover, honestly: a crash inside the store
11996 /// commit between an event's claim and the resume retires the
11997 /// subscription with nothing left to re-register it, and that window
11998 /// stays what it always was — recorded rather than closed, with later
11999 /// events for the same correlation dead-lettering as its symptom.
12000 pub(crate) async fn redeliver_claimed(
12001 &self,
12002 limit: usize,
12003 ) -> Result<crate::runtime::Redelivered, RuntimeError> {
12004 let Some(events) = self.events.as_ref() else {
12005 return Ok(crate::runtime::Redelivered::default());
12006 };
12007 let parked = events
12008 .parked_waits(limit)
12009 .await
12010 .map_err(RuntimeError::from_store)?;
12011 let examined = parked.len();
12012 let mut delivered = 0usize;
12013 for sub in parked {
12014 let Some(buffered) = events
12015 .claim_for(&sub, now_for_admission())
12016 .await
12017 .map_err(RuntimeError::from_store)?
12018 else {
12019 continue;
12020 };
12021 match self
12022 .resume_subscription(events, sub.clone(), &buffered.event)
12023 .await
12024 {
12025 Ok(Delivery::Resumed { .. }) => delivered += 1,
12026 // Still held live: the owner is working, and will either
12027 // consume the claim itself or conclude and be found next tick.
12028 Ok(_) => {}
12029 Err(error) => {
12030 tracing::error!(
12031 run = %sub.run,
12032 %error,
12033 "a claimed event's redelivery failed; retried next tick",
12034 );
12035 }
12036 }
12037 }
12038 Ok(crate::runtime::Redelivered {
12039 finished: delivered,
12040 examined,
12041 })
12042 }
12043
12044 /// Retire events that nobody claimed within `grace`.
12045 ///
12046 /// A non-empty dead-letter list means a correlation key is wrong somewhere:
12047 /// the message arrived, was held, and no run ever asked for it. That is the
12048 /// failure which otherwise presents as a process silently never completing,
12049 /// so it is worth alerting on rather than logging.
12050 /// `grace` is a `std::time::Duration` for the reason
12051 /// [`StepCtx::deadline`](crate::runtime::StepCtx::deadline)'s `warn_before`
12052 /// is: a negative grace window is meaningless, and the signed type could
12053 /// express it — a cutoff moved *forward* of now, retiring events that had
12054 /// not yet had their chance. It is also the `Duration` the caller has.
12055 pub async fn sweep_events(&self, grace: std::time::Duration) -> Result<usize, RuntimeError> {
12056 let events = self
12057 .events
12058 .as_ref()
12059 .ok_or_else(|| RuntimeError::PlanContract("this runtime has no event store".into()))?;
12060 // Saturating rather than fallible, at both steps: a grace window
12061 // beyond what the calendar can hold means "retire nothing", and
12062 // refusing the call would be a worse answer to a caller asking for a
12063 // longer hold. The subtraction has to saturate too — `time` panics on
12064 // underflow, so converting the window to `Duration::MAX` and then
12065 // subtracting it aborts the tick that also breaches obligations and
12066 // recovers abandoned runs. The first instant the calendar can name is
12067 // a cutoff nothing is older than, which is exactly "retire nothing".
12068 let grace = time::Duration::try_from(grace).unwrap_or(time::Duration::MAX);
12069 let cutoff = now_for_admission()
12070 .checked_sub(grace)
12071 .unwrap_or_else(crate::core::first_instant);
12072 let retired = events
12073 .sweep_unclaimed(cutoff, "no run claimed this event within the grace window")
12074 .await
12075 .map_err(RuntimeError::from_store)?;
12076
12077 if retired > 0 {
12078 // A non-empty dead-letter list means a correlation key is wrong
12079 // somewhere: the message arrived, was held, and no run ever asked
12080 // for it. That is the failure which otherwise presents as a process
12081 // silently never completing.
12082 tracing::error!(target: telemetry::DEAD_LETTERED, count = retired, %cutoff);
12083 self.meter
12084 .count_by(metrics::DEAD_LETTERS, "", retired as u64);
12085 }
12086 Ok(retired)
12087 }
12088}
12089
12090/// Every `RunStatus`, so adding one forces a decision everywhere one is owed.
12091///
12092/// Written out rather than derived, because Rust cannot enumerate a
12093/// data-carrying enum. What keeps it honest is **not** the count assertions in
12094/// its consumers: those fire when somebody edits this list, which is the case
12095/// that needs no help. A variant added to the enum and forgotten here changes
12096/// nothing they can see, and every agreement test then walks a list that no
12097/// longer describes the type — passing over the one variant nobody decided
12098/// about.
12099///
12100/// `every_run_status_variant_is_listed` in `tests/guards` is what closes that:
12101/// it reads the enum's variants and this function's arms out of the source and
12102/// holds them to each other, which is the only check available for a list a
12103/// language cannot derive.
12104///
12105/// Crate-visible so those consumers share **one** list. Two copies would be two
12106/// places to remember, and the second would be the one that went stale — which
12107/// is the same failure the tests using it exist to catch.
12108#[cfg(test)]
12109pub(crate) fn every_status() -> Vec<RunStatus> {
12110 use crate::core::{BudgetExceeded, CorrelationKey, SuspendReason, Timestamp};
12111 vec![
12112 RunStatus::Succeeded,
12113 RunStatus::Failed("because".into()),
12114 RunStatus::Suspended(SuspendReason::AwaitingEvent {
12115 kind: "reply".into(),
12116 correlation: vec![CorrelationKey::new("claim", "CLM-1")],
12117 until: Timestamp::from_unix_timestamp(1_760_000_000).expect("time"),
12118 }),
12119 RunStatus::Exhausted(BudgetExceeded::Steps { allowed: 1 }),
12120 RunStatus::Quarantined("unknown outcome".into()),
12121 RunStatus::Replanning("try again".into()),
12122 RunStatus::Cancelled {
12123 actor: crate::core::Operator::asserted("ops").expect("a name"),
12124 reason: "stop".into(),
12125 },
12126 RunStatus::Abandoned {
12127 actor: crate::core::Operator::asserted("ops").expect("a name"),
12128 reason: "nobody could establish what happened".into(),
12129 },
12130 RunStatus::Swept,
12131 RunStatus::BrokeGlass {
12132 actor: crate::core::Operator::asserted("ops").expect("a name"),
12133 reason: "INC-42".into(),
12134 },
12135 RunStatus::HaltLifted {
12136 actor: crate::core::Operator::asserted("ops").expect("a name"),
12137 },
12138 RunStatus::HoldReleased {
12139 actor: crate::core::Operator::asserted("ops").expect("a name"),
12140 },
12141 RunStatus::Withheld {
12142 subject: "alice".into(),
12143 reason: "credential withdrawn".into(),
12144 },
12145 RunStatus::Observed,
12146 ]
12147}
12148
12149#[cfg(test)]
12150mod resume_agreement_tests {
12151 use super::{QuotaPass, RunStatus, resume_is_closed};
12152 use crate::journal::RecordKind;
12153
12154 // Through the crate re-export rather than `super::`, so the one path both
12155 // consumers use is exercised in every build. Reaching it directly here
12156 // would leave the re-export unused whenever `a2a-server` is off — which is
12157 // a lint failure in exactly the feature configuration nobody runs locally.
12158 use crate::runtime::every_status;
12159
12160 /// **A conclusion that sealed the journal may never be resumed.**
12161 ///
12162 /// `RunStatus::seals` decides whether a conclusion freezes the run and
12163 /// publishes a Merkle leaf; `resume_is_closed` decides whether a recorded
12164 /// conclusion may be continued. A status that did the first and permits the
12165 /// second would grow the history past the leaf every later checkpoint
12166 /// attests — the failure the "a conclusion is not a closure" work removed
12167 /// for `failed`, reachable again the moment the two disagree.
12168 ///
12169 /// The executor's own comment claimed "a test pins the agreement" while no
12170 /// such test existed, which is the unfalsifiable-guarantee shape this
12171 /// project treats as a defect in itself: the rule was real, checked by
12172 /// nobody, and deletable in silence.
12173 #[test]
12174 fn a_sealing_conclusion_is_never_resumable() {
12175 let statuses = every_status();
12176 assert_eq!(
12177 statuses.len(),
12178 14,
12179 "a RunStatus variant was added or removed — decide whether it seals \
12180 and whether a resume may continue from it, then update this list"
12181 );
12182
12183 for status in &statuses {
12184 let records = sealed_as(status.as_str());
12185 let verdict = resume_is_closed(&records);
12186 if status.seals() {
12187 assert!(
12188 verdict.is_some(),
12189 "'{}' seals the journal and enters the Merkle log, yet a resume \
12190 is permitted from it — the resume would grow the history past \
12191 the leaf every later checkpoint attests",
12192 status.as_str()
12193 );
12194 }
12195 }
12196 }
12197
12198 /// The availability half: the conclusions that stay open really do.
12199 ///
12200 /// Asserted separately and by name rather than as the converse of the rule
12201 /// above, because the converse is false — `Suspended` and `Replanning` do
12202 /// not seal either, and neither is ever a recorded conclusion.
12203 ///
12204 /// `withheld` is here because it is a **pause**: the authority a run acts
12205 /// under was withdrawn, its mutations stand, and lifting the halt is what
12206 /// lets it continue. A withheld run that could not be resumed would make the
12207 /// withdrawal a destruction rather than a stop.
12208 #[test]
12209 fn a_failed_or_exhausted_run_may_still_be_resumed() {
12210 for outcome in ["failed", "exhausted", super::WITHHELD_OUTCOME] {
12211 assert!(
12212 resume_is_closed(&sealed_as(outcome)).is_none(),
12213 "'{outcome}' is a conclusion a resume must be able to continue \
12214 from — its completed effects are read back from history, which \
12215 is the point of having a journal"
12216 );
12217 }
12218 }
12219
12220 /// `SEALED_OUTCOMES` and `RunStatus::seals` are one rule in two spellings,
12221 /// and this holds them together variant by variant.
12222 ///
12223 /// The list exists so a caller wanting every sealed run — the export CLI —
12224 /// does not restate the rule as string literals in a binary, which is where
12225 /// a new sealing outcome would have been silently dropped from exactly the
12226 /// artifact an auditor asks for. Both directions matter: a sealing status
12227 /// missing from the list loses runs, and a non-sealing status present in it
12228 /// makes the export ask the outcome index about a backlog that drains.
12229 #[test]
12230 fn the_sealed_outcome_list_agrees_with_the_sealing_rule() {
12231 for status in super::every_status() {
12232 assert_eq!(
12233 super::SEALED_OUTCOMES.contains(&status.as_str()),
12234 status.seals(),
12235 "'{}' disagrees between RunStatus::seals and SEALED_OUTCOMES — \
12236 one rule, two spellings, and the export reads the list",
12237 status.as_str()
12238 );
12239 }
12240 }
12241
12242 /// **Every outcome this build writes is one its own reader recognises.**
12243 ///
12244 /// The reader is a `match` over `&str` with a catch-all, so it goes on
12245 /// compiling forever while the writer's list grows — and the catch-all is
12246 /// not benign: it answers `Quarantined`, so a run this plane sealed itself
12247 /// reads back as *the runtime could not decide what happened*. Two endings
12248 /// sat there, `swept` and `broke-glass`, and a break-glass crossing — the
12249 /// one control whose entire content is who crossed and why — answered the
12250 /// operator surface with the words "this build does not recognise it" in
12251 /// place of the reason.
12252 ///
12253 /// Held from the writer's side, and the writer's side is
12254 /// [`every_status`](super::every_status) — every variant the type can
12255 /// take. `OUTCOMES_OF_RECORD` is the *export's* list and deliberately
12256 /// omits the endings that stay resumable, so a guard reading it asks about
12257 /// a subset of what the writer writes and passes while the rest fall
12258 /// through the catch-all. `withheld` sat there: the one ending whose whole
12259 /// content is *whose credential was withdrawn*, answering the operator API
12260 /// with "this build does not recognise it".
12261 #[test]
12262 fn every_outcome_this_build_writes_is_one_it_can_read_back() {
12263 for written in super::every_status() {
12264 // A suspension is not a conclusion; it has its own record and its
12265 // own arm in `observed_status`.
12266 if written.is_suspended() {
12267 continue;
12268 }
12269 let outcome = written.as_str();
12270 let status = super::recorded_status(
12271 outcome,
12272 written.own_account().as_deref(),
12273 match &written {
12274 RunStatus::Exhausted(limit) => Some(limit),
12275 _ => None,
12276 },
12277 &sealed_as(outcome),
12278 );
12279 assert_eq!(
12280 status.as_str(),
12281 outcome,
12282 "this build concludes runs as '{outcome}' and reads that back as \
12283 '{}' — a catch-all answering `Quarantined` about a record this \
12284 plane wrote itself",
12285 status.as_str()
12286 );
12287 }
12288 }
12289
12290 /// **An ending a person caused reports the words that person gave.**
12291 ///
12292 /// The conclusion does not carry them: one sentence, one record. So the
12293 /// reader has to reach the attribution record for the reason exactly as it
12294 /// already does for the actor, and a reader that only learned the second
12295 /// half would answer an incident review with a blank where the operator
12296 /// wrote why.
12297 #[test]
12298 fn an_ending_a_person_caused_reports_their_own_words() {
12299 for (outcome, expected) in [
12300 ("cancelled", "stop"),
12301 ("abandoned", "nobody could establish what happened"),
12302 (super::BREAK_GLASS_OUTCOME, "INC-42"),
12303 (super::WITHHELD_OUTCOME, "credential withdrawn"),
12304 ] {
12305 // `None`, as the conclusion this build writes now carries.
12306 let status = super::recorded_status(outcome, None, None, &sealed_as(outcome));
12307 assert_eq!(
12308 status.reason().as_deref(),
12309 Some(expected),
12310 "'{outcome}' lost the reason the person gave: {status:?}"
12311 );
12312 }
12313 }
12314
12315 /// **An ending nobody is named on is quarantined, not given a name.**
12316 ///
12317 /// The chain this build writes always carries the attribution record before
12318 /// the conclusion; a chain that carries only the conclusion is a crash
12319 /// between the two appends, or a chain somebody edited. Either way the
12320 /// record is not trustworthy, and *not trustworthy* is a different bucket
12321 /// from *an operator called unknown did this*.
12322 ///
12323 /// A fabricated actor would be worse than a missing one: indistinguishable
12324 /// from a real operator with that name, and served through the operator API
12325 /// as fact.
12326 #[test]
12327 fn an_ending_with_no_attribution_record_is_not_given_a_name() {
12328 for outcome in ["cancelled", "abandoned", super::BREAK_GLASS_OUTCOME] {
12329 let status =
12330 super::recorded_status(outcome, Some("why"), None, &concluded_only(outcome));
12331 assert!(
12332 matches!(status, RunStatus::Quarantined(_)),
12333 "'{outcome}' with no record of who asked read back as {status:?} — \
12334 an operator act with nobody on it was given a name"
12335 );
12336 }
12337 }
12338
12339 /// The offline sweep covers every ending, plus the backlog that is not one.
12340 ///
12341 /// Held to `SEALED_OUTCOMES` rather than restated, because the failure this
12342 /// prevents is silent in exactly the artifact nobody re-reads: an ending
12343 /// added to the runtime and not to the export drops those runs from the
12344 /// file an auditor is handed, and the file looks complete.
12345 #[test]
12346 fn the_offline_sweep_covers_every_ending_and_the_quarantine_backlog() {
12347 for outcome in super::SEALED_OUTCOMES {
12348 assert!(
12349 super::OUTCOMES_OF_RECORD.contains(outcome),
12350 "'{outcome}' is an ending this plane can reach and the export's \
12351 default sweep would silently omit it"
12352 );
12353 }
12354 assert!(
12355 super::OUTCOMES_OF_RECORD.contains(&"quarantined"),
12356 "a quarantine is the plane's outstanding unresolved risk and the one \
12357 thing an auditor is most likely to have come for"
12358 );
12359 for open in ["failed", "exhausted", "suspended"] {
12360 assert!(
12361 !super::OUTCOMES_OF_RECORD.contains(&open),
12362 "'{open}' is work in progress that a resume moves on its own — \
12363 exporting it is a snapshot of a moving target, not a record"
12364 );
12365 }
12366 }
12367
12368 /// An outcome this build cannot interpret is never permission to resume.
12369 #[test]
12370 fn an_unrecognised_conclusion_fails_closed() {
12371 let verdict = resume_is_closed(&sealed_as("swept"));
12372 assert!(
12373 matches!(verdict, Some(RunStatus::Quarantined(_))),
12374 "an unrecognised conclusion must quarantine rather than resume: {verdict:?}"
12375 );
12376 }
12377
12378 /// A chain carrying the conclusion and **not** the attribution record.
12379 ///
12380 /// The damaged shape, which no healthy write produces: a crash between the
12381 /// two appends, or an edit. Built by taking the real chain and dropping
12382 /// everything before the conclusion, so it cannot drift from what
12383 /// [`sealed_as`] produces.
12384 fn concluded_only(outcome: &str) -> Vec<crate::journal::Record> {
12385 let full = sealed_as(outcome);
12386 full.into_iter()
12387 .filter(|r| matches!(r.kind(), RecordKind::RunConcluded { .. }))
12388 .collect()
12389 }
12390
12391 /// The chain this build actually writes for an ending.
12392 ///
12393 /// **Including the attribution record**, where the ending has one. A
12394 /// conclusion is never the only record of a cancellation, an abandonment or
12395 /// a crossing — the runtime appends who asked first — so a fixture holding
12396 /// the conclusion alone asserts a round trip over a chain this plane does
12397 /// not produce, and would keep passing if the reader began inventing a
12398 /// name for the missing half.
12399 fn sealed_as(outcome: &str) -> Vec<crate::journal::Record> {
12400 use crate::core::{Digest, Epoch};
12401 use crate::journal::{Record, RecordBody};
12402
12403 let run = crate::core::RunId::generate();
12404 let who = || crate::core::Operator::asserted("ops").expect("a name");
12405 let mut kinds: Vec<RecordKind> = match outcome {
12406 "cancelled" => vec![RecordKind::RunCancelled {
12407 actor: who(),
12408 reason: "stop".into(),
12409 }],
12410 "abandoned" => vec![RecordKind::QuarantineDecided {
12411 decider: who(),
12412 reason: "nobody could establish what happened".into(),
12413 decision: crate::core::QuarantineDecision::Abandon,
12414 }],
12415 super::BREAK_GLASS_OUTCOME => vec![RecordKind::BreakGlass {
12416 actor: who(),
12417 roles: vec!["oncall".into()],
12418 reason: "INC-42".into(),
12419 }],
12420 super::HALT_LIFTED_OUTCOME => vec![RecordKind::HaltLifted {
12421 scope: "tenant".into(),
12422 by: who(),
12423 at: crate::core::Timestamp::UNIX_EPOCH,
12424 reason: "INC-42".into(),
12425 thrown_by: who(),
12426 thrown_at: crate::core::Timestamp::UNIX_EPOCH,
12427 }],
12428 super::HOLD_RELEASED_OUTCOME => vec![RecordKind::HoldReleased {
12429 by: who(),
12430 at: crate::core::Timestamp::UNIX_EPOCH,
12431 placed_by: who(),
12432 placed_at: crate::core::Timestamp::UNIX_EPOCH,
12433 }],
12434 super::WITHHELD_OUTCOME => vec![RecordKind::AuthorityWithheld {
12435 subject: "alice".into(),
12436 reason: "credential withdrawn".into(),
12437 by: who(),
12438 }],
12439 _ => Vec::new(),
12440 };
12441 kinds.push(RecordKind::RunConcluded {
12442 outcome: outcome.to_owned(),
12443 chain_head: Digest::of(b""),
12444 reason: None,
12445 exhaustion: None,
12446 live_spend: crate::core::Spend::default(),
12447 });
12448
12449 let mut prev = Digest::of(b"");
12450 let mut out = Vec::new();
12451 for (i, kind) in kinds.into_iter().enumerate() {
12452 let body = RecordBody {
12453 seq: u64::try_from(i + 1).expect("a small fixture"),
12454 run,
12455 case: None,
12456 step: None,
12457 phase: super::Phase::Forward,
12458 epoch: Epoch::default(),
12459 v: 1,
12460 effect_key: None,
12461 kind,
12462 };
12463 let record = Record::seal(body, prev).expect("a sealed record");
12464 prev = record.hash;
12465 out.push(record);
12466 }
12467 out
12468 }
12469
12470 /// One execution pass is authorized and accrued against one period even
12471 /// when it finishes after a billing boundary.
12472 #[test]
12473 fn a_quota_pass_keeps_the_period_it_started_in() {
12474 let quota = crate::quota::TenantQuota {
12475 max_tokens_per_period: Some(1_000),
12476 period: crate::quota::Period::Daily,
12477 ..crate::quota::TenantQuota::default()
12478 };
12479 let before_midnight = crate::core::Timestamp::from_unix_timestamp(86_399).unwrap();
12480 let after_midnight = crate::core::Timestamp::from_unix_timestamp(86_400).unwrap();
12481
12482 let pass = QuotaPass::at("a, before_midnight, true);
12483 assert_eq!(
12484 pass.period(),
12485 Some(quota.period.key_for(before_midnight).as_str())
12486 );
12487 assert_ne!(
12488 pass.period(),
12489 Some(quota.period.key_for(after_midnight).as_str()),
12490 "settlement recomputed the period at completion, so this pass was \
12491 authorized against one ledger and charged to another"
12492 );
12493 }
12494}