Skip to main content

stateknot_runtime/
invocation_executor.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Durable-before-dispatch model and tool attempt execution.
5
6use std::{
7    error::Error as StdError,
8    fmt,
9    future::poll_fn,
10    sync::Arc,
11    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
12};
13
14use serde_json::json;
15use stateknot_core::{
16    AgentResultProvenance, AttemptId, BoundedJson, BoxFuture, BudgetRemaining, CancellationSignal,
17    Digest, DurationMillis, EventId, ExecutionCount, Failure, FailureCategory, FailureCode,
18    FailureId, FailureMessage, FailureOrigin, GraphSchemaValidationError, InvocationId,
19    JournalAppend, JournalEvent, JournalEventIntent, JournalEventKind, JournalExpectation,
20    JournalPayload, ModelContext, ModelContextError, ModelError, ModelErrorPhase,
21    ModelErrorProvenance, ModelEvent, ModelEventAccumulator, ModelInvocation,
22    ModelInvocationStatus, ModelInvocationTransition, ModelRequest, ModelResponse,
23    ModelResponseMode, ModelStopReason, RetryAdvice, RunFence, SchemaReference, Timestamp,
24    ToolContext, ToolContextError, ToolDescriptor, ToolError, ToolErrorPhase, ToolErrorProvenance,
25    ToolExternalEffect, ToolInputValidationError, ToolInvocation, ToolInvocationError,
26    ToolInvocationState, ToolInvocationStatus, ToolInvocationTransition, ToolProgressSink,
27    ToolReconciliationContext, ToolReconciliationContextError, ToolReconciliationObservation,
28    ToolReconciliationObservationError, ToolReconciliationProbeError, ToolResult, ToolRisk,
29    ToolStopReason,
30};
31use stateknot_store_postgres::{
32    ModelInvocationCommitOutcome, PostgresStore, StoreError, ToolInvocationCommitOutcome,
33};
34use thiserror::Error;
35
36use crate::{
37    JsonSchemaRegistry, ModelProviderRegistry, ModelProviderRegistryError,
38    StandardInvocationExecutionSchemaError, ToolProviderRegistry, ToolProviderRegistryError,
39    standard_invocation_execution_event_schema,
40};
41
42const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
43
44/// Trusted wall/monotonic clock pair captured at one process observation.
45#[derive(Clone, Copy, Debug)]
46pub struct InvocationClockObservation {
47    observed_at: Timestamp,
48    observed_instant: Instant,
49}
50
51impl InvocationClockObservation {
52    /// Constructs one trusted paired observation.
53    #[must_use]
54    pub const fn new(observed_at: Timestamp, observed_instant: Instant) -> Self {
55        Self {
56            observed_at,
57            observed_instant,
58        }
59    }
60
61    /// Returns the wall-clock component used for durable budget decisions.
62    #[must_use]
63    pub const fn observed_at(self) -> Timestamp {
64        self.observed_at
65    }
66
67    /// Returns the paired monotonic observation used for active deadlines.
68    #[must_use]
69    pub const fn observed_instant(self) -> Instant {
70        self.observed_instant
71    }
72}
73
74/// Trusted clock source for attempt contexts.
75pub trait InvocationClock: Send + Sync + 'static {
76    /// Captures paired wall and monotonic observations.
77    fn observe(&self) -> Result<InvocationClockObservation, InvocationClockError>;
78}
79
80/// Production system/monotonic clock implementation.
81#[derive(Clone, Copy, Debug, Default)]
82pub struct SystemInvocationClock;
83
84impl InvocationClock for SystemInvocationClock {
85    fn observe(&self) -> Result<InvocationClockObservation, InvocationClockError> {
86        let observed_instant = Instant::now();
87        let elapsed = SystemTime::now()
88            .duration_since(UNIX_EPOCH)
89            .map_err(|_| InvocationClockError::BeforeUnixEpoch)?;
90        let micros = i64::try_from(elapsed.as_micros())
91            .map_err(|_| InvocationClockError::TimestampOutOfRange)?;
92        let observed_at = Timestamp::from_unix_micros(micros)
93            .map_err(|_| InvocationClockError::TimestampOutOfRange)?;
94        Ok(InvocationClockObservation::new(
95            observed_at,
96            observed_instant,
97        ))
98    }
99}
100
101/// Failure to capture a representable runtime clock observation.
102#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
103#[non_exhaustive]
104pub enum InvocationClockError {
105    /// Host wall time preceded the Unix epoch.
106    #[error("invocation system clock precedes the Unix epoch")]
107    BeforeUnixEpoch,
108    /// Host wall time exceeded the canonical timestamp range.
109    #[error("invocation system clock is outside the supported timestamp range")]
110    TimestampOutOfRange,
111}
112
113/// Closed external boundary whose next-attempt budget is requested.
114#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
115#[non_exhaustive]
116pub enum InvocationBoundaryKind {
117    /// One model provider attempt.
118    Model,
119    /// One tool implementation attempt.
120    Tool,
121}
122
123/// Payload-free trusted request to an admission/accounting implementation.
124#[derive(Clone, Debug)]
125pub struct InvocationBudgetContext {
126    provenance: AgentResultProvenance,
127    boundary: InvocationBoundaryKind,
128    invocation_id: InvocationId,
129    attempt_id: AttemptId,
130    intent_digest: Digest,
131    observed_at: Timestamp,
132}
133
134impl InvocationBudgetContext {
135    /// Returns immutable admitted run provenance loaded from storage.
136    #[must_use]
137    pub const fn provenance(&self) -> &AgentResultProvenance {
138        &self.provenance
139    }
140
141    /// Returns the external boundary being admitted.
142    #[must_use]
143    pub const fn boundary(&self) -> InvocationBoundaryKind {
144        self.boundary
145    }
146
147    /// Returns the stable logical invocation identity.
148    #[must_use]
149    pub const fn invocation_id(&self) -> InvocationId {
150        self.invocation_id
151    }
152
153    /// Returns the proposed physical attempt identity.
154    #[must_use]
155    pub const fn attempt_id(&self) -> AttemptId {
156        self.attempt_id
157    }
158
159    /// Returns the exact durable invocation-intent checksum.
160    #[must_use]
161    pub const fn intent_digest(&self) -> Digest {
162        self.intent_digest
163    }
164
165    /// Returns the trusted clock used to evaluate remaining capacity.
166    #[must_use]
167    pub const fn observed_at(&self) -> Timestamp {
168        self.observed_at
169    }
170}
171
172/// Trusted admission/accounting source for exact remaining run capacity.
173///
174/// Implementations load the immutable resolved budget and cumulative durable
175/// DIRECT-only usage by `provenance`, recheck the intent/attempt admission policy, and
176/// return a finite [`BudgetRemaining`] evaluated at `observed_at`. The executor
177/// never accepts a caller-authored remaining-budget value directly. The executor
178/// deducts verified child subtree charges exactly once, then binds their account
179/// digest to the durable start transaction. Providers must not include children.
180pub trait InvocationBudgetProvider: Send + Sync + 'static {
181    /// Resolves exact remaining capacity before a durable attempt start.
182    fn remaining(
183        &self,
184        context: InvocationBudgetContext,
185    ) -> BoxFuture<'_, Result<BudgetRemaining, InvocationBudgetProviderError>>;
186}
187
188type PrivateBudgetSource = dyn StdError + Send + Sync + 'static;
189
190fn deduct_child_budget(
191    remaining: BudgetRemaining,
192    account: Option<&stateknot_core::ChildRunBudgetAccount>,
193) -> Result<BudgetRemaining, StoreError> {
194    let Some(account) = account else {
195        return Ok(remaining);
196    };
197    if account
198        .children()
199        .iter()
200        .any(|entry| entry.settlement().is_none())
201    {
202        return Err(StoreError::UnsettledChildRuns);
203    }
204    remaining
205        .deduct_cumulative(
206            &account
207                .delegated_usage()
208                .map_err(|_| StoreError::IncompleteChildAccounting)?,
209        )
210        .map_err(|_| StoreError::IncompleteChildAccounting)
211}
212
213/// Payload-redacted trusted accounting-provider failure.
214#[derive(Clone)]
215pub struct InvocationBudgetProviderError {
216    private_source: Arc<PrivateBudgetSource>,
217}
218
219impl InvocationBudgetProviderError {
220    /// Wraps one private admission/accounting diagnostic.
221    #[must_use]
222    pub fn new<E>(source: E) -> Self
223    where
224        E: StdError + Send + Sync + 'static,
225    {
226        Self {
227            private_source: Arc::new(source),
228        }
229    }
230
231    /// Returns the private source to trusted in-process diagnostics only.
232    #[must_use]
233    pub fn private_source(&self) -> &(dyn StdError + Send + Sync + 'static) {
234        self.private_source.as_ref()
235    }
236}
237
238impl fmt::Debug for InvocationBudgetProviderError {
239    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
240        formatter
241            .debug_struct("InvocationBudgetProviderError")
242            .field("has_private_source", &true)
243            .finish_non_exhaustive()
244    }
245}
246
247impl fmt::Display for InvocationBudgetProviderError {
248    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
249        formatter.write_str("invocation budget provider failed")
250    }
251}
252
253impl StdError for InvocationBudgetProviderError {}
254
255/// Runtime-owned sink for already validated semantic model stream events.
256///
257/// Implementations must durably deduplicate by `(attempt_id, sequence)` before
258/// exposing events externally. A successful future asserts ordered acceptance;
259/// the final accumulated response remains authoritative only after its separate
260/// invocation-ledger commit.
261pub trait ModelEventSink: Send + Sync + 'static {
262    /// Accepts one identity-bound, contiguous semantic event.
263    fn emit(&self, event: ModelEvent) -> BoxFuture<'_, Result<(), ModelEventSinkError>>;
264}
265
266type PrivateEventSinkSource = dyn StdError + Send + Sync + 'static;
267
268/// Payload-redacted model event sink failure.
269#[derive(Clone)]
270pub struct ModelEventSinkError {
271    private_source: Arc<PrivateEventSinkSource>,
272}
273
274impl ModelEventSinkError {
275    /// Wraps one private durable-sink diagnostic.
276    #[must_use]
277    pub fn new<E>(source: E) -> Self
278    where
279        E: StdError + Send + Sync + 'static,
280    {
281        Self {
282            private_source: Arc::new(source),
283        }
284    }
285
286    /// Returns the private source to trusted in-process diagnostics only.
287    #[must_use]
288    pub fn private_source(&self) -> &(dyn StdError + Send + Sync + 'static) {
289        self.private_source.as_ref()
290    }
291}
292
293impl fmt::Debug for ModelEventSinkError {
294    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
295        formatter
296            .debug_struct("ModelEventSinkError")
297            .field("has_private_source", &true)
298            .finish_non_exhaustive()
299    }
300}
301
302impl fmt::Display for ModelEventSinkError {
303    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
304        formatter.write_str("model event sink failed")
305    }
306}
307
308impl StdError for ModelEventSinkError {}
309
310/// Stable start and terminal event identities for one physical attempt.
311#[derive(Clone, Copy, Debug, Eq, PartialEq)]
312pub struct InvocationAttemptEventIds {
313    start: EventId,
314    terminal: EventId,
315}
316
317impl InvocationAttemptEventIds {
318    /// Binds two distinct preallocated event identities.
319    ///
320    /// # Errors
321    ///
322    /// Rejects identity reuse between start and terminal facts.
323    pub fn new(start: EventId, terminal: EventId) -> Result<Self, InvocationAttemptEventIdsError> {
324        if start == terminal {
325            return Err(InvocationAttemptEventIdsError::Duplicate);
326        }
327        Ok(Self { start, terminal })
328    }
329
330    /// Generates two `UUIDv7` identities for a newly retained handoff.
331    #[must_use]
332    pub fn generate() -> Self {
333        Self {
334            start: EventId::generate(),
335            terminal: EventId::generate(),
336        }
337    }
338
339    /// Returns the durable-before-dispatch start event identity.
340    #[must_use]
341    pub const fn start(self) -> EventId {
342        self.start
343    }
344
345    /// Returns the stable terminal response/error event identity.
346    #[must_use]
347    pub const fn terminal(self) -> EventId {
348        self.terminal
349    }
350}
351
352/// Invalid attempt event identity pair.
353#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
354#[non_exhaustive]
355pub enum InvocationAttemptEventIdsError {
356    /// Start and terminal facts reused one event identity.
357    #[error("invocation start and terminal event identities must be distinct")]
358    Duplicate,
359}
360
361/// Retained model-attempt handoff safe to retry before physical dispatch.
362#[derive(Clone)]
363pub struct ModelAttemptHandoff {
364    fence: RunFence,
365    invocation: ModelInvocation,
366    attempt_id: AttemptId,
367    events: InvocationAttemptEventIds,
368    cancellation: CancellationSignal,
369    stream_sink: Option<Arc<dyn ModelEventSink>>,
370}
371
372impl ModelAttemptHandoff {
373    /// Constructs and scope-checks one model attempt handoff.
374    ///
375    /// A streaming request requires a durable event sink; a complete request
376    /// rejects one so event-delivery configuration cannot be silently ignored.
377    ///
378    /// # Errors
379    ///
380    /// Rejects crossed scope, a non-startable invocation state, or an invalid
381    /// response-mode/sink pairing.
382    pub fn new(
383        fence: RunFence,
384        invocation: ModelInvocation,
385        attempt_id: AttemptId,
386        events: InvocationAttemptEventIds,
387        cancellation: CancellationSignal,
388        stream_sink: Option<Arc<dyn ModelEventSink>>,
389    ) -> Result<Self, InvocationAttemptHandoffError> {
390        validate_model_handoff(&fence, &invocation, stream_sink.is_some())?;
391        Ok(Self {
392            fence,
393            invocation,
394            attempt_id,
395            events,
396            cancellation,
397            stream_sink,
398        })
399    }
400
401    /// Returns the exact retained worker fence.
402    #[must_use]
403    pub const fn fence(&self) -> &RunFence {
404        &self.fence
405    }
406
407    /// Returns the prepared or safely retryable durable invocation revision.
408    #[must_use]
409    pub const fn invocation(&self) -> &ModelInvocation {
410        &self.invocation
411    }
412
413    /// Returns the proposed unique physical provider attempt.
414    #[must_use]
415    pub const fn attempt_id(&self) -> AttemptId {
416        self.attempt_id
417    }
418
419    /// Returns stable start/terminal journal identities.
420    #[must_use]
421    pub const fn events(&self) -> InvocationAttemptEventIds {
422        self.events
423    }
424}
425
426impl fmt::Debug for ModelAttemptHandoff {
427    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
428        formatter
429            .debug_struct("ModelAttemptHandoff")
430            .field("fence", &self.fence)
431            .field("invocation_head", &self.invocation.head())
432            .field("attempt_id", &self.attempt_id)
433            .field("events", &self.events)
434            .field("streaming", &self.stream_sink.is_some())
435            .finish_non_exhaustive()
436    }
437}
438
439/// Retained tool-attempt handoff safe to retry before physical dispatch.
440#[derive(Clone)]
441pub struct ToolAttemptHandoff {
442    fence: RunFence,
443    invocation: ToolInvocation,
444    attempt_id: AttemptId,
445    events: InvocationAttemptEventIds,
446    cancellation: CancellationSignal,
447    progress_sink: Option<Arc<dyn ToolProgressSink>>,
448}
449
450impl ToolAttemptHandoff {
451    /// Constructs and scope-checks one tool attempt handoff.
452    ///
453    /// # Errors
454    ///
455    /// Rejects crossed scope or a non-startable durable invocation state.
456    pub fn new(
457        fence: RunFence,
458        invocation: ToolInvocation,
459        attempt_id: AttemptId,
460        events: InvocationAttemptEventIds,
461        cancellation: CancellationSignal,
462        progress_sink: Option<Arc<dyn ToolProgressSink>>,
463    ) -> Result<Self, InvocationAttemptHandoffError> {
464        validate_tool_handoff(&fence, &invocation)?;
465        Ok(Self {
466            fence,
467            invocation,
468            attempt_id,
469            events,
470            cancellation,
471            progress_sink,
472        })
473    }
474
475    /// Returns the exact retained worker fence.
476    #[must_use]
477    pub const fn fence(&self) -> &RunFence {
478        &self.fence
479    }
480
481    /// Returns the prepared or safely retryable durable invocation revision.
482    #[must_use]
483    pub const fn invocation(&self) -> &ToolInvocation {
484        &self.invocation
485    }
486
487    /// Returns the proposed unique physical tool attempt.
488    #[must_use]
489    pub const fn attempt_id(&self) -> AttemptId {
490        self.attempt_id
491    }
492
493    /// Returns stable start/terminal journal identities.
494    #[must_use]
495    pub const fn events(&self) -> InvocationAttemptEventIds {
496        self.events
497    }
498}
499
500impl fmt::Debug for ToolAttemptHandoff {
501    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
502        formatter
503            .debug_struct("ToolAttemptHandoff")
504            .field("fence", &self.fence)
505            .field("invocation_head", &self.invocation.head())
506            .field("attempt_id", &self.attempt_id)
507            .field("events", &self.events)
508            .field("progress", &self.progress_sink.is_some())
509            .finish_non_exhaustive()
510    }
511}
512
513/// Invalid durable attempt handoff.
514#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
515#[non_exhaustive]
516pub enum InvocationAttemptHandoffError {
517    /// Fence and invocation crossed tenant/run scope.
518    #[error("invocation attempt handoff crosses its retained worker fence")]
519    ScopeMismatch,
520    /// Invocation is executing or already terminal.
521    #[error("invocation attempt handoff does not reference a startable revision")]
522    InvocationNotStartable,
523    /// Streaming request omitted its required durable semantic-event sink.
524    #[error("streaming model attempt requires a durable event sink")]
525    MissingStreamSink,
526    /// Complete-response request supplied an unused stream sink.
527    #[error("complete model attempt must not supply a stream event sink")]
528    UnexpectedStreamSink,
529}
530
531fn validate_model_handoff(
532    fence: &RunFence,
533    invocation: &ModelInvocation,
534    has_stream_sink: bool,
535) -> Result<(), InvocationAttemptHandoffError> {
536    if invocation.intent().tenant_id() != fence.tenant_id()
537        || invocation.intent().run_id() != fence.run_id()
538        || invocation.journal_head().tenant_id() != fence.tenant_id()
539        || invocation.journal_head().run_id() != fence.run_id()
540    {
541        return Err(InvocationAttemptHandoffError::ScopeMismatch);
542    }
543    if !matches!(
544        invocation.status(),
545        ModelInvocationStatus::Prepared | ModelInvocationStatus::Failed
546    ) {
547        return Err(InvocationAttemptHandoffError::InvocationNotStartable);
548    }
549    match (
550        invocation.intent().request().response_mode(),
551        has_stream_sink,
552    ) {
553        (ModelResponseMode::Streaming, false) => {
554            Err(InvocationAttemptHandoffError::MissingStreamSink)
555        }
556        (ModelResponseMode::Complete, true) => {
557            Err(InvocationAttemptHandoffError::UnexpectedStreamSink)
558        }
559        (ModelResponseMode::Complete, false) | (ModelResponseMode::Streaming, true) => Ok(()),
560    }
561}
562
563fn validate_tool_handoff(
564    fence: &RunFence,
565    invocation: &ToolInvocation,
566) -> Result<(), InvocationAttemptHandoffError> {
567    if invocation.intent().tenant_id() != fence.tenant_id()
568        || invocation.intent().run_id() != fence.run_id()
569        || invocation.journal_head().tenant_id() != fence.tenant_id()
570        || invocation.journal_head().run_id() != fence.run_id()
571    {
572        return Err(InvocationAttemptHandoffError::ScopeMismatch);
573    }
574    if !matches!(
575        invocation.status(),
576        ToolInvocationStatus::Prepared | ToolInvocationStatus::Failed
577    ) {
578        return Err(InvocationAttemptHandoffError::InvocationNotStartable);
579    }
580    Ok(())
581}
582
583/// Bounded identical-mutation retry policy for invocation ledger commits.
584#[derive(Clone, Copy, Debug, Eq, PartialEq)]
585pub struct DurableInvocationExecutorOptions {
586    maximum_mutation_attempts: u8,
587    mutation_retry_initial_delay: Duration,
588}
589
590impl DurableInvocationExecutorOptions {
591    /// Absolute identical transaction-attempt ceiling.
592    pub const HARD_MAXIMUM_MUTATION_ATTEMPTS: u8 = 10;
593
594    /// Constructs a bounded lost-acknowledgement retry policy.
595    ///
596    /// # Errors
597    ///
598    /// Rejects zero/excessive attempts or a zero/greater-than-one-second delay.
599    pub fn new(
600        maximum_mutation_attempts: u8,
601        mutation_retry_initial_delay: Duration,
602    ) -> Result<Self, DurableInvocationExecutorOptionsError> {
603        if maximum_mutation_attempts == 0
604            || maximum_mutation_attempts > Self::HARD_MAXIMUM_MUTATION_ATTEMPTS
605        {
606            return Err(DurableInvocationExecutorOptionsError::InvalidMutationAttempts);
607        }
608        if mutation_retry_initial_delay.is_zero()
609            || mutation_retry_initial_delay > MAX_MUTATION_RETRY_DELAY
610        {
611            return Err(DurableInvocationExecutorOptionsError::InvalidMutationRetryDelay);
612        }
613        Ok(Self {
614            maximum_mutation_attempts,
615            mutation_retry_initial_delay,
616        })
617    }
618
619    /// Returns maximum identical database mutation attempts.
620    #[must_use]
621    pub const fn maximum_mutation_attempts(self) -> u8 {
622        self.maximum_mutation_attempts
623    }
624
625    /// Returns the first exponential retry delay.
626    #[must_use]
627    pub const fn mutation_retry_initial_delay(self) -> Duration {
628        self.mutation_retry_initial_delay
629    }
630}
631
632impl Default for DurableInvocationExecutorOptions {
633    fn default() -> Self {
634        Self {
635            maximum_mutation_attempts: 3,
636            mutation_retry_initial_delay: Duration::from_millis(25),
637        }
638    }
639}
640
641/// Invalid durable invocation executor retry policy.
642#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
643#[non_exhaustive]
644pub enum DurableInvocationExecutorOptionsError {
645    /// Mutation attempts were zero or above the hard ceiling.
646    #[error("invocation executor mutation attempt count is invalid")]
647    InvalidMutationAttempts,
648    /// Initial mutation delay was zero or above one second.
649    #[error("invocation executor mutation retry delay is invalid")]
650    InvalidMutationRetryDelay,
651}
652
653/// Whether a newly dispatched model attempt committed a response or error.
654#[derive(Clone, Copy, Debug, Eq, PartialEq)]
655#[non_exhaustive]
656pub enum ModelAttemptTerminalKind {
657    /// Complete unary or accumulated streaming response committed.
658    Response,
659    /// Adapter/provider/supervisor failure evidence committed.
660    Error,
661}
662
663/// Closed result of one model attempt executor call.
664#[derive(Clone, Debug)]
665#[non_exhaustive]
666pub enum ModelAttemptOutcome {
667    /// This call performed one physical dispatch and committed its terminal fact.
668    Dispatched {
669        /// Whether response or error evidence won.
670        terminal: ModelAttemptTerminalKind,
671        /// Exact terminal invocation revision.
672        invocation: ModelInvocation,
673    },
674    /// The start event had already committed, so no provider call was repeated.
675    Recovered {
676        /// Current exact invocation revision loaded after duplicate suppression.
677        invocation: ModelInvocation,
678    },
679}
680
681/// Whether a newly dispatched tool attempt committed a result or error.
682#[derive(Clone, Copy, Debug, Eq, PartialEq)]
683#[non_exhaustive]
684pub enum ToolAttemptTerminalKind {
685    /// Validated tool result committed.
686    Result,
687    /// Known or ambiguous failure evidence committed.
688    Error,
689}
690
691/// Closed result of one tool attempt executor call.
692#[derive(Clone, Debug)]
693#[non_exhaustive]
694pub enum ToolAttemptOutcome {
695    /// This call performed one physical dispatch and committed its terminal fact.
696    Dispatched {
697        /// Whether result or error evidence won.
698        terminal: ToolAttemptTerminalKind,
699        /// Exact terminal invocation revision.
700        invocation: ToolInvocation,
701    },
702    /// The start event had already committed, so no tool call was repeated.
703    Recovered {
704        /// Current exact invocation revision loaded after duplicate suppression.
705        invocation: ToolInvocation,
706    },
707}
708
709/// Closed result of the durable-before-dispatch half of one tool attempt.
710#[derive(Debug)]
711#[non_exhaustive]
712pub enum ToolAttemptStartOutcome {
713    /// A fresh executing revision committed and this caller owns dispatch.
714    Started(Box<ToolDispatchHandoff>),
715    /// The stable start identity already advanced; provider I/O is not allowed.
716    Recovered {
717        /// Current exact invocation revision after duplicate suppression.
718        invocation: Box<ToolInvocation>,
719    },
720}
721
722impl ToolAttemptStartOutcome {
723    /// Returns the current invocation revision in either outcome.
724    #[must_use]
725    pub fn invocation(&self) -> &ToolInvocation {
726        match self {
727            Self::Started(handoff) => handoff.invocation(),
728            Self::Recovered { invocation } => invocation.as_ref(),
729        }
730    }
731}
732
733/// Fresh durable tool start retained for exactly one provider dispatch.
734///
735/// This type exists so a trusted coordinator can serialize multiple start
736/// facts, overlap only policy-safe provider calls, and then serialize retained
737/// terminal evidence in semantic order. Dropping it never calls a provider,
738/// but leaves the durable attempt executing for higher-fence recovery.
739pub struct ToolDispatchHandoff {
740    fence: RunFence,
741    invocation: ToolInvocation,
742    terminal_event_id: EventId,
743    provider: Arc<dyn stateknot_core::ErasedTool>,
744    context: ToolContext,
745}
746
747impl ToolDispatchHandoff {
748    /// Returns the exact live fence that committed the start.
749    #[must_use]
750    pub const fn fence(&self) -> &RunFence {
751        &self.fence
752    }
753
754    /// Returns the fresh executing invocation revision.
755    #[must_use]
756    pub const fn invocation(&self) -> &ToolInvocation {
757        &self.invocation
758    }
759}
760
761impl fmt::Debug for ToolDispatchHandoff {
762    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
763        formatter
764            .debug_struct("ToolDispatchHandoff")
765            .field("fence", &self.fence)
766            .field("invocation_head", &self.invocation.head())
767            .field("terminal_event_id", &self.terminal_event_id)
768            .finish_non_exhaustive()
769    }
770}
771
772/// Retained request to probe one exact unknown tool attempt.
773///
774/// The event identity is stable across node and process retries. A probe that
775/// remains pending performs no mutation; authoritative evidence is committed
776/// through the existing atomic reconciliation transition.
777#[derive(Clone)]
778pub struct ToolReconciliationAttemptHandoff {
779    fence: RunFence,
780    invocation: ToolInvocation,
781    event_id: EventId,
782    cancellation: CancellationSignal,
783}
784
785impl ToolReconciliationAttemptHandoff {
786    /// Constructs a scope- and capability-bound reconciliation probe.
787    ///
788    /// # Errors
789    ///
790    /// Rejects crossed run scope, non-unknown invocation state, a missing
791    /// physical attempt, or a descriptor without status-query support.
792    pub fn new(
793        fence: RunFence,
794        invocation: ToolInvocation,
795        event_id: EventId,
796        cancellation: CancellationSignal,
797    ) -> Result<Self, ToolReconciliationAttemptHandoffError> {
798        validate_tool_reconciliation_attempt_handoff(&fence, &invocation)?;
799        Ok(Self {
800            fence,
801            invocation,
802            event_id,
803            cancellation,
804        })
805    }
806
807    /// Returns the live worker fence used for reads and commit.
808    #[must_use]
809    pub const fn fence(&self) -> &RunFence {
810        &self.fence
811    }
812
813    /// Returns the exact unknown invocation revision.
814    #[must_use]
815    pub const fn invocation(&self) -> &ToolInvocation {
816        &self.invocation
817    }
818
819    /// Returns the stable audit-event identity.
820    #[must_use]
821    pub const fn event_id(&self) -> EventId {
822        self.event_id
823    }
824
825    /// Rebinds the probe to a newer live fence for the same run.
826    ///
827    /// # Errors
828    ///
829    /// Rejects a fence from another tenant or run.
830    pub fn rebind_fence(
831        mut self,
832        fence: RunFence,
833    ) -> Result<Self, ToolReconciliationAttemptHandoffError> {
834        if self.invocation.intent().tenant_id() != fence.tenant_id()
835            || self.invocation.intent().run_id() != fence.run_id()
836        {
837            return Err(ToolReconciliationAttemptHandoffError::ScopeMismatch);
838        }
839        self.fence = fence;
840        Ok(self)
841    }
842}
843
844impl fmt::Debug for ToolReconciliationAttemptHandoff {
845    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
846        formatter
847            .debug_struct("ToolReconciliationAttemptHandoff")
848            .field("fence", &self.fence)
849            .field("invocation_head", &self.invocation.head())
850            .field("event_id", &self.event_id)
851            .field("cancelled", &self.cancellation.is_cancelled())
852            .finish_non_exhaustive()
853    }
854}
855
856/// Invalid request to execute an automated reconciliation probe.
857#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
858#[non_exhaustive]
859pub enum ToolReconciliationAttemptHandoffError {
860    /// Fence and invocation crossed tenant/run or journal scope.
861    #[error("tool reconciliation attempt crosses its retained worker fence")]
862    ScopeMismatch,
863    /// The invocation is not currently unknown.
864    #[error("tool reconciliation attempt does not reference an unknown invocation")]
865    InvocationNotUnknown,
866    /// The unknown invocation does not identify its physical attempt.
867    #[error("tool reconciliation attempt has no physical attempt identity")]
868    MissingAttempt,
869    /// The descriptor does not declare status-query support.
870    #[error("tool reconciliation attempt is not declared by the descriptor")]
871    StatusQueryUnsupported,
872}
873
874fn validate_tool_reconciliation_attempt_handoff(
875    fence: &RunFence,
876    invocation: &ToolInvocation,
877) -> Result<(), ToolReconciliationAttemptHandoffError> {
878    if invocation.intent().tenant_id() != fence.tenant_id()
879        || invocation.intent().run_id() != fence.run_id()
880        || invocation.journal_head().tenant_id() != fence.tenant_id()
881        || invocation.journal_head().run_id() != fence.run_id()
882    {
883        return Err(ToolReconciliationAttemptHandoffError::ScopeMismatch);
884    }
885    if invocation.status() != ToolInvocationStatus::Unknown {
886        return Err(ToolReconciliationAttemptHandoffError::InvocationNotUnknown);
887    }
888    if invocation.attempt_id().is_none() {
889        return Err(ToolReconciliationAttemptHandoffError::MissingAttempt);
890    }
891    if !invocation
892        .intent()
893        .descriptor()
894        .semantics()
895        .supports_status_query()
896    {
897        return Err(ToolReconciliationAttemptHandoffError::StatusQueryUnsupported);
898    }
899    Ok(())
900}
901
902/// Result of one bounded provider-backed reconciliation probe.
903#[derive(Clone, Debug)]
904#[non_exhaustive]
905pub enum ToolReconciliationAttemptOutcome {
906    /// Authoritative evidence is now durable, or was already durable.
907    Resolved {
908        /// Result or known-error evidence classification.
909        kind: ToolReconciliationKind,
910        /// Current terminal invocation revision.
911        invocation: ToolInvocation,
912    },
913    /// No authoritative evidence exists yet; no ledger mutation occurred.
914    Pending {
915        /// Earliest safe time for another bounded probe.
916        retry_after: DurationMillis,
917        /// Current unchanged unknown invocation revision.
918        invocation: ToolInvocation,
919    },
920}
921
922impl ToolReconciliationAttemptOutcome {
923    /// Returns the current invocation revision.
924    #[must_use]
925    pub const fn invocation(&self) -> &ToolInvocation {
926        match self {
927            Self::Resolved { invocation, .. } | Self::Pending { invocation, .. } => invocation,
928        }
929    }
930}
931
932/// Authoritative evidence classification for one ambiguous tool attempt.
933#[derive(Clone, Copy, Debug, Eq, PartialEq)]
934#[non_exhaustive]
935pub enum ToolReconciliationKind {
936    /// Authoritative evidence supplies the exact successful result.
937    Result,
938    /// Authoritative evidence supplies a failure and external-effect fact.
939    Error,
940}
941
942#[derive(Clone)]
943enum ToolReconciliationEvidence {
944    Result(ToolResult),
945    Error(ToolError),
946}
947
948/// Retained, validation-bound evidence for reconciling an unknown tool attempt.
949///
950/// Construction binds the evidence to an exact unknown invocation revision and
951/// physical attempt. Debug output deliberately excludes result/error payloads.
952/// Committing this handoff performs no tool or provider I/O.
953#[derive(Clone)]
954pub struct ToolReconciliationHandoff {
955    fence: RunFence,
956    invocation: ToolInvocation,
957    event_id: EventId,
958    evidence: ToolReconciliationEvidence,
959}
960
961impl ToolReconciliationHandoff {
962    /// Binds authoritative successful-result evidence to an unknown attempt.
963    ///
964    /// # Errors
965    ///
966    /// Rejects crossed tenant/run scope, a non-unknown invocation, or result
967    /// evidence that violates the exact durable invocation contract.
968    pub fn result(
969        fence: RunFence,
970        invocation: ToolInvocation,
971        event_id: EventId,
972        result: ToolResult,
973    ) -> Result<Self, ToolReconciliationHandoffError> {
974        validate_tool_reconciliation_scope(&fence, &invocation)?;
975        invocation
976            .validate_reconciliation_result(&result)
977            .map_err(ToolReconciliationHandoffError::invalid_evidence)?;
978        Ok(Self {
979            fence,
980            invocation,
981            event_id,
982            evidence: ToolReconciliationEvidence::Result(result),
983        })
984    }
985
986    /// Binds authoritative failure/effect evidence to an unknown attempt.
987    ///
988    /// `Unknown` effect evidence may intentionally retain the unresolved state;
989    /// an authoritative known effect resolves it to `Failed`.
990    ///
991    /// # Errors
992    ///
993    /// Rejects crossed tenant/run scope, a non-unknown invocation, or error
994    /// evidence that violates the exact durable invocation contract.
995    pub fn error(
996        fence: RunFence,
997        invocation: ToolInvocation,
998        event_id: EventId,
999        error: ToolError,
1000    ) -> Result<Self, ToolReconciliationHandoffError> {
1001        validate_tool_reconciliation_scope(&fence, &invocation)?;
1002        invocation
1003            .validate_reconciliation_error(&error)
1004            .map_err(ToolReconciliationHandoffError::invalid_evidence)?;
1005        Ok(Self {
1006            fence,
1007            invocation,
1008            event_id,
1009            evidence: ToolReconciliationEvidence::Error(error),
1010        })
1011    }
1012
1013    /// Returns the authoritative reconciliation classification.
1014    #[must_use]
1015    pub const fn kind(&self) -> ToolReconciliationKind {
1016        match &self.evidence {
1017            ToolReconciliationEvidence::Result(_) => ToolReconciliationKind::Result,
1018            ToolReconciliationEvidence::Error(_) => ToolReconciliationKind::Error,
1019        }
1020    }
1021
1022    /// Returns the worker fence attached to this reconciliation mutation.
1023    #[must_use]
1024    pub const fn fence(&self) -> &RunFence {
1025        &self.fence
1026    }
1027
1028    /// Returns the exact unknown invocation revision being reconciled.
1029    #[must_use]
1030    pub const fn invocation(&self) -> &ToolInvocation {
1031        &self.invocation
1032    }
1033
1034    /// Returns the stable journal event identity retained across commit retries.
1035    #[must_use]
1036    pub const fn event_id(&self) -> EventId {
1037        self.event_id
1038    }
1039
1040    /// Rebinds retained evidence to a newer live fence in the same run.
1041    ///
1042    /// This changes no evidence bytes and never calls the tool. The durable
1043    /// store validates current fence ownership at commit time.
1044    ///
1045    /// # Errors
1046    ///
1047    /// Rejects a fence from another tenant or run.
1048    pub fn rebind_fence(mut self, fence: RunFence) -> Result<Self, ToolReconciliationHandoffError> {
1049        validate_tool_reconciliation_scope(&fence, &self.invocation)?;
1050        self.fence = fence;
1051        Ok(self)
1052    }
1053}
1054
1055impl fmt::Debug for ToolReconciliationHandoff {
1056    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1057        formatter
1058            .debug_struct("ToolReconciliationHandoff")
1059            .field("fence", &self.fence)
1060            .field("invocation_head", &self.invocation.head())
1061            .field("event_id", &self.event_id)
1062            .field("kind", &self.kind())
1063            .finish_non_exhaustive()
1064    }
1065}
1066
1067/// Invalid binding between reconciliation evidence and durable invocation state.
1068#[derive(Clone, Debug, Eq, Error, PartialEq)]
1069#[non_exhaustive]
1070pub enum ToolReconciliationHandoffError {
1071    /// Fence and invocation crossed tenant/run or journal scope.
1072    #[error("tool reconciliation handoff crosses its retained worker fence")]
1073    ScopeMismatch,
1074    /// Result/error evidence did not belong to the exact unknown attempt.
1075    #[error("tool reconciliation evidence violates the durable invocation contract: {source}")]
1076    InvalidEvidence {
1077        /// Exact public-safe core invariant violation.
1078        #[source]
1079        source: ToolInvocationError,
1080    },
1081}
1082
1083impl ToolReconciliationHandoffError {
1084    fn invalid_evidence(source: ToolInvocationError) -> Self {
1085        Self::InvalidEvidence { source }
1086    }
1087}
1088
1089fn validate_tool_reconciliation_scope(
1090    fence: &RunFence,
1091    invocation: &ToolInvocation,
1092) -> Result<(), ToolReconciliationHandoffError> {
1093    if invocation.intent().tenant_id() != fence.tenant_id()
1094        || invocation.intent().run_id() != fence.run_id()
1095        || invocation.journal_head().tenant_id() != fence.tenant_id()
1096        || invocation.journal_head().run_id() != fence.run_id()
1097    {
1098        return Err(ToolReconciliationHandoffError::ScopeMismatch);
1099    }
1100    Ok(())
1101}
1102
1103/// Result of durably committing authoritative tool reconciliation evidence.
1104#[derive(Clone, Debug)]
1105#[non_exhaustive]
1106pub enum ToolReconciliationOutcome {
1107    /// A new reconciliation event and invocation revision committed atomically.
1108    Committed {
1109        /// Reconciliation evidence classification.
1110        kind: ToolReconciliationKind,
1111        /// Newly committed immutable journal event.
1112        event: JournalEvent,
1113        /// Newly committed terminal or deliberately still-unknown revision.
1114        invocation: ToolInvocation,
1115    },
1116    /// The exact reconciliation event and mutation had already committed.
1117    Idempotent {
1118        /// Reconciliation evidence classification.
1119        kind: ToolReconciliationKind,
1120        /// Previously committed immutable journal event.
1121        event: JournalEvent,
1122        /// Previously committed terminal or deliberately still-unknown revision.
1123        invocation: ToolInvocation,
1124    },
1125}
1126
1127impl ToolReconciliationOutcome {
1128    /// Returns the reconciliation evidence classification.
1129    #[must_use]
1130    pub const fn kind(&self) -> ToolReconciliationKind {
1131        match self {
1132            Self::Committed { kind, .. } | Self::Idempotent { kind, .. } => *kind,
1133        }
1134    }
1135
1136    /// Returns the exact reconciliation journal event.
1137    #[must_use]
1138    pub const fn event(&self) -> &JournalEvent {
1139        match self {
1140            Self::Committed { event, .. } | Self::Idempotent { event, .. } => event,
1141        }
1142    }
1143
1144    /// Returns the resulting durable invocation revision.
1145    #[must_use]
1146    pub const fn invocation(&self) -> &ToolInvocation {
1147        match self {
1148            Self::Committed { invocation, .. } | Self::Idempotent { invocation, .. } => invocation,
1149        }
1150    }
1151
1152    /// Returns whether this exact reconciliation had already committed.
1153    #[must_use]
1154    pub const fn is_idempotent(&self) -> bool {
1155        matches!(self, Self::Idempotent { .. })
1156    }
1157}
1158
1159#[derive(Clone)]
1160#[allow(clippy::large_enum_variant)]
1161enum ModelTerminalEvidence {
1162    Response(ModelResponse),
1163    Error(ModelError),
1164}
1165
1166/// Retained terminal model evidence after a database commit failure.
1167///
1168/// Debug output excludes the response/error payload. Pass this value back to
1169/// [`DurableInvocationExecutor::commit_model_terminal`] without modification;
1170/// no provider request is performed by that method.
1171#[derive(Clone)]
1172pub struct ModelTerminalCommitHandoff {
1173    fence: RunFence,
1174    invocation: ModelInvocation,
1175    event_id: EventId,
1176    evidence: ModelTerminalEvidence,
1177}
1178
1179impl fmt::Debug for ModelTerminalCommitHandoff {
1180    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1181        formatter
1182            .debug_struct("ModelTerminalCommitHandoff")
1183            .field("fence", &self.fence)
1184            .field("invocation_head", &self.invocation.head())
1185            .field("event_id", &self.event_id)
1186            .field("terminal", &self.kind())
1187            .finish_non_exhaustive()
1188    }
1189}
1190
1191impl ModelTerminalCommitHandoff {
1192    /// Returns response/error classification without exposing payload content.
1193    #[must_use]
1194    pub const fn kind(&self) -> ModelAttemptTerminalKind {
1195        match &self.evidence {
1196            ModelTerminalEvidence::Response(_) => ModelAttemptTerminalKind::Response,
1197            ModelTerminalEvidence::Error(_) => ModelAttemptTerminalKind::Error,
1198        }
1199    }
1200
1201    /// Returns the worker fence currently attached to terminal evidence.
1202    #[must_use]
1203    pub const fn fence(&self) -> &RunFence {
1204        &self.fence
1205    }
1206
1207    /// Rebinds retained evidence to a newer live fence in the same run.
1208    ///
1209    /// This is required when a provider call outlives its original lease. The
1210    /// store remains authoritative: it validates that the replacement fence is
1211    /// currently live before accepting the already-produced terminal evidence.
1212    ///
1213    /// # Errors
1214    ///
1215    /// Rejects a fence from another tenant or run.
1216    pub fn rebind_fence(mut self, fence: RunFence) -> Result<Self, InvocationAttemptHandoffError> {
1217        if self.invocation.intent().tenant_id() != fence.tenant_id()
1218            || self.invocation.intent().run_id() != fence.run_id()
1219        {
1220            return Err(InvocationAttemptHandoffError::ScopeMismatch);
1221        }
1222        self.fence = fence;
1223        Ok(self)
1224    }
1225}
1226
1227#[derive(Clone)]
1228enum ToolTerminalEvidence {
1229    Result(ToolResult),
1230    Error(ToolError),
1231}
1232
1233/// Retained terminal tool evidence after a database commit failure.
1234///
1235/// Debug output excludes tool result/error payloads. Retrying this handoff never
1236/// calls the tool again.
1237#[derive(Clone)]
1238pub struct ToolTerminalCommitHandoff {
1239    fence: RunFence,
1240    invocation: ToolInvocation,
1241    event_id: EventId,
1242    evidence: ToolTerminalEvidence,
1243}
1244
1245impl fmt::Debug for ToolTerminalCommitHandoff {
1246    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1247        formatter
1248            .debug_struct("ToolTerminalCommitHandoff")
1249            .field("fence", &self.fence)
1250            .field("invocation_head", &self.invocation.head())
1251            .field("event_id", &self.event_id)
1252            .field("terminal", &self.kind())
1253            .finish_non_exhaustive()
1254    }
1255}
1256
1257impl ToolTerminalCommitHandoff {
1258    /// Returns result/error classification without exposing payload content.
1259    #[must_use]
1260    pub const fn kind(&self) -> ToolAttemptTerminalKind {
1261        match &self.evidence {
1262            ToolTerminalEvidence::Result(_) => ToolAttemptTerminalKind::Result,
1263            ToolTerminalEvidence::Error(_) => ToolAttemptTerminalKind::Error,
1264        }
1265    }
1266
1267    /// Returns the worker fence currently attached to terminal evidence.
1268    #[must_use]
1269    pub const fn fence(&self) -> &RunFence {
1270        &self.fence
1271    }
1272
1273    /// Rebinds retained evidence to a newer live fence in the same run.
1274    ///
1275    /// This changes no result/error bytes and never calls the tool again. The
1276    /// durable store validates current fence ownership at commit time.
1277    ///
1278    /// # Errors
1279    ///
1280    /// Rejects a fence from another tenant or run.
1281    pub fn rebind_fence(mut self, fence: RunFence) -> Result<Self, InvocationAttemptHandoffError> {
1282        if self.invocation.intent().tenant_id() != fence.tenant_id()
1283            || self.invocation.intent().run_id() != fence.run_id()
1284        {
1285            return Err(InvocationAttemptHandoffError::ScopeMismatch);
1286        }
1287        self.fence = fence;
1288        Ok(self)
1289    }
1290}
1291
1292/// Payload-safe cause of a retained terminal invocation commit failure.
1293#[derive(Debug, Error)]
1294#[non_exhaustive]
1295pub enum InvocationTerminalCommitFailure {
1296    /// The durable store rejected or could not complete the mutation.
1297    #[error(transparent)]
1298    Store(#[from] StoreError),
1299    /// Runtime-owned standard event data could not be encoded or validated.
1300    #[error("terminal invocation event payload is invalid")]
1301    InvalidEventPayload,
1302    /// Runtime-owned worker journal metadata violated its invariant.
1303    #[error("terminal invocation journal append is invalid")]
1304    InvalidJournalAppend,
1305    /// The run lost its durable journal anchor before terminal evidence could commit.
1306    #[error("terminal invocation run journal is unavailable")]
1307    RunJournalUnavailable,
1308    /// Retained terminal evidence did not reference an executing attempt.
1309    #[error("terminal invocation does not reference an executing attempt")]
1310    InvalidInvocationState,
1311    /// The store returned a future commit outcome unknown to this runtime.
1312    #[error("terminal invocation store outcome is unsupported by this runtime")]
1313    UnsupportedStoreOutcome,
1314}
1315
1316/// Terminal model commit failure retaining exact retry evidence.
1317#[derive(Debug, Error)]
1318#[error("model terminal invocation commit failed: {source}")]
1319pub struct ModelTerminalCommitError {
1320    #[source]
1321    source: InvocationTerminalCommitFailure,
1322    recovery: Box<ModelTerminalCommitHandoff>,
1323}
1324
1325impl ModelTerminalCommitError {
1326    /// Returns the payload-redacted store failure.
1327    #[must_use]
1328    pub const fn source_error(&self) -> &InvocationTerminalCommitFailure {
1329        &self.source
1330    }
1331
1332    /// Recovers the exact response/error commit handoff for a no-dispatch retry.
1333    #[must_use]
1334    pub fn into_recovery(self) -> ModelTerminalCommitHandoff {
1335        *self.recovery
1336    }
1337}
1338
1339/// Terminal tool commit failure retaining exact retry evidence.
1340#[derive(Debug, Error)]
1341#[error("tool terminal invocation commit failed: {source}")]
1342pub struct ToolTerminalCommitError {
1343    #[source]
1344    source: InvocationTerminalCommitFailure,
1345    recovery: Box<ToolTerminalCommitHandoff>,
1346}
1347
1348impl ToolTerminalCommitError {
1349    /// Returns the payload-redacted store failure.
1350    #[must_use]
1351    pub const fn source_error(&self) -> &InvocationTerminalCommitFailure {
1352        &self.source
1353    }
1354
1355    /// Recovers the exact result/error commit handoff for a no-dispatch retry.
1356    #[must_use]
1357    pub fn into_recovery(self) -> ToolTerminalCommitHandoff {
1358        *self.recovery
1359    }
1360}
1361
1362/// Payload-safe cause of a durable tool reconciliation commit failure.
1363#[derive(Debug, Error)]
1364#[non_exhaustive]
1365pub enum ToolReconciliationCommitFailure {
1366    /// The durable store rejected or could not complete the mutation.
1367    #[error(transparent)]
1368    Store(#[from] StoreError),
1369    /// Successful evidence was rejected by or missing from the frozen registry.
1370    #[error("tool reconciliation output schema validation failed: {source}")]
1371    OutputSchema {
1372        /// Public-safe unavailable/rejected classification.
1373        #[source]
1374        source: GraphSchemaValidationError,
1375    },
1376    /// Runtime-owned standard event data could not be encoded or validated.
1377    #[error("tool reconciliation event payload is invalid")]
1378    InvalidEventPayload,
1379    /// Runtime-owned worker journal metadata violated its invariant.
1380    #[error("tool reconciliation journal append is invalid")]
1381    InvalidJournalAppend,
1382    /// The run lost its durable journal anchor before evidence could commit.
1383    #[error("tool reconciliation run journal is unavailable")]
1384    RunJournalUnavailable,
1385    /// Retained evidence no longer references an unknown physical attempt.
1386    #[error("tool reconciliation does not reference an unknown physical attempt")]
1387    InvalidInvocationState,
1388    /// The store returned a future commit outcome unknown to this runtime.
1389    #[error("tool reconciliation store outcome is unsupported by this runtime")]
1390    UnsupportedStoreOutcome,
1391}
1392
1393/// Reconciliation failure retaining the exact evidence for a no-dispatch retry.
1394#[derive(Debug, Error)]
1395#[error("tool reconciliation commit failed: {source}")]
1396pub struct ToolReconciliationCommitError {
1397    #[source]
1398    source: ToolReconciliationCommitFailure,
1399    recovery: Box<ToolReconciliationHandoff>,
1400}
1401
1402impl ToolReconciliationCommitError {
1403    /// Returns the payload-redacted commit failure.
1404    #[must_use]
1405    pub const fn source_error(&self) -> &ToolReconciliationCommitFailure {
1406        &self.source
1407    }
1408
1409    /// Recovers the exact evidence handoff for an identical no-dispatch retry.
1410    #[must_use]
1411    pub fn into_recovery(self) -> ToolReconciliationHandoff {
1412        *self.recovery
1413    }
1414}
1415
1416/// Failure before, during, or after one automated reconciliation probe.
1417#[derive(Debug, Error)]
1418#[non_exhaustive]
1419pub enum ToolReconciliationAttemptExecutionError {
1420    /// The retained probe request violated scope or state invariants.
1421    #[error(transparent)]
1422    Handoff(#[from] ToolReconciliationAttemptHandoffError),
1423    /// No exact executable tool binding was installed.
1424    #[error(transparent)]
1425    Registry(#[from] ToolProviderRegistryError),
1426    /// The exact binding leaves status reconciliation to an external operator.
1427    #[error("installed tool binding does not implement automated reconciliation")]
1428    ProviderReconciliationUnsupported,
1429    /// Durable state could not be loaded.
1430    #[error(transparent)]
1431    Store(#[from] StoreError),
1432    /// Stored run provenance crossed the retained worker fence.
1433    #[error("stored run provenance does not match the tool reconciliation fence")]
1434    RunProvenanceMismatch,
1435    /// Runtime clock observation failed.
1436    #[error(transparent)]
1437    Clock(#[from] InvocationClockError),
1438    /// The descriptor-bounded fallback deadline was not representable.
1439    #[error("tool reconciliation deadline is outside the supported timestamp range")]
1440    DeadlineOutOfRange,
1441    /// A finite reconciliation context could not be constructed.
1442    #[error(transparent)]
1443    Context(#[from] ToolReconciliationContextError),
1444    /// Cancellation or the run/probe deadline won.
1445    #[error("tool reconciliation stopped: {reason:?}")]
1446    Stopped {
1447        /// Deterministic stop reason.
1448        reason: ToolStopReason,
1449    },
1450    /// The provider's probe itself failed without resolving the original call.
1451    #[error(transparent)]
1452    Probe(#[from] ToolReconciliationProbeError),
1453    /// Provider polling advice violated hard resource bounds.
1454    #[error(transparent)]
1455    Observation(#[from] ToolReconciliationObservationError),
1456    /// A probe returned another ambiguous outcome instead of `Pending`.
1457    #[error("tool reconciliation returned unresolved error evidence")]
1458    UnresolvedErrorEvidence,
1459    /// A newer provider observation variant is unsupported by this runtime.
1460    #[error("tool reconciliation observation is unsupported by this runtime")]
1461    UnsupportedObservation,
1462    /// Result or known-error evidence violated the exact unknown invocation.
1463    #[error(transparent)]
1464    Evidence(#[from] ToolReconciliationHandoffError),
1465    /// The invocation changed to another durable intent.
1466    #[error("recovered tool reconciliation invocation does not match the retained intent")]
1467    RecoveredInvocationMismatch,
1468    /// The invocation no longer references the original physical attempt.
1469    #[error("tool reconciliation invocation advanced under another attempt or state")]
1470    InvocationAdvanced,
1471    /// Authoritative evidence was produced but its atomic commit failed.
1472    #[error(transparent)]
1473    Commit(#[from] ToolReconciliationCommitError),
1474}
1475
1476impl ToolReconciliationAttemptExecutionError {
1477    /// Returns explicit node-level recovery advice for this probe failure.
1478    ///
1479    /// Retryability is inherited only from an explicit provider failure or a
1480    /// retryable durable-store classification. Contract and integrity failures
1481    /// remain terminal.
1482    #[must_use]
1483    pub fn retry_advice(&self) -> RetryAdvice {
1484        let store_retry = || RetryAdvice::SafeAfter {
1485            delay: DurationMillis::new(100).expect("positive static retry delay"),
1486        };
1487        match self {
1488            Self::Probe(source) => source.failure().retry_advice(),
1489            Self::Store(source) if source.is_retryable() => store_retry(),
1490            Self::Commit(source)
1491                if matches!(
1492                    source.source_error(),
1493                    ToolReconciliationCommitFailure::Store(store) if store.is_retryable()
1494                ) =>
1495            {
1496                store_retry()
1497            }
1498            Self::Handoff(_)
1499            | Self::Registry(_)
1500            | Self::ProviderReconciliationUnsupported
1501            | Self::Store(_)
1502            | Self::RunProvenanceMismatch
1503            | Self::Clock(_)
1504            | Self::DeadlineOutOfRange
1505            | Self::Context(_)
1506            | Self::Stopped { .. }
1507            | Self::Observation(_)
1508            | Self::UnresolvedErrorEvidence
1509            | Self::UnsupportedObservation
1510            | Self::Evidence(_)
1511            | Self::RecoveredInvocationMismatch
1512            | Self::InvocationAdvanced
1513            | Self::Commit(_) => RetryAdvice::Never,
1514        }
1515    }
1516}
1517
1518/// First-party durable model/tool attempt executor.
1519#[derive(Clone)]
1520pub struct DurableInvocationExecutor {
1521    store: PostgresStore,
1522    schemas: JsonSchemaRegistry,
1523    event_schema: SchemaReference,
1524    models: ModelProviderRegistry,
1525    tools: ToolProviderRegistry,
1526    budget: Arc<dyn InvocationBudgetProvider>,
1527    clock: Arc<dyn InvocationClock>,
1528    options: DurableInvocationExecutorOptions,
1529}
1530
1531impl DurableInvocationExecutor {
1532    /// Returns whether the exact installed binding implements reconciliation.
1533    ///
1534    /// # Errors
1535    ///
1536    /// Fails if no exact provider binding exists for the descriptor snapshot.
1537    pub fn supports_tool_reconciliation(
1538        &self,
1539        descriptor: &ToolDescriptor,
1540    ) -> Result<bool, ToolProviderRegistryError> {
1541        self.tools
1542            .resolve(descriptor)
1543            .map(|provider| provider.supports_reconciliation())
1544    }
1545
1546    /// Builds an executor using the production system/monotonic clock.
1547    ///
1548    /// # Errors
1549    ///
1550    /// Fails unless the exact embedded invocation journal schema is installed
1551    /// in the frozen offline registry.
1552    #[allow(clippy::too_many_arguments)]
1553    pub fn new(
1554        store: PostgresStore,
1555        schemas: JsonSchemaRegistry,
1556        models: ModelProviderRegistry,
1557        tools: ToolProviderRegistry,
1558        budget: Arc<dyn InvocationBudgetProvider>,
1559        options: DurableInvocationExecutorOptions,
1560    ) -> Result<Self, DurableInvocationExecutorBuildError> {
1561        Self::with_clock(
1562            store,
1563            schemas,
1564            models,
1565            tools,
1566            budget,
1567            Arc::new(SystemInvocationClock),
1568            options,
1569        )
1570    }
1571
1572    /// Builds an executor with an injected trusted paired clock.
1573    ///
1574    /// # Errors
1575    ///
1576    /// Fails unless the exact embedded invocation journal schema is installed.
1577    #[allow(clippy::too_many_arguments)]
1578    pub fn with_clock(
1579        store: PostgresStore,
1580        schemas: JsonSchemaRegistry,
1581        models: ModelProviderRegistry,
1582        tools: ToolProviderRegistry,
1583        budget: Arc<dyn InvocationBudgetProvider>,
1584        clock: Arc<dyn InvocationClock>,
1585        options: DurableInvocationExecutorOptions,
1586    ) -> Result<Self, DurableInvocationExecutorBuildError> {
1587        let (event_schema, _) = standard_invocation_execution_event_schema()?;
1588        if !schemas.contains(&event_schema) {
1589            return Err(DurableInvocationExecutorBuildError::MissingEventSchema);
1590        }
1591        Ok(Self {
1592            store,
1593            schemas,
1594            event_schema,
1595            models,
1596            tools,
1597            budget,
1598            clock,
1599            options,
1600        })
1601    }
1602
1603    async fn prepare_model_context(
1604        &self,
1605        handoff: &ModelAttemptHandoff,
1606    ) -> Result<(ModelContext, Option<Digest>), ModelAttemptExecutionError> {
1607        let run = self
1608            .store
1609            .load_run(handoff.fence.tenant_id(), handoff.fence.run_id())
1610            .await?;
1611        let provenance = run.lifecycle().provenance().clone();
1612        validate_run_provenance(&provenance, &handoff.fence)?;
1613        let account = self
1614            .store
1615            .load_child_budget_account(handoff.fence.tenant_id(), handoff.fence.run_id())
1616            .await?;
1617        let child_budget = account
1618            .as_ref()
1619            .map(stateknot_core::ChildRunBudgetAccount::digest);
1620        let observation = self.clock.observe()?;
1621        let remaining = self
1622            .budget
1623            .remaining(InvocationBudgetContext {
1624                provenance: provenance.clone(),
1625                boundary: InvocationBoundaryKind::Model,
1626                invocation_id: handoff.invocation.intent().invocation_id(),
1627                attempt_id: handoff.attempt_id,
1628                intent_digest: handoff.invocation.intent().intent_digest(),
1629                observed_at: observation.observed_at,
1630            })
1631            .await?;
1632        validate_model_budget(&remaining, handoff.invocation.intent().request())?;
1633        let remaining = deduct_child_budget(remaining, account.as_ref()).map_err(|error| {
1634            ModelAttemptExecutionError::BudgetProvider(InvocationBudgetProviderError::new(error))
1635        })?;
1636        validate_model_budget(&remaining, handoff.invocation.intent().request())?;
1637        let context = ModelContext::new(
1638            provenance.tenant_id().clone(),
1639            provenance.run_id(),
1640            provenance.thread_id(),
1641            handoff.attempt_id,
1642            remaining,
1643            observation.observed_at,
1644            observation.observed_instant,
1645            handoff.cancellation.clone(),
1646        )?;
1647        if let Some(reason) = context.stop_reason_at(Instant::now()) {
1648            return Err(ModelAttemptExecutionError::StoppedBeforeStart { reason });
1649        }
1650        Ok((context, child_budget))
1651    }
1652
1653    async fn prepare_tool_context(
1654        &self,
1655        handoff: &ToolAttemptHandoff,
1656    ) -> Result<(ToolContext, Option<Digest>), ToolAttemptExecutionError> {
1657        let run = self
1658            .store
1659            .load_run(handoff.fence.tenant_id(), handoff.fence.run_id())
1660            .await?;
1661        let provenance = run.lifecycle().provenance().clone();
1662        validate_tool_run_provenance(&provenance, &handoff.fence)?;
1663        let account = self
1664            .store
1665            .load_child_budget_account(handoff.fence.tenant_id(), handoff.fence.run_id())
1666            .await?;
1667        let child_budget = account
1668            .as_ref()
1669            .map(stateknot_core::ChildRunBudgetAccount::digest);
1670        let observation = self.clock.observe()?;
1671        let remaining = self
1672            .budget
1673            .remaining(InvocationBudgetContext {
1674                provenance: provenance.clone(),
1675                boundary: InvocationBoundaryKind::Tool,
1676                invocation_id: handoff.invocation.intent().invocation_id(),
1677                attempt_id: handoff.attempt_id,
1678                intent_digest: handoff.invocation.intent().intent_digest(),
1679                observed_at: observation.observed_at,
1680            })
1681            .await?;
1682        let remaining = deduct_child_budget(remaining, account.as_ref()).map_err(|error| {
1683            ToolAttemptExecutionError::BudgetProvider(InvocationBudgetProviderError::new(error))
1684        })?;
1685        validate_tool_budget(
1686            &remaining,
1687            handoff.invocation.intent().descriptor().semantics().risk(),
1688        )?;
1689        let context = Self::build_tool_context(handoff, &provenance, remaining, observation)?;
1690        handoff
1691            .invocation
1692            .intent()
1693            .input()
1694            .validate_for(&context, handoff.invocation.intent().descriptor())?;
1695        if let Some(reason) = context.stop_reason_at(Instant::now()) {
1696            return Err(ToolAttemptExecutionError::StoppedBeforeStart { reason });
1697        }
1698        Ok((context, child_budget))
1699    }
1700
1701    fn build_tool_context(
1702        handoff: &ToolAttemptHandoff,
1703        provenance: &AgentResultProvenance,
1704        remaining: BudgetRemaining,
1705        observation: InvocationClockObservation,
1706    ) -> Result<ToolContext, ToolContextError> {
1707        let intent = handoff.invocation.intent();
1708        let context = if let Some(progress_sink) = &handoff.progress_sink {
1709            ToolContext::new_with_progress(
1710                provenance.tenant_id().clone(),
1711                provenance.run_id(),
1712                provenance.thread_id(),
1713                intent.invocation_id(),
1714                handoff.attempt_id,
1715                intent.descriptor(),
1716                remaining,
1717                intent.effective_limits().timeout(),
1718                observation.observed_at,
1719                observation.observed_instant,
1720                handoff.cancellation.clone(),
1721                Arc::clone(progress_sink),
1722            )?
1723        } else {
1724            ToolContext::new(
1725                provenance.tenant_id().clone(),
1726                provenance.run_id(),
1727                provenance.thread_id(),
1728                intent.invocation_id(),
1729                handoff.attempt_id,
1730                intent.descriptor(),
1731                remaining,
1732                intent.effective_limits().timeout(),
1733                observation.observed_at,
1734                observation.observed_instant,
1735                handoff.cancellation.clone(),
1736            )?
1737        };
1738        Ok(context
1739            .with_durable_origin_event(handoff.events.start())
1740            .narrow_artifact_limits(intent.effective_limits()))
1741    }
1742
1743    async fn prepare_tool_reconciliation_context(
1744        &self,
1745        handoff: &ToolReconciliationAttemptHandoff,
1746        invocation: &ToolInvocation,
1747    ) -> Result<ToolReconciliationContext, ToolReconciliationAttemptExecutionError> {
1748        let run = self
1749            .store
1750            .load_run(handoff.fence.tenant_id(), handoff.fence.run_id())
1751            .await?;
1752        let provenance = run.lifecycle().provenance();
1753        if provenance.tenant_id() != handoff.fence.tenant_id()
1754            || provenance.run_id() != handoff.fence.run_id()
1755        {
1756            return Err(ToolReconciliationAttemptExecutionError::RunProvenanceMismatch);
1757        }
1758        let observation = self.clock.observe()?;
1759        let intent = invocation.intent();
1760        let timeout_micros = intent
1761            .effective_limits()
1762            .timeout()
1763            .as_i64()
1764            .checked_mul(1_000)
1765            .ok_or(ToolReconciliationAttemptExecutionError::DeadlineOutOfRange)?;
1766        let fallback_deadline_micros = observation
1767            .observed_at()
1768            .unix_micros()
1769            .checked_add(timeout_micros)
1770            .ok_or(ToolReconciliationAttemptExecutionError::DeadlineOutOfRange)?;
1771        let fallback_deadline = Timestamp::from_unix_micros(fallback_deadline_micros)
1772            .map_err(|_| ToolReconciliationAttemptExecutionError::DeadlineOutOfRange)?;
1773        let run_deadline = match self
1774            .store
1775            .load_agent_admission(handoff.fence.tenant_id(), handoff.fence.run_id())
1776            .await
1777        {
1778            Ok(stored) => stored.admission().intent().budget().deadline(),
1779            Err(StoreError::AgentAdmissionNotFound) => fallback_deadline,
1780            Err(source) => return Err(source.into()),
1781        };
1782        let recovery_handle = match invocation.state() {
1783            ToolInvocationState::Unknown { error } => error.recovery_handle().cloned(),
1784            _ => None,
1785        };
1786        let origin_event_id = invocation
1787            .previous()
1788            .filter(|head| head.status() == stateknot_core::ToolInvocationStatus::Executing)
1789            .map(|head| head.journal_head().event_id())
1790            .ok_or(ToolReconciliationAttemptExecutionError::InvocationAdvanced)?;
1791        let context = ToolReconciliationContext::new(
1792            provenance.tenant_id().clone(),
1793            provenance.run_id(),
1794            provenance.thread_id(),
1795            intent.invocation_id(),
1796            invocation
1797                .attempt_id()
1798                .ok_or(ToolReconciliationAttemptExecutionError::InvocationAdvanced)?,
1799            intent.descriptor(),
1800            intent.effective_limits().timeout(),
1801            observation.observed_at(),
1802            observation.observed_instant(),
1803            run_deadline,
1804            handoff.cancellation.clone(),
1805        )?
1806        .with_durable_recovery(origin_event_id, recovery_handle)
1807        .narrow_artifact_limits(intent.effective_limits());
1808        if let Some(reason) = context.stop_reason_at(Instant::now()) {
1809            return Err(ToolReconciliationAttemptExecutionError::Stopped { reason });
1810        }
1811        Ok(context)
1812    }
1813
1814    async fn recover_model_if_started(
1815        &self,
1816        handoff: &ModelAttemptHandoff,
1817    ) -> Result<Option<ModelAttemptOutcome>, ModelAttemptExecutionError> {
1818        let current = self
1819            .store
1820            .load_model_invocation(
1821                handoff.fence.tenant_id(),
1822                handoff.fence.run_id(),
1823                handoff.invocation.intent().invocation_id(),
1824            )
1825            .await?;
1826        if current.intent() != handoff.invocation.intent() {
1827            return Err(ModelAttemptExecutionError::RecoveredInvocationMismatch);
1828        }
1829        if current.head() == handoff.invocation.head() {
1830            return Ok(None);
1831        }
1832        if current.attempt_id() != Some(handoff.attempt_id) {
1833            return Err(ModelAttemptExecutionError::InvocationAdvanced);
1834        }
1835        Ok(Some(ModelAttemptOutcome::Recovered {
1836            invocation: current,
1837        }))
1838    }
1839
1840    async fn recover_tool_if_started(
1841        &self,
1842        handoff: &ToolAttemptHandoff,
1843    ) -> Result<Option<ToolAttemptOutcome>, ToolAttemptExecutionError> {
1844        let current = self
1845            .store
1846            .load_tool_invocation(
1847                handoff.fence.tenant_id(),
1848                handoff.fence.run_id(),
1849                handoff.invocation.intent().invocation_id(),
1850            )
1851            .await?;
1852        if current.intent() != handoff.invocation.intent() {
1853            return Err(ToolAttemptExecutionError::RecoveredInvocationMismatch);
1854        }
1855        if current.head() == handoff.invocation.head() {
1856            return Ok(None);
1857        }
1858        if current.attempt_id() != Some(handoff.attempt_id) {
1859            return Err(ToolAttemptExecutionError::InvocationAdvanced);
1860        }
1861        Ok(Some(ToolAttemptOutcome::Recovered {
1862            invocation: current,
1863        }))
1864    }
1865
1866    /// Executes at most one physical model provider exchange.
1867    ///
1868    /// `StartAttempt` commits before provider I/O. If that event is found
1869    /// idempotently, this method loads current state and never calls the model.
1870    /// A terminal database failure returns the exact response/error recovery
1871    /// handoff so callers can retry persistence without repeating provider I/O.
1872    pub fn execute_model(
1873        &self,
1874        handoff: ModelAttemptHandoff,
1875    ) -> BoxFuture<'_, Result<ModelAttemptOutcome, ModelAttemptExecutionError>> {
1876        Box::pin(self.execute_model_inner(handoff))
1877    }
1878
1879    async fn execute_model_inner(
1880        &self,
1881        handoff: ModelAttemptHandoff,
1882    ) -> Result<ModelAttemptOutcome, ModelAttemptExecutionError> {
1883        validate_model_handoff(
1884            &handoff.fence,
1885            &handoff.invocation,
1886            handoff.stream_sink.is_some(),
1887        )?;
1888        if let Some(recovered) = self.recover_model_if_started(&handoff).await? {
1889            return Ok(recovered);
1890        }
1891        let provider = self
1892            .models
1893            .resolve(handoff.invocation.intent().descriptor())?;
1894        let (context, child_budget) = self.prepare_model_context(&handoff).await?;
1895
1896        let start_transition = ModelInvocationTransition::StartAttempt {
1897            attempt_id: handoff.attempt_id,
1898        };
1899        let start_payload = self.event_payload(
1900            "model-attempt-started",
1901            "model_attempt_started",
1902            "model",
1903            handoff.invocation.intent().invocation_id(),
1904            handoff.attempt_id,
1905            handoff.invocation.intent().intent_digest(),
1906        )?;
1907        let start_append = worker_append(
1908            &handoff.fence,
1909            handoff.invocation.journal_head().clone(),
1910            handoff.events.start,
1911            start_payload,
1912        )
1913        .map_err(|_| ModelAttemptExecutionError::JournalAppend)?;
1914        let start = self
1915            .advance_model_with_retry(
1916                start_append,
1917                &handoff.invocation,
1918                start_transition,
1919                child_budget,
1920            )
1921            .await?;
1922        let executing = match start {
1923            ModelInvocationCommitOutcome::Committed { invocation, .. } => invocation,
1924            ModelInvocationCommitOutcome::Idempotent { .. } => {
1925                let current = self
1926                    .store
1927                    .load_model_invocation(
1928                        handoff.fence.tenant_id(),
1929                        handoff.fence.run_id(),
1930                        handoff.invocation.intent().invocation_id(),
1931                    )
1932                    .await?;
1933                if current.intent() != handoff.invocation.intent() {
1934                    return Err(ModelAttemptExecutionError::RecoveredInvocationMismatch);
1935                }
1936                if current.attempt_id() != Some(handoff.attempt_id) {
1937                    return Err(ModelAttemptExecutionError::InvocationAdvanced);
1938                }
1939                return Ok(ModelAttemptOutcome::Recovered {
1940                    invocation: current,
1941                });
1942            }
1943            _ => return Err(ModelAttemptExecutionError::UnsupportedStoreOutcome),
1944        };
1945
1946        let evidence = match context.stop_reason_at(Instant::now()) {
1947            Some(reason) => ModelTerminalEvidence::Error(model_stop_error(
1948                &context,
1949                executing.intent().descriptor(),
1950                reason,
1951            )),
1952            None => {
1953                self.dispatch_model(
1954                    provider,
1955                    context,
1956                    executing.intent().request().clone(),
1957                    handoff.stream_sink,
1958                )
1959                .await
1960            }
1961        };
1962        let terminal = ModelTerminalCommitHandoff {
1963            fence: handoff.fence,
1964            invocation: executing,
1965            event_id: handoff.events.terminal,
1966            evidence,
1967        };
1968        self.commit_model_terminal(terminal)
1969            .await
1970            .map_err(ModelAttemptExecutionError::Terminal)
1971    }
1972
1973    /// Commits retained terminal model evidence without provider I/O.
1974    pub fn commit_model_terminal(
1975        &self,
1976        handoff: ModelTerminalCommitHandoff,
1977    ) -> BoxFuture<'_, Result<ModelAttemptOutcome, ModelTerminalCommitError>> {
1978        Box::pin(self.commit_model_terminal_inner(handoff))
1979    }
1980
1981    async fn commit_model_terminal_inner(
1982        &self,
1983        handoff: ModelTerminalCommitHandoff,
1984    ) -> Result<ModelAttemptOutcome, ModelTerminalCommitError> {
1985        let terminal_kind = handoff.kind();
1986        let (operation, event_kind, transition) = match &handoff.evidence {
1987            ModelTerminalEvidence::Response(response) => (
1988                "model_response_committed",
1989                "model-response-committed",
1990                ModelInvocationTransition::RecordResponse {
1991                    response: response.clone(),
1992                },
1993            ),
1994            ModelTerminalEvidence::Error(error) => (
1995                "model_error_committed",
1996                "model-error-committed",
1997                ModelInvocationTransition::RecordError {
1998                    error: error.clone(),
1999                },
2000            ),
2001        };
2002        let Some(attempt_id) = handoff.invocation.attempt_id() else {
2003            return Err(ModelTerminalCommitError {
2004                source: InvocationTerminalCommitFailure::InvalidInvocationState,
2005                recovery: Box::new(handoff),
2006            });
2007        };
2008        let Ok(payload) = self.event_payload(
2009            event_kind,
2010            operation,
2011            "model",
2012            handoff.invocation.intent().invocation_id(),
2013            attempt_id,
2014            handoff.invocation.intent().intent_digest(),
2015        ) else {
2016            return Err(ModelTerminalCommitError {
2017                source: InvocationTerminalCommitFailure::InvalidEventPayload,
2018                recovery: Box::new(handoff),
2019            });
2020        };
2021        let run = match self
2022            .store
2023            .load_run(
2024                handoff.invocation.intent().tenant_id(),
2025                handoff.invocation.intent().run_id(),
2026            )
2027            .await
2028        {
2029            Ok(run) => run,
2030            Err(source) => {
2031                return Err(ModelTerminalCommitError {
2032                    source: InvocationTerminalCommitFailure::Store(source),
2033                    recovery: Box::new(handoff),
2034                });
2035            }
2036        };
2037        let Some(journal_head) = run.journal_head().cloned() else {
2038            return Err(ModelTerminalCommitError {
2039                source: InvocationTerminalCommitFailure::RunJournalUnavailable,
2040                recovery: Box::new(handoff),
2041            });
2042        };
2043        let Ok(append) = worker_append(&handoff.fence, journal_head, handoff.event_id, payload)
2044        else {
2045            return Err(ModelTerminalCommitError {
2046                source: InvocationTerminalCommitFailure::InvalidJournalAppend,
2047                recovery: Box::new(handoff),
2048            });
2049        };
2050        match self
2051            .advance_model_with_retry(append, &handoff.invocation, transition, None)
2052            .await
2053        {
2054            Ok(outcome) => Ok(ModelAttemptOutcome::Dispatched {
2055                terminal: terminal_kind,
2056                invocation: outcome.invocation().clone(),
2057            }),
2058            Err(source) => Err(ModelTerminalCommitError {
2059                source: InvocationTerminalCommitFailure::Store(source),
2060                recovery: Box::new(handoff),
2061            }),
2062        }
2063    }
2064
2065    /// Executes at most one physical tool implementation call.
2066    ///
2067    /// Start duplicate suppression and retained terminal recovery have the same
2068    /// no-repeat semantics as [`Self::execute_model`].
2069    pub fn execute_tool(
2070        &self,
2071        handoff: ToolAttemptHandoff,
2072    ) -> BoxFuture<'_, Result<ToolAttemptOutcome, ToolAttemptExecutionError>> {
2073        Box::pin(self.execute_tool_inner(handoff))
2074    }
2075
2076    async fn execute_tool_inner(
2077        &self,
2078        handoff: ToolAttemptHandoff,
2079    ) -> Result<ToolAttemptOutcome, ToolAttemptExecutionError> {
2080        let started = self.start_tool_attempt_inner(handoff).await?;
2081        let ToolAttemptStartOutcome::Started(dispatch) = started else {
2082            let ToolAttemptStartOutcome::Recovered { invocation } = started else {
2083                unreachable!("closed tool attempt start outcome")
2084            };
2085            return Ok(ToolAttemptOutcome::Recovered {
2086                invocation: *invocation,
2087            });
2088        };
2089        let terminal = self.dispatch_started_tool_inner(*dispatch).await;
2090        self.commit_tool_terminal(terminal)
2091            .await
2092            .map_err(ToolAttemptExecutionError::Terminal)
2093    }
2094
2095    /// Commits a tool attempt start without performing provider I/O.
2096    ///
2097    /// Only [`ToolAttemptStartOutcome::Started`] grants dispatch authority.
2098    /// This split is intended for bounded ordered coordinators; ordinary
2099    /// callers should prefer [`Self::execute_tool`].
2100    pub fn start_tool_attempt(
2101        &self,
2102        handoff: ToolAttemptHandoff,
2103    ) -> BoxFuture<'_, Result<ToolAttemptStartOutcome, ToolAttemptExecutionError>> {
2104        Box::pin(self.start_tool_attempt_inner(handoff))
2105    }
2106
2107    async fn start_tool_attempt_inner(
2108        &self,
2109        handoff: ToolAttemptHandoff,
2110    ) -> Result<ToolAttemptStartOutcome, ToolAttemptExecutionError> {
2111        validate_tool_handoff(&handoff.fence, &handoff.invocation)?;
2112        if let Some(recovered) = self.recover_tool_if_started(&handoff).await? {
2113            let ToolAttemptOutcome::Recovered { invocation } = recovered else {
2114                return Err(ToolAttemptExecutionError::UnsupportedStoreOutcome);
2115            };
2116            return Ok(ToolAttemptStartOutcome::Recovered {
2117                invocation: Box::new(invocation),
2118            });
2119        }
2120        let provider = self
2121            .tools
2122            .resolve(handoff.invocation.intent().descriptor())?;
2123        let (context, child_budget) = self.prepare_tool_context(&handoff).await?;
2124
2125        let start_payload = self.event_payload(
2126            "tool-attempt-started",
2127            "tool_attempt_started",
2128            "tool",
2129            handoff.invocation.intent().invocation_id(),
2130            handoff.attempt_id,
2131            handoff.invocation.intent().intent_digest(),
2132        )?;
2133        let start_append = worker_append(
2134            &handoff.fence,
2135            handoff.invocation.journal_head().clone(),
2136            handoff.events.start,
2137            start_payload,
2138        )
2139        .map_err(|_| ToolAttemptExecutionError::JournalAppend)?;
2140        let start = self
2141            .advance_tool_with_retry(
2142                start_append,
2143                &handoff.invocation,
2144                ToolInvocationTransition::StartAttempt {
2145                    attempt_id: handoff.attempt_id,
2146                },
2147                child_budget,
2148            )
2149            .await?;
2150        let executing = match start {
2151            ToolInvocationCommitOutcome::Committed { invocation, .. } => invocation,
2152            ToolInvocationCommitOutcome::Idempotent { .. } => {
2153                let current = self
2154                    .store
2155                    .load_tool_invocation(
2156                        handoff.fence.tenant_id(),
2157                        handoff.fence.run_id(),
2158                        handoff.invocation.intent().invocation_id(),
2159                    )
2160                    .await?;
2161                if current.intent() != handoff.invocation.intent() {
2162                    return Err(ToolAttemptExecutionError::RecoveredInvocationMismatch);
2163                }
2164                if current.attempt_id() != Some(handoff.attempt_id) {
2165                    return Err(ToolAttemptExecutionError::InvocationAdvanced);
2166                }
2167                return Ok(ToolAttemptStartOutcome::Recovered {
2168                    invocation: Box::new(current),
2169                });
2170            }
2171            _ => return Err(ToolAttemptExecutionError::UnsupportedStoreOutcome),
2172        };
2173
2174        Ok(ToolAttemptStartOutcome::Started(Box::new(
2175            ToolDispatchHandoff {
2176                fence: handoff.fence,
2177                invocation: executing,
2178                terminal_event_id: handoff.events.terminal,
2179                provider,
2180                context,
2181            },
2182        )))
2183    }
2184
2185    /// Performs exactly one provider call for a fresh durable start and retains
2186    /// its terminal evidence without committing it.
2187    ///
2188    /// The returned handoff must be committed with
2189    /// [`Self::commit_tool_terminal`]. Separating provider completion from the
2190    /// commit lets a coordinator restore proposal-order journal semantics even
2191    /// when read-only calls finish out of order.
2192    pub fn dispatch_started_tool(
2193        &self,
2194        handoff: ToolDispatchHandoff,
2195    ) -> BoxFuture<'_, ToolTerminalCommitHandoff> {
2196        Box::pin(self.dispatch_started_tool_inner(handoff))
2197    }
2198
2199    async fn dispatch_started_tool_inner(
2200        &self,
2201        handoff: ToolDispatchHandoff,
2202    ) -> ToolTerminalCommitHandoff {
2203        let evidence = if let Some(reason) = handoff.context.stop_reason_at(Instant::now()) {
2204            ToolTerminalEvidence::Error(tool_stop_error(
2205                &handoff.context,
2206                handoff.invocation.intent().descriptor(),
2207                reason,
2208            ))
2209        } else {
2210            let input = handoff.invocation.intent().input().clone();
2211            self.dispatch_tool(handoff.provider, handoff.context, input)
2212                .await
2213        };
2214        ToolTerminalCommitHandoff {
2215            fence: handoff.fence,
2216            invocation: handoff.invocation,
2217            event_id: handoff.terminal_event_id,
2218            evidence,
2219        }
2220    }
2221
2222    /// Commits retained terminal tool evidence without calling the tool.
2223    pub fn commit_tool_terminal(
2224        &self,
2225        handoff: ToolTerminalCommitHandoff,
2226    ) -> BoxFuture<'_, Result<ToolAttemptOutcome, ToolTerminalCommitError>> {
2227        Box::pin(self.commit_tool_terminal_inner(handoff))
2228    }
2229
2230    async fn commit_tool_terminal_inner(
2231        &self,
2232        handoff: ToolTerminalCommitHandoff,
2233    ) -> Result<ToolAttemptOutcome, ToolTerminalCommitError> {
2234        let terminal_kind = handoff.kind();
2235        let (operation, event_kind, transition) = match &handoff.evidence {
2236            ToolTerminalEvidence::Result(result) => (
2237                "tool_result_committed",
2238                "tool-result-committed",
2239                ToolInvocationTransition::RecordResult {
2240                    result: result.clone(),
2241                },
2242            ),
2243            ToolTerminalEvidence::Error(error) => (
2244                "tool_error_committed",
2245                "tool-error-committed",
2246                ToolInvocationTransition::RecordError {
2247                    error: error.clone(),
2248                },
2249            ),
2250        };
2251        let Some(attempt_id) = handoff.invocation.attempt_id() else {
2252            return Err(ToolTerminalCommitError {
2253                source: InvocationTerminalCommitFailure::InvalidInvocationState,
2254                recovery: Box::new(handoff),
2255            });
2256        };
2257        let Ok(payload) = self.event_payload(
2258            event_kind,
2259            operation,
2260            "tool",
2261            handoff.invocation.intent().invocation_id(),
2262            attempt_id,
2263            handoff.invocation.intent().intent_digest(),
2264        ) else {
2265            return Err(ToolTerminalCommitError {
2266                source: InvocationTerminalCommitFailure::InvalidEventPayload,
2267                recovery: Box::new(handoff),
2268            });
2269        };
2270        let run = match self
2271            .store
2272            .load_run(
2273                handoff.invocation.intent().tenant_id(),
2274                handoff.invocation.intent().run_id(),
2275            )
2276            .await
2277        {
2278            Ok(run) => run,
2279            Err(source) => {
2280                return Err(ToolTerminalCommitError {
2281                    source: InvocationTerminalCommitFailure::Store(source),
2282                    recovery: Box::new(handoff),
2283                });
2284            }
2285        };
2286        let Some(journal_head) = run.journal_head().cloned() else {
2287            return Err(ToolTerminalCommitError {
2288                source: InvocationTerminalCommitFailure::RunJournalUnavailable,
2289                recovery: Box::new(handoff),
2290            });
2291        };
2292        let Ok(append) = worker_append(&handoff.fence, journal_head, handoff.event_id, payload)
2293        else {
2294            return Err(ToolTerminalCommitError {
2295                source: InvocationTerminalCommitFailure::InvalidJournalAppend,
2296                recovery: Box::new(handoff),
2297            });
2298        };
2299        match self
2300            .advance_tool_with_retry(append, &handoff.invocation, transition, None)
2301            .await
2302        {
2303            Ok(outcome) => Ok(ToolAttemptOutcome::Dispatched {
2304                terminal: terminal_kind,
2305                invocation: outcome.invocation().clone(),
2306            }),
2307            Err(source) => Err(ToolTerminalCommitError {
2308                source: InvocationTerminalCommitFailure::Store(source),
2309                recovery: Box::new(handoff),
2310            }),
2311        }
2312    }
2313
2314    /// Performs at most one bounded provider reconciliation probe and commits
2315    /// any authoritative evidence atomically.
2316    ///
2317    /// The current invocation is reloaded before provider I/O. If the stable
2318    /// reconciliation event already won, this returns the terminal revision
2319    /// without another query or replay. `Pending` performs no durable mutation;
2320    /// callers should persist a delayed node retry using the supplied interval.
2321    pub fn reconcile_tool(
2322        &self,
2323        handoff: ToolReconciliationAttemptHandoff,
2324    ) -> BoxFuture<
2325        '_,
2326        Result<ToolReconciliationAttemptOutcome, ToolReconciliationAttemptExecutionError>,
2327    > {
2328        Box::pin(self.reconcile_tool_inner(handoff))
2329    }
2330
2331    async fn reconcile_tool_inner(
2332        &self,
2333        handoff: ToolReconciliationAttemptHandoff,
2334    ) -> Result<ToolReconciliationAttemptOutcome, ToolReconciliationAttemptExecutionError> {
2335        validate_tool_reconciliation_attempt_handoff(&handoff.fence, &handoff.invocation)?;
2336        let current = self
2337            .store
2338            .load_tool_invocation(
2339                handoff.fence.tenant_id(),
2340                handoff.fence.run_id(),
2341                handoff.invocation.intent().invocation_id(),
2342            )
2343            .await?;
2344        if current.intent() != handoff.invocation.intent() {
2345            return Err(ToolReconciliationAttemptExecutionError::RecoveredInvocationMismatch);
2346        }
2347        if current.attempt_id() != handoff.invocation.attempt_id() {
2348            return Err(ToolReconciliationAttemptExecutionError::InvocationAdvanced);
2349        }
2350        match current.state() {
2351            ToolInvocationState::Committed { .. } => {
2352                return Ok(ToolReconciliationAttemptOutcome::Resolved {
2353                    kind: ToolReconciliationKind::Result,
2354                    invocation: current,
2355                });
2356            }
2357            ToolInvocationState::Failed { .. } => {
2358                return Ok(ToolReconciliationAttemptOutcome::Resolved {
2359                    kind: ToolReconciliationKind::Error,
2360                    invocation: current,
2361                });
2362            }
2363            ToolInvocationState::Unknown { .. } => {}
2364            ToolInvocationState::Prepared | ToolInvocationState::Executing { .. } => {
2365                return Err(ToolReconciliationAttemptExecutionError::InvocationAdvanced);
2366            }
2367        }
2368
2369        let provider = self.tools.resolve(current.intent().descriptor())?;
2370        if !provider.supports_reconciliation() {
2371            return Err(ToolReconciliationAttemptExecutionError::ProviderReconciliationUnsupported);
2372        }
2373        let context = self
2374            .prepare_tool_reconciliation_context(&handoff, &current)
2375            .await?;
2376        let input = current.intent().input().clone();
2377        let deadline = tokio::time::Instant::from_std(context.deadline_instant());
2378        let observation = tokio::select! {
2379            biased;
2380            () = context.cancellation().cancelled() => {
2381                return Err(ToolReconciliationAttemptExecutionError::Stopped {
2382                    reason: ToolStopReason::Cancelled,
2383                });
2384            }
2385            () = tokio::time::sleep_until(deadline) => {
2386                return Err(ToolReconciliationAttemptExecutionError::Stopped {
2387                    reason: ToolStopReason::DeadlineExceeded,
2388                });
2389            }
2390            observation = provider.reconcile(context.clone(), input) => observation?,
2391        };
2392        observation.validate()?;
2393
2394        let evidence = match observation {
2395            ToolReconciliationObservation::Pending { retry_after } => {
2396                return Ok(ToolReconciliationAttemptOutcome::Pending {
2397                    retry_after,
2398                    invocation: current,
2399                });
2400            }
2401            ToolReconciliationObservation::Result(result) => {
2402                ToolReconciliationHandoff::result(handoff.fence, current, handoff.event_id, result)?
2403            }
2404            ToolReconciliationObservation::Error(error) => {
2405                if error.external_effect() == ToolExternalEffect::Unknown {
2406                    return Err(ToolReconciliationAttemptExecutionError::UnresolvedErrorEvidence);
2407                }
2408                ToolReconciliationHandoff::error(handoff.fence, current, handoff.event_id, error)?
2409            }
2410            _ => {
2411                return Err(ToolReconciliationAttemptExecutionError::UnsupportedObservation);
2412            }
2413        };
2414        let outcome = self.commit_tool_reconciliation(evidence).await?;
2415        Ok(ToolReconciliationAttemptOutcome::Resolved {
2416            kind: outcome.kind(),
2417            invocation: outcome.invocation().clone(),
2418        })
2419    }
2420
2421    /// Atomically commits authoritative evidence for an unknown tool attempt.
2422    ///
2423    /// This is a trusted worker/operations boundary. It validates successful
2424    /// output against the frozen local schema registry, appends a distinct audit
2425    /// event, and advances the invocation in one fenced `PostgreSQL` transaction.
2426    /// It never resolves or calls a tool provider.
2427    pub fn commit_tool_reconciliation(
2428        &self,
2429        handoff: ToolReconciliationHandoff,
2430    ) -> BoxFuture<'_, Result<ToolReconciliationOutcome, ToolReconciliationCommitError>> {
2431        Box::pin(self.commit_tool_reconciliation_inner(handoff))
2432    }
2433
2434    async fn commit_tool_reconciliation_inner(
2435        &self,
2436        handoff: ToolReconciliationHandoff,
2437    ) -> Result<ToolReconciliationOutcome, ToolReconciliationCommitError> {
2438        let kind = handoff.kind();
2439        let (operation, event_kind, transition) = match self.reconciliation_transition(&handoff) {
2440            Ok(prepared) => prepared,
2441            Err(source) => {
2442                return Err(ToolReconciliationCommitError {
2443                    source,
2444                    recovery: Box::new(handoff),
2445                });
2446            }
2447        };
2448        let Some(attempt_id) = handoff.invocation.attempt_id() else {
2449            return Err(ToolReconciliationCommitError {
2450                source: ToolReconciliationCommitFailure::InvalidInvocationState,
2451                recovery: Box::new(handoff),
2452            });
2453        };
2454        let Ok(payload) = self.event_payload(
2455            event_kind,
2456            operation,
2457            "tool",
2458            handoff.invocation.intent().invocation_id(),
2459            attempt_id,
2460            handoff.invocation.intent().intent_digest(),
2461        ) else {
2462            return Err(ToolReconciliationCommitError {
2463                source: ToolReconciliationCommitFailure::InvalidEventPayload,
2464                recovery: Box::new(handoff),
2465            });
2466        };
2467        let run = match self
2468            .store
2469            .load_run(
2470                handoff.invocation.intent().tenant_id(),
2471                handoff.invocation.intent().run_id(),
2472            )
2473            .await
2474        {
2475            Ok(run) => run,
2476            Err(source) => {
2477                return Err(ToolReconciliationCommitError {
2478                    source: ToolReconciliationCommitFailure::Store(source),
2479                    recovery: Box::new(handoff),
2480                });
2481            }
2482        };
2483        let Some(journal_head) = run.journal_head().cloned() else {
2484            return Err(ToolReconciliationCommitError {
2485                source: ToolReconciliationCommitFailure::RunJournalUnavailable,
2486                recovery: Box::new(handoff),
2487            });
2488        };
2489        let Ok(append) = worker_append(&handoff.fence, journal_head, handoff.event_id, payload)
2490        else {
2491            return Err(ToolReconciliationCommitError {
2492                source: ToolReconciliationCommitFailure::InvalidJournalAppend,
2493                recovery: Box::new(handoff),
2494            });
2495        };
2496        match self
2497            .advance_tool_with_retry(append, &handoff.invocation, transition, None)
2498            .await
2499        {
2500            Ok(ToolInvocationCommitOutcome::Committed { event, invocation }) => {
2501                Ok(ToolReconciliationOutcome::Committed {
2502                    kind,
2503                    event,
2504                    invocation,
2505                })
2506            }
2507            Ok(ToolInvocationCommitOutcome::Idempotent { event, invocation }) => {
2508                Ok(ToolReconciliationOutcome::Idempotent {
2509                    kind,
2510                    event,
2511                    invocation,
2512                })
2513            }
2514            Ok(_) => Err(ToolReconciliationCommitError {
2515                source: ToolReconciliationCommitFailure::UnsupportedStoreOutcome,
2516                recovery: Box::new(handoff),
2517            }),
2518            Err(source) => Err(ToolReconciliationCommitError {
2519                source: ToolReconciliationCommitFailure::Store(source),
2520                recovery: Box::new(handoff),
2521            }),
2522        }
2523    }
2524
2525    fn reconciliation_transition(
2526        &self,
2527        handoff: &ToolReconciliationHandoff,
2528    ) -> Result<
2529        (&'static str, &'static str, ToolInvocationTransition),
2530        ToolReconciliationCommitFailure,
2531    > {
2532        match &handoff.evidence {
2533            ToolReconciliationEvidence::Result(result) => {
2534                self.schemas
2535                    .validate_bounded(result.output_schema(), result.output())
2536                    .map_err(|source| ToolReconciliationCommitFailure::OutputSchema { source })?;
2537                Ok((
2538                    "tool_result_committed",
2539                    "tool-reconciliation-result-committed",
2540                    ToolInvocationTransition::ReconcileResult {
2541                        result: result.clone(),
2542                    },
2543                ))
2544            }
2545            ToolReconciliationEvidence::Error(error) => Ok((
2546                "tool_error_committed",
2547                "tool-reconciliation-error-committed",
2548                ToolInvocationTransition::ReconcileError {
2549                    error: error.clone(),
2550                },
2551            )),
2552        }
2553    }
2554
2555    async fn dispatch_model(
2556        &self,
2557        provider: Arc<dyn stateknot_core::Model>,
2558        context: ModelContext,
2559        request: ModelRequest,
2560        stream_sink: Option<Arc<dyn ModelEventSink>>,
2561    ) -> ModelTerminalEvidence {
2562        match request.response_mode() {
2563            ModelResponseMode::Complete => {
2564                Self::dispatch_complete_model(provider, context, request).await
2565            }
2566            ModelResponseMode::Streaming => {
2567                Self::dispatch_streaming_model(provider, context, request, stream_sink).await
2568            }
2569        }
2570    }
2571
2572    async fn dispatch_complete_model(
2573        provider: Arc<dyn stateknot_core::Model>,
2574        context: ModelContext,
2575        request: ModelRequest,
2576    ) -> ModelTerminalEvidence {
2577        let result = tokio::select! {
2578            biased;
2579            () = context.cancellation().cancelled() => {
2580                return ModelTerminalEvidence::Error(model_stop_error(
2581                    &context,
2582                    provider.descriptor(),
2583                    ModelStopReason::Cancelled,
2584                ));
2585            }
2586            () = tokio::time::sleep_until(tokio::time::Instant::from_std(context.deadline_instant())) => {
2587                return ModelTerminalEvidence::Error(model_stop_error(
2588                    &context,
2589                    provider.descriptor(),
2590                    ModelStopReason::DeadlineExceeded,
2591                ));
2592            }
2593            result = provider.invoke(context.clone(), request.clone()) => result,
2594        };
2595        match result {
2596            Ok(response)
2597                if response.provenance().attempt_id() == context.attempt_id()
2598                    && response
2599                        .validate_for(provider.descriptor(), &request)
2600                        .is_ok() =>
2601            {
2602                ModelTerminalEvidence::Response(response)
2603            }
2604            Ok(_) => ModelTerminalEvidence::Error(model_contract_error(
2605                &context,
2606                provider.descriptor(),
2607                ModelErrorPhase::Response,
2608                "runtime.model.invalid_response",
2609                "Model provider returned a response outside its registered contract.",
2610            )),
2611            Err(error)
2612                if error
2613                    .validate_for(&context, provider.descriptor(), &request)
2614                    .is_ok() =>
2615            {
2616                ModelTerminalEvidence::Error(error)
2617            }
2618            Err(_) => ModelTerminalEvidence::Error(model_contract_error(
2619                &context,
2620                provider.descriptor(),
2621                ModelErrorPhase::Response,
2622                "runtime.model.invalid_error",
2623                "Model provider returned failure evidence outside its registered contract.",
2624            )),
2625        }
2626    }
2627
2628    async fn dispatch_streaming_model(
2629        provider: Arc<dyn stateknot_core::Model>,
2630        context: ModelContext,
2631        request: ModelRequest,
2632        stream_sink: Option<Arc<dyn ModelEventSink>>,
2633    ) -> ModelTerminalEvidence {
2634        let Some(sink) = stream_sink else {
2635            return ModelTerminalEvidence::Error(model_contract_error(
2636                &context,
2637                provider.descriptor(),
2638                ModelErrorPhase::Preparation,
2639                "runtime.model.missing_stream_sink",
2640                "Streaming model execution has no durable event sink.",
2641            ));
2642        };
2643        let Ok(mut accumulator) =
2644            ModelEventAccumulator::new(context.attempt_id(), provider.descriptor(), &request)
2645        else {
2646            return ModelTerminalEvidence::Error(model_contract_error(
2647                &context,
2648                provider.descriptor(),
2649                ModelErrorPhase::Preparation,
2650                "runtime.model.invalid_stream_setup",
2651                "Streaming model execution could not satisfy its registered contract.",
2652            ));
2653        };
2654        let mut stream = provider.stream(context.clone(), request.clone());
2655        loop {
2656            let next = tokio::select! {
2657                biased;
2658                () = context.cancellation().cancelled() => {
2659                    return ModelTerminalEvidence::Error(model_stop_error(
2660                        &context,
2661                        provider.descriptor(),
2662                        ModelStopReason::Cancelled,
2663                    ));
2664                }
2665                () = tokio::time::sleep_until(tokio::time::Instant::from_std(context.deadline_instant())) => {
2666                    return ModelTerminalEvidence::Error(model_stop_error(
2667                        &context,
2668                        provider.descriptor(),
2669                        ModelStopReason::DeadlineExceeded,
2670                    ));
2671                }
2672                next = poll_fn(|task| stream.as_mut().poll_next(task)) => next,
2673            };
2674            match next {
2675                Some(Ok(event)) => {
2676                    if let Err(evidence) = Self::accept_model_stream_event(
2677                        provider.as_ref(),
2678                        &context,
2679                        &mut accumulator,
2680                        sink.as_ref(),
2681                        event,
2682                    )
2683                    .await
2684                    {
2685                        return evidence;
2686                    }
2687                }
2688                Some(Err(error))
2689                    if error
2690                        .validate_for(&context, provider.descriptor(), &request)
2691                        .is_ok() =>
2692                {
2693                    return ModelTerminalEvidence::Error(error);
2694                }
2695                Some(Err(_)) => {
2696                    return ModelTerminalEvidence::Error(model_contract_error(
2697                        &context,
2698                        provider.descriptor(),
2699                        ModelErrorPhase::Stream,
2700                        "runtime.model.invalid_stream_error",
2701                        "Model provider returned invalid stream failure evidence.",
2702                    ));
2703                }
2704                None => return finish_model_stream(&context, provider.descriptor(), accumulator),
2705            }
2706        }
2707    }
2708
2709    async fn accept_model_stream_event(
2710        provider: &dyn stateknot_core::Model,
2711        context: &ModelContext,
2712        accumulator: &mut ModelEventAccumulator<'_>,
2713        sink: &dyn ModelEventSink,
2714        event: ModelEvent,
2715    ) -> Result<(), ModelTerminalEvidence> {
2716        if accumulator.push(event.clone()).is_err() {
2717            return Err(ModelTerminalEvidence::Error(model_contract_error(
2718                context,
2719                provider.descriptor(),
2720                ModelErrorPhase::Stream,
2721                "runtime.model.invalid_stream_event",
2722                "Model provider emitted an invalid semantic stream event.",
2723            )));
2724        }
2725        let emitted = tokio::select! {
2726            biased;
2727            () = context.cancellation().cancelled() => {
2728                return Err(ModelTerminalEvidence::Error(model_stop_error(
2729                    context,
2730                    provider.descriptor(),
2731                    ModelStopReason::Cancelled,
2732                )));
2733            }
2734            () = tokio::time::sleep_until(tokio::time::Instant::from_std(context.deadline_instant())) => {
2735                return Err(ModelTerminalEvidence::Error(model_stop_error(
2736                    context,
2737                    provider.descriptor(),
2738                    ModelStopReason::DeadlineExceeded,
2739                )));
2740            }
2741            emitted = sink.emit(event) => emitted,
2742        };
2743        emitted.map_err(|_| {
2744            ModelTerminalEvidence::Error(model_contract_error(
2745                context,
2746                provider.descriptor(),
2747                ModelErrorPhase::Stream,
2748                "runtime.model.stream_sink_failed",
2749                "The durable model stream sink did not accept an event.",
2750            ))
2751        })
2752    }
2753
2754    async fn dispatch_tool(
2755        &self,
2756        provider: Arc<dyn stateknot_core::ErasedTool>,
2757        context: ToolContext,
2758        input: stateknot_core::ToolInput,
2759    ) -> ToolTerminalEvidence {
2760        let result = tokio::select! {
2761            biased;
2762            () = context.cancellation().cancelled() => {
2763                return ToolTerminalEvidence::Error(tool_stop_error(
2764                    &context,
2765                    provider.descriptor(),
2766                    ToolStopReason::Cancelled,
2767                ));
2768            }
2769            () = tokio::time::sleep_until(tokio::time::Instant::from_std(context.deadline_instant())) => {
2770                return ToolTerminalEvidence::Error(tool_stop_error(
2771                    &context,
2772                    provider.descriptor(),
2773                    ToolStopReason::DeadlineExceeded,
2774                ));
2775            }
2776            result = provider.call(context.clone(), input) => result,
2777        };
2778        match result {
2779            Ok(result) if result.validate_for(&context, provider.descriptor()).is_ok() => {
2780                ToolTerminalEvidence::Result(result)
2781            }
2782            Ok(_) => ToolTerminalEvidence::Error(tool_contract_error(
2783                &context,
2784                provider.descriptor(),
2785                ToolErrorPhase::Result,
2786                true,
2787                "runtime.tool.invalid_result",
2788                "Tool returned a result outside its registered contract.",
2789            )),
2790            Err(error) if error.validate_for(&context, provider.descriptor()).is_ok() => {
2791                ToolTerminalEvidence::Error(error)
2792            }
2793            Err(_) => ToolTerminalEvidence::Error(tool_contract_error(
2794                &context,
2795                provider.descriptor(),
2796                ToolErrorPhase::Execution,
2797                false,
2798                "runtime.tool.invalid_error",
2799                "Tool returned failure evidence outside its registered contract.",
2800            )),
2801        }
2802    }
2803
2804    async fn advance_model_with_retry(
2805        &self,
2806        append: JournalAppend,
2807        expected: &ModelInvocation,
2808        transition: ModelInvocationTransition,
2809        child_budget: Option<Digest>,
2810    ) -> Result<ModelInvocationCommitOutcome, StoreError> {
2811        let mut attempt = 1_u8;
2812        loop {
2813            match self
2814                .store
2815                .advance_model_invocation_with_child_budget(
2816                    append.clone(),
2817                    &expected.head(),
2818                    transition.clone(),
2819                    child_budget,
2820                )
2821                .await
2822            {
2823                Ok(outcome) => return Ok(outcome),
2824                Err(error)
2825                    if attempt < self.options.maximum_mutation_attempts()
2826                        && error.is_retryable() =>
2827                {
2828                    tokio::time::sleep(exponential_backoff(
2829                        self.options.mutation_retry_initial_delay(),
2830                        attempt,
2831                    ))
2832                    .await;
2833                    attempt = attempt.saturating_add(1);
2834                }
2835                Err(error) => return Err(error),
2836            }
2837        }
2838    }
2839
2840    async fn advance_tool_with_retry(
2841        &self,
2842        append: JournalAppend,
2843        expected: &ToolInvocation,
2844        transition: ToolInvocationTransition,
2845        child_budget: Option<Digest>,
2846    ) -> Result<ToolInvocationCommitOutcome, StoreError> {
2847        let mut attempt = 1_u8;
2848        loop {
2849            match self
2850                .store
2851                .advance_tool_invocation_with_child_budget(
2852                    append.clone(),
2853                    &expected.head(),
2854                    transition.clone(),
2855                    child_budget,
2856                )
2857                .await
2858            {
2859                Ok(outcome) => return Ok(outcome),
2860                Err(error)
2861                    if attempt < self.options.maximum_mutation_attempts()
2862                        && error.is_retryable() =>
2863                {
2864                    tokio::time::sleep(exponential_backoff(
2865                        self.options.mutation_retry_initial_delay(),
2866                        attempt,
2867                    ))
2868                    .await;
2869                    attempt = attempt.saturating_add(1);
2870                }
2871                Err(error) => return Err(error),
2872            }
2873        }
2874    }
2875
2876    fn event_payload(
2877        &self,
2878        kind: &'static str,
2879        operation: &'static str,
2880        binding_kind: &'static str,
2881        invocation_id: InvocationId,
2882        attempt_id: AttemptId,
2883        intent_digest: Digest,
2884    ) -> Result<JournalPayload, InvocationEventPayloadError> {
2885        let data = BoundedJson::try_from_value(json!({
2886            "operation": operation,
2887            "binding_kind": binding_kind,
2888            "invocation_id": invocation_id.to_string(),
2889            "attempt_id": attempt_id.to_string(),
2890            "intent_digest": digest_hex(intent_digest)
2891        }))
2892        .map_err(|_| InvocationEventPayloadError::Invalid)?;
2893        self.schemas
2894            .validate_bounded(&self.event_schema, &data)
2895            .map_err(|_| InvocationEventPayloadError::Invalid)?;
2896        JournalPayload::new(
2897            self.event_schema.clone(),
2898            JournalEventKind::new(kind).map_err(|_| InvocationEventPayloadError::Invalid)?,
2899            data,
2900        )
2901        .map_err(|_| InvocationEventPayloadError::Invalid)
2902    }
2903}
2904
2905/// Startup failure for the durable invocation executor.
2906#[derive(Clone, Debug, Eq, Error, PartialEq)]
2907#[non_exhaustive]
2908pub enum DurableInvocationExecutorBuildError {
2909    /// Embedded release schema was malformed.
2910    #[error(transparent)]
2911    EventSchema(#[from] StandardInvocationExecutionSchemaError),
2912    /// Frozen offline schema registry omitted the exact standard event schema.
2913    #[error("invocation execution event schema is absent from the frozen registry")]
2914    MissingEventSchema,
2915}
2916
2917/// Failure before or around one model attempt dispatch.
2918#[derive(Debug, Error)]
2919#[non_exhaustive]
2920pub enum ModelAttemptExecutionError {
2921    /// Handoff invariants changed before execution.
2922    #[error(transparent)]
2923    Handoff(#[from] InvocationAttemptHandoffError),
2924    /// No exact provider binding was installed.
2925    #[error(transparent)]
2926    Registry(#[from] ModelProviderRegistryError),
2927    /// Store read/start mutation failed before provider I/O.
2928    #[error(transparent)]
2929    Store(#[from] StoreError),
2930    /// Trusted run provenance crossed the retained fence.
2931    #[error("stored run provenance does not match the model attempt fence")]
2932    RunProvenanceMismatch,
2933    /// Runtime clock observation failed.
2934    #[error(transparent)]
2935    Clock(#[from] InvocationClockError),
2936    /// Trusted admission/accounting provider failed.
2937    #[error(transparent)]
2938    BudgetProvider(#[from] InvocationBudgetProviderError),
2939    /// Remaining capacity cannot admit the exact request snapshot.
2940    #[error("remaining run budget cannot admit the model attempt")]
2941    BudgetInsufficient,
2942    /// Finite model context could not be constructed.
2943    #[error(transparent)]
2944    Context(#[from] ModelContextError),
2945    /// Cancellation or deadline won before the durable start.
2946    #[error("model attempt stopped before durable start: {reason:?}")]
2947    StoppedBeforeStart {
2948        /// Deterministic stop reason.
2949        reason: ModelStopReason,
2950    },
2951    /// Standard public-safe journal payload could not be constructed.
2952    #[error("model invocation journal payload is invalid")]
2953    EventPayload(#[from] InvocationEventPayloadError),
2954    /// Worker journal append could not be constructed.
2955    #[error("model invocation worker journal append is invalid")]
2956    JournalAppend,
2957    /// Idempotent start recovery loaded another durable intent.
2958    #[error("recovered model invocation does not match the retained intent")]
2959    RecoveredInvocationMismatch,
2960    /// Another physical attempt advanced the invocation from this handoff.
2961    #[error("model invocation advanced under another physical attempt")]
2962    InvocationAdvanced,
2963    /// The store returned a future commit outcome unknown to this runtime.
2964    #[error("model invocation store outcome is unsupported by this runtime")]
2965    UnsupportedStoreOutcome,
2966    /// Provider I/O completed but terminal persistence needs a no-dispatch retry.
2967    #[error(transparent)]
2968    Terminal(#[from] ModelTerminalCommitError),
2969}
2970
2971/// Failure before or around one tool attempt dispatch.
2972#[derive(Debug, Error)]
2973#[non_exhaustive]
2974pub enum ToolAttemptExecutionError {
2975    /// Handoff invariants changed before execution.
2976    #[error(transparent)]
2977    Handoff(#[from] InvocationAttemptHandoffError),
2978    /// No exact executable tool binding was installed.
2979    #[error(transparent)]
2980    Registry(#[from] ToolProviderRegistryError),
2981    /// Store read/start mutation failed before tool I/O.
2982    #[error(transparent)]
2983    Store(#[from] StoreError),
2984    /// Trusted run provenance crossed the retained fence.
2985    #[error("stored run provenance does not match the tool attempt fence")]
2986    RunProvenanceMismatch,
2987    /// Runtime clock observation failed.
2988    #[error(transparent)]
2989    Clock(#[from] InvocationClockError),
2990    /// Trusted admission/accounting provider failed.
2991    #[error(transparent)]
2992    BudgetProvider(#[from] InvocationBudgetProviderError),
2993    /// Remaining capacity cannot admit the exact tool risk class.
2994    #[error("remaining run budget cannot admit the tool attempt")]
2995    BudgetInsufficient,
2996    /// Finite tool context could not be constructed.
2997    #[error(transparent)]
2998    Context(#[from] ToolContextError),
2999    /// Durable input no longer fits the trusted context/descriptor snapshot.
3000    #[error(transparent)]
3001    Input(#[from] ToolInputValidationError),
3002    /// Cancellation or deadline won before the durable start.
3003    #[error("tool attempt stopped before durable start: {reason:?}")]
3004    StoppedBeforeStart {
3005        /// Deterministic stop reason.
3006        reason: ToolStopReason,
3007    },
3008    /// Standard public-safe journal payload could not be constructed.
3009    #[error("tool invocation journal payload is invalid")]
3010    EventPayload(#[from] InvocationEventPayloadError),
3011    /// Worker journal append could not be constructed.
3012    #[error("tool invocation worker journal append is invalid")]
3013    JournalAppend,
3014    /// Idempotent start recovery loaded another durable intent.
3015    #[error("recovered tool invocation does not match the retained intent")]
3016    RecoveredInvocationMismatch,
3017    /// Another physical attempt advanced the invocation from this handoff.
3018    #[error("tool invocation advanced under another physical attempt")]
3019    InvocationAdvanced,
3020    /// The store returned a future commit outcome unknown to this runtime.
3021    #[error("tool invocation store outcome is unsupported by this runtime")]
3022    UnsupportedStoreOutcome,
3023    /// Tool I/O completed but terminal persistence needs a no-dispatch retry.
3024    #[error(transparent)]
3025    Terminal(#[from] ToolTerminalCommitError),
3026}
3027
3028/// Invalid standard invocation journal event data.
3029#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
3030#[non_exhaustive]
3031pub enum InvocationEventPayloadError {
3032    /// Bounds, pinned schema, event kind, or payload construction failed.
3033    #[error("standard invocation event payload is invalid")]
3034    Invalid,
3035}
3036
3037#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
3038#[error("invocation worker journal append is invalid")]
3039struct InvocationJournalAppendError;
3040
3041fn validate_run_provenance(
3042    provenance: &AgentResultProvenance,
3043    fence: &RunFence,
3044) -> Result<(), ModelAttemptExecutionError> {
3045    if provenance.tenant_id() != fence.tenant_id() || provenance.run_id() != fence.run_id() {
3046        return Err(ModelAttemptExecutionError::RunProvenanceMismatch);
3047    }
3048    Ok(())
3049}
3050
3051fn validate_tool_run_provenance(
3052    provenance: &AgentResultProvenance,
3053    fence: &RunFence,
3054) -> Result<(), ToolAttemptExecutionError> {
3055    if provenance.tenant_id() != fence.tenant_id() || provenance.run_id() != fence.run_id() {
3056        return Err(ToolAttemptExecutionError::RunProvenanceMismatch);
3057    }
3058    Ok(())
3059}
3060
3061fn validate_model_budget(
3062    remaining: &BudgetRemaining,
3063    request: &ModelRequest,
3064) -> Result<(), ModelAttemptExecutionError> {
3065    if remaining.model_attempts() == ExecutionCount::ZERO
3066        || remaining.model_turns() == ExecutionCount::ZERO
3067        || remaining.input_tokens() < request.limits().max_input_tokens()
3068        || remaining.output_tokens() < request.limits().max_output_tokens()
3069        || remaining.input_bytes() < request.content_bytes()
3070    {
3071        return Err(ModelAttemptExecutionError::BudgetInsufficient);
3072    }
3073    Ok(())
3074}
3075
3076fn validate_tool_budget(
3077    remaining: &BudgetRemaining,
3078    risk: ToolRisk,
3079) -> Result<(), ToolAttemptExecutionError> {
3080    if remaining.tool_calls() == ExecutionCount::ZERO
3081        || (risk != ToolRisk::ReadOnly && remaining.write_calls() == ExecutionCount::ZERO)
3082    {
3083        return Err(ToolAttemptExecutionError::BudgetInsufficient);
3084    }
3085    Ok(())
3086}
3087
3088fn worker_append(
3089    fence: &RunFence,
3090    head: stateknot_core::JournalHead,
3091    event_id: EventId,
3092    payload: JournalPayload,
3093) -> Result<JournalAppend, InvocationJournalAppendError> {
3094    let intent = JournalEventIntent::worker(
3095        fence.tenant_id().clone(),
3096        fence.run_id(),
3097        event_id,
3098        fence.clone(),
3099        payload,
3100    )
3101    .map_err(|_| InvocationJournalAppendError)?;
3102    JournalAppend::new(JournalExpectation::exact(head), intent)
3103        .map_err(|_| InvocationJournalAppendError)
3104}
3105
3106fn finish_model_stream(
3107    context: &ModelContext,
3108    descriptor: &stateknot_core::ModelDescriptor,
3109    accumulator: ModelEventAccumulator<'_>,
3110) -> ModelTerminalEvidence {
3111    match accumulator.finish() {
3112        Ok(response) => ModelTerminalEvidence::Response(response),
3113        Err(_) => ModelTerminalEvidence::Error(model_contract_error(
3114            context,
3115            descriptor,
3116            ModelErrorPhase::Stream,
3117            "runtime.model.incomplete_stream",
3118            "Model provider stream ended without a valid terminal event.",
3119        )),
3120    }
3121}
3122
3123fn model_stop_error(
3124    context: &ModelContext,
3125    descriptor: &stateknot_core::ModelDescriptor,
3126    reason: ModelStopReason,
3127) -> ModelError {
3128    let (category, code, message) = match reason {
3129        ModelStopReason::Cancelled => (
3130            FailureCategory::Cancelled,
3131            "runtime.model.cancelled",
3132            "Model attempt stopped after cancellation was requested.",
3133        ),
3134        ModelStopReason::DeadlineExceeded => (
3135            FailureCategory::DeadlineExceeded,
3136            "runtime.model.deadline_exceeded",
3137            "Model attempt exceeded its effective deadline.",
3138        ),
3139        _ => (
3140            FailureCategory::Internal,
3141            "runtime.model.unsupported_stop_reason",
3142            "Model attempt stopped for a reason unsupported by this runtime.",
3143        ),
3144    };
3145    ModelError::new(
3146        runtime_failure(category, code, message, RetryAdvice::Never),
3147        ModelErrorPhase::Dispatch,
3148        ModelErrorProvenance::new(
3149            context.attempt_id(),
3150            descriptor.metadata().identity().clone(),
3151            None,
3152            None,
3153            None,
3154        ),
3155        None,
3156    )
3157}
3158
3159fn model_contract_error(
3160    context: &ModelContext,
3161    descriptor: &stateknot_core::ModelDescriptor,
3162    phase: ModelErrorPhase,
3163    code: &'static str,
3164    message: &'static str,
3165) -> ModelError {
3166    ModelError::new(
3167        runtime_failure(FailureCategory::Internal, code, message, RetryAdvice::Never),
3168        phase,
3169        ModelErrorProvenance::new(
3170            context.attempt_id(),
3171            descriptor.metadata().identity().clone(),
3172            None,
3173            None,
3174            None,
3175        ),
3176        None,
3177    )
3178}
3179
3180fn tool_stop_error(
3181    context: &ToolContext,
3182    descriptor: &stateknot_core::ToolDescriptor,
3183    reason: ToolStopReason,
3184) -> ToolError {
3185    let risk = descriptor.semantics().risk();
3186    let (failure, effect) = if risk == ToolRisk::ReadOnly {
3187        let (category, code, message) = match reason {
3188            ToolStopReason::Cancelled => (
3189                FailureCategory::Cancelled,
3190                "runtime.tool.cancelled",
3191                "Tool attempt stopped after cancellation was requested.",
3192            ),
3193            ToolStopReason::DeadlineExceeded => (
3194                FailureCategory::DeadlineExceeded,
3195                "runtime.tool.deadline_exceeded",
3196                "Tool attempt exceeded its effective deadline.",
3197            ),
3198            _ => (
3199                FailureCategory::Internal,
3200                "runtime.tool.unsupported_stop_reason",
3201                "Tool attempt stopped for a reason unsupported by this runtime.",
3202            ),
3203        };
3204        (
3205            runtime_failure(category, code, message, RetryAdvice::Never),
3206            ToolExternalEffect::NotApplicable,
3207        )
3208    } else {
3209        let (code, message) = match reason {
3210            ToolStopReason::Cancelled => (
3211                "runtime.tool.cancelled_outcome_unknown",
3212                "A cancelled tool write may have changed external state and requires reconciliation.",
3213            ),
3214            ToolStopReason::DeadlineExceeded => (
3215                "runtime.tool.deadline_outcome_unknown",
3216                "A timed-out tool write may have changed external state and requires reconciliation.",
3217            ),
3218            _ => (
3219                "runtime.tool.unsupported_stop_reason_outcome_unknown",
3220                "A stopped tool write may have changed external state and requires reconciliation.",
3221            ),
3222        };
3223        (
3224            runtime_failure(
3225                FailureCategory::AmbiguousExternalOutcome,
3226                code,
3227                message,
3228                RetryAdvice::ReconcileFirst,
3229            ),
3230            ToolExternalEffect::Unknown,
3231        )
3232    };
3233    ToolError::new(
3234        failure,
3235        ToolErrorPhase::Execution,
3236        effect,
3237        ToolErrorProvenance::for_invocation(context, descriptor),
3238    )
3239    .expect("runtime stop evidence is constructed from valid risk/effect pairs")
3240}
3241
3242fn tool_contract_error(
3243    context: &ToolContext,
3244    descriptor: &stateknot_core::ToolDescriptor,
3245    phase: ToolErrorPhase,
3246    nominal_success: bool,
3247    code: &'static str,
3248    message: &'static str,
3249) -> ToolError {
3250    let risk = descriptor.semantics().risk();
3251    let (category, advice, effect) = match risk {
3252        ToolRisk::ReadOnly => (
3253            FailureCategory::Internal,
3254            RetryAdvice::Never,
3255            ToolExternalEffect::NotApplicable,
3256        ),
3257        ToolRisk::IdempotentWrite | ToolRisk::NonIdempotentWrite if nominal_success => (
3258            FailureCategory::Internal,
3259            RetryAdvice::Never,
3260            ToolExternalEffect::Applied,
3261        ),
3262        ToolRisk::IdempotentWrite | ToolRisk::NonIdempotentWrite => (
3263            FailureCategory::AmbiguousExternalOutcome,
3264            RetryAdvice::ReconcileFirst,
3265            ToolExternalEffect::Unknown,
3266        ),
3267    };
3268    ToolError::new(
3269        runtime_failure(category, code, message, advice),
3270        phase,
3271        effect,
3272        ToolErrorProvenance::for_invocation(context, descriptor),
3273    )
3274    .expect("runtime adapter-contract evidence is constructed from valid risk/effect pairs")
3275}
3276
3277fn runtime_failure(
3278    category: FailureCategory,
3279    code: &'static str,
3280    message: &'static str,
3281    advice: RetryAdvice,
3282) -> Failure {
3283    Failure::new(
3284        FailureId::generate(),
3285        category,
3286        FailureCode::new(code).expect("runtime failure code is static and valid"),
3287        FailureOrigin::new("stateknot.runtime")
3288            .expect("runtime failure origin is static and valid"),
3289        FailureMessage::new(message).expect("runtime failure message is static and valid"),
3290        advice,
3291    )
3292    .expect("runtime failure category and advice are statically compatible")
3293}
3294
3295fn exponential_backoff(initial: Duration, attempt: u8) -> Duration {
3296    let multiplier = 1_u32
3297        .checked_shl(u32::from(attempt.saturating_sub(1)))
3298        .unwrap_or(u32::MAX);
3299    initial
3300        .checked_mul(multiplier)
3301        .unwrap_or(MAX_MUTATION_RETRY_DELAY)
3302        .min(MAX_MUTATION_RETRY_DELAY)
3303}
3304
3305fn digest_hex(digest: Digest) -> String {
3306    const HEX: &[u8; 16] = b"0123456789abcdef";
3307    let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
3308    for byte in digest.as_bytes() {
3309        value.push(char::from(HEX[usize::from(byte >> 4)]));
3310        value.push(char::from(HEX[usize::from(byte & 0x0f)]));
3311    }
3312    value
3313}
3314
3315#[cfg(test)]
3316mod tests {
3317    use super::*;
3318
3319    #[test]
3320    fn event_ids_and_retry_options_reject_unsafe_shapes() {
3321        let event = EventId::generate();
3322        assert_eq!(
3323            InvocationAttemptEventIds::new(event, event),
3324            Err(InvocationAttemptEventIdsError::Duplicate)
3325        );
3326        assert_eq!(
3327            DurableInvocationExecutorOptions::new(0, Duration::from_millis(1)),
3328            Err(DurableInvocationExecutorOptionsError::InvalidMutationAttempts)
3329        );
3330        assert_eq!(
3331            DurableInvocationExecutorOptions::new(1, Duration::ZERO),
3332            Err(DurableInvocationExecutorOptionsError::InvalidMutationRetryDelay)
3333        );
3334    }
3335
3336    #[test]
3337    fn system_clock_produces_a_canonical_paired_observation() {
3338        let before = Instant::now();
3339        let observation = SystemInvocationClock.observe().unwrap();
3340        assert!(observation.observed_at().unix_micros() > 0);
3341        assert!(observation.observed_instant() >= before);
3342    }
3343}