1use std::{
7 error::Error as StdError,
8 fmt,
9 future::poll_fn,
10 sync::Arc,
11 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
12};
13
14use serde_json::json;
15use stateknot_core::{
16 AgentResultProvenance, AttemptId, BoundedJson, BoxFuture, BudgetRemaining, CancellationSignal,
17 Digest, DurationMillis, EventId, ExecutionCount, Failure, FailureCategory, FailureCode,
18 FailureId, FailureMessage, FailureOrigin, GraphSchemaValidationError, InvocationId,
19 JournalAppend, JournalEvent, JournalEventIntent, JournalEventKind, JournalExpectation,
20 JournalPayload, ModelContext, ModelContextError, ModelError, ModelErrorPhase,
21 ModelErrorProvenance, ModelEvent, ModelEventAccumulator, ModelInvocation,
22 ModelInvocationStatus, ModelInvocationTransition, ModelRequest, ModelResponse,
23 ModelResponseMode, ModelStopReason, RetryAdvice, RunFence, SchemaReference, Timestamp,
24 ToolContext, ToolContextError, ToolDescriptor, ToolError, ToolErrorPhase, ToolErrorProvenance,
25 ToolExternalEffect, ToolInputValidationError, ToolInvocation, ToolInvocationError,
26 ToolInvocationState, ToolInvocationStatus, ToolInvocationTransition, ToolProgressSink,
27 ToolReconciliationContext, ToolReconciliationContextError, ToolReconciliationObservation,
28 ToolReconciliationObservationError, ToolReconciliationProbeError, ToolResult, ToolRisk,
29 ToolStopReason,
30};
31use stateknot_store_postgres::{
32 ModelInvocationCommitOutcome, PostgresStore, StoreError, ToolInvocationCommitOutcome,
33};
34use thiserror::Error;
35
36use crate::{
37 JsonSchemaRegistry, ModelProviderRegistry, ModelProviderRegistryError,
38 StandardInvocationExecutionSchemaError, ToolProviderRegistry, ToolProviderRegistryError,
39 standard_invocation_execution_event_schema,
40};
41
42const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
43
44#[derive(Clone, Copy, Debug)]
46pub struct InvocationClockObservation {
47 observed_at: Timestamp,
48 observed_instant: Instant,
49}
50
51impl InvocationClockObservation {
52 #[must_use]
54 pub const fn new(observed_at: Timestamp, observed_instant: Instant) -> Self {
55 Self {
56 observed_at,
57 observed_instant,
58 }
59 }
60
61 #[must_use]
63 pub const fn observed_at(self) -> Timestamp {
64 self.observed_at
65 }
66
67 #[must_use]
69 pub const fn observed_instant(self) -> Instant {
70 self.observed_instant
71 }
72}
73
74pub trait InvocationClock: Send + Sync + 'static {
76 fn observe(&self) -> Result<InvocationClockObservation, InvocationClockError>;
78}
79
80#[derive(Clone, Copy, Debug, Default)]
82pub struct SystemInvocationClock;
83
84impl InvocationClock for SystemInvocationClock {
85 fn observe(&self) -> Result<InvocationClockObservation, InvocationClockError> {
86 let observed_instant = Instant::now();
87 let elapsed = SystemTime::now()
88 .duration_since(UNIX_EPOCH)
89 .map_err(|_| InvocationClockError::BeforeUnixEpoch)?;
90 let micros = i64::try_from(elapsed.as_micros())
91 .map_err(|_| InvocationClockError::TimestampOutOfRange)?;
92 let observed_at = Timestamp::from_unix_micros(micros)
93 .map_err(|_| InvocationClockError::TimestampOutOfRange)?;
94 Ok(InvocationClockObservation::new(
95 observed_at,
96 observed_instant,
97 ))
98 }
99}
100
101#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
103#[non_exhaustive]
104pub enum InvocationClockError {
105 #[error("invocation system clock precedes the Unix epoch")]
107 BeforeUnixEpoch,
108 #[error("invocation system clock is outside the supported timestamp range")]
110 TimestampOutOfRange,
111}
112
113#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
115#[non_exhaustive]
116pub enum InvocationBoundaryKind {
117 Model,
119 Tool,
121}
122
123#[derive(Clone, Debug)]
125pub struct InvocationBudgetContext {
126 provenance: AgentResultProvenance,
127 boundary: InvocationBoundaryKind,
128 invocation_id: InvocationId,
129 attempt_id: AttemptId,
130 intent_digest: Digest,
131 observed_at: Timestamp,
132}
133
134impl InvocationBudgetContext {
135 #[must_use]
137 pub const fn provenance(&self) -> &AgentResultProvenance {
138 &self.provenance
139 }
140
141 #[must_use]
143 pub const fn boundary(&self) -> InvocationBoundaryKind {
144 self.boundary
145 }
146
147 #[must_use]
149 pub const fn invocation_id(&self) -> InvocationId {
150 self.invocation_id
151 }
152
153 #[must_use]
155 pub const fn attempt_id(&self) -> AttemptId {
156 self.attempt_id
157 }
158
159 #[must_use]
161 pub const fn intent_digest(&self) -> Digest {
162 self.intent_digest
163 }
164
165 #[must_use]
167 pub const fn observed_at(&self) -> Timestamp {
168 self.observed_at
169 }
170}
171
172pub trait InvocationBudgetProvider: Send + Sync + 'static {
181 fn remaining(
183 &self,
184 context: InvocationBudgetContext,
185 ) -> BoxFuture<'_, Result<BudgetRemaining, InvocationBudgetProviderError>>;
186}
187
188type PrivateBudgetSource = dyn StdError + Send + Sync + 'static;
189
190fn deduct_child_budget(
191 remaining: BudgetRemaining,
192 account: Option<&stateknot_core::ChildRunBudgetAccount>,
193) -> Result<BudgetRemaining, StoreError> {
194 let Some(account) = account else {
195 return Ok(remaining);
196 };
197 if account
198 .children()
199 .iter()
200 .any(|entry| entry.settlement().is_none())
201 {
202 return Err(StoreError::UnsettledChildRuns);
203 }
204 remaining
205 .deduct_cumulative(
206 &account
207 .delegated_usage()
208 .map_err(|_| StoreError::IncompleteChildAccounting)?,
209 )
210 .map_err(|_| StoreError::IncompleteChildAccounting)
211}
212
213#[derive(Clone)]
215pub struct InvocationBudgetProviderError {
216 private_source: Arc<PrivateBudgetSource>,
217}
218
219impl InvocationBudgetProviderError {
220 #[must_use]
222 pub fn new<E>(source: E) -> Self
223 where
224 E: StdError + Send + Sync + 'static,
225 {
226 Self {
227 private_source: Arc::new(source),
228 }
229 }
230
231 #[must_use]
233 pub fn private_source(&self) -> &(dyn StdError + Send + Sync + 'static) {
234 self.private_source.as_ref()
235 }
236}
237
238impl fmt::Debug for InvocationBudgetProviderError {
239 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
240 formatter
241 .debug_struct("InvocationBudgetProviderError")
242 .field("has_private_source", &true)
243 .finish_non_exhaustive()
244 }
245}
246
247impl fmt::Display for InvocationBudgetProviderError {
248 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
249 formatter.write_str("invocation budget provider failed")
250 }
251}
252
253impl StdError for InvocationBudgetProviderError {}
254
255pub trait ModelEventSink: Send + Sync + 'static {
262 fn emit(&self, event: ModelEvent) -> BoxFuture<'_, Result<(), ModelEventSinkError>>;
264}
265
266type PrivateEventSinkSource = dyn StdError + Send + Sync + 'static;
267
268#[derive(Clone)]
270pub struct ModelEventSinkError {
271 private_source: Arc<PrivateEventSinkSource>,
272}
273
274impl ModelEventSinkError {
275 #[must_use]
277 pub fn new<E>(source: E) -> Self
278 where
279 E: StdError + Send + Sync + 'static,
280 {
281 Self {
282 private_source: Arc::new(source),
283 }
284 }
285
286 #[must_use]
288 pub fn private_source(&self) -> &(dyn StdError + Send + Sync + 'static) {
289 self.private_source.as_ref()
290 }
291}
292
293impl fmt::Debug for ModelEventSinkError {
294 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
295 formatter
296 .debug_struct("ModelEventSinkError")
297 .field("has_private_source", &true)
298 .finish_non_exhaustive()
299 }
300}
301
302impl fmt::Display for ModelEventSinkError {
303 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
304 formatter.write_str("model event sink failed")
305 }
306}
307
308impl StdError for ModelEventSinkError {}
309
310#[derive(Clone, Copy, Debug, Eq, PartialEq)]
312pub struct InvocationAttemptEventIds {
313 start: EventId,
314 terminal: EventId,
315}
316
317impl InvocationAttemptEventIds {
318 pub fn new(start: EventId, terminal: EventId) -> Result<Self, InvocationAttemptEventIdsError> {
324 if start == terminal {
325 return Err(InvocationAttemptEventIdsError::Duplicate);
326 }
327 Ok(Self { start, terminal })
328 }
329
330 #[must_use]
332 pub fn generate() -> Self {
333 Self {
334 start: EventId::generate(),
335 terminal: EventId::generate(),
336 }
337 }
338
339 #[must_use]
341 pub const fn start(self) -> EventId {
342 self.start
343 }
344
345 #[must_use]
347 pub const fn terminal(self) -> EventId {
348 self.terminal
349 }
350}
351
352#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
354#[non_exhaustive]
355pub enum InvocationAttemptEventIdsError {
356 #[error("invocation start and terminal event identities must be distinct")]
358 Duplicate,
359}
360
361#[derive(Clone)]
363pub struct ModelAttemptHandoff {
364 fence: RunFence,
365 invocation: ModelInvocation,
366 attempt_id: AttemptId,
367 events: InvocationAttemptEventIds,
368 cancellation: CancellationSignal,
369 stream_sink: Option<Arc<dyn ModelEventSink>>,
370}
371
372impl ModelAttemptHandoff {
373 pub fn new(
383 fence: RunFence,
384 invocation: ModelInvocation,
385 attempt_id: AttemptId,
386 events: InvocationAttemptEventIds,
387 cancellation: CancellationSignal,
388 stream_sink: Option<Arc<dyn ModelEventSink>>,
389 ) -> Result<Self, InvocationAttemptHandoffError> {
390 validate_model_handoff(&fence, &invocation, stream_sink.is_some())?;
391 Ok(Self {
392 fence,
393 invocation,
394 attempt_id,
395 events,
396 cancellation,
397 stream_sink,
398 })
399 }
400
401 #[must_use]
403 pub const fn fence(&self) -> &RunFence {
404 &self.fence
405 }
406
407 #[must_use]
409 pub const fn invocation(&self) -> &ModelInvocation {
410 &self.invocation
411 }
412
413 #[must_use]
415 pub const fn attempt_id(&self) -> AttemptId {
416 self.attempt_id
417 }
418
419 #[must_use]
421 pub const fn events(&self) -> InvocationAttemptEventIds {
422 self.events
423 }
424}
425
426impl fmt::Debug for ModelAttemptHandoff {
427 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
428 formatter
429 .debug_struct("ModelAttemptHandoff")
430 .field("fence", &self.fence)
431 .field("invocation_head", &self.invocation.head())
432 .field("attempt_id", &self.attempt_id)
433 .field("events", &self.events)
434 .field("streaming", &self.stream_sink.is_some())
435 .finish_non_exhaustive()
436 }
437}
438
439#[derive(Clone)]
441pub struct ToolAttemptHandoff {
442 fence: RunFence,
443 invocation: ToolInvocation,
444 attempt_id: AttemptId,
445 events: InvocationAttemptEventIds,
446 cancellation: CancellationSignal,
447 progress_sink: Option<Arc<dyn ToolProgressSink>>,
448}
449
450impl ToolAttemptHandoff {
451 pub fn new(
457 fence: RunFence,
458 invocation: ToolInvocation,
459 attempt_id: AttemptId,
460 events: InvocationAttemptEventIds,
461 cancellation: CancellationSignal,
462 progress_sink: Option<Arc<dyn ToolProgressSink>>,
463 ) -> Result<Self, InvocationAttemptHandoffError> {
464 validate_tool_handoff(&fence, &invocation)?;
465 Ok(Self {
466 fence,
467 invocation,
468 attempt_id,
469 events,
470 cancellation,
471 progress_sink,
472 })
473 }
474
475 #[must_use]
477 pub const fn fence(&self) -> &RunFence {
478 &self.fence
479 }
480
481 #[must_use]
483 pub const fn invocation(&self) -> &ToolInvocation {
484 &self.invocation
485 }
486
487 #[must_use]
489 pub const fn attempt_id(&self) -> AttemptId {
490 self.attempt_id
491 }
492
493 #[must_use]
495 pub const fn events(&self) -> InvocationAttemptEventIds {
496 self.events
497 }
498}
499
500impl fmt::Debug for ToolAttemptHandoff {
501 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
502 formatter
503 .debug_struct("ToolAttemptHandoff")
504 .field("fence", &self.fence)
505 .field("invocation_head", &self.invocation.head())
506 .field("attempt_id", &self.attempt_id)
507 .field("events", &self.events)
508 .field("progress", &self.progress_sink.is_some())
509 .finish_non_exhaustive()
510 }
511}
512
513#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
515#[non_exhaustive]
516pub enum InvocationAttemptHandoffError {
517 #[error("invocation attempt handoff crosses its retained worker fence")]
519 ScopeMismatch,
520 #[error("invocation attempt handoff does not reference a startable revision")]
522 InvocationNotStartable,
523 #[error("streaming model attempt requires a durable event sink")]
525 MissingStreamSink,
526 #[error("complete model attempt must not supply a stream event sink")]
528 UnexpectedStreamSink,
529}
530
531fn validate_model_handoff(
532 fence: &RunFence,
533 invocation: &ModelInvocation,
534 has_stream_sink: bool,
535) -> Result<(), InvocationAttemptHandoffError> {
536 if invocation.intent().tenant_id() != fence.tenant_id()
537 || invocation.intent().run_id() != fence.run_id()
538 || invocation.journal_head().tenant_id() != fence.tenant_id()
539 || invocation.journal_head().run_id() != fence.run_id()
540 {
541 return Err(InvocationAttemptHandoffError::ScopeMismatch);
542 }
543 if !matches!(
544 invocation.status(),
545 ModelInvocationStatus::Prepared | ModelInvocationStatus::Failed
546 ) {
547 return Err(InvocationAttemptHandoffError::InvocationNotStartable);
548 }
549 match (
550 invocation.intent().request().response_mode(),
551 has_stream_sink,
552 ) {
553 (ModelResponseMode::Streaming, false) => {
554 Err(InvocationAttemptHandoffError::MissingStreamSink)
555 }
556 (ModelResponseMode::Complete, true) => {
557 Err(InvocationAttemptHandoffError::UnexpectedStreamSink)
558 }
559 (ModelResponseMode::Complete, false) | (ModelResponseMode::Streaming, true) => Ok(()),
560 }
561}
562
563fn validate_tool_handoff(
564 fence: &RunFence,
565 invocation: &ToolInvocation,
566) -> Result<(), InvocationAttemptHandoffError> {
567 if invocation.intent().tenant_id() != fence.tenant_id()
568 || invocation.intent().run_id() != fence.run_id()
569 || invocation.journal_head().tenant_id() != fence.tenant_id()
570 || invocation.journal_head().run_id() != fence.run_id()
571 {
572 return Err(InvocationAttemptHandoffError::ScopeMismatch);
573 }
574 if !matches!(
575 invocation.status(),
576 ToolInvocationStatus::Prepared | ToolInvocationStatus::Failed
577 ) {
578 return Err(InvocationAttemptHandoffError::InvocationNotStartable);
579 }
580 Ok(())
581}
582
583#[derive(Clone, Copy, Debug, Eq, PartialEq)]
585pub struct DurableInvocationExecutorOptions {
586 maximum_mutation_attempts: u8,
587 mutation_retry_initial_delay: Duration,
588}
589
590impl DurableInvocationExecutorOptions {
591 pub const HARD_MAXIMUM_MUTATION_ATTEMPTS: u8 = 10;
593
594 pub fn new(
600 maximum_mutation_attempts: u8,
601 mutation_retry_initial_delay: Duration,
602 ) -> Result<Self, DurableInvocationExecutorOptionsError> {
603 if maximum_mutation_attempts == 0
604 || maximum_mutation_attempts > Self::HARD_MAXIMUM_MUTATION_ATTEMPTS
605 {
606 return Err(DurableInvocationExecutorOptionsError::InvalidMutationAttempts);
607 }
608 if mutation_retry_initial_delay.is_zero()
609 || mutation_retry_initial_delay > MAX_MUTATION_RETRY_DELAY
610 {
611 return Err(DurableInvocationExecutorOptionsError::InvalidMutationRetryDelay);
612 }
613 Ok(Self {
614 maximum_mutation_attempts,
615 mutation_retry_initial_delay,
616 })
617 }
618
619 #[must_use]
621 pub const fn maximum_mutation_attempts(self) -> u8 {
622 self.maximum_mutation_attempts
623 }
624
625 #[must_use]
627 pub const fn mutation_retry_initial_delay(self) -> Duration {
628 self.mutation_retry_initial_delay
629 }
630}
631
632impl Default for DurableInvocationExecutorOptions {
633 fn default() -> Self {
634 Self {
635 maximum_mutation_attempts: 3,
636 mutation_retry_initial_delay: Duration::from_millis(25),
637 }
638 }
639}
640
641#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
643#[non_exhaustive]
644pub enum DurableInvocationExecutorOptionsError {
645 #[error("invocation executor mutation attempt count is invalid")]
647 InvalidMutationAttempts,
648 #[error("invocation executor mutation retry delay is invalid")]
650 InvalidMutationRetryDelay,
651}
652
653#[derive(Clone, Copy, Debug, Eq, PartialEq)]
655#[non_exhaustive]
656pub enum ModelAttemptTerminalKind {
657 Response,
659 Error,
661}
662
663#[derive(Clone, Debug)]
665#[non_exhaustive]
666pub enum ModelAttemptOutcome {
667 Dispatched {
669 terminal: ModelAttemptTerminalKind,
671 invocation: ModelInvocation,
673 },
674 Recovered {
676 invocation: ModelInvocation,
678 },
679}
680
681#[derive(Clone, Copy, Debug, Eq, PartialEq)]
683#[non_exhaustive]
684pub enum ToolAttemptTerminalKind {
685 Result,
687 Error,
689}
690
691#[derive(Clone, Debug)]
693#[non_exhaustive]
694pub enum ToolAttemptOutcome {
695 Dispatched {
697 terminal: ToolAttemptTerminalKind,
699 invocation: ToolInvocation,
701 },
702 Recovered {
704 invocation: ToolInvocation,
706 },
707}
708
709#[derive(Debug)]
711#[non_exhaustive]
712pub enum ToolAttemptStartOutcome {
713 Started(Box<ToolDispatchHandoff>),
715 Recovered {
717 invocation: Box<ToolInvocation>,
719 },
720}
721
722impl ToolAttemptStartOutcome {
723 #[must_use]
725 pub fn invocation(&self) -> &ToolInvocation {
726 match self {
727 Self::Started(handoff) => handoff.invocation(),
728 Self::Recovered { invocation } => invocation.as_ref(),
729 }
730 }
731}
732
733pub struct ToolDispatchHandoff {
740 fence: RunFence,
741 invocation: ToolInvocation,
742 terminal_event_id: EventId,
743 provider: Arc<dyn stateknot_core::ErasedTool>,
744 context: ToolContext,
745}
746
747impl ToolDispatchHandoff {
748 #[must_use]
750 pub const fn fence(&self) -> &RunFence {
751 &self.fence
752 }
753
754 #[must_use]
756 pub const fn invocation(&self) -> &ToolInvocation {
757 &self.invocation
758 }
759}
760
761impl fmt::Debug for ToolDispatchHandoff {
762 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
763 formatter
764 .debug_struct("ToolDispatchHandoff")
765 .field("fence", &self.fence)
766 .field("invocation_head", &self.invocation.head())
767 .field("terminal_event_id", &self.terminal_event_id)
768 .finish_non_exhaustive()
769 }
770}
771
772#[derive(Clone)]
778pub struct ToolReconciliationAttemptHandoff {
779 fence: RunFence,
780 invocation: ToolInvocation,
781 event_id: EventId,
782 cancellation: CancellationSignal,
783}
784
785impl ToolReconciliationAttemptHandoff {
786 pub fn new(
793 fence: RunFence,
794 invocation: ToolInvocation,
795 event_id: EventId,
796 cancellation: CancellationSignal,
797 ) -> Result<Self, ToolReconciliationAttemptHandoffError> {
798 validate_tool_reconciliation_attempt_handoff(&fence, &invocation)?;
799 Ok(Self {
800 fence,
801 invocation,
802 event_id,
803 cancellation,
804 })
805 }
806
807 #[must_use]
809 pub const fn fence(&self) -> &RunFence {
810 &self.fence
811 }
812
813 #[must_use]
815 pub const fn invocation(&self) -> &ToolInvocation {
816 &self.invocation
817 }
818
819 #[must_use]
821 pub const fn event_id(&self) -> EventId {
822 self.event_id
823 }
824
825 pub fn rebind_fence(
831 mut self,
832 fence: RunFence,
833 ) -> Result<Self, ToolReconciliationAttemptHandoffError> {
834 if self.invocation.intent().tenant_id() != fence.tenant_id()
835 || self.invocation.intent().run_id() != fence.run_id()
836 {
837 return Err(ToolReconciliationAttemptHandoffError::ScopeMismatch);
838 }
839 self.fence = fence;
840 Ok(self)
841 }
842}
843
844impl fmt::Debug for ToolReconciliationAttemptHandoff {
845 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
846 formatter
847 .debug_struct("ToolReconciliationAttemptHandoff")
848 .field("fence", &self.fence)
849 .field("invocation_head", &self.invocation.head())
850 .field("event_id", &self.event_id)
851 .field("cancelled", &self.cancellation.is_cancelled())
852 .finish_non_exhaustive()
853 }
854}
855
856#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
858#[non_exhaustive]
859pub enum ToolReconciliationAttemptHandoffError {
860 #[error("tool reconciliation attempt crosses its retained worker fence")]
862 ScopeMismatch,
863 #[error("tool reconciliation attempt does not reference an unknown invocation")]
865 InvocationNotUnknown,
866 #[error("tool reconciliation attempt has no physical attempt identity")]
868 MissingAttempt,
869 #[error("tool reconciliation attempt is not declared by the descriptor")]
871 StatusQueryUnsupported,
872}
873
874fn validate_tool_reconciliation_attempt_handoff(
875 fence: &RunFence,
876 invocation: &ToolInvocation,
877) -> Result<(), ToolReconciliationAttemptHandoffError> {
878 if invocation.intent().tenant_id() != fence.tenant_id()
879 || invocation.intent().run_id() != fence.run_id()
880 || invocation.journal_head().tenant_id() != fence.tenant_id()
881 || invocation.journal_head().run_id() != fence.run_id()
882 {
883 return Err(ToolReconciliationAttemptHandoffError::ScopeMismatch);
884 }
885 if invocation.status() != ToolInvocationStatus::Unknown {
886 return Err(ToolReconciliationAttemptHandoffError::InvocationNotUnknown);
887 }
888 if invocation.attempt_id().is_none() {
889 return Err(ToolReconciliationAttemptHandoffError::MissingAttempt);
890 }
891 if !invocation
892 .intent()
893 .descriptor()
894 .semantics()
895 .supports_status_query()
896 {
897 return Err(ToolReconciliationAttemptHandoffError::StatusQueryUnsupported);
898 }
899 Ok(())
900}
901
902#[derive(Clone, Debug)]
904#[non_exhaustive]
905pub enum ToolReconciliationAttemptOutcome {
906 Resolved {
908 kind: ToolReconciliationKind,
910 invocation: ToolInvocation,
912 },
913 Pending {
915 retry_after: DurationMillis,
917 invocation: ToolInvocation,
919 },
920}
921
922impl ToolReconciliationAttemptOutcome {
923 #[must_use]
925 pub const fn invocation(&self) -> &ToolInvocation {
926 match self {
927 Self::Resolved { invocation, .. } | Self::Pending { invocation, .. } => invocation,
928 }
929 }
930}
931
932#[derive(Clone, Copy, Debug, Eq, PartialEq)]
934#[non_exhaustive]
935pub enum ToolReconciliationKind {
936 Result,
938 Error,
940}
941
942#[derive(Clone)]
943enum ToolReconciliationEvidence {
944 Result(ToolResult),
945 Error(ToolError),
946}
947
948#[derive(Clone)]
954pub struct ToolReconciliationHandoff {
955 fence: RunFence,
956 invocation: ToolInvocation,
957 event_id: EventId,
958 evidence: ToolReconciliationEvidence,
959}
960
961impl ToolReconciliationHandoff {
962 pub fn result(
969 fence: RunFence,
970 invocation: ToolInvocation,
971 event_id: EventId,
972 result: ToolResult,
973 ) -> Result<Self, ToolReconciliationHandoffError> {
974 validate_tool_reconciliation_scope(&fence, &invocation)?;
975 invocation
976 .validate_reconciliation_result(&result)
977 .map_err(ToolReconciliationHandoffError::invalid_evidence)?;
978 Ok(Self {
979 fence,
980 invocation,
981 event_id,
982 evidence: ToolReconciliationEvidence::Result(result),
983 })
984 }
985
986 pub fn error(
996 fence: RunFence,
997 invocation: ToolInvocation,
998 event_id: EventId,
999 error: ToolError,
1000 ) -> Result<Self, ToolReconciliationHandoffError> {
1001 validate_tool_reconciliation_scope(&fence, &invocation)?;
1002 invocation
1003 .validate_reconciliation_error(&error)
1004 .map_err(ToolReconciliationHandoffError::invalid_evidence)?;
1005 Ok(Self {
1006 fence,
1007 invocation,
1008 event_id,
1009 evidence: ToolReconciliationEvidence::Error(error),
1010 })
1011 }
1012
1013 #[must_use]
1015 pub const fn kind(&self) -> ToolReconciliationKind {
1016 match &self.evidence {
1017 ToolReconciliationEvidence::Result(_) => ToolReconciliationKind::Result,
1018 ToolReconciliationEvidence::Error(_) => ToolReconciliationKind::Error,
1019 }
1020 }
1021
1022 #[must_use]
1024 pub const fn fence(&self) -> &RunFence {
1025 &self.fence
1026 }
1027
1028 #[must_use]
1030 pub const fn invocation(&self) -> &ToolInvocation {
1031 &self.invocation
1032 }
1033
1034 #[must_use]
1036 pub const fn event_id(&self) -> EventId {
1037 self.event_id
1038 }
1039
1040 pub fn rebind_fence(mut self, fence: RunFence) -> Result<Self, ToolReconciliationHandoffError> {
1049 validate_tool_reconciliation_scope(&fence, &self.invocation)?;
1050 self.fence = fence;
1051 Ok(self)
1052 }
1053}
1054
1055impl fmt::Debug for ToolReconciliationHandoff {
1056 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1057 formatter
1058 .debug_struct("ToolReconciliationHandoff")
1059 .field("fence", &self.fence)
1060 .field("invocation_head", &self.invocation.head())
1061 .field("event_id", &self.event_id)
1062 .field("kind", &self.kind())
1063 .finish_non_exhaustive()
1064 }
1065}
1066
1067#[derive(Clone, Debug, Eq, Error, PartialEq)]
1069#[non_exhaustive]
1070pub enum ToolReconciliationHandoffError {
1071 #[error("tool reconciliation handoff crosses its retained worker fence")]
1073 ScopeMismatch,
1074 #[error("tool reconciliation evidence violates the durable invocation contract: {source}")]
1076 InvalidEvidence {
1077 #[source]
1079 source: ToolInvocationError,
1080 },
1081}
1082
1083impl ToolReconciliationHandoffError {
1084 fn invalid_evidence(source: ToolInvocationError) -> Self {
1085 Self::InvalidEvidence { source }
1086 }
1087}
1088
1089fn validate_tool_reconciliation_scope(
1090 fence: &RunFence,
1091 invocation: &ToolInvocation,
1092) -> Result<(), ToolReconciliationHandoffError> {
1093 if invocation.intent().tenant_id() != fence.tenant_id()
1094 || invocation.intent().run_id() != fence.run_id()
1095 || invocation.journal_head().tenant_id() != fence.tenant_id()
1096 || invocation.journal_head().run_id() != fence.run_id()
1097 {
1098 return Err(ToolReconciliationHandoffError::ScopeMismatch);
1099 }
1100 Ok(())
1101}
1102
1103#[derive(Clone, Debug)]
1105#[non_exhaustive]
1106pub enum ToolReconciliationOutcome {
1107 Committed {
1109 kind: ToolReconciliationKind,
1111 event: JournalEvent,
1113 invocation: ToolInvocation,
1115 },
1116 Idempotent {
1118 kind: ToolReconciliationKind,
1120 event: JournalEvent,
1122 invocation: ToolInvocation,
1124 },
1125}
1126
1127impl ToolReconciliationOutcome {
1128 #[must_use]
1130 pub const fn kind(&self) -> ToolReconciliationKind {
1131 match self {
1132 Self::Committed { kind, .. } | Self::Idempotent { kind, .. } => *kind,
1133 }
1134 }
1135
1136 #[must_use]
1138 pub const fn event(&self) -> &JournalEvent {
1139 match self {
1140 Self::Committed { event, .. } | Self::Idempotent { event, .. } => event,
1141 }
1142 }
1143
1144 #[must_use]
1146 pub const fn invocation(&self) -> &ToolInvocation {
1147 match self {
1148 Self::Committed { invocation, .. } | Self::Idempotent { invocation, .. } => invocation,
1149 }
1150 }
1151
1152 #[must_use]
1154 pub const fn is_idempotent(&self) -> bool {
1155 matches!(self, Self::Idempotent { .. })
1156 }
1157}
1158
1159#[derive(Clone)]
1160#[allow(clippy::large_enum_variant)]
1161enum ModelTerminalEvidence {
1162 Response(ModelResponse),
1163 Error(ModelError),
1164}
1165
1166#[derive(Clone)]
1172pub struct ModelTerminalCommitHandoff {
1173 fence: RunFence,
1174 invocation: ModelInvocation,
1175 event_id: EventId,
1176 evidence: ModelTerminalEvidence,
1177}
1178
1179impl fmt::Debug for ModelTerminalCommitHandoff {
1180 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1181 formatter
1182 .debug_struct("ModelTerminalCommitHandoff")
1183 .field("fence", &self.fence)
1184 .field("invocation_head", &self.invocation.head())
1185 .field("event_id", &self.event_id)
1186 .field("terminal", &self.kind())
1187 .finish_non_exhaustive()
1188 }
1189}
1190
1191impl ModelTerminalCommitHandoff {
1192 #[must_use]
1194 pub const fn kind(&self) -> ModelAttemptTerminalKind {
1195 match &self.evidence {
1196 ModelTerminalEvidence::Response(_) => ModelAttemptTerminalKind::Response,
1197 ModelTerminalEvidence::Error(_) => ModelAttemptTerminalKind::Error,
1198 }
1199 }
1200
1201 #[must_use]
1203 pub const fn fence(&self) -> &RunFence {
1204 &self.fence
1205 }
1206
1207 pub fn rebind_fence(mut self, fence: RunFence) -> Result<Self, InvocationAttemptHandoffError> {
1217 if self.invocation.intent().tenant_id() != fence.tenant_id()
1218 || self.invocation.intent().run_id() != fence.run_id()
1219 {
1220 return Err(InvocationAttemptHandoffError::ScopeMismatch);
1221 }
1222 self.fence = fence;
1223 Ok(self)
1224 }
1225}
1226
1227#[derive(Clone)]
1228enum ToolTerminalEvidence {
1229 Result(ToolResult),
1230 Error(ToolError),
1231}
1232
1233#[derive(Clone)]
1238pub struct ToolTerminalCommitHandoff {
1239 fence: RunFence,
1240 invocation: ToolInvocation,
1241 event_id: EventId,
1242 evidence: ToolTerminalEvidence,
1243}
1244
1245impl fmt::Debug for ToolTerminalCommitHandoff {
1246 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1247 formatter
1248 .debug_struct("ToolTerminalCommitHandoff")
1249 .field("fence", &self.fence)
1250 .field("invocation_head", &self.invocation.head())
1251 .field("event_id", &self.event_id)
1252 .field("terminal", &self.kind())
1253 .finish_non_exhaustive()
1254 }
1255}
1256
1257impl ToolTerminalCommitHandoff {
1258 #[must_use]
1260 pub const fn kind(&self) -> ToolAttemptTerminalKind {
1261 match &self.evidence {
1262 ToolTerminalEvidence::Result(_) => ToolAttemptTerminalKind::Result,
1263 ToolTerminalEvidence::Error(_) => ToolAttemptTerminalKind::Error,
1264 }
1265 }
1266
1267 #[must_use]
1269 pub const fn fence(&self) -> &RunFence {
1270 &self.fence
1271 }
1272
1273 pub fn rebind_fence(mut self, fence: RunFence) -> Result<Self, InvocationAttemptHandoffError> {
1282 if self.invocation.intent().tenant_id() != fence.tenant_id()
1283 || self.invocation.intent().run_id() != fence.run_id()
1284 {
1285 return Err(InvocationAttemptHandoffError::ScopeMismatch);
1286 }
1287 self.fence = fence;
1288 Ok(self)
1289 }
1290}
1291
1292#[derive(Debug, Error)]
1294#[non_exhaustive]
1295pub enum InvocationTerminalCommitFailure {
1296 #[error(transparent)]
1298 Store(#[from] StoreError),
1299 #[error("terminal invocation event payload is invalid")]
1301 InvalidEventPayload,
1302 #[error("terminal invocation journal append is invalid")]
1304 InvalidJournalAppend,
1305 #[error("terminal invocation run journal is unavailable")]
1307 RunJournalUnavailable,
1308 #[error("terminal invocation does not reference an executing attempt")]
1310 InvalidInvocationState,
1311 #[error("terminal invocation store outcome is unsupported by this runtime")]
1313 UnsupportedStoreOutcome,
1314}
1315
1316#[derive(Debug, Error)]
1318#[error("model terminal invocation commit failed: {source}")]
1319pub struct ModelTerminalCommitError {
1320 #[source]
1321 source: InvocationTerminalCommitFailure,
1322 recovery: Box<ModelTerminalCommitHandoff>,
1323}
1324
1325impl ModelTerminalCommitError {
1326 #[must_use]
1328 pub const fn source_error(&self) -> &InvocationTerminalCommitFailure {
1329 &self.source
1330 }
1331
1332 #[must_use]
1334 pub fn into_recovery(self) -> ModelTerminalCommitHandoff {
1335 *self.recovery
1336 }
1337}
1338
1339#[derive(Debug, Error)]
1341#[error("tool terminal invocation commit failed: {source}")]
1342pub struct ToolTerminalCommitError {
1343 #[source]
1344 source: InvocationTerminalCommitFailure,
1345 recovery: Box<ToolTerminalCommitHandoff>,
1346}
1347
1348impl ToolTerminalCommitError {
1349 #[must_use]
1351 pub const fn source_error(&self) -> &InvocationTerminalCommitFailure {
1352 &self.source
1353 }
1354
1355 #[must_use]
1357 pub fn into_recovery(self) -> ToolTerminalCommitHandoff {
1358 *self.recovery
1359 }
1360}
1361
1362#[derive(Debug, Error)]
1364#[non_exhaustive]
1365pub enum ToolReconciliationCommitFailure {
1366 #[error(transparent)]
1368 Store(#[from] StoreError),
1369 #[error("tool reconciliation output schema validation failed: {source}")]
1371 OutputSchema {
1372 #[source]
1374 source: GraphSchemaValidationError,
1375 },
1376 #[error("tool reconciliation event payload is invalid")]
1378 InvalidEventPayload,
1379 #[error("tool reconciliation journal append is invalid")]
1381 InvalidJournalAppend,
1382 #[error("tool reconciliation run journal is unavailable")]
1384 RunJournalUnavailable,
1385 #[error("tool reconciliation does not reference an unknown physical attempt")]
1387 InvalidInvocationState,
1388 #[error("tool reconciliation store outcome is unsupported by this runtime")]
1390 UnsupportedStoreOutcome,
1391}
1392
1393#[derive(Debug, Error)]
1395#[error("tool reconciliation commit failed: {source}")]
1396pub struct ToolReconciliationCommitError {
1397 #[source]
1398 source: ToolReconciliationCommitFailure,
1399 recovery: Box<ToolReconciliationHandoff>,
1400}
1401
1402impl ToolReconciliationCommitError {
1403 #[must_use]
1405 pub const fn source_error(&self) -> &ToolReconciliationCommitFailure {
1406 &self.source
1407 }
1408
1409 #[must_use]
1411 pub fn into_recovery(self) -> ToolReconciliationHandoff {
1412 *self.recovery
1413 }
1414}
1415
1416#[derive(Debug, Error)]
1418#[non_exhaustive]
1419pub enum ToolReconciliationAttemptExecutionError {
1420 #[error(transparent)]
1422 Handoff(#[from] ToolReconciliationAttemptHandoffError),
1423 #[error(transparent)]
1425 Registry(#[from] ToolProviderRegistryError),
1426 #[error("installed tool binding does not implement automated reconciliation")]
1428 ProviderReconciliationUnsupported,
1429 #[error(transparent)]
1431 Store(#[from] StoreError),
1432 #[error("stored run provenance does not match the tool reconciliation fence")]
1434 RunProvenanceMismatch,
1435 #[error(transparent)]
1437 Clock(#[from] InvocationClockError),
1438 #[error("tool reconciliation deadline is outside the supported timestamp range")]
1440 DeadlineOutOfRange,
1441 #[error(transparent)]
1443 Context(#[from] ToolReconciliationContextError),
1444 #[error("tool reconciliation stopped: {reason:?}")]
1446 Stopped {
1447 reason: ToolStopReason,
1449 },
1450 #[error(transparent)]
1452 Probe(#[from] ToolReconciliationProbeError),
1453 #[error(transparent)]
1455 Observation(#[from] ToolReconciliationObservationError),
1456 #[error("tool reconciliation returned unresolved error evidence")]
1458 UnresolvedErrorEvidence,
1459 #[error("tool reconciliation observation is unsupported by this runtime")]
1461 UnsupportedObservation,
1462 #[error(transparent)]
1464 Evidence(#[from] ToolReconciliationHandoffError),
1465 #[error("recovered tool reconciliation invocation does not match the retained intent")]
1467 RecoveredInvocationMismatch,
1468 #[error("tool reconciliation invocation advanced under another attempt or state")]
1470 InvocationAdvanced,
1471 #[error(transparent)]
1473 Commit(#[from] ToolReconciliationCommitError),
1474}
1475
1476impl ToolReconciliationAttemptExecutionError {
1477 #[must_use]
1483 pub fn retry_advice(&self) -> RetryAdvice {
1484 let store_retry = || RetryAdvice::SafeAfter {
1485 delay: DurationMillis::new(100).expect("positive static retry delay"),
1486 };
1487 match self {
1488 Self::Probe(source) => source.failure().retry_advice(),
1489 Self::Store(source) if source.is_retryable() => store_retry(),
1490 Self::Commit(source)
1491 if matches!(
1492 source.source_error(),
1493 ToolReconciliationCommitFailure::Store(store) if store.is_retryable()
1494 ) =>
1495 {
1496 store_retry()
1497 }
1498 Self::Handoff(_)
1499 | Self::Registry(_)
1500 | Self::ProviderReconciliationUnsupported
1501 | Self::Store(_)
1502 | Self::RunProvenanceMismatch
1503 | Self::Clock(_)
1504 | Self::DeadlineOutOfRange
1505 | Self::Context(_)
1506 | Self::Stopped { .. }
1507 | Self::Observation(_)
1508 | Self::UnresolvedErrorEvidence
1509 | Self::UnsupportedObservation
1510 | Self::Evidence(_)
1511 | Self::RecoveredInvocationMismatch
1512 | Self::InvocationAdvanced
1513 | Self::Commit(_) => RetryAdvice::Never,
1514 }
1515 }
1516}
1517
1518#[derive(Clone)]
1520pub struct DurableInvocationExecutor {
1521 store: PostgresStore,
1522 schemas: JsonSchemaRegistry,
1523 event_schema: SchemaReference,
1524 models: ModelProviderRegistry,
1525 tools: ToolProviderRegistry,
1526 budget: Arc<dyn InvocationBudgetProvider>,
1527 clock: Arc<dyn InvocationClock>,
1528 options: DurableInvocationExecutorOptions,
1529}
1530
1531impl DurableInvocationExecutor {
1532 pub fn supports_tool_reconciliation(
1538 &self,
1539 descriptor: &ToolDescriptor,
1540 ) -> Result<bool, ToolProviderRegistryError> {
1541 self.tools
1542 .resolve(descriptor)
1543 .map(|provider| provider.supports_reconciliation())
1544 }
1545
1546 #[allow(clippy::too_many_arguments)]
1553 pub fn new(
1554 store: PostgresStore,
1555 schemas: JsonSchemaRegistry,
1556 models: ModelProviderRegistry,
1557 tools: ToolProviderRegistry,
1558 budget: Arc<dyn InvocationBudgetProvider>,
1559 options: DurableInvocationExecutorOptions,
1560 ) -> Result<Self, DurableInvocationExecutorBuildError> {
1561 Self::with_clock(
1562 store,
1563 schemas,
1564 models,
1565 tools,
1566 budget,
1567 Arc::new(SystemInvocationClock),
1568 options,
1569 )
1570 }
1571
1572 #[allow(clippy::too_many_arguments)]
1578 pub fn with_clock(
1579 store: PostgresStore,
1580 schemas: JsonSchemaRegistry,
1581 models: ModelProviderRegistry,
1582 tools: ToolProviderRegistry,
1583 budget: Arc<dyn InvocationBudgetProvider>,
1584 clock: Arc<dyn InvocationClock>,
1585 options: DurableInvocationExecutorOptions,
1586 ) -> Result<Self, DurableInvocationExecutorBuildError> {
1587 let (event_schema, _) = standard_invocation_execution_event_schema()?;
1588 if !schemas.contains(&event_schema) {
1589 return Err(DurableInvocationExecutorBuildError::MissingEventSchema);
1590 }
1591 Ok(Self {
1592 store,
1593 schemas,
1594 event_schema,
1595 models,
1596 tools,
1597 budget,
1598 clock,
1599 options,
1600 })
1601 }
1602
1603 async fn prepare_model_context(
1604 &self,
1605 handoff: &ModelAttemptHandoff,
1606 ) -> Result<(ModelContext, Option<Digest>), ModelAttemptExecutionError> {
1607 let run = self
1608 .store
1609 .load_run(handoff.fence.tenant_id(), handoff.fence.run_id())
1610 .await?;
1611 let provenance = run.lifecycle().provenance().clone();
1612 validate_run_provenance(&provenance, &handoff.fence)?;
1613 let account = self
1614 .store
1615 .load_child_budget_account(handoff.fence.tenant_id(), handoff.fence.run_id())
1616 .await?;
1617 let child_budget = account
1618 .as_ref()
1619 .map(stateknot_core::ChildRunBudgetAccount::digest);
1620 let observation = self.clock.observe()?;
1621 let remaining = self
1622 .budget
1623 .remaining(InvocationBudgetContext {
1624 provenance: provenance.clone(),
1625 boundary: InvocationBoundaryKind::Model,
1626 invocation_id: handoff.invocation.intent().invocation_id(),
1627 attempt_id: handoff.attempt_id,
1628 intent_digest: handoff.invocation.intent().intent_digest(),
1629 observed_at: observation.observed_at,
1630 })
1631 .await?;
1632 validate_model_budget(&remaining, handoff.invocation.intent().request())?;
1633 let remaining = deduct_child_budget(remaining, account.as_ref()).map_err(|error| {
1634 ModelAttemptExecutionError::BudgetProvider(InvocationBudgetProviderError::new(error))
1635 })?;
1636 validate_model_budget(&remaining, handoff.invocation.intent().request())?;
1637 let context = ModelContext::new(
1638 provenance.tenant_id().clone(),
1639 provenance.run_id(),
1640 provenance.thread_id(),
1641 handoff.attempt_id,
1642 remaining,
1643 observation.observed_at,
1644 observation.observed_instant,
1645 handoff.cancellation.clone(),
1646 )?;
1647 if let Some(reason) = context.stop_reason_at(Instant::now()) {
1648 return Err(ModelAttemptExecutionError::StoppedBeforeStart { reason });
1649 }
1650 Ok((context, child_budget))
1651 }
1652
1653 async fn prepare_tool_context(
1654 &self,
1655 handoff: &ToolAttemptHandoff,
1656 ) -> Result<(ToolContext, Option<Digest>), ToolAttemptExecutionError> {
1657 let run = self
1658 .store
1659 .load_run(handoff.fence.tenant_id(), handoff.fence.run_id())
1660 .await?;
1661 let provenance = run.lifecycle().provenance().clone();
1662 validate_tool_run_provenance(&provenance, &handoff.fence)?;
1663 let account = self
1664 .store
1665 .load_child_budget_account(handoff.fence.tenant_id(), handoff.fence.run_id())
1666 .await?;
1667 let child_budget = account
1668 .as_ref()
1669 .map(stateknot_core::ChildRunBudgetAccount::digest);
1670 let observation = self.clock.observe()?;
1671 let remaining = self
1672 .budget
1673 .remaining(InvocationBudgetContext {
1674 provenance: provenance.clone(),
1675 boundary: InvocationBoundaryKind::Tool,
1676 invocation_id: handoff.invocation.intent().invocation_id(),
1677 attempt_id: handoff.attempt_id,
1678 intent_digest: handoff.invocation.intent().intent_digest(),
1679 observed_at: observation.observed_at,
1680 })
1681 .await?;
1682 let remaining = deduct_child_budget(remaining, account.as_ref()).map_err(|error| {
1683 ToolAttemptExecutionError::BudgetProvider(InvocationBudgetProviderError::new(error))
1684 })?;
1685 validate_tool_budget(
1686 &remaining,
1687 handoff.invocation.intent().descriptor().semantics().risk(),
1688 )?;
1689 let context = Self::build_tool_context(handoff, &provenance, remaining, observation)?;
1690 handoff
1691 .invocation
1692 .intent()
1693 .input()
1694 .validate_for(&context, handoff.invocation.intent().descriptor())?;
1695 if let Some(reason) = context.stop_reason_at(Instant::now()) {
1696 return Err(ToolAttemptExecutionError::StoppedBeforeStart { reason });
1697 }
1698 Ok((context, child_budget))
1699 }
1700
1701 fn build_tool_context(
1702 handoff: &ToolAttemptHandoff,
1703 provenance: &AgentResultProvenance,
1704 remaining: BudgetRemaining,
1705 observation: InvocationClockObservation,
1706 ) -> Result<ToolContext, ToolContextError> {
1707 let intent = handoff.invocation.intent();
1708 let context = if let Some(progress_sink) = &handoff.progress_sink {
1709 ToolContext::new_with_progress(
1710 provenance.tenant_id().clone(),
1711 provenance.run_id(),
1712 provenance.thread_id(),
1713 intent.invocation_id(),
1714 handoff.attempt_id,
1715 intent.descriptor(),
1716 remaining,
1717 intent.effective_limits().timeout(),
1718 observation.observed_at,
1719 observation.observed_instant,
1720 handoff.cancellation.clone(),
1721 Arc::clone(progress_sink),
1722 )?
1723 } else {
1724 ToolContext::new(
1725 provenance.tenant_id().clone(),
1726 provenance.run_id(),
1727 provenance.thread_id(),
1728 intent.invocation_id(),
1729 handoff.attempt_id,
1730 intent.descriptor(),
1731 remaining,
1732 intent.effective_limits().timeout(),
1733 observation.observed_at,
1734 observation.observed_instant,
1735 handoff.cancellation.clone(),
1736 )?
1737 };
1738 Ok(context
1739 .with_durable_origin_event(handoff.events.start())
1740 .narrow_artifact_limits(intent.effective_limits()))
1741 }
1742
1743 async fn prepare_tool_reconciliation_context(
1744 &self,
1745 handoff: &ToolReconciliationAttemptHandoff,
1746 invocation: &ToolInvocation,
1747 ) -> Result<ToolReconciliationContext, ToolReconciliationAttemptExecutionError> {
1748 let run = self
1749 .store
1750 .load_run(handoff.fence.tenant_id(), handoff.fence.run_id())
1751 .await?;
1752 let provenance = run.lifecycle().provenance();
1753 if provenance.tenant_id() != handoff.fence.tenant_id()
1754 || provenance.run_id() != handoff.fence.run_id()
1755 {
1756 return Err(ToolReconciliationAttemptExecutionError::RunProvenanceMismatch);
1757 }
1758 let observation = self.clock.observe()?;
1759 let intent = invocation.intent();
1760 let timeout_micros = intent
1761 .effective_limits()
1762 .timeout()
1763 .as_i64()
1764 .checked_mul(1_000)
1765 .ok_or(ToolReconciliationAttemptExecutionError::DeadlineOutOfRange)?;
1766 let fallback_deadline_micros = observation
1767 .observed_at()
1768 .unix_micros()
1769 .checked_add(timeout_micros)
1770 .ok_or(ToolReconciliationAttemptExecutionError::DeadlineOutOfRange)?;
1771 let fallback_deadline = Timestamp::from_unix_micros(fallback_deadline_micros)
1772 .map_err(|_| ToolReconciliationAttemptExecutionError::DeadlineOutOfRange)?;
1773 let run_deadline = match self
1774 .store
1775 .load_agent_admission(handoff.fence.tenant_id(), handoff.fence.run_id())
1776 .await
1777 {
1778 Ok(stored) => stored.admission().intent().budget().deadline(),
1779 Err(StoreError::AgentAdmissionNotFound) => fallback_deadline,
1780 Err(source) => return Err(source.into()),
1781 };
1782 let recovery_handle = match invocation.state() {
1783 ToolInvocationState::Unknown { error } => error.recovery_handle().cloned(),
1784 _ => None,
1785 };
1786 let origin_event_id = invocation
1787 .previous()
1788 .filter(|head| head.status() == stateknot_core::ToolInvocationStatus::Executing)
1789 .map(|head| head.journal_head().event_id())
1790 .ok_or(ToolReconciliationAttemptExecutionError::InvocationAdvanced)?;
1791 let context = ToolReconciliationContext::new(
1792 provenance.tenant_id().clone(),
1793 provenance.run_id(),
1794 provenance.thread_id(),
1795 intent.invocation_id(),
1796 invocation
1797 .attempt_id()
1798 .ok_or(ToolReconciliationAttemptExecutionError::InvocationAdvanced)?,
1799 intent.descriptor(),
1800 intent.effective_limits().timeout(),
1801 observation.observed_at(),
1802 observation.observed_instant(),
1803 run_deadline,
1804 handoff.cancellation.clone(),
1805 )?
1806 .with_durable_recovery(origin_event_id, recovery_handle)
1807 .narrow_artifact_limits(intent.effective_limits());
1808 if let Some(reason) = context.stop_reason_at(Instant::now()) {
1809 return Err(ToolReconciliationAttemptExecutionError::Stopped { reason });
1810 }
1811 Ok(context)
1812 }
1813
1814 async fn recover_model_if_started(
1815 &self,
1816 handoff: &ModelAttemptHandoff,
1817 ) -> Result<Option<ModelAttemptOutcome>, ModelAttemptExecutionError> {
1818 let current = self
1819 .store
1820 .load_model_invocation(
1821 handoff.fence.tenant_id(),
1822 handoff.fence.run_id(),
1823 handoff.invocation.intent().invocation_id(),
1824 )
1825 .await?;
1826 if current.intent() != handoff.invocation.intent() {
1827 return Err(ModelAttemptExecutionError::RecoveredInvocationMismatch);
1828 }
1829 if current.head() == handoff.invocation.head() {
1830 return Ok(None);
1831 }
1832 if current.attempt_id() != Some(handoff.attempt_id) {
1833 return Err(ModelAttemptExecutionError::InvocationAdvanced);
1834 }
1835 Ok(Some(ModelAttemptOutcome::Recovered {
1836 invocation: current,
1837 }))
1838 }
1839
1840 async fn recover_tool_if_started(
1841 &self,
1842 handoff: &ToolAttemptHandoff,
1843 ) -> Result<Option<ToolAttemptOutcome>, ToolAttemptExecutionError> {
1844 let current = self
1845 .store
1846 .load_tool_invocation(
1847 handoff.fence.tenant_id(),
1848 handoff.fence.run_id(),
1849 handoff.invocation.intent().invocation_id(),
1850 )
1851 .await?;
1852 if current.intent() != handoff.invocation.intent() {
1853 return Err(ToolAttemptExecutionError::RecoveredInvocationMismatch);
1854 }
1855 if current.head() == handoff.invocation.head() {
1856 return Ok(None);
1857 }
1858 if current.attempt_id() != Some(handoff.attempt_id) {
1859 return Err(ToolAttemptExecutionError::InvocationAdvanced);
1860 }
1861 Ok(Some(ToolAttemptOutcome::Recovered {
1862 invocation: current,
1863 }))
1864 }
1865
1866 pub fn execute_model(
1873 &self,
1874 handoff: ModelAttemptHandoff,
1875 ) -> BoxFuture<'_, Result<ModelAttemptOutcome, ModelAttemptExecutionError>> {
1876 Box::pin(self.execute_model_inner(handoff))
1877 }
1878
1879 async fn execute_model_inner(
1880 &self,
1881 handoff: ModelAttemptHandoff,
1882 ) -> Result<ModelAttemptOutcome, ModelAttemptExecutionError> {
1883 validate_model_handoff(
1884 &handoff.fence,
1885 &handoff.invocation,
1886 handoff.stream_sink.is_some(),
1887 )?;
1888 if let Some(recovered) = self.recover_model_if_started(&handoff).await? {
1889 return Ok(recovered);
1890 }
1891 let provider = self
1892 .models
1893 .resolve(handoff.invocation.intent().descriptor())?;
1894 let (context, child_budget) = self.prepare_model_context(&handoff).await?;
1895
1896 let start_transition = ModelInvocationTransition::StartAttempt {
1897 attempt_id: handoff.attempt_id,
1898 };
1899 let start_payload = self.event_payload(
1900 "model-attempt-started",
1901 "model_attempt_started",
1902 "model",
1903 handoff.invocation.intent().invocation_id(),
1904 handoff.attempt_id,
1905 handoff.invocation.intent().intent_digest(),
1906 )?;
1907 let start_append = worker_append(
1908 &handoff.fence,
1909 handoff.invocation.journal_head().clone(),
1910 handoff.events.start,
1911 start_payload,
1912 )
1913 .map_err(|_| ModelAttemptExecutionError::JournalAppend)?;
1914 let start = self
1915 .advance_model_with_retry(
1916 start_append,
1917 &handoff.invocation,
1918 start_transition,
1919 child_budget,
1920 )
1921 .await?;
1922 let executing = match start {
1923 ModelInvocationCommitOutcome::Committed { invocation, .. } => invocation,
1924 ModelInvocationCommitOutcome::Idempotent { .. } => {
1925 let current = self
1926 .store
1927 .load_model_invocation(
1928 handoff.fence.tenant_id(),
1929 handoff.fence.run_id(),
1930 handoff.invocation.intent().invocation_id(),
1931 )
1932 .await?;
1933 if current.intent() != handoff.invocation.intent() {
1934 return Err(ModelAttemptExecutionError::RecoveredInvocationMismatch);
1935 }
1936 if current.attempt_id() != Some(handoff.attempt_id) {
1937 return Err(ModelAttemptExecutionError::InvocationAdvanced);
1938 }
1939 return Ok(ModelAttemptOutcome::Recovered {
1940 invocation: current,
1941 });
1942 }
1943 _ => return Err(ModelAttemptExecutionError::UnsupportedStoreOutcome),
1944 };
1945
1946 let evidence = match context.stop_reason_at(Instant::now()) {
1947 Some(reason) => ModelTerminalEvidence::Error(model_stop_error(
1948 &context,
1949 executing.intent().descriptor(),
1950 reason,
1951 )),
1952 None => {
1953 self.dispatch_model(
1954 provider,
1955 context,
1956 executing.intent().request().clone(),
1957 handoff.stream_sink,
1958 )
1959 .await
1960 }
1961 };
1962 let terminal = ModelTerminalCommitHandoff {
1963 fence: handoff.fence,
1964 invocation: executing,
1965 event_id: handoff.events.terminal,
1966 evidence,
1967 };
1968 self.commit_model_terminal(terminal)
1969 .await
1970 .map_err(ModelAttemptExecutionError::Terminal)
1971 }
1972
1973 pub fn commit_model_terminal(
1975 &self,
1976 handoff: ModelTerminalCommitHandoff,
1977 ) -> BoxFuture<'_, Result<ModelAttemptOutcome, ModelTerminalCommitError>> {
1978 Box::pin(self.commit_model_terminal_inner(handoff))
1979 }
1980
1981 async fn commit_model_terminal_inner(
1982 &self,
1983 handoff: ModelTerminalCommitHandoff,
1984 ) -> Result<ModelAttemptOutcome, ModelTerminalCommitError> {
1985 let terminal_kind = handoff.kind();
1986 let (operation, event_kind, transition) = match &handoff.evidence {
1987 ModelTerminalEvidence::Response(response) => (
1988 "model_response_committed",
1989 "model-response-committed",
1990 ModelInvocationTransition::RecordResponse {
1991 response: response.clone(),
1992 },
1993 ),
1994 ModelTerminalEvidence::Error(error) => (
1995 "model_error_committed",
1996 "model-error-committed",
1997 ModelInvocationTransition::RecordError {
1998 error: error.clone(),
1999 },
2000 ),
2001 };
2002 let Some(attempt_id) = handoff.invocation.attempt_id() else {
2003 return Err(ModelTerminalCommitError {
2004 source: InvocationTerminalCommitFailure::InvalidInvocationState,
2005 recovery: Box::new(handoff),
2006 });
2007 };
2008 let Ok(payload) = self.event_payload(
2009 event_kind,
2010 operation,
2011 "model",
2012 handoff.invocation.intent().invocation_id(),
2013 attempt_id,
2014 handoff.invocation.intent().intent_digest(),
2015 ) else {
2016 return Err(ModelTerminalCommitError {
2017 source: InvocationTerminalCommitFailure::InvalidEventPayload,
2018 recovery: Box::new(handoff),
2019 });
2020 };
2021 let run = match self
2022 .store
2023 .load_run(
2024 handoff.invocation.intent().tenant_id(),
2025 handoff.invocation.intent().run_id(),
2026 )
2027 .await
2028 {
2029 Ok(run) => run,
2030 Err(source) => {
2031 return Err(ModelTerminalCommitError {
2032 source: InvocationTerminalCommitFailure::Store(source),
2033 recovery: Box::new(handoff),
2034 });
2035 }
2036 };
2037 let Some(journal_head) = run.journal_head().cloned() else {
2038 return Err(ModelTerminalCommitError {
2039 source: InvocationTerminalCommitFailure::RunJournalUnavailable,
2040 recovery: Box::new(handoff),
2041 });
2042 };
2043 let Ok(append) = worker_append(&handoff.fence, journal_head, handoff.event_id, payload)
2044 else {
2045 return Err(ModelTerminalCommitError {
2046 source: InvocationTerminalCommitFailure::InvalidJournalAppend,
2047 recovery: Box::new(handoff),
2048 });
2049 };
2050 match self
2051 .advance_model_with_retry(append, &handoff.invocation, transition, None)
2052 .await
2053 {
2054 Ok(outcome) => Ok(ModelAttemptOutcome::Dispatched {
2055 terminal: terminal_kind,
2056 invocation: outcome.invocation().clone(),
2057 }),
2058 Err(source) => Err(ModelTerminalCommitError {
2059 source: InvocationTerminalCommitFailure::Store(source),
2060 recovery: Box::new(handoff),
2061 }),
2062 }
2063 }
2064
2065 pub fn execute_tool(
2070 &self,
2071 handoff: ToolAttemptHandoff,
2072 ) -> BoxFuture<'_, Result<ToolAttemptOutcome, ToolAttemptExecutionError>> {
2073 Box::pin(self.execute_tool_inner(handoff))
2074 }
2075
2076 async fn execute_tool_inner(
2077 &self,
2078 handoff: ToolAttemptHandoff,
2079 ) -> Result<ToolAttemptOutcome, ToolAttemptExecutionError> {
2080 let started = self.start_tool_attempt_inner(handoff).await?;
2081 let ToolAttemptStartOutcome::Started(dispatch) = started else {
2082 let ToolAttemptStartOutcome::Recovered { invocation } = started else {
2083 unreachable!("closed tool attempt start outcome")
2084 };
2085 return Ok(ToolAttemptOutcome::Recovered {
2086 invocation: *invocation,
2087 });
2088 };
2089 let terminal = self.dispatch_started_tool_inner(*dispatch).await;
2090 self.commit_tool_terminal(terminal)
2091 .await
2092 .map_err(ToolAttemptExecutionError::Terminal)
2093 }
2094
2095 pub fn start_tool_attempt(
2101 &self,
2102 handoff: ToolAttemptHandoff,
2103 ) -> BoxFuture<'_, Result<ToolAttemptStartOutcome, ToolAttemptExecutionError>> {
2104 Box::pin(self.start_tool_attempt_inner(handoff))
2105 }
2106
2107 async fn start_tool_attempt_inner(
2108 &self,
2109 handoff: ToolAttemptHandoff,
2110 ) -> Result<ToolAttemptStartOutcome, ToolAttemptExecutionError> {
2111 validate_tool_handoff(&handoff.fence, &handoff.invocation)?;
2112 if let Some(recovered) = self.recover_tool_if_started(&handoff).await? {
2113 let ToolAttemptOutcome::Recovered { invocation } = recovered else {
2114 return Err(ToolAttemptExecutionError::UnsupportedStoreOutcome);
2115 };
2116 return Ok(ToolAttemptStartOutcome::Recovered {
2117 invocation: Box::new(invocation),
2118 });
2119 }
2120 let provider = self
2121 .tools
2122 .resolve(handoff.invocation.intent().descriptor())?;
2123 let (context, child_budget) = self.prepare_tool_context(&handoff).await?;
2124
2125 let start_payload = self.event_payload(
2126 "tool-attempt-started",
2127 "tool_attempt_started",
2128 "tool",
2129 handoff.invocation.intent().invocation_id(),
2130 handoff.attempt_id,
2131 handoff.invocation.intent().intent_digest(),
2132 )?;
2133 let start_append = worker_append(
2134 &handoff.fence,
2135 handoff.invocation.journal_head().clone(),
2136 handoff.events.start,
2137 start_payload,
2138 )
2139 .map_err(|_| ToolAttemptExecutionError::JournalAppend)?;
2140 let start = self
2141 .advance_tool_with_retry(
2142 start_append,
2143 &handoff.invocation,
2144 ToolInvocationTransition::StartAttempt {
2145 attempt_id: handoff.attempt_id,
2146 },
2147 child_budget,
2148 )
2149 .await?;
2150 let executing = match start {
2151 ToolInvocationCommitOutcome::Committed { invocation, .. } => invocation,
2152 ToolInvocationCommitOutcome::Idempotent { .. } => {
2153 let current = self
2154 .store
2155 .load_tool_invocation(
2156 handoff.fence.tenant_id(),
2157 handoff.fence.run_id(),
2158 handoff.invocation.intent().invocation_id(),
2159 )
2160 .await?;
2161 if current.intent() != handoff.invocation.intent() {
2162 return Err(ToolAttemptExecutionError::RecoveredInvocationMismatch);
2163 }
2164 if current.attempt_id() != Some(handoff.attempt_id) {
2165 return Err(ToolAttemptExecutionError::InvocationAdvanced);
2166 }
2167 return Ok(ToolAttemptStartOutcome::Recovered {
2168 invocation: Box::new(current),
2169 });
2170 }
2171 _ => return Err(ToolAttemptExecutionError::UnsupportedStoreOutcome),
2172 };
2173
2174 Ok(ToolAttemptStartOutcome::Started(Box::new(
2175 ToolDispatchHandoff {
2176 fence: handoff.fence,
2177 invocation: executing,
2178 terminal_event_id: handoff.events.terminal,
2179 provider,
2180 context,
2181 },
2182 )))
2183 }
2184
2185 pub fn dispatch_started_tool(
2193 &self,
2194 handoff: ToolDispatchHandoff,
2195 ) -> BoxFuture<'_, ToolTerminalCommitHandoff> {
2196 Box::pin(self.dispatch_started_tool_inner(handoff))
2197 }
2198
2199 async fn dispatch_started_tool_inner(
2200 &self,
2201 handoff: ToolDispatchHandoff,
2202 ) -> ToolTerminalCommitHandoff {
2203 let evidence = if let Some(reason) = handoff.context.stop_reason_at(Instant::now()) {
2204 ToolTerminalEvidence::Error(tool_stop_error(
2205 &handoff.context,
2206 handoff.invocation.intent().descriptor(),
2207 reason,
2208 ))
2209 } else {
2210 let input = handoff.invocation.intent().input().clone();
2211 self.dispatch_tool(handoff.provider, handoff.context, input)
2212 .await
2213 };
2214 ToolTerminalCommitHandoff {
2215 fence: handoff.fence,
2216 invocation: handoff.invocation,
2217 event_id: handoff.terminal_event_id,
2218 evidence,
2219 }
2220 }
2221
2222 pub fn commit_tool_terminal(
2224 &self,
2225 handoff: ToolTerminalCommitHandoff,
2226 ) -> BoxFuture<'_, Result<ToolAttemptOutcome, ToolTerminalCommitError>> {
2227 Box::pin(self.commit_tool_terminal_inner(handoff))
2228 }
2229
2230 async fn commit_tool_terminal_inner(
2231 &self,
2232 handoff: ToolTerminalCommitHandoff,
2233 ) -> Result<ToolAttemptOutcome, ToolTerminalCommitError> {
2234 let terminal_kind = handoff.kind();
2235 let (operation, event_kind, transition) = match &handoff.evidence {
2236 ToolTerminalEvidence::Result(result) => (
2237 "tool_result_committed",
2238 "tool-result-committed",
2239 ToolInvocationTransition::RecordResult {
2240 result: result.clone(),
2241 },
2242 ),
2243 ToolTerminalEvidence::Error(error) => (
2244 "tool_error_committed",
2245 "tool-error-committed",
2246 ToolInvocationTransition::RecordError {
2247 error: error.clone(),
2248 },
2249 ),
2250 };
2251 let Some(attempt_id) = handoff.invocation.attempt_id() else {
2252 return Err(ToolTerminalCommitError {
2253 source: InvocationTerminalCommitFailure::InvalidInvocationState,
2254 recovery: Box::new(handoff),
2255 });
2256 };
2257 let Ok(payload) = self.event_payload(
2258 event_kind,
2259 operation,
2260 "tool",
2261 handoff.invocation.intent().invocation_id(),
2262 attempt_id,
2263 handoff.invocation.intent().intent_digest(),
2264 ) else {
2265 return Err(ToolTerminalCommitError {
2266 source: InvocationTerminalCommitFailure::InvalidEventPayload,
2267 recovery: Box::new(handoff),
2268 });
2269 };
2270 let run = match self
2271 .store
2272 .load_run(
2273 handoff.invocation.intent().tenant_id(),
2274 handoff.invocation.intent().run_id(),
2275 )
2276 .await
2277 {
2278 Ok(run) => run,
2279 Err(source) => {
2280 return Err(ToolTerminalCommitError {
2281 source: InvocationTerminalCommitFailure::Store(source),
2282 recovery: Box::new(handoff),
2283 });
2284 }
2285 };
2286 let Some(journal_head) = run.journal_head().cloned() else {
2287 return Err(ToolTerminalCommitError {
2288 source: InvocationTerminalCommitFailure::RunJournalUnavailable,
2289 recovery: Box::new(handoff),
2290 });
2291 };
2292 let Ok(append) = worker_append(&handoff.fence, journal_head, handoff.event_id, payload)
2293 else {
2294 return Err(ToolTerminalCommitError {
2295 source: InvocationTerminalCommitFailure::InvalidJournalAppend,
2296 recovery: Box::new(handoff),
2297 });
2298 };
2299 match self
2300 .advance_tool_with_retry(append, &handoff.invocation, transition, None)
2301 .await
2302 {
2303 Ok(outcome) => Ok(ToolAttemptOutcome::Dispatched {
2304 terminal: terminal_kind,
2305 invocation: outcome.invocation().clone(),
2306 }),
2307 Err(source) => Err(ToolTerminalCommitError {
2308 source: InvocationTerminalCommitFailure::Store(source),
2309 recovery: Box::new(handoff),
2310 }),
2311 }
2312 }
2313
2314 pub fn reconcile_tool(
2322 &self,
2323 handoff: ToolReconciliationAttemptHandoff,
2324 ) -> BoxFuture<
2325 '_,
2326 Result<ToolReconciliationAttemptOutcome, ToolReconciliationAttemptExecutionError>,
2327 > {
2328 Box::pin(self.reconcile_tool_inner(handoff))
2329 }
2330
2331 async fn reconcile_tool_inner(
2332 &self,
2333 handoff: ToolReconciliationAttemptHandoff,
2334 ) -> Result<ToolReconciliationAttemptOutcome, ToolReconciliationAttemptExecutionError> {
2335 validate_tool_reconciliation_attempt_handoff(&handoff.fence, &handoff.invocation)?;
2336 let current = self
2337 .store
2338 .load_tool_invocation(
2339 handoff.fence.tenant_id(),
2340 handoff.fence.run_id(),
2341 handoff.invocation.intent().invocation_id(),
2342 )
2343 .await?;
2344 if current.intent() != handoff.invocation.intent() {
2345 return Err(ToolReconciliationAttemptExecutionError::RecoveredInvocationMismatch);
2346 }
2347 if current.attempt_id() != handoff.invocation.attempt_id() {
2348 return Err(ToolReconciliationAttemptExecutionError::InvocationAdvanced);
2349 }
2350 match current.state() {
2351 ToolInvocationState::Committed { .. } => {
2352 return Ok(ToolReconciliationAttemptOutcome::Resolved {
2353 kind: ToolReconciliationKind::Result,
2354 invocation: current,
2355 });
2356 }
2357 ToolInvocationState::Failed { .. } => {
2358 return Ok(ToolReconciliationAttemptOutcome::Resolved {
2359 kind: ToolReconciliationKind::Error,
2360 invocation: current,
2361 });
2362 }
2363 ToolInvocationState::Unknown { .. } => {}
2364 ToolInvocationState::Prepared | ToolInvocationState::Executing { .. } => {
2365 return Err(ToolReconciliationAttemptExecutionError::InvocationAdvanced);
2366 }
2367 }
2368
2369 let provider = self.tools.resolve(current.intent().descriptor())?;
2370 if !provider.supports_reconciliation() {
2371 return Err(ToolReconciliationAttemptExecutionError::ProviderReconciliationUnsupported);
2372 }
2373 let context = self
2374 .prepare_tool_reconciliation_context(&handoff, ¤t)
2375 .await?;
2376 let input = current.intent().input().clone();
2377 let deadline = tokio::time::Instant::from_std(context.deadline_instant());
2378 let observation = tokio::select! {
2379 biased;
2380 () = context.cancellation().cancelled() => {
2381 return Err(ToolReconciliationAttemptExecutionError::Stopped {
2382 reason: ToolStopReason::Cancelled,
2383 });
2384 }
2385 () = tokio::time::sleep_until(deadline) => {
2386 return Err(ToolReconciliationAttemptExecutionError::Stopped {
2387 reason: ToolStopReason::DeadlineExceeded,
2388 });
2389 }
2390 observation = provider.reconcile(context.clone(), input) => observation?,
2391 };
2392 observation.validate()?;
2393
2394 let evidence = match observation {
2395 ToolReconciliationObservation::Pending { retry_after } => {
2396 return Ok(ToolReconciliationAttemptOutcome::Pending {
2397 retry_after,
2398 invocation: current,
2399 });
2400 }
2401 ToolReconciliationObservation::Result(result) => {
2402 ToolReconciliationHandoff::result(handoff.fence, current, handoff.event_id, result)?
2403 }
2404 ToolReconciliationObservation::Error(error) => {
2405 if error.external_effect() == ToolExternalEffect::Unknown {
2406 return Err(ToolReconciliationAttemptExecutionError::UnresolvedErrorEvidence);
2407 }
2408 ToolReconciliationHandoff::error(handoff.fence, current, handoff.event_id, error)?
2409 }
2410 _ => {
2411 return Err(ToolReconciliationAttemptExecutionError::UnsupportedObservation);
2412 }
2413 };
2414 let outcome = self.commit_tool_reconciliation(evidence).await?;
2415 Ok(ToolReconciliationAttemptOutcome::Resolved {
2416 kind: outcome.kind(),
2417 invocation: outcome.invocation().clone(),
2418 })
2419 }
2420
2421 pub fn commit_tool_reconciliation(
2428 &self,
2429 handoff: ToolReconciliationHandoff,
2430 ) -> BoxFuture<'_, Result<ToolReconciliationOutcome, ToolReconciliationCommitError>> {
2431 Box::pin(self.commit_tool_reconciliation_inner(handoff))
2432 }
2433
2434 async fn commit_tool_reconciliation_inner(
2435 &self,
2436 handoff: ToolReconciliationHandoff,
2437 ) -> Result<ToolReconciliationOutcome, ToolReconciliationCommitError> {
2438 let kind = handoff.kind();
2439 let (operation, event_kind, transition) = match self.reconciliation_transition(&handoff) {
2440 Ok(prepared) => prepared,
2441 Err(source) => {
2442 return Err(ToolReconciliationCommitError {
2443 source,
2444 recovery: Box::new(handoff),
2445 });
2446 }
2447 };
2448 let Some(attempt_id) = handoff.invocation.attempt_id() else {
2449 return Err(ToolReconciliationCommitError {
2450 source: ToolReconciliationCommitFailure::InvalidInvocationState,
2451 recovery: Box::new(handoff),
2452 });
2453 };
2454 let Ok(payload) = self.event_payload(
2455 event_kind,
2456 operation,
2457 "tool",
2458 handoff.invocation.intent().invocation_id(),
2459 attempt_id,
2460 handoff.invocation.intent().intent_digest(),
2461 ) else {
2462 return Err(ToolReconciliationCommitError {
2463 source: ToolReconciliationCommitFailure::InvalidEventPayload,
2464 recovery: Box::new(handoff),
2465 });
2466 };
2467 let run = match self
2468 .store
2469 .load_run(
2470 handoff.invocation.intent().tenant_id(),
2471 handoff.invocation.intent().run_id(),
2472 )
2473 .await
2474 {
2475 Ok(run) => run,
2476 Err(source) => {
2477 return Err(ToolReconciliationCommitError {
2478 source: ToolReconciliationCommitFailure::Store(source),
2479 recovery: Box::new(handoff),
2480 });
2481 }
2482 };
2483 let Some(journal_head) = run.journal_head().cloned() else {
2484 return Err(ToolReconciliationCommitError {
2485 source: ToolReconciliationCommitFailure::RunJournalUnavailable,
2486 recovery: Box::new(handoff),
2487 });
2488 };
2489 let Ok(append) = worker_append(&handoff.fence, journal_head, handoff.event_id, payload)
2490 else {
2491 return Err(ToolReconciliationCommitError {
2492 source: ToolReconciliationCommitFailure::InvalidJournalAppend,
2493 recovery: Box::new(handoff),
2494 });
2495 };
2496 match self
2497 .advance_tool_with_retry(append, &handoff.invocation, transition, None)
2498 .await
2499 {
2500 Ok(ToolInvocationCommitOutcome::Committed { event, invocation }) => {
2501 Ok(ToolReconciliationOutcome::Committed {
2502 kind,
2503 event,
2504 invocation,
2505 })
2506 }
2507 Ok(ToolInvocationCommitOutcome::Idempotent { event, invocation }) => {
2508 Ok(ToolReconciliationOutcome::Idempotent {
2509 kind,
2510 event,
2511 invocation,
2512 })
2513 }
2514 Ok(_) => Err(ToolReconciliationCommitError {
2515 source: ToolReconciliationCommitFailure::UnsupportedStoreOutcome,
2516 recovery: Box::new(handoff),
2517 }),
2518 Err(source) => Err(ToolReconciliationCommitError {
2519 source: ToolReconciliationCommitFailure::Store(source),
2520 recovery: Box::new(handoff),
2521 }),
2522 }
2523 }
2524
2525 fn reconciliation_transition(
2526 &self,
2527 handoff: &ToolReconciliationHandoff,
2528 ) -> Result<
2529 (&'static str, &'static str, ToolInvocationTransition),
2530 ToolReconciliationCommitFailure,
2531 > {
2532 match &handoff.evidence {
2533 ToolReconciliationEvidence::Result(result) => {
2534 self.schemas
2535 .validate_bounded(result.output_schema(), result.output())
2536 .map_err(|source| ToolReconciliationCommitFailure::OutputSchema { source })?;
2537 Ok((
2538 "tool_result_committed",
2539 "tool-reconciliation-result-committed",
2540 ToolInvocationTransition::ReconcileResult {
2541 result: result.clone(),
2542 },
2543 ))
2544 }
2545 ToolReconciliationEvidence::Error(error) => Ok((
2546 "tool_error_committed",
2547 "tool-reconciliation-error-committed",
2548 ToolInvocationTransition::ReconcileError {
2549 error: error.clone(),
2550 },
2551 )),
2552 }
2553 }
2554
2555 async fn dispatch_model(
2556 &self,
2557 provider: Arc<dyn stateknot_core::Model>,
2558 context: ModelContext,
2559 request: ModelRequest,
2560 stream_sink: Option<Arc<dyn ModelEventSink>>,
2561 ) -> ModelTerminalEvidence {
2562 match request.response_mode() {
2563 ModelResponseMode::Complete => {
2564 Self::dispatch_complete_model(provider, context, request).await
2565 }
2566 ModelResponseMode::Streaming => {
2567 Self::dispatch_streaming_model(provider, context, request, stream_sink).await
2568 }
2569 }
2570 }
2571
2572 async fn dispatch_complete_model(
2573 provider: Arc<dyn stateknot_core::Model>,
2574 context: ModelContext,
2575 request: ModelRequest,
2576 ) -> ModelTerminalEvidence {
2577 let result = tokio::select! {
2578 biased;
2579 () = context.cancellation().cancelled() => {
2580 return ModelTerminalEvidence::Error(model_stop_error(
2581 &context,
2582 provider.descriptor(),
2583 ModelStopReason::Cancelled,
2584 ));
2585 }
2586 () = tokio::time::sleep_until(tokio::time::Instant::from_std(context.deadline_instant())) => {
2587 return ModelTerminalEvidence::Error(model_stop_error(
2588 &context,
2589 provider.descriptor(),
2590 ModelStopReason::DeadlineExceeded,
2591 ));
2592 }
2593 result = provider.invoke(context.clone(), request.clone()) => result,
2594 };
2595 match result {
2596 Ok(response)
2597 if response.provenance().attempt_id() == context.attempt_id()
2598 && response
2599 .validate_for(provider.descriptor(), &request)
2600 .is_ok() =>
2601 {
2602 ModelTerminalEvidence::Response(response)
2603 }
2604 Ok(_) => ModelTerminalEvidence::Error(model_contract_error(
2605 &context,
2606 provider.descriptor(),
2607 ModelErrorPhase::Response,
2608 "runtime.model.invalid_response",
2609 "Model provider returned a response outside its registered contract.",
2610 )),
2611 Err(error)
2612 if error
2613 .validate_for(&context, provider.descriptor(), &request)
2614 .is_ok() =>
2615 {
2616 ModelTerminalEvidence::Error(error)
2617 }
2618 Err(_) => ModelTerminalEvidence::Error(model_contract_error(
2619 &context,
2620 provider.descriptor(),
2621 ModelErrorPhase::Response,
2622 "runtime.model.invalid_error",
2623 "Model provider returned failure evidence outside its registered contract.",
2624 )),
2625 }
2626 }
2627
2628 async fn dispatch_streaming_model(
2629 provider: Arc<dyn stateknot_core::Model>,
2630 context: ModelContext,
2631 request: ModelRequest,
2632 stream_sink: Option<Arc<dyn ModelEventSink>>,
2633 ) -> ModelTerminalEvidence {
2634 let Some(sink) = stream_sink else {
2635 return ModelTerminalEvidence::Error(model_contract_error(
2636 &context,
2637 provider.descriptor(),
2638 ModelErrorPhase::Preparation,
2639 "runtime.model.missing_stream_sink",
2640 "Streaming model execution has no durable event sink.",
2641 ));
2642 };
2643 let Ok(mut accumulator) =
2644 ModelEventAccumulator::new(context.attempt_id(), provider.descriptor(), &request)
2645 else {
2646 return ModelTerminalEvidence::Error(model_contract_error(
2647 &context,
2648 provider.descriptor(),
2649 ModelErrorPhase::Preparation,
2650 "runtime.model.invalid_stream_setup",
2651 "Streaming model execution could not satisfy its registered contract.",
2652 ));
2653 };
2654 let mut stream = provider.stream(context.clone(), request.clone());
2655 loop {
2656 let next = tokio::select! {
2657 biased;
2658 () = context.cancellation().cancelled() => {
2659 return ModelTerminalEvidence::Error(model_stop_error(
2660 &context,
2661 provider.descriptor(),
2662 ModelStopReason::Cancelled,
2663 ));
2664 }
2665 () = tokio::time::sleep_until(tokio::time::Instant::from_std(context.deadline_instant())) => {
2666 return ModelTerminalEvidence::Error(model_stop_error(
2667 &context,
2668 provider.descriptor(),
2669 ModelStopReason::DeadlineExceeded,
2670 ));
2671 }
2672 next = poll_fn(|task| stream.as_mut().poll_next(task)) => next,
2673 };
2674 match next {
2675 Some(Ok(event)) => {
2676 if let Err(evidence) = Self::accept_model_stream_event(
2677 provider.as_ref(),
2678 &context,
2679 &mut accumulator,
2680 sink.as_ref(),
2681 event,
2682 )
2683 .await
2684 {
2685 return evidence;
2686 }
2687 }
2688 Some(Err(error))
2689 if error
2690 .validate_for(&context, provider.descriptor(), &request)
2691 .is_ok() =>
2692 {
2693 return ModelTerminalEvidence::Error(error);
2694 }
2695 Some(Err(_)) => {
2696 return ModelTerminalEvidence::Error(model_contract_error(
2697 &context,
2698 provider.descriptor(),
2699 ModelErrorPhase::Stream,
2700 "runtime.model.invalid_stream_error",
2701 "Model provider returned invalid stream failure evidence.",
2702 ));
2703 }
2704 None => return finish_model_stream(&context, provider.descriptor(), accumulator),
2705 }
2706 }
2707 }
2708
2709 async fn accept_model_stream_event(
2710 provider: &dyn stateknot_core::Model,
2711 context: &ModelContext,
2712 accumulator: &mut ModelEventAccumulator<'_>,
2713 sink: &dyn ModelEventSink,
2714 event: ModelEvent,
2715 ) -> Result<(), ModelTerminalEvidence> {
2716 if accumulator.push(event.clone()).is_err() {
2717 return Err(ModelTerminalEvidence::Error(model_contract_error(
2718 context,
2719 provider.descriptor(),
2720 ModelErrorPhase::Stream,
2721 "runtime.model.invalid_stream_event",
2722 "Model provider emitted an invalid semantic stream event.",
2723 )));
2724 }
2725 let emitted = tokio::select! {
2726 biased;
2727 () = context.cancellation().cancelled() => {
2728 return Err(ModelTerminalEvidence::Error(model_stop_error(
2729 context,
2730 provider.descriptor(),
2731 ModelStopReason::Cancelled,
2732 )));
2733 }
2734 () = tokio::time::sleep_until(tokio::time::Instant::from_std(context.deadline_instant())) => {
2735 return Err(ModelTerminalEvidence::Error(model_stop_error(
2736 context,
2737 provider.descriptor(),
2738 ModelStopReason::DeadlineExceeded,
2739 )));
2740 }
2741 emitted = sink.emit(event) => emitted,
2742 };
2743 emitted.map_err(|_| {
2744 ModelTerminalEvidence::Error(model_contract_error(
2745 context,
2746 provider.descriptor(),
2747 ModelErrorPhase::Stream,
2748 "runtime.model.stream_sink_failed",
2749 "The durable model stream sink did not accept an event.",
2750 ))
2751 })
2752 }
2753
2754 async fn dispatch_tool(
2755 &self,
2756 provider: Arc<dyn stateknot_core::ErasedTool>,
2757 context: ToolContext,
2758 input: stateknot_core::ToolInput,
2759 ) -> ToolTerminalEvidence {
2760 let result = tokio::select! {
2761 biased;
2762 () = context.cancellation().cancelled() => {
2763 return ToolTerminalEvidence::Error(tool_stop_error(
2764 &context,
2765 provider.descriptor(),
2766 ToolStopReason::Cancelled,
2767 ));
2768 }
2769 () = tokio::time::sleep_until(tokio::time::Instant::from_std(context.deadline_instant())) => {
2770 return ToolTerminalEvidence::Error(tool_stop_error(
2771 &context,
2772 provider.descriptor(),
2773 ToolStopReason::DeadlineExceeded,
2774 ));
2775 }
2776 result = provider.call(context.clone(), input) => result,
2777 };
2778 match result {
2779 Ok(result) if result.validate_for(&context, provider.descriptor()).is_ok() => {
2780 ToolTerminalEvidence::Result(result)
2781 }
2782 Ok(_) => ToolTerminalEvidence::Error(tool_contract_error(
2783 &context,
2784 provider.descriptor(),
2785 ToolErrorPhase::Result,
2786 true,
2787 "runtime.tool.invalid_result",
2788 "Tool returned a result outside its registered contract.",
2789 )),
2790 Err(error) if error.validate_for(&context, provider.descriptor()).is_ok() => {
2791 ToolTerminalEvidence::Error(error)
2792 }
2793 Err(_) => ToolTerminalEvidence::Error(tool_contract_error(
2794 &context,
2795 provider.descriptor(),
2796 ToolErrorPhase::Execution,
2797 false,
2798 "runtime.tool.invalid_error",
2799 "Tool returned failure evidence outside its registered contract.",
2800 )),
2801 }
2802 }
2803
2804 async fn advance_model_with_retry(
2805 &self,
2806 append: JournalAppend,
2807 expected: &ModelInvocation,
2808 transition: ModelInvocationTransition,
2809 child_budget: Option<Digest>,
2810 ) -> Result<ModelInvocationCommitOutcome, StoreError> {
2811 let mut attempt = 1_u8;
2812 loop {
2813 match self
2814 .store
2815 .advance_model_invocation_with_child_budget(
2816 append.clone(),
2817 &expected.head(),
2818 transition.clone(),
2819 child_budget,
2820 )
2821 .await
2822 {
2823 Ok(outcome) => return Ok(outcome),
2824 Err(error)
2825 if attempt < self.options.maximum_mutation_attempts()
2826 && error.is_retryable() =>
2827 {
2828 tokio::time::sleep(exponential_backoff(
2829 self.options.mutation_retry_initial_delay(),
2830 attempt,
2831 ))
2832 .await;
2833 attempt = attempt.saturating_add(1);
2834 }
2835 Err(error) => return Err(error),
2836 }
2837 }
2838 }
2839
2840 async fn advance_tool_with_retry(
2841 &self,
2842 append: JournalAppend,
2843 expected: &ToolInvocation,
2844 transition: ToolInvocationTransition,
2845 child_budget: Option<Digest>,
2846 ) -> Result<ToolInvocationCommitOutcome, StoreError> {
2847 let mut attempt = 1_u8;
2848 loop {
2849 match self
2850 .store
2851 .advance_tool_invocation_with_child_budget(
2852 append.clone(),
2853 &expected.head(),
2854 transition.clone(),
2855 child_budget,
2856 )
2857 .await
2858 {
2859 Ok(outcome) => return Ok(outcome),
2860 Err(error)
2861 if attempt < self.options.maximum_mutation_attempts()
2862 && error.is_retryable() =>
2863 {
2864 tokio::time::sleep(exponential_backoff(
2865 self.options.mutation_retry_initial_delay(),
2866 attempt,
2867 ))
2868 .await;
2869 attempt = attempt.saturating_add(1);
2870 }
2871 Err(error) => return Err(error),
2872 }
2873 }
2874 }
2875
2876 fn event_payload(
2877 &self,
2878 kind: &'static str,
2879 operation: &'static str,
2880 binding_kind: &'static str,
2881 invocation_id: InvocationId,
2882 attempt_id: AttemptId,
2883 intent_digest: Digest,
2884 ) -> Result<JournalPayload, InvocationEventPayloadError> {
2885 let data = BoundedJson::try_from_value(json!({
2886 "operation": operation,
2887 "binding_kind": binding_kind,
2888 "invocation_id": invocation_id.to_string(),
2889 "attempt_id": attempt_id.to_string(),
2890 "intent_digest": digest_hex(intent_digest)
2891 }))
2892 .map_err(|_| InvocationEventPayloadError::Invalid)?;
2893 self.schemas
2894 .validate_bounded(&self.event_schema, &data)
2895 .map_err(|_| InvocationEventPayloadError::Invalid)?;
2896 JournalPayload::new(
2897 self.event_schema.clone(),
2898 JournalEventKind::new(kind).map_err(|_| InvocationEventPayloadError::Invalid)?,
2899 data,
2900 )
2901 .map_err(|_| InvocationEventPayloadError::Invalid)
2902 }
2903}
2904
2905#[derive(Clone, Debug, Eq, Error, PartialEq)]
2907#[non_exhaustive]
2908pub enum DurableInvocationExecutorBuildError {
2909 #[error(transparent)]
2911 EventSchema(#[from] StandardInvocationExecutionSchemaError),
2912 #[error("invocation execution event schema is absent from the frozen registry")]
2914 MissingEventSchema,
2915}
2916
2917#[derive(Debug, Error)]
2919#[non_exhaustive]
2920pub enum ModelAttemptExecutionError {
2921 #[error(transparent)]
2923 Handoff(#[from] InvocationAttemptHandoffError),
2924 #[error(transparent)]
2926 Registry(#[from] ModelProviderRegistryError),
2927 #[error(transparent)]
2929 Store(#[from] StoreError),
2930 #[error("stored run provenance does not match the model attempt fence")]
2932 RunProvenanceMismatch,
2933 #[error(transparent)]
2935 Clock(#[from] InvocationClockError),
2936 #[error(transparent)]
2938 BudgetProvider(#[from] InvocationBudgetProviderError),
2939 #[error("remaining run budget cannot admit the model attempt")]
2941 BudgetInsufficient,
2942 #[error(transparent)]
2944 Context(#[from] ModelContextError),
2945 #[error("model attempt stopped before durable start: {reason:?}")]
2947 StoppedBeforeStart {
2948 reason: ModelStopReason,
2950 },
2951 #[error("model invocation journal payload is invalid")]
2953 EventPayload(#[from] InvocationEventPayloadError),
2954 #[error("model invocation worker journal append is invalid")]
2956 JournalAppend,
2957 #[error("recovered model invocation does not match the retained intent")]
2959 RecoveredInvocationMismatch,
2960 #[error("model invocation advanced under another physical attempt")]
2962 InvocationAdvanced,
2963 #[error("model invocation store outcome is unsupported by this runtime")]
2965 UnsupportedStoreOutcome,
2966 #[error(transparent)]
2968 Terminal(#[from] ModelTerminalCommitError),
2969}
2970
2971#[derive(Debug, Error)]
2973#[non_exhaustive]
2974pub enum ToolAttemptExecutionError {
2975 #[error(transparent)]
2977 Handoff(#[from] InvocationAttemptHandoffError),
2978 #[error(transparent)]
2980 Registry(#[from] ToolProviderRegistryError),
2981 #[error(transparent)]
2983 Store(#[from] StoreError),
2984 #[error("stored run provenance does not match the tool attempt fence")]
2986 RunProvenanceMismatch,
2987 #[error(transparent)]
2989 Clock(#[from] InvocationClockError),
2990 #[error(transparent)]
2992 BudgetProvider(#[from] InvocationBudgetProviderError),
2993 #[error("remaining run budget cannot admit the tool attempt")]
2995 BudgetInsufficient,
2996 #[error(transparent)]
2998 Context(#[from] ToolContextError),
2999 #[error(transparent)]
3001 Input(#[from] ToolInputValidationError),
3002 #[error("tool attempt stopped before durable start: {reason:?}")]
3004 StoppedBeforeStart {
3005 reason: ToolStopReason,
3007 },
3008 #[error("tool invocation journal payload is invalid")]
3010 EventPayload(#[from] InvocationEventPayloadError),
3011 #[error("tool invocation worker journal append is invalid")]
3013 JournalAppend,
3014 #[error("recovered tool invocation does not match the retained intent")]
3016 RecoveredInvocationMismatch,
3017 #[error("tool invocation advanced under another physical attempt")]
3019 InvocationAdvanced,
3020 #[error("tool invocation store outcome is unsupported by this runtime")]
3022 UnsupportedStoreOutcome,
3023 #[error(transparent)]
3025 Terminal(#[from] ToolTerminalCommitError),
3026}
3027
3028#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
3030#[non_exhaustive]
3031pub enum InvocationEventPayloadError {
3032 #[error("standard invocation event payload is invalid")]
3034 Invalid,
3035}
3036
3037#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
3038#[error("invocation worker journal append is invalid")]
3039struct InvocationJournalAppendError;
3040
3041fn validate_run_provenance(
3042 provenance: &AgentResultProvenance,
3043 fence: &RunFence,
3044) -> Result<(), ModelAttemptExecutionError> {
3045 if provenance.tenant_id() != fence.tenant_id() || provenance.run_id() != fence.run_id() {
3046 return Err(ModelAttemptExecutionError::RunProvenanceMismatch);
3047 }
3048 Ok(())
3049}
3050
3051fn validate_tool_run_provenance(
3052 provenance: &AgentResultProvenance,
3053 fence: &RunFence,
3054) -> Result<(), ToolAttemptExecutionError> {
3055 if provenance.tenant_id() != fence.tenant_id() || provenance.run_id() != fence.run_id() {
3056 return Err(ToolAttemptExecutionError::RunProvenanceMismatch);
3057 }
3058 Ok(())
3059}
3060
3061fn validate_model_budget(
3062 remaining: &BudgetRemaining,
3063 request: &ModelRequest,
3064) -> Result<(), ModelAttemptExecutionError> {
3065 if remaining.model_attempts() == ExecutionCount::ZERO
3066 || remaining.model_turns() == ExecutionCount::ZERO
3067 || remaining.input_tokens() < request.limits().max_input_tokens()
3068 || remaining.output_tokens() < request.limits().max_output_tokens()
3069 || remaining.input_bytes() < request.content_bytes()
3070 {
3071 return Err(ModelAttemptExecutionError::BudgetInsufficient);
3072 }
3073 Ok(())
3074}
3075
3076fn validate_tool_budget(
3077 remaining: &BudgetRemaining,
3078 risk: ToolRisk,
3079) -> Result<(), ToolAttemptExecutionError> {
3080 if remaining.tool_calls() == ExecutionCount::ZERO
3081 || (risk != ToolRisk::ReadOnly && remaining.write_calls() == ExecutionCount::ZERO)
3082 {
3083 return Err(ToolAttemptExecutionError::BudgetInsufficient);
3084 }
3085 Ok(())
3086}
3087
3088fn worker_append(
3089 fence: &RunFence,
3090 head: stateknot_core::JournalHead,
3091 event_id: EventId,
3092 payload: JournalPayload,
3093) -> Result<JournalAppend, InvocationJournalAppendError> {
3094 let intent = JournalEventIntent::worker(
3095 fence.tenant_id().clone(),
3096 fence.run_id(),
3097 event_id,
3098 fence.clone(),
3099 payload,
3100 )
3101 .map_err(|_| InvocationJournalAppendError)?;
3102 JournalAppend::new(JournalExpectation::exact(head), intent)
3103 .map_err(|_| InvocationJournalAppendError)
3104}
3105
3106fn finish_model_stream(
3107 context: &ModelContext,
3108 descriptor: &stateknot_core::ModelDescriptor,
3109 accumulator: ModelEventAccumulator<'_>,
3110) -> ModelTerminalEvidence {
3111 match accumulator.finish() {
3112 Ok(response) => ModelTerminalEvidence::Response(response),
3113 Err(_) => ModelTerminalEvidence::Error(model_contract_error(
3114 context,
3115 descriptor,
3116 ModelErrorPhase::Stream,
3117 "runtime.model.incomplete_stream",
3118 "Model provider stream ended without a valid terminal event.",
3119 )),
3120 }
3121}
3122
3123fn model_stop_error(
3124 context: &ModelContext,
3125 descriptor: &stateknot_core::ModelDescriptor,
3126 reason: ModelStopReason,
3127) -> ModelError {
3128 let (category, code, message) = match reason {
3129 ModelStopReason::Cancelled => (
3130 FailureCategory::Cancelled,
3131 "runtime.model.cancelled",
3132 "Model attempt stopped after cancellation was requested.",
3133 ),
3134 ModelStopReason::DeadlineExceeded => (
3135 FailureCategory::DeadlineExceeded,
3136 "runtime.model.deadline_exceeded",
3137 "Model attempt exceeded its effective deadline.",
3138 ),
3139 _ => (
3140 FailureCategory::Internal,
3141 "runtime.model.unsupported_stop_reason",
3142 "Model attempt stopped for a reason unsupported by this runtime.",
3143 ),
3144 };
3145 ModelError::new(
3146 runtime_failure(category, code, message, RetryAdvice::Never),
3147 ModelErrorPhase::Dispatch,
3148 ModelErrorProvenance::new(
3149 context.attempt_id(),
3150 descriptor.metadata().identity().clone(),
3151 None,
3152 None,
3153 None,
3154 ),
3155 None,
3156 )
3157}
3158
3159fn model_contract_error(
3160 context: &ModelContext,
3161 descriptor: &stateknot_core::ModelDescriptor,
3162 phase: ModelErrorPhase,
3163 code: &'static str,
3164 message: &'static str,
3165) -> ModelError {
3166 ModelError::new(
3167 runtime_failure(FailureCategory::Internal, code, message, RetryAdvice::Never),
3168 phase,
3169 ModelErrorProvenance::new(
3170 context.attempt_id(),
3171 descriptor.metadata().identity().clone(),
3172 None,
3173 None,
3174 None,
3175 ),
3176 None,
3177 )
3178}
3179
3180fn tool_stop_error(
3181 context: &ToolContext,
3182 descriptor: &stateknot_core::ToolDescriptor,
3183 reason: ToolStopReason,
3184) -> ToolError {
3185 let risk = descriptor.semantics().risk();
3186 let (failure, effect) = if risk == ToolRisk::ReadOnly {
3187 let (category, code, message) = match reason {
3188 ToolStopReason::Cancelled => (
3189 FailureCategory::Cancelled,
3190 "runtime.tool.cancelled",
3191 "Tool attempt stopped after cancellation was requested.",
3192 ),
3193 ToolStopReason::DeadlineExceeded => (
3194 FailureCategory::DeadlineExceeded,
3195 "runtime.tool.deadline_exceeded",
3196 "Tool attempt exceeded its effective deadline.",
3197 ),
3198 _ => (
3199 FailureCategory::Internal,
3200 "runtime.tool.unsupported_stop_reason",
3201 "Tool attempt stopped for a reason unsupported by this runtime.",
3202 ),
3203 };
3204 (
3205 runtime_failure(category, code, message, RetryAdvice::Never),
3206 ToolExternalEffect::NotApplicable,
3207 )
3208 } else {
3209 let (code, message) = match reason {
3210 ToolStopReason::Cancelled => (
3211 "runtime.tool.cancelled_outcome_unknown",
3212 "A cancelled tool write may have changed external state and requires reconciliation.",
3213 ),
3214 ToolStopReason::DeadlineExceeded => (
3215 "runtime.tool.deadline_outcome_unknown",
3216 "A timed-out tool write may have changed external state and requires reconciliation.",
3217 ),
3218 _ => (
3219 "runtime.tool.unsupported_stop_reason_outcome_unknown",
3220 "A stopped tool write may have changed external state and requires reconciliation.",
3221 ),
3222 };
3223 (
3224 runtime_failure(
3225 FailureCategory::AmbiguousExternalOutcome,
3226 code,
3227 message,
3228 RetryAdvice::ReconcileFirst,
3229 ),
3230 ToolExternalEffect::Unknown,
3231 )
3232 };
3233 ToolError::new(
3234 failure,
3235 ToolErrorPhase::Execution,
3236 effect,
3237 ToolErrorProvenance::for_invocation(context, descriptor),
3238 )
3239 .expect("runtime stop evidence is constructed from valid risk/effect pairs")
3240}
3241
3242fn tool_contract_error(
3243 context: &ToolContext,
3244 descriptor: &stateknot_core::ToolDescriptor,
3245 phase: ToolErrorPhase,
3246 nominal_success: bool,
3247 code: &'static str,
3248 message: &'static str,
3249) -> ToolError {
3250 let risk = descriptor.semantics().risk();
3251 let (category, advice, effect) = match risk {
3252 ToolRisk::ReadOnly => (
3253 FailureCategory::Internal,
3254 RetryAdvice::Never,
3255 ToolExternalEffect::NotApplicable,
3256 ),
3257 ToolRisk::IdempotentWrite | ToolRisk::NonIdempotentWrite if nominal_success => (
3258 FailureCategory::Internal,
3259 RetryAdvice::Never,
3260 ToolExternalEffect::Applied,
3261 ),
3262 ToolRisk::IdempotentWrite | ToolRisk::NonIdempotentWrite => (
3263 FailureCategory::AmbiguousExternalOutcome,
3264 RetryAdvice::ReconcileFirst,
3265 ToolExternalEffect::Unknown,
3266 ),
3267 };
3268 ToolError::new(
3269 runtime_failure(category, code, message, advice),
3270 phase,
3271 effect,
3272 ToolErrorProvenance::for_invocation(context, descriptor),
3273 )
3274 .expect("runtime adapter-contract evidence is constructed from valid risk/effect pairs")
3275}
3276
3277fn runtime_failure(
3278 category: FailureCategory,
3279 code: &'static str,
3280 message: &'static str,
3281 advice: RetryAdvice,
3282) -> Failure {
3283 Failure::new(
3284 FailureId::generate(),
3285 category,
3286 FailureCode::new(code).expect("runtime failure code is static and valid"),
3287 FailureOrigin::new("stateknot.runtime")
3288 .expect("runtime failure origin is static and valid"),
3289 FailureMessage::new(message).expect("runtime failure message is static and valid"),
3290 advice,
3291 )
3292 .expect("runtime failure category and advice are statically compatible")
3293}
3294
3295fn exponential_backoff(initial: Duration, attempt: u8) -> Duration {
3296 let multiplier = 1_u32
3297 .checked_shl(u32::from(attempt.saturating_sub(1)))
3298 .unwrap_or(u32::MAX);
3299 initial
3300 .checked_mul(multiplier)
3301 .unwrap_or(MAX_MUTATION_RETRY_DELAY)
3302 .min(MAX_MUTATION_RETRY_DELAY)
3303}
3304
3305fn digest_hex(digest: Digest) -> String {
3306 const HEX: &[u8; 16] = b"0123456789abcdef";
3307 let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
3308 for byte in digest.as_bytes() {
3309 value.push(char::from(HEX[usize::from(byte >> 4)]));
3310 value.push(char::from(HEX[usize::from(byte & 0x0f)]));
3311 }
3312 value
3313}
3314
3315#[cfg(test)]
3316mod tests {
3317 use super::*;
3318
3319 #[test]
3320 fn event_ids_and_retry_options_reject_unsafe_shapes() {
3321 let event = EventId::generate();
3322 assert_eq!(
3323 InvocationAttemptEventIds::new(event, event),
3324 Err(InvocationAttemptEventIdsError::Duplicate)
3325 );
3326 assert_eq!(
3327 DurableInvocationExecutorOptions::new(0, Duration::from_millis(1)),
3328 Err(DurableInvocationExecutorOptionsError::InvalidMutationAttempts)
3329 );
3330 assert_eq!(
3331 DurableInvocationExecutorOptions::new(1, Duration::ZERO),
3332 Err(DurableInvocationExecutorOptionsError::InvalidMutationRetryDelay)
3333 );
3334 }
3335
3336 #[test]
3337 fn system_clock_produces_a_canonical_paired_observation() {
3338 let before = Instant::now();
3339 let observation = SystemInvocationClock.observe().unwrap();
3340 assert!(observation.observed_at().unix_micros() > 0);
3341 assert!(observation.observed_instant() >= before);
3342 }
3343}