1use std::{collections::BTreeSet, fmt, sync::Arc};
13
14use schemars::JsonSchema;
15use serde::{Deserialize, Serialize};
16use serde_json::{Value, json};
17use stateknot_core::{
18 AgentArtifacts, AgentDescriptor, AgentInstructions, AgentStructuredOutputStrategy,
19 AgentToolConcurrency, AttemptId, BoundedJson, BoxFuture, BudgetUsage, ByteCount,
20 CapabilityIdentity, Checkpoint, CheckpointState, CompiledGraph, ContentMetadata, ContentPart,
21 ContentSource, ContentTrust, Digest, DurationMillis, EventId, ExecutionCount, Failure,
22 FailureCategory, FailureCode, FailureId, FailureMessage, FailureOrigin, GraphCompileError,
23 GraphExecutionLimits, GraphNode, GraphReducer, GraphReducerError, GraphReducerInput,
24 GraphReducerReference, GraphReference, GraphRoute, GraphRoutes, Instruction,
25 InstructionIdentity, InstructionName, InstructionProvenance, InvocationId, JournalAppend,
26 JournalEventIntent, JournalEventKind, JournalExpectation, JournalPayload, JsonContent,
27 KnownCosts, Message, MessageId, MessageParts, MessageProducer, MessageProvenance, MessageRole,
28 ModelError, ModelErrorPhase, ModelFinishReason, ModelInvocation, ModelInvocationIntent,
29 ModelInvocationState, ModelOutputItem, ModelRequest, ModelRequestLimits, ModelTextOutputFormat,
30 ModelToolCallProposal, ModelToolChoice, ModelToolOutcome, ModelToolSelection, ModelTranscript,
31 ModelTranscriptTurn, NodeActivation, NodeControl, NodeId, NodeInvocationBinding,
32 NodeInvocationBindings, NodeStateChange, NodeStateUpdate, NodeTerminalOutput, ReadyNodes,
33 RedactionState, RetryAdvice, RouteId, SchemaId, SchemaReference, SecurityLabel, Superstep,
34 TextContent, TokenCount, ToolInput, ToolInvocation, ToolInvocationIntent, ToolInvocationState,
35 ToolRisk, Version,
36};
37use stateknot_store_postgres::{NodeAttemptHistoryPageSize, PostgresStore, StoreError};
38use thiserror::Error;
39
40use crate::{
41 DurableInvocationExecutor, ExecutableGraphRegistryBuilder, ExecutableGraphRegistryError,
42 GraphCancellationEvidence, GraphCancellationEvidenceContext, GraphFailureEvidence,
43 GraphFailureEvidenceContext, GraphLifecycleEvidenceError, GraphLifecycleEvidenceProvider,
44 GraphNodeContext, GraphNodeExecution, GraphNodeExecutionError, GraphNodeExecutor,
45 GraphTerminalEvidence, GraphTerminalEvidenceContext, InvocationAttemptEventIds,
46 JsonSchemaRegistry, JsonSchemaRegistryBuilder, JsonSchemaRegistryError, ModelAttemptHandoff,
47 ModelAttemptOutcome, ToolAttemptHandoff, ToolAttemptOutcome, ToolAttemptStartOutcome,
48 ToolReconciliationAttemptExecutionError, ToolReconciliationAttemptHandoff,
49 ToolReconciliationAttemptOutcome, ToolTerminalCommitHandoff,
50 standard_invocation_execution_event_schema,
51};
52
53const IMPLEMENTATION_VERSION: &str = "stateknot.provider-native-agent.v1";
54const MAX_CHECKPOINT_INVOCATION_REFERENCES: u64 = 4096;
55const OUTPUT_REPAIR_INSTRUCTION_NAME: &str = "stateknot.output_repair";
56const OUTPUT_REPAIR_INSTRUCTION_TEXT: &str = "A prior model attempt did not produce exactly one JSON value that satisfies the required JSON Schema. Return exactly one JSON value that conforms to the requested schema. Do not call tools. Do not include prose, Markdown, or additional content.";
57
58pub const PROVIDER_NATIVE_MODEL_NODE_ID: &str = "agent.model";
60pub const PROVIDER_NATIVE_TOOLS_NODE_ID: &str = "agent.tools";
62pub const PROVIDER_NATIVE_TOOLS_ROUTE_ID: &str = "tools";
64
65#[derive(Clone, Debug, Eq, JsonSchema, PartialEq, Serialize, Deserialize)]
67#[serde(deny_unknown_fields)]
68pub struct AgentToolPolicyReference {
69 identity: CapabilityIdentity,
70 definition_digest: Digest,
71}
72
73impl AgentToolPolicyReference {
74 #[must_use]
76 pub const fn new(identity: CapabilityIdentity, definition_digest: Digest) -> Self {
77 Self {
78 identity,
79 definition_digest,
80 }
81 }
82
83 #[must_use]
85 pub const fn identity(&self) -> &CapabilityIdentity {
86 &self.identity
87 }
88
89 #[must_use]
91 pub const fn definition_digest(&self) -> Digest {
92 self.definition_digest
93 }
94}
95
96#[derive(Clone, Debug)]
98pub struct AgentToolPolicyContext {
99 agent: CapabilityIdentity,
100 admission_digest: Digest,
101 model_invocation_id: InvocationId,
102 proposal_index: usize,
103 proposal: ModelToolCallProposal,
104 action_digest: Digest,
105}
106
107impl AgentToolPolicyContext {
108 #[must_use]
110 pub const fn agent(&self) -> &CapabilityIdentity {
111 &self.agent
112 }
113
114 #[must_use]
116 pub const fn admission_digest(&self) -> Digest {
117 self.admission_digest
118 }
119
120 #[must_use]
122 pub const fn model_invocation_id(&self) -> InvocationId {
123 self.model_invocation_id
124 }
125
126 #[must_use]
128 pub const fn proposal_index(&self) -> usize {
129 self.proposal_index
130 }
131
132 #[must_use]
134 pub const fn proposal(&self) -> &ModelToolCallProposal {
135 &self.proposal
136 }
137
138 #[must_use]
140 pub const fn action_digest(&self) -> Digest {
141 self.action_digest
142 }
143}
144
145#[derive(Clone, Debug)]
147#[non_exhaustive]
148#[allow(clippy::large_enum_variant)]
149pub enum AgentToolPolicyDecision {
150 Allow {
152 evidence_digest: Digest,
154 },
155 Deny {
157 failure: Failure,
159 },
160}
161
162#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
164#[non_exhaustive]
165pub enum AgentToolPolicyError {
166 #[error("agent tool policy is temporarily unavailable")]
168 TemporarilyUnavailable,
169 #[error("agent tool policy returned invalid evidence")]
171 InvalidEvidence,
172}
173
174pub trait AgentToolPolicy: Send + Sync + 'static {
180 fn reference(&self) -> &AgentToolPolicyReference;
182
183 fn evaluate(
185 &self,
186 context: AgentToolPolicyContext,
187 ) -> BoxFuture<'_, Result<AgentToolPolicyDecision, AgentToolPolicyError>>;
188}
189
190#[derive(Clone, Debug, Eq, JsonSchema, PartialEq, Serialize, Deserialize)]
192#[serde(deny_unknown_fields)]
193pub struct AgentInvocationAccountingReference {
194 identity: CapabilityIdentity,
195 definition_digest: Digest,
196}
197
198impl AgentInvocationAccountingReference {
199 #[must_use]
201 pub const fn new(identity: CapabilityIdentity, definition_digest: Digest) -> Self {
202 Self {
203 identity,
204 definition_digest,
205 }
206 }
207
208 #[must_use]
210 pub const fn identity(&self) -> &CapabilityIdentity {
211 &self.identity
212 }
213
214 #[must_use]
216 pub const fn definition_digest(&self) -> Digest {
217 self.definition_digest
218 }
219}
220
221#[derive(Clone, Debug)]
223#[non_exhaustive]
224pub enum AgentInvocationCharge {
225 Known(KnownCosts),
227 Unpriced,
229}
230
231pub trait AgentInvocationAccounting: Send + Sync + 'static {
239 fn reference(&self) -> &AgentInvocationAccountingReference;
241
242 fn model_charge(&self, invocation: &ModelInvocation) -> AgentInvocationCharge;
244
245 fn tool_charge(&self, invocation: &ToolInvocation) -> AgentInvocationCharge;
247}
248
249#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
251#[serde(deny_unknown_fields)]
252#[allow(clippy::struct_field_names)]
253pub struct ProviderNativeModelPlan {
254 invocation_id: InvocationId,
255 attempt_id: AttemptId,
256 prepared_event_id: EventId,
257 attempt_start_event_id: EventId,
258 attempt_terminal_event_id: EventId,
259}
260
261impl ProviderNativeModelPlan {
262 fn generate() -> Self {
263 Self {
264 invocation_id: InvocationId::generate(),
265 attempt_id: AttemptId::generate(),
266 prepared_event_id: EventId::generate(),
267 attempt_start_event_id: EventId::generate(),
268 attempt_terminal_event_id: EventId::generate(),
269 }
270 }
271
272 #[must_use]
274 pub const fn invocation_id(&self) -> InvocationId {
275 self.invocation_id
276 }
277
278 #[must_use]
280 pub const fn attempt_id(&self) -> AttemptId {
281 self.attempt_id
282 }
283
284 #[must_use]
286 pub const fn prepared_event_id(&self) -> EventId {
287 self.prepared_event_id
288 }
289
290 #[must_use]
292 pub const fn attempt_start_event_id(&self) -> EventId {
293 self.attempt_start_event_id
294 }
295
296 #[must_use]
298 pub const fn attempt_terminal_event_id(&self) -> EventId {
299 self.attempt_terminal_event_id
300 }
301}
302
303#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
305#[serde(deny_unknown_fields)]
306pub struct ProviderNativeToolPlan {
307 proposal_index: u16,
308 invocation_id: InvocationId,
309 attempt_id: AttemptId,
310 prepared_event_id: EventId,
311 attempt_start_event_id: EventId,
312 attempt_terminal_event_id: EventId,
313 action_digest: Digest,
314 policy_evidence_digest: Digest,
315}
316
317impl ProviderNativeToolPlan {
318 fn generate(
319 proposal_index: u16,
320 action_digest: Digest,
321 policy_evidence_digest: Digest,
322 ) -> Self {
323 Self {
324 proposal_index,
325 invocation_id: InvocationId::generate(),
326 attempt_id: AttemptId::generate(),
327 prepared_event_id: EventId::generate(),
328 attempt_start_event_id: EventId::generate(),
329 attempt_terminal_event_id: EventId::generate(),
330 action_digest,
331 policy_evidence_digest,
332 }
333 }
334
335 #[must_use]
337 pub const fn proposal_index(&self) -> u16 {
338 self.proposal_index
339 }
340
341 #[must_use]
343 pub const fn invocation_id(&self) -> InvocationId {
344 self.invocation_id
345 }
346
347 #[must_use]
349 pub const fn attempt_id(&self) -> AttemptId {
350 self.attempt_id
351 }
352
353 #[must_use]
355 pub const fn prepared_event_id(&self) -> EventId {
356 self.prepared_event_id
357 }
358
359 #[must_use]
361 pub const fn attempt_start_event_id(&self) -> EventId {
362 self.attempt_start_event_id
363 }
364
365 #[must_use]
367 pub const fn attempt_terminal_event_id(&self) -> EventId {
368 self.attempt_terminal_event_id
369 }
370
371 #[must_use]
378 pub fn reconciliation_event_id(&self) -> EventId {
379 let mut material =
380 b"stateknot.provider-native-agent.tool-reconciliation-event.v1\0".to_vec();
381 material.extend_from_slice(self.invocation_id.as_uuid().as_bytes());
382 material.extend_from_slice(self.attempt_id.as_uuid().as_bytes());
383 material.extend_from_slice(self.attempt_terminal_event_id.as_uuid().as_bytes());
384 material.extend_from_slice(self.action_digest.as_bytes());
385 material.extend_from_slice(self.policy_evidence_digest.as_bytes());
386 let digest = Digest::sha256(material);
387
388 let mut bytes = [0_u8; 16];
389 bytes[..6].copy_from_slice(&self.attempt_terminal_event_id.as_uuid().as_bytes()[..6]);
390 bytes[6..].copy_from_slice(&digest.as_bytes()[..10]);
391 bytes[6] = (bytes[6] & 0x0f) | 0x70;
392 bytes[8] = (bytes[8] & 0x3f) | 0x80;
393 EventId::from_uuid(uuid::Uuid::from_bytes(bytes))
394 .expect("derived reconciliation event bytes are a valid UUIDv7")
395 }
396
397 #[must_use]
399 pub const fn action_digest(&self) -> Digest {
400 self.action_digest
401 }
402
403 #[must_use]
405 pub const fn policy_evidence_digest(&self) -> Digest {
406 self.policy_evidence_digest
407 }
408}
409
410#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
412#[serde(deny_unknown_fields)]
413pub struct ProviderNativeCompletedTurn {
414 model_invocation_id: InvocationId,
415 tool_invocation_ids: Vec<InvocationId>,
416}
417
418impl ProviderNativeCompletedTurn {
419 #[must_use]
421 pub const fn model_invocation_id(&self) -> InvocationId {
422 self.model_invocation_id
423 }
424
425 #[must_use]
427 pub fn tool_invocation_ids(&self) -> &[InvocationId] {
428 &self.tool_invocation_ids
429 }
430
431 #[must_use]
437 pub fn requires_output_repair(&self) -> bool {
438 self.tool_invocation_ids.is_empty()
439 }
440}
441
442#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
444#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
445pub enum ProviderNativeAgentPhase {
446 Model {
448 plan: ProviderNativeModelPlan,
450 },
451 Tools {
453 model_invocation_id: InvocationId,
455 plans: Vec<ProviderNativeToolPlan>,
457 },
458}
459
460#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
462#[serde(deny_unknown_fields)]
463pub struct ProviderNativeAgentState {
464 contract_digest: Digest,
465 input_message_id: MessageId,
466 completed_turns: Vec<ProviderNativeCompletedTurn>,
467 phase: ProviderNativeAgentPhase,
468}
469
470impl ProviderNativeAgentState {
471 #[must_use]
473 pub const fn contract_digest(&self) -> Digest {
474 self.contract_digest
475 }
476
477 #[must_use]
479 pub const fn input_message_id(&self) -> MessageId {
480 self.input_message_id
481 }
482
483 #[must_use]
485 pub fn completed_turns(&self) -> &[ProviderNativeCompletedTurn] {
486 &self.completed_turns
487 }
488
489 #[must_use]
491 pub const fn phase(&self) -> &ProviderNativeAgentPhase {
492 &self.phase
493 }
494}
495
496#[derive(Clone)]
498pub struct ProviderNativeAgentGraph {
499 descriptor: AgentDescriptor,
500 policy: Arc<dyn AgentToolPolicy>,
501 accounting: Arc<dyn AgentInvocationAccounting>,
502 input_security_label: SecurityLabel,
503 contract_digest: Digest,
504 state_schema: SchemaReference,
505 state_schema_document: Value,
506 graph: CompiledGraph,
507 model_node_id: NodeId,
508 tools_node_id: NodeId,
509 tools_route_id: RouteId,
510 output_repair_instruction: Option<Instruction>,
511}
512
513impl ProviderNativeAgentGraph {
514 pub const MAX_CHECKPOINT_INVOCATION_REFERENCES: u64 = MAX_CHECKPOINT_INVOCATION_REFERENCES;
516
517 pub fn compile(
524 descriptor: AgentDescriptor,
525 graph_identity: CapabilityIdentity,
526 reducer_identity: CapabilityIdentity,
527 state_schema_id: SchemaId,
528 input_security_label: SecurityLabel,
529 policy: Arc<dyn AgentToolPolicy>,
530 accounting: Arc<dyn AgentInvocationAccounting>,
531 ) -> Result<Self, ProviderNativeAgentGraphBuildError> {
532 validate_supported_execution(&descriptor)?;
533 let output_repair_instruction = build_output_repair_instruction(&descriptor)?;
534 let contract_digest = composition_digest(
535 &descriptor,
536 policy.reference(),
537 accounting.reference(),
538 &input_security_label,
539 )?;
540 let maximum_turns = descriptor.execution().max_model_turns().get();
541 let maximum_calls = descriptor.execution().max_tool_calls_per_turn().get();
542 let maximum_references = maximum_turns
543 .checked_mul(maximum_calls)
544 .ok_or(ProviderNativeAgentGraphBuildError::CheckpointReferenceLimit)?;
545 if maximum_references > MAX_CHECKPOINT_INVOCATION_REFERENCES {
546 return Err(ProviderNativeAgentGraphBuildError::CheckpointReferenceLimit);
547 }
548
549 let (state_schema, state_schema_document) = build_state_schema(
550 state_schema_id,
551 contract_digest,
552 maximum_turns,
553 maximum_calls,
554 descriptor.execution().max_output_repair_turns().get(),
555 )?;
556 let reducer = GraphReducerReference::new(reducer_identity, contract_digest);
557 let model_node_id = NodeId::new(PROVIDER_NATIVE_MODEL_NODE_ID)
558 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?;
559 let tools_node_id = NodeId::new(PROVIDER_NATIVE_TOOLS_NODE_ID)
560 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?;
561 let tools_route_id = RouteId::new(PROVIDER_NATIVE_TOOLS_ROUTE_ID)
562 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?;
563 let model_continue_to = if output_repair_instruction.is_some() {
564 Some(
565 ReadyNodes::try_new([model_node_id.clone()])
566 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?,
567 )
568 } else {
569 None
570 };
571 let model = GraphNode::new(
572 model_node_id.clone(),
573 model_continue_to,
574 GraphRoutes::try_new([GraphRoute::new(
575 tools_route_id.clone(),
576 ReadyNodes::try_new([tools_node_id.clone()])
577 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?,
578 )
579 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?])
580 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?,
581 None,
582 true,
583 )
584 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?;
585 let tools = GraphNode::new(
586 tools_node_id.clone(),
587 Some(
588 ReadyNodes::try_new([model_node_id.clone()])
589 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?,
590 ),
591 GraphRoutes::empty(),
592 None,
593 false,
594 )
595 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?;
596 let maximum_supersteps = maximum_turns
597 .checked_mul(2)
598 .ok_or(ProviderNativeAgentGraphBuildError::SuperstepLimit)?;
599 let graph = CompiledGraph::compile(
600 graph_identity,
601 descriptor.input_schema().clone(),
602 state_schema.clone(),
603 state_schema.clone(),
604 descriptor.output_schema().clone(),
605 reducer,
606 ReadyNodes::try_new([model_node_id.clone()])
607 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?,
608 [model, tools],
609 GraphExecutionLimits::new(
610 Superstep::new(maximum_supersteps)
611 .map_err(|_| ProviderNativeAgentGraphBuildError::SuperstepLimit)?,
612 1,
613 )
614 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?,
615 )?;
616 Ok(Self {
617 descriptor,
618 policy,
619 accounting,
620 input_security_label,
621 contract_digest,
622 state_schema,
623 state_schema_document,
624 graph,
625 model_node_id,
626 tools_node_id,
627 tools_route_id,
628 output_repair_instruction,
629 })
630 }
631
632 #[must_use]
634 pub const fn graph(&self) -> &CompiledGraph {
635 &self.graph
636 }
637
638 #[must_use]
640 pub const fn descriptor(&self) -> &AgentDescriptor {
641 &self.descriptor
642 }
643
644 #[must_use]
646 pub const fn contract_digest(&self) -> Digest {
647 self.contract_digest
648 }
649
650 pub fn register_schema(
652 &self,
653 builder: &mut JsonSchemaRegistryBuilder,
654 ) -> Result<(), JsonSchemaRegistryError> {
655 builder.register(
656 self.state_schema.clone(),
657 self.state_schema_document.clone(),
658 )
659 }
660
661 pub fn register_executable(
669 &self,
670 builder: &mut ExecutableGraphRegistryBuilder,
671 store: PostgresStore,
672 invocation_executor: DurableInvocationExecutor,
673 schemas: JsonSchemaRegistry,
674 ) -> Result<(), ProviderNativeAgentRegistrationError> {
675 let (invocation_event_schema, _) = standard_invocation_execution_event_schema()
676 .map_err(|_| ProviderNativeAgentRegistrationError::EmbeddedEventSchema)?;
677 if !schemas.contains(&self.state_schema)
678 || !schemas.contains(self.descriptor.input_schema())
679 || !schemas.contains(self.descriptor.output_schema())
680 || !schemas.contains(&invocation_event_schema)
681 || self.descriptor.tools().iter().any(|tool| {
682 !schemas.contains(tool.input_schema()) || !schemas.contains(tool.output_schema())
683 })
684 {
685 return Err(ProviderNativeAgentRegistrationError::MissingSchema);
686 }
687 let definition = Arc::new(self.clone());
688 let shared = Arc::new(ProviderNativeBindings {
689 definition: Arc::clone(&definition),
690 store,
691 invocation_executor,
692 schemas,
693 invocation_event_schema,
694 });
695 let reducer: Arc<dyn GraphReducer> = Arc::new(ProviderNativeReducer {
696 reference: self.graph.reducer().clone(),
697 graph: Arc::clone(&definition),
698 });
699 builder.register_reducer(reducer)?;
700 builder.register_node(Arc::new(ProviderNativeModelNode {
701 graph: self.graph.reference(),
702 node_id: self.model_node_id.clone(),
703 shared: Arc::clone(&shared),
704 }))?;
705 builder.register_node(Arc::new(ProviderNativeToolsNode {
706 graph: self.graph.reference(),
707 node_id: self.tools_node_id.clone(),
708 shared,
709 }))?;
710 builder.register_graph(self.graph.clone())?;
711 Ok(())
712 }
713
714 pub fn initial_state(&self) -> Result<CheckpointState, ProviderNativeAgentStateError> {
716 self.encode_checkpoint_state(&ProviderNativeAgentState {
717 contract_digest: self.contract_digest,
718 input_message_id: MessageId::generate(),
719 completed_turns: Vec::new(),
720 phase: ProviderNativeAgentPhase::Model {
721 plan: ProviderNativeModelPlan::generate(),
722 },
723 })
724 }
725
726 pub fn restore_state(
732 &self,
733 state: &CheckpointState,
734 ) -> Result<ProviderNativeAgentState, ProviderNativeAgentStateError> {
735 if state.schema() != &self.state_schema {
736 return Err(ProviderNativeAgentStateError::Integrity);
737 }
738 let decoded = decode_state(state.data())?;
739 validate_state(&decoded, self)?;
740 Ok(decoded)
741 }
742
743 fn encode_checkpoint_state(
744 &self,
745 state: &ProviderNativeAgentState,
746 ) -> Result<CheckpointState, ProviderNativeAgentStateError> {
747 validate_state(state, self)?;
748 let value = serde_json::to_value(state)
749 .map_err(|_| ProviderNativeAgentStateError::Serialization)?;
750 let bounded = BoundedJson::try_from_value(value)
751 .map_err(|_| ProviderNativeAgentStateError::ResourceLimit)?;
752 CheckpointState::new(self.state_schema.clone(), bounded)
753 .map_err(|_| ProviderNativeAgentStateError::Integrity)
754 }
755
756 fn encode_update(
757 &self,
758 state: &ProviderNativeAgentState,
759 ) -> Result<NodeStateUpdate, ProviderNativeAgentNodeError> {
760 validate_state(state, self).map_err(|_| ProviderNativeAgentNodeError::InvalidState)?;
761 let value =
762 serde_json::to_value(state).map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
763 let bounded = BoundedJson::try_from_value(value)
764 .map_err(|_| ProviderNativeAgentNodeError::InvalidState)?;
765 NodeStateUpdate::new(self.state_schema.clone(), bounded)
766 .map_err(|_| ProviderNativeAgentNodeError::Integrity)
767 }
768
769 fn decode_checkpoint(
770 &self,
771 state: &CheckpointState,
772 ) -> Result<ProviderNativeAgentState, ProviderNativeAgentNodeError> {
773 if state.schema() != &self.state_schema {
774 return Err(ProviderNativeAgentNodeError::InvalidState);
775 }
776 let decoded =
777 decode_state(state.data()).map_err(|_| ProviderNativeAgentNodeError::InvalidState)?;
778 validate_state(&decoded, self).map_err(|_| ProviderNativeAgentNodeError::InvalidState)?;
779 Ok(decoded)
780 }
781}
782
783impl fmt::Debug for ProviderNativeAgentGraph {
784 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
785 formatter
786 .debug_struct("ProviderNativeAgentGraph")
787 .field("graph", &self.graph.reference())
788 .field("agent", self.descriptor.metadata().identity())
789 .field("policy", self.policy.reference())
790 .field("accounting", self.accounting.reference())
791 .field("contract_digest", &self.contract_digest)
792 .finish_non_exhaustive()
793 }
794}
795
796#[derive(Clone)]
805pub struct ProviderNativeAgentLifecycleEvidence {
806 definition: Arc<ProviderNativeAgentGraph>,
807 store: PostgresStore,
808}
809
810impl ProviderNativeAgentLifecycleEvidence {
811 #[must_use]
813 pub fn new(definition: ProviderNativeAgentGraph, store: PostgresStore) -> Self {
814 Self {
815 definition: Arc::new(definition),
816 store,
817 }
818 }
819
820 async fn terminal_evidence_inner(
821 &self,
822 context: GraphTerminalEvidenceContext,
823 ) -> Result<GraphTerminalEvidence, GraphLifecycleEvidenceError> {
824 let (stored, checkpoint, state) = self
825 .load_context(context.provenance(), context.graph(), context.checkpoint())
826 .await?;
827 if context.output_schema() != self.definition.descriptor.output_schema() {
828 return Err(GraphLifecycleEvidenceError::Corrupt);
829 }
830 let ProviderNativeAgentPhase::Model { plan } = state.phase() else {
831 return Err(GraphLifecycleEvidenceError::Corrupt);
832 };
833 let (transcript, prior_usage, output_repair_ordinal) = reconstruct_transcript(
834 &self.store,
835 checkpoint.tenant_id(),
836 checkpoint.run_id(),
837 &state,
838 &self.definition,
839 )
840 .await
841 .map_err(evidence_node_error)?;
842 let expected_request = build_model_request(
843 &stored,
844 &self.definition,
845 state.input_message_id(),
846 transcript,
847 &prior_usage,
848 output_repair_ordinal,
849 )
850 .map_err(evidence_node_error)?;
851 let invocation = self
852 .store
853 .load_model_invocation(
854 checkpoint.tenant_id(),
855 checkpoint.run_id(),
856 plan.invocation_id(),
857 )
858 .await
859 .map_err(|error| evidence_store_error(&error))?;
860 validate_terminal_model_binding(
861 &invocation,
862 plan,
863 &checkpoint,
864 &self.definition.model_node_id,
865 &expected_request,
866 )?;
867 let response = match invocation.state() {
868 ModelInvocationState::Committed { response }
869 if response.finish_reason() == ModelFinishReason::Completed =>
870 {
871 response
872 }
873 _ => return Err(GraphLifecycleEvidenceError::Corrupt),
874 };
875 let output = completed_json_output(response).map_err(evidence_node_error)?;
876 let terminal = NodeTerminalOutput::new(context.output_schema().clone(), output)
877 .map_err(|_| GraphLifecycleEvidenceError::Corrupt)?;
878 if terminal.digest() != context.output_digest() {
879 return Err(GraphLifecycleEvidenceError::Corrupt);
880 }
881 let delta = model_usage(&invocation, response, self.definition.accounting.as_ref())
882 .map_err(evidence_node_error)?;
883 let usage = prior_usage
884 .checked_accumulate(&delta)
885 .map_err(|_| GraphLifecycleEvidenceError::Corrupt)?;
886 let intent = stored.admission().intent();
887 Ok(GraphTerminalEvidence::new(
888 intent.descriptor().clone(),
889 intent.request().clone(),
890 intent.budget().clone(),
891 AgentArtifacts::empty(),
892 usage,
893 ))
894 }
895
896 async fn failure_evidence_inner(
897 &self,
898 context: GraphFailureEvidenceContext,
899 ) -> Result<GraphFailureEvidence, GraphLifecycleEvidenceError> {
900 if context.blockers().in_flight() != 0 || context.blockers().unsupported() != 0 {
901 return Err(GraphLifecycleEvidenceError::Unavailable);
902 }
903 let (_, checkpoint, state) = self
904 .load_context(context.provenance(), context.graph(), context.checkpoint())
905 .await?;
906 let usage =
907 recover_failure_usage(&self.store, &self.definition, &checkpoint, &state).await?;
908 let failure = if context.blockers().superstep_limit_reached() {
909 crate::lifecycle::superstep_limit_failure()
910 } else {
911 load_current_node_failure(&self.store, &checkpoint).await?
912 };
913 Ok(GraphFailureEvidence::new(failure, usage))
914 }
915
916 async fn cancellation_evidence_inner(
917 &self,
918 context: GraphCancellationEvidenceContext,
919 ) -> Result<GraphCancellationEvidence, GraphLifecycleEvidenceError> {
920 let (_, checkpoint, state) = self
921 .load_context(context.provenance(), context.graph(), context.checkpoint())
922 .await?;
923 let usage =
924 recover_failure_usage(&self.store, &self.definition, &checkpoint, &state).await?;
925 Ok(GraphCancellationEvidence::new(usage))
926 }
927
928 async fn load_context(
929 &self,
930 provenance: &stateknot_core::AgentResultProvenance,
931 graph: &GraphReference,
932 head: &stateknot_core::CheckpointHead,
933 ) -> Result<
934 (
935 stateknot_store_postgres::StoredAgentAdmission,
936 Checkpoint,
937 ProviderNativeAgentState,
938 ),
939 GraphLifecycleEvidenceError,
940 > {
941 if graph != &self.definition.graph.reference()
942 || head.graph() != graph
943 || provenance.tenant_id() != head.tenant_id()
944 || provenance.run_id() != head.run_id()
945 {
946 return Err(GraphLifecycleEvidenceError::Corrupt);
947 }
948 let checkpoint = self
949 .store
950 .load_checkpoint(head.tenant_id(), head.run_id(), head.checkpoint_id())
951 .await
952 .map_err(|error| evidence_store_error(&error))?;
953 if checkpoint.head() != *head {
954 return Err(GraphLifecycleEvidenceError::Corrupt);
955 }
956 let stored = self
957 .store
958 .load_agent_admission(head.tenant_id(), head.run_id())
959 .await
960 .map_err(|error| evidence_store_error(&error))?;
961 validate_admission(&stored, &self.definition)
962 .map_err(|_| GraphLifecycleEvidenceError::Corrupt)?;
963 if stored.admission().intent().provenance() != provenance {
964 return Err(GraphLifecycleEvidenceError::Corrupt);
965 }
966 let state = self
967 .definition
968 .restore_state(checkpoint.state())
969 .map_err(|_| GraphLifecycleEvidenceError::Corrupt)?;
970 Ok((stored, checkpoint, state))
971 }
972}
973
974impl GraphLifecycleEvidenceProvider for ProviderNativeAgentLifecycleEvidence {
975 fn terminal_evidence(
976 &self,
977 context: GraphTerminalEvidenceContext,
978 ) -> BoxFuture<'_, Result<GraphTerminalEvidence, GraphLifecycleEvidenceError>> {
979 Box::pin(self.terminal_evidence_inner(context))
980 }
981
982 fn failure_evidence(
983 &self,
984 context: GraphFailureEvidenceContext,
985 ) -> BoxFuture<'_, Result<GraphFailureEvidence, GraphLifecycleEvidenceError>> {
986 Box::pin(self.failure_evidence_inner(context))
987 }
988
989 fn cancellation_evidence(
990 &self,
991 context: GraphCancellationEvidenceContext,
992 ) -> BoxFuture<'_, Result<GraphCancellationEvidence, GraphLifecycleEvidenceError>> {
993 Box::pin(self.cancellation_evidence_inner(context))
994 }
995}
996
997impl fmt::Debug for ProviderNativeAgentLifecycleEvidence {
998 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
999 formatter
1000 .debug_struct("ProviderNativeAgentLifecycleEvidence")
1001 .field("graph", &self.definition.graph.reference())
1002 .finish_non_exhaustive()
1003 }
1004}
1005
1006#[derive(Debug, Error)]
1008#[non_exhaustive]
1009pub enum ProviderNativeAgentGraphBuildError {
1010 #[error("provider-native agent graph requires model-native structured output")]
1012 StructuredOutputUnsupported,
1013 #[error("provider-native agent instructions use the reserved output-repair identity")]
1015 OutputRepairInstructionConflict,
1016 #[error("provider-native agent has no instruction capacity for output repair")]
1018 OutputRepairInstructionCapacity,
1019 #[error("output repair with Tools requires model support for tool selection none")]
1021 OutputRepairToolSelectionUnsupported,
1022 #[error("provider-native agent graph allows at most 256 tool calls per turn")]
1024 ToolCallsPerTurnLimit,
1025 #[error("provider-native agent graph exceeds the 4096 checkpoint invocation-reference limit")]
1027 CheckpointReferenceLimit,
1028 #[error("provider-native agent graph superstep limit is not representable")]
1030 SuperstepLimit,
1031 #[error("provider-native agent graph embedded definition is invalid")]
1033 StaticDefinition,
1034 #[error("provider-native agent graph state schema is invalid")]
1036 StateSchema,
1037 #[error("provider-native agent graph contract cannot be canonicalized")]
1039 Canonicalization,
1040 #[error(transparent)]
1042 Graph(#[from] GraphCompileError),
1043}
1044
1045#[derive(Debug, Error)]
1047#[non_exhaustive]
1048pub enum ProviderNativeAgentRegistrationError {
1049 #[error("provider-native agent invocation event schema is invalid")]
1051 EmbeddedEventSchema,
1052 #[error("provider-native agent requires schemas missing from the registry")]
1054 MissingSchema,
1055 #[error(transparent)]
1057 Registry(#[from] ExecutableGraphRegistryError),
1058}
1059
1060#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
1062#[non_exhaustive]
1063pub enum ProviderNativeAgentStateError {
1064 #[error("provider-native agent state contract digest does not match")]
1066 ContractMismatch,
1067 #[error("provider-native agent state exceeds its configured bounds")]
1069 Bounds,
1070 #[error("provider-native agent state transition is invalid")]
1072 Transition,
1073 #[error("provider-native agent state cannot be serialized")]
1075 Serialization,
1076 #[error("provider-native agent state exceeds the checkpoint resource limit")]
1078 ResourceLimit,
1079 #[error("provider-native agent checkpoint state integrity failed")]
1081 Integrity,
1082}
1083
1084#[derive(Clone)]
1085struct ProviderNativeReducer {
1086 reference: GraphReducerReference,
1087 graph: Arc<ProviderNativeAgentGraph>,
1088}
1089
1090impl GraphReducer for ProviderNativeReducer {
1091 fn reference(&self) -> &GraphReducerReference {
1092 &self.reference
1093 }
1094
1095 fn reduce(
1096 &self,
1097 state: &BoundedJson,
1098 updates: &[GraphReducerInput<'_>],
1099 ) -> Result<BoundedJson, GraphReducerError> {
1100 let current = decode_state(state).map_err(|_| GraphReducerError::Rejected)?;
1101 validate_state(¤t, &self.graph).map_err(|_| GraphReducerError::Rejected)?;
1102 if updates.is_empty() {
1103 return Ok(state.clone());
1104 }
1105 if updates.len() != 1 {
1106 return Err(GraphReducerError::Rejected);
1107 }
1108 let update = updates[0];
1109 let next = decode_state(update.update().data()).map_err(|_| GraphReducerError::Rejected)?;
1110 validate_state(&next, &self.graph).map_err(|_| GraphReducerError::Rejected)?;
1111 validate_transition(update.node_id(), ¤t, &next, &self.graph)
1112 .map_err(|_| GraphReducerError::Rejected)?;
1113 Ok(update.update().data().clone())
1114 }
1115}
1116
1117#[derive(Clone)]
1118struct ProviderNativeBindings {
1119 definition: Arc<ProviderNativeAgentGraph>,
1120 store: PostgresStore,
1121 invocation_executor: DurableInvocationExecutor,
1122 schemas: JsonSchemaRegistry,
1123 invocation_event_schema: SchemaReference,
1124}
1125
1126struct ProviderNativeModelNode {
1127 graph: GraphReference,
1128 node_id: NodeId,
1129 shared: Arc<ProviderNativeBindings>,
1130}
1131
1132impl GraphNodeExecutor for ProviderNativeModelNode {
1133 fn graph(&self) -> &GraphReference {
1134 &self.graph
1135 }
1136
1137 fn node_id(&self) -> &NodeId {
1138 &self.node_id
1139 }
1140
1141 fn execute(
1142 &self,
1143 context: GraphNodeContext,
1144 ) -> BoxFuture<'_, Result<GraphNodeExecution, GraphNodeExecutionError>> {
1145 Box::pin(async move {
1146 Box::pin(self.execute_model(context))
1147 .await
1148 .map_err(ProviderNativeAgentExecutionError::into_graph_error)
1149 })
1150 }
1151}
1152
1153impl ProviderNativeModelNode {
1154 async fn execute_model(
1155 &self,
1156 context: GraphNodeContext,
1157 ) -> Result<GraphNodeExecution, ProviderNativeAgentExecutionError> {
1158 let observation = Box::pin(self.observe_model(&context)).await?;
1159 let usage = observation.usage.clone();
1160 self.finish_model(observation)
1161 .await
1162 .map_err(|source| ProviderNativeAgentExecutionError::observed(source, usage))
1163 }
1164
1165 async fn observe_model(
1166 &self,
1167 context: &GraphNodeContext,
1168 ) -> Result<ProviderNativeModelObservation, ProviderNativeAgentExecutionError> {
1169 let state = self
1170 .shared
1171 .definition
1172 .decode_checkpoint(context.checkpoint().state())?;
1173 let plan = match &state.phase {
1174 ProviderNativeAgentPhase::Model { plan } => plan.clone(),
1175 ProviderNativeAgentPhase::Tools { .. } => {
1176 return Err(ProviderNativeAgentNodeError::InvalidState.into());
1177 }
1178 };
1179 let stored = self
1180 .shared
1181 .store
1182 .load_agent_admission(
1183 context.attempt().fence().tenant_id(),
1184 context.attempt().fence().run_id(),
1185 )
1186 .await
1187 .map_err(ProviderNativeAgentNodeError::Store)?;
1188 validate_admission(&stored, &self.shared.definition)?;
1189 let (transcript, prior_usage, output_repair_ordinal) = reconstruct_transcript(
1190 &self.shared.store,
1191 context.attempt().fence().tenant_id(),
1192 context.attempt().fence().run_id(),
1193 &state,
1194 &self.shared.definition,
1195 )
1196 .await?;
1197 let request = build_model_request(
1198 &stored,
1199 &self.shared.definition,
1200 state.input_message_id,
1201 transcript,
1202 &prior_usage,
1203 output_repair_ordinal,
1204 )?;
1205 let expected_intent = ModelInvocationIntent::new(
1206 context.attempt().activation().clone(),
1207 plan.invocation_id,
1208 self.shared.definition.descriptor.model().clone(),
1209 request,
1210 )
1211 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1212 let invocation = self
1213 .load_or_prepare_model(context, &plan, expected_intent)
1214 .await?;
1215 let invocation = self
1216 .dispatch_or_recover_model(context, &plan, invocation)
1217 .await?;
1218 let ProviderNativeTerminalModelEvidence {
1219 response,
1220 usage,
1221 failure,
1222 } = self.terminal_model_evidence(&invocation)?;
1223 if let Some(failure) = failure {
1224 return Err(ProviderNativeAgentExecutionError::observed(
1225 ProviderNativeAgentNodeError::ModelFailed(failure),
1226 usage,
1227 ));
1228 }
1229 let binding = NodeInvocationBinding::from_model(&invocation)
1230 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1231 let bindings = NodeInvocationBindings::try_new(context.attempt().activation(), [binding])
1232 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1233 Ok(ProviderNativeModelObservation {
1234 state,
1235 stored,
1236 plan,
1237 response,
1238 usage,
1239 bindings,
1240 })
1241 }
1242
1243 fn terminal_model_evidence(
1244 &self,
1245 invocation: &ModelInvocation,
1246 ) -> Result<ProviderNativeTerminalModelEvidence, ProviderNativeAgentNodeError> {
1247 let evidence = match invocation.state() {
1248 ModelInvocationState::Committed { response } => ProviderNativeTerminalModelEvidence {
1249 response: Some(response.clone()),
1250 usage: model_usage(
1251 invocation,
1252 response,
1253 self.shared.definition.accounting.as_ref(),
1254 )?,
1255 failure: None,
1256 },
1257 ModelInvocationState::Failed { error }
1258 if repairable_model_output_error(error)
1259 && self
1260 .shared
1261 .definition
1262 .descriptor
1263 .execution()
1264 .max_output_repair_turns()
1265 != ExecutionCount::ZERO =>
1266 {
1267 ProviderNativeTerminalModelEvidence {
1268 response: None,
1269 usage: failed_model_usage(
1270 invocation,
1271 error,
1272 self.shared.definition.accounting.as_ref(),
1273 true,
1274 )?,
1275 failure: None,
1276 }
1277 }
1278 ModelInvocationState::Failed { error } => {
1279 let usage = failed_model_usage(
1280 invocation,
1281 error,
1282 self.shared.definition.accounting.as_ref(),
1283 false,
1284 )?;
1285 ProviderNativeTerminalModelEvidence {
1286 response: None,
1287 usage,
1288 failure: Some(Box::new(error.failure().clone())),
1289 }
1290 }
1291 ModelInvocationState::Prepared | ModelInvocationState::Executing { .. } => {
1292 return Err(ProviderNativeAgentNodeError::UncertainInvocation);
1293 }
1294 };
1295 Ok(evidence)
1296 }
1297
1298 async fn load_or_prepare_model(
1299 &self,
1300 context: &GraphNodeContext,
1301 plan: &ProviderNativeModelPlan,
1302 expected_intent: ModelInvocationIntent,
1303 ) -> Result<ModelInvocation, ProviderNativeAgentNodeError> {
1304 let invocation = match self
1305 .shared
1306 .store
1307 .load_model_invocation(
1308 context.attempt().fence().tenant_id(),
1309 context.attempt().fence().run_id(),
1310 plan.invocation_id,
1311 )
1312 .await
1313 {
1314 Ok(invocation) => {
1315 if invocation.intent() != &expected_intent {
1316 return Err(ProviderNativeAgentNodeError::Integrity);
1317 }
1318 invocation
1319 }
1320 Err(StoreError::ModelInvocationNotFound) => {
1321 let payload = invocation_payload(
1322 &self.shared.schemas,
1323 &self.shared.invocation_event_schema,
1324 "model-invocation-prepared",
1325 "model_invocation_prepared",
1326 "model",
1327 plan.invocation_id,
1328 plan.attempt_id,
1329 expected_intent.intent_digest(),
1330 )?;
1331 let append = worker_append(
1332 context.attempt().fence(),
1333 context.attempt().journal_head().clone(),
1334 plan.prepared_event_id,
1335 payload,
1336 )?;
1337 self.shared
1338 .store
1339 .prepare_model_invocation(append, expected_intent)
1340 .await
1341 .map_err(ProviderNativeAgentNodeError::Store)?
1342 .invocation()
1343 .clone()
1344 }
1345 Err(error) => return Err(ProviderNativeAgentNodeError::Store(error)),
1346 };
1347 Ok(invocation)
1348 }
1349
1350 async fn dispatch_or_recover_model(
1351 &self,
1352 context: &GraphNodeContext,
1353 plan: &ProviderNativeModelPlan,
1354 invocation: ModelInvocation,
1355 ) -> Result<ModelInvocation, ProviderNativeAgentNodeError> {
1356 let invocation = match invocation.state() {
1357 ModelInvocationState::Prepared => {
1358 let events = InvocationAttemptEventIds::new(
1359 plan.attempt_start_event_id,
1360 plan.attempt_terminal_event_id,
1361 )
1362 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1363 let handoff = ModelAttemptHandoff::new(
1364 context.attempt().fence().clone(),
1365 invocation,
1366 plan.attempt_id,
1367 events,
1368 context.cancellation().clone(),
1369 None,
1370 )
1371 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1372 match self
1373 .shared
1374 .invocation_executor
1375 .execute_model(handoff)
1376 .await
1377 .map_err(|_| ProviderNativeAgentNodeError::InvocationExecutor)?
1378 {
1379 ModelAttemptOutcome::Dispatched { invocation, .. }
1380 | ModelAttemptOutcome::Recovered { invocation } => invocation,
1381 }
1382 }
1383 ModelInvocationState::Executing { .. } => {
1384 Err(ProviderNativeAgentNodeError::UncertainInvocation)?
1385 }
1386 ModelInvocationState::Committed { .. } | ModelInvocationState::Failed { .. } => {
1387 invocation
1388 }
1389 };
1390 Ok(invocation)
1391 }
1392
1393 async fn finish_model(
1394 &self,
1395 mut observation: ProviderNativeModelObservation,
1396 ) -> Result<GraphNodeExecution, ProviderNativeAgentNodeError> {
1397 let Some(response) = observation.response.as_ref() else {
1398 return self.output_repair_execution(observation);
1399 };
1400 match response.finish_reason() {
1401 ModelFinishReason::Completed => self.completed_model_execution(observation),
1402 ModelFinishReason::ToolCalls if output_repair_active(&observation.state) => {
1403 self.output_repair_execution(observation)
1404 }
1405 ModelFinishReason::ToolCalls => {
1406 let turn = observation
1407 .state
1408 .completed_turns
1409 .len()
1410 .checked_add(1)
1411 .ok_or(ProviderNativeAgentNodeError::Budget)?;
1412 let maximum_turns = usize::try_from(
1413 self.shared
1414 .definition
1415 .descriptor
1416 .execution()
1417 .max_model_turns()
1418 .get(),
1419 )
1420 .map_err(|_| ProviderNativeAgentNodeError::Budget)?;
1421 if turn >= maximum_turns {
1422 return Err(ProviderNativeAgentNodeError::Budget);
1423 }
1424 let plans = self
1425 .authorize_tool_plans(&observation.stored, &observation.plan, response)
1426 .await?;
1427 observation.state.phase = ProviderNativeAgentPhase::Tools {
1428 model_invocation_id: observation.plan.invocation_id,
1429 plans,
1430 };
1431 let update = self.shared.definition.encode_update(&observation.state)?;
1432 Ok(GraphNodeExecution::new(
1433 NodeStateChange::Update { update },
1434 NodeControl::Route {
1435 route_id: self.shared.definition.tools_route_id.clone(),
1436 },
1437 observation.bindings,
1438 observation.usage,
1439 ))
1440 }
1441 _ => Err(ProviderNativeAgentNodeError::IncompleteModelOutput),
1442 }
1443 }
1444
1445 fn completed_model_execution(
1446 &self,
1447 observation: ProviderNativeModelObservation,
1448 ) -> Result<GraphNodeExecution, ProviderNativeAgentNodeError> {
1449 let response = observation
1450 .response
1451 .as_ref()
1452 .ok_or(ProviderNativeAgentNodeError::Integrity)?;
1453 let output = match completed_json_output(response).and_then(|output| {
1454 self.shared
1455 .schemas
1456 .validate_bounded(self.shared.definition.descriptor.output_schema(), &output)
1457 .map(|()| output)
1458 .map_err(|_| ProviderNativeAgentNodeError::InvalidModelOutput)
1459 }) {
1460 Ok(output) => output,
1461 Err(ProviderNativeAgentNodeError::InvalidModelOutput) => {
1462 return self.output_repair_execution(observation);
1463 }
1464 Err(error) => return Err(error),
1465 };
1466 let terminal = NodeTerminalOutput::new(
1467 self.shared.definition.descriptor.output_schema().clone(),
1468 output,
1469 )
1470 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1471 Ok(GraphNodeExecution::new(
1472 NodeStateChange::Unchanged,
1473 NodeControl::Terminal { output: terminal },
1474 observation.bindings,
1475 observation.usage,
1476 ))
1477 }
1478
1479 fn output_repair_execution(
1480 &self,
1481 mut observation: ProviderNativeModelObservation,
1482 ) -> Result<GraphNodeExecution, ProviderNativeAgentNodeError> {
1483 let completed_repairs = output_repair_ordinal(&observation.state)?;
1484 let maximum_repairs = self
1485 .shared
1486 .definition
1487 .descriptor
1488 .execution()
1489 .max_output_repair_turns();
1490 if completed_repairs >= maximum_repairs {
1491 return Err(if maximum_repairs == ExecutionCount::ZERO {
1492 ProviderNativeAgentNodeError::InvalidModelOutput
1493 } else {
1494 ProviderNativeAgentNodeError::OutputRepairExhausted
1495 });
1496 }
1497 let completed_model_turns = u64::try_from(observation.state.completed_turns.len())
1498 .map_err(|_| ProviderNativeAgentNodeError::Budget)?
1499 .checked_add(1)
1500 .ok_or(ProviderNativeAgentNodeError::Budget)?;
1501 if completed_model_turns
1502 >= self
1503 .shared
1504 .definition
1505 .descriptor
1506 .execution()
1507 .max_model_turns()
1508 .get()
1509 {
1510 return Err(ProviderNativeAgentNodeError::Budget);
1511 }
1512 observation
1513 .state
1514 .completed_turns
1515 .push(ProviderNativeCompletedTurn {
1516 model_invocation_id: observation.plan.invocation_id,
1517 tool_invocation_ids: Vec::new(),
1518 });
1519 observation.state.phase = ProviderNativeAgentPhase::Model {
1520 plan: ProviderNativeModelPlan::generate(),
1521 };
1522 let update = self.shared.definition.encode_update(&observation.state)?;
1523 Ok(GraphNodeExecution::new(
1524 NodeStateChange::Update { update },
1525 NodeControl::Continue,
1526 observation.bindings,
1527 observation.usage,
1528 ))
1529 }
1530
1531 async fn authorize_tool_plans(
1532 &self,
1533 stored: &stateknot_store_postgres::StoredAgentAdmission,
1534 plan: &ProviderNativeModelPlan,
1535 response: &stateknot_core::ModelResponse,
1536 ) -> Result<Vec<ProviderNativeToolPlan>, ProviderNativeAgentNodeError> {
1537 let mut plans = Vec::with_capacity(response.tool_call_count());
1538 for (index, proposal) in response.tool_calls().enumerate() {
1539 if proposal.provider_call_id().is_none() {
1540 return Err(ProviderNativeAgentNodeError::InvalidModelOutput);
1541 }
1542 let digest = action_digest(
1543 stored.admission().digest(),
1544 plan.invocation_id,
1545 index,
1546 proposal,
1547 )?;
1548 let policy_context = AgentToolPolicyContext {
1549 agent: self
1550 .shared
1551 .definition
1552 .descriptor
1553 .metadata()
1554 .identity()
1555 .clone(),
1556 admission_digest: stored.admission().digest(),
1557 model_invocation_id: plan.invocation_id,
1558 proposal_index: index,
1559 proposal: proposal.clone(),
1560 action_digest: digest,
1561 };
1562 let decision = self
1563 .shared
1564 .definition
1565 .policy
1566 .evaluate(policy_context)
1567 .await
1568 .map_err(|_| ProviderNativeAgentNodeError::Policy)?;
1569 match decision {
1570 AgentToolPolicyDecision::Allow { evidence_digest } => {
1571 let proposal_index = u16::try_from(index)
1572 .map_err(|_| ProviderNativeAgentNodeError::InvalidModelOutput)?;
1573 plans.push(ProviderNativeToolPlan::generate(
1574 proposal_index,
1575 digest,
1576 evidence_digest,
1577 ));
1578 }
1579 AgentToolPolicyDecision::Deny { failure } => {
1580 return Err(ProviderNativeAgentNodeError::PolicyDenied(Box::new(
1581 failure,
1582 )));
1583 }
1584 }
1585 }
1586 if plans.is_empty() {
1587 return Err(ProviderNativeAgentNodeError::InvalidModelOutput);
1588 }
1589 Ok(plans)
1590 }
1591}
1592
1593struct ProviderNativeModelObservation {
1594 state: ProviderNativeAgentState,
1595 stored: stateknot_store_postgres::StoredAgentAdmission,
1596 plan: ProviderNativeModelPlan,
1597 response: Option<stateknot_core::ModelResponse>,
1598 usage: BudgetUsage,
1599 bindings: NodeInvocationBindings,
1600}
1601
1602struct ProviderNativeTerminalModelEvidence {
1603 response: Option<stateknot_core::ModelResponse>,
1604 usage: BudgetUsage,
1605 failure: Option<Box<Failure>>,
1606}
1607
1608struct ProviderNativeToolsNode {
1609 graph: GraphReference,
1610 node_id: NodeId,
1611 shared: Arc<ProviderNativeBindings>,
1612}
1613
1614impl GraphNodeExecutor for ProviderNativeToolsNode {
1615 fn graph(&self) -> &GraphReference {
1616 &self.graph
1617 }
1618
1619 fn node_id(&self) -> &NodeId {
1620 &self.node_id
1621 }
1622
1623 fn execute(
1624 &self,
1625 context: GraphNodeContext,
1626 ) -> BoxFuture<'_, Result<GraphNodeExecution, GraphNodeExecutionError>> {
1627 Box::pin(async move {
1628 Box::pin(self.execute_tools(context))
1629 .await
1630 .map_err(ProviderNativeAgentExecutionError::into_graph_error)
1631 })
1632 }
1633}
1634
1635impl ProviderNativeToolsNode {
1636 #[allow(clippy::too_many_lines)]
1637 async fn execute_tools(
1638 &self,
1639 context: GraphNodeContext,
1640 ) -> Result<GraphNodeExecution, ProviderNativeAgentExecutionError> {
1641 let mut observation = self.observe_tools(&context).await?;
1642
1643 let mut head = context.attempt().journal_head().clone();
1644 let mut outcomes = Vec::with_capacity(observation.plans.len());
1645 let mut bindings = Vec::with_capacity(observation.plans.len());
1646 let mut usage = BudgetUsage::zero();
1647 let proposals = observation
1648 .response
1649 .tool_calls()
1650 .cloned()
1651 .collect::<Vec<_>>();
1652 let concurrency = self
1653 .shared
1654 .definition
1655 .descriptor
1656 .execution()
1657 .tool_concurrency();
1658 let mut index = 0_usize;
1659 while index < proposals.len() {
1660 let parallel_end = match concurrency {
1661 AgentToolConcurrency::Sequential {} => index + 1,
1662 AgentToolConcurrency::ParallelReadOnly { max_concurrency } => {
1663 let maximum = usize::try_from(max_concurrency.get())
1664 .map_err(|_| ProviderNativeAgentNodeError::Budget)?;
1665 let mut end = index;
1666 while end < proposals.len()
1667 && end.saturating_sub(index) < maximum
1668 && self.proposal_tool_risk(&proposals[end])? == ToolRisk::ReadOnly
1669 {
1670 end = end.saturating_add(1);
1671 }
1672 end.max(index + 1)
1673 }
1674 };
1675 let steps = if parallel_end.saturating_sub(index) > 1 {
1676 Box::pin(self.execute_read_only_tool_wave(
1677 &context,
1678 &observation.stored,
1679 observation.model_invocation_id,
1680 head.clone(),
1681 index,
1682 &proposals[index..parallel_end],
1683 &observation.plans[index..parallel_end],
1684 ))
1685 .await
1686 .map_err(|source| {
1687 ProviderNativeAgentExecutionError::observed(source, usage.clone())
1688 })?
1689 } else {
1690 vec![
1691 self.execute_tool_plan(
1692 &context,
1693 &observation.stored,
1694 observation.model_invocation_id,
1695 head.clone(),
1696 index,
1697 &proposals[index],
1698 &observation.plans[index],
1699 )
1700 .await
1701 .map_err(|source| {
1702 ProviderNativeAgentExecutionError::observed(source, usage.clone())
1703 })?,
1704 ]
1705 };
1706 for step in steps {
1707 usage = usage.checked_accumulate(&step.usage).map_err(|_| {
1708 ProviderNativeAgentExecutionError::observed(
1709 ProviderNativeAgentNodeError::Budget,
1710 usage.clone(),
1711 )
1712 })?;
1713 bindings.push(step.binding);
1714 head = latest_journal_head(head, step.journal_head).map_err(|source| {
1715 ProviderNativeAgentExecutionError::observed(source, usage.clone())
1716 })?;
1717 outcomes.push(step.outcome);
1718 }
1719 index = parallel_end;
1720 }
1721 ModelTranscriptTurn::new(observation.response, outcomes).map_err(|_| {
1722 ProviderNativeAgentExecutionError::observed(
1723 ProviderNativeAgentNodeError::Integrity,
1724 usage.clone(),
1725 )
1726 })?;
1727 observation
1728 .state
1729 .completed_turns
1730 .push(ProviderNativeCompletedTurn {
1731 model_invocation_id: observation.model_invocation_id,
1732 tool_invocation_ids: observation
1733 .plans
1734 .iter()
1735 .map(|plan| plan.invocation_id)
1736 .collect(),
1737 });
1738 observation.state.phase = ProviderNativeAgentPhase::Model {
1739 plan: ProviderNativeModelPlan::generate(),
1740 };
1741 let update = self
1742 .shared
1743 .definition
1744 .encode_update(&observation.state)
1745 .map_err(|source| ProviderNativeAgentExecutionError::observed(source, usage.clone()))?;
1746 let bindings = NodeInvocationBindings::try_new(context.attempt().activation(), bindings)
1747 .map_err(|_| {
1748 ProviderNativeAgentExecutionError::observed(
1749 ProviderNativeAgentNodeError::Integrity,
1750 usage.clone(),
1751 )
1752 })?;
1753 Ok(GraphNodeExecution::new(
1754 NodeStateChange::Update { update },
1755 NodeControl::Continue,
1756 bindings,
1757 usage,
1758 ))
1759 }
1760
1761 async fn observe_tools(
1762 &self,
1763 context: &GraphNodeContext,
1764 ) -> Result<ProviderNativeToolsObservation, ProviderNativeAgentNodeError> {
1765 let state = self
1766 .shared
1767 .definition
1768 .decode_checkpoint(context.checkpoint().state())?;
1769 let (model_invocation_id, plans) = match &state.phase {
1770 ProviderNativeAgentPhase::Tools {
1771 model_invocation_id,
1772 plans,
1773 } => (*model_invocation_id, plans.clone()),
1774 ProviderNativeAgentPhase::Model { .. } => {
1775 return Err(ProviderNativeAgentNodeError::InvalidState);
1776 }
1777 };
1778 let stored = self
1779 .shared
1780 .store
1781 .load_agent_admission(
1782 context.attempt().fence().tenant_id(),
1783 context.attempt().fence().run_id(),
1784 )
1785 .await
1786 .map_err(ProviderNativeAgentNodeError::Store)?;
1787 validate_admission(&stored, &self.shared.definition)?;
1788 let model = self
1789 .shared
1790 .store
1791 .load_model_invocation(
1792 context.attempt().fence().tenant_id(),
1793 context.attempt().fence().run_id(),
1794 model_invocation_id,
1795 )
1796 .await
1797 .map_err(ProviderNativeAgentNodeError::Store)?;
1798 let response = match model.state() {
1799 ModelInvocationState::Committed { response }
1800 if response.finish_reason() == ModelFinishReason::ToolCalls =>
1801 {
1802 response.clone()
1803 }
1804 _ => return Err(ProviderNativeAgentNodeError::Integrity),
1805 };
1806 if response.tool_call_count() != plans.len() {
1807 return Err(ProviderNativeAgentNodeError::Integrity);
1808 }
1809 Ok(ProviderNativeToolsObservation {
1810 state,
1811 stored,
1812 model_invocation_id,
1813 plans,
1814 response,
1815 })
1816 }
1817
1818 #[allow(clippy::too_many_arguments)]
1819 async fn execute_tool_plan(
1820 &self,
1821 context: &GraphNodeContext,
1822 stored: &stateknot_store_postgres::StoredAgentAdmission,
1823 model_invocation_id: InvocationId,
1824 head: stateknot_core::JournalHead,
1825 index: usize,
1826 proposal: &ModelToolCallProposal,
1827 plan: &ProviderNativeToolPlan,
1828 ) -> Result<ProviderNativeToolStep, ProviderNativeAgentNodeError> {
1829 let (provider_call_id, expected_intent) =
1830 self.prepare_tool_intent(context, stored, model_invocation_id, index, proposal, plan)?;
1831 let invocation = self
1832 .load_or_prepare_tool(context, head, plan, expected_intent)
1833 .await?;
1834 let invocation = self
1835 .dispatch_or_recover_tool(context, plan, invocation)
1836 .await?;
1837 self.completed_tool_step(provider_call_id, &invocation)
1838 }
1839
1840 fn completed_tool_step(
1841 &self,
1842 provider_call_id: stateknot_core::ModelProviderToolCallId,
1843 invocation: &ToolInvocation,
1844 ) -> Result<ProviderNativeToolStep, ProviderNativeAgentNodeError> {
1845 let outcome = match invocation.state() {
1846 ToolInvocationState::Committed { result } => {
1847 ModelToolOutcome::succeeded(provider_call_id, result.clone())
1848 }
1849 ToolInvocationState::Failed { error } => {
1850 ModelToolOutcome::failed(provider_call_id, error)
1851 }
1852 ToolInvocationState::Prepared
1853 | ToolInvocationState::Executing { .. }
1854 | ToolInvocationState::Unknown { .. } => {
1855 return Err(ProviderNativeAgentNodeError::UncertainInvocation);
1856 }
1857 };
1858 let usage = tool_usage(invocation, self.shared.definition.accounting.as_ref())?;
1859 let binding = NodeInvocationBinding::from_tool(invocation)
1860 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1861 Ok(ProviderNativeToolStep {
1862 outcome,
1863 binding,
1864 journal_head: invocation.journal_head().clone(),
1865 usage,
1866 })
1867 }
1868
1869 fn proposal_tool_risk(
1870 &self,
1871 proposal: &ModelToolCallProposal,
1872 ) -> Result<ToolRisk, ProviderNativeAgentNodeError> {
1873 self.shared
1874 .definition
1875 .descriptor
1876 .tools()
1877 .iter()
1878 .find(|tool| tool.metadata().identity() == proposal.tool())
1879 .map(|tool| tool.semantics().risk())
1880 .ok_or(ProviderNativeAgentNodeError::InvalidModelOutput)
1881 }
1882
1883 #[allow(clippy::too_many_arguments)]
1884 async fn execute_read_only_tool_wave(
1885 &self,
1886 context: &GraphNodeContext,
1887 stored: &stateknot_store_postgres::StoredAgentAdmission,
1888 model_invocation_id: InvocationId,
1889 mut head: stateknot_core::JournalHead,
1890 first_index: usize,
1891 proposals: &[ModelToolCallProposal],
1892 plans: &[ProviderNativeToolPlan],
1893 ) -> Result<Vec<ProviderNativeToolStep>, ProviderNativeAgentNodeError> {
1894 if proposals.len() < 2 || proposals.len() != plans.len() {
1895 return Err(ProviderNativeAgentNodeError::Integrity);
1896 }
1897 let mut entries = Vec::with_capacity(proposals.len());
1898 let mut launch_error = None;
1899 for (offset, (proposal, plan)) in proposals.iter().zip(plans).enumerate() {
1900 let index = first_index
1901 .checked_add(offset)
1902 .ok_or(ProviderNativeAgentNodeError::Budget)?;
1903 let prepared = Box::pin(self.launch_read_only_tool(
1904 context,
1905 stored,
1906 model_invocation_id,
1907 head.clone(),
1908 index,
1909 proposal,
1910 plan,
1911 ))
1912 .await;
1913 match prepared {
1914 Ok((entry, next_head)) => {
1915 entries.push(entry);
1916 head = next_head;
1917 }
1918 Err(error) => {
1919 launch_error = Some(error);
1920 break;
1921 }
1922 }
1923 }
1924
1925 let settled = self.settle_read_only_tool_wave(entries).await;
1926 if let Some(error) = launch_error {
1927 let _ = settled;
1930 return Err(error);
1931 }
1932 settled
1933 }
1934
1935 #[allow(clippy::too_many_arguments)]
1936 async fn launch_read_only_tool(
1937 &self,
1938 context: &GraphNodeContext,
1939 stored: &stateknot_store_postgres::StoredAgentAdmission,
1940 model_invocation_id: InvocationId,
1941 mut head: stateknot_core::JournalHead,
1942 index: usize,
1943 proposal: &ModelToolCallProposal,
1944 plan: &ProviderNativeToolPlan,
1945 ) -> Result<
1946 (ProviderNativeToolWaveEntry, stateknot_core::JournalHead),
1947 ProviderNativeAgentNodeError,
1948 > {
1949 let (provider_call_id, expected_intent) =
1950 self.prepare_tool_intent(context, stored, model_invocation_id, index, proposal, plan)?;
1951 if expected_intent.descriptor().semantics().risk() != ToolRisk::ReadOnly {
1952 return Err(ProviderNativeAgentNodeError::Integrity);
1953 }
1954 let invocation = self
1955 .load_or_prepare_tool(context, head.clone(), plan, expected_intent)
1956 .await?;
1957 head = latest_journal_head(head, invocation.journal_head().clone())?;
1958 if !matches!(invocation.state(), ToolInvocationState::Prepared) {
1959 let invocation = self
1960 .dispatch_or_recover_tool(context, plan, invocation)
1961 .await?;
1962 head = latest_journal_head(head, invocation.journal_head().clone())?;
1963 return Ok((
1964 ProviderNativeToolWaveEntry::Resolved {
1965 provider_call_id,
1966 invocation: Box::new(invocation),
1967 },
1968 head,
1969 ));
1970 }
1971 let events = InvocationAttemptEventIds::new(
1972 plan.attempt_start_event_id,
1973 plan.attempt_terminal_event_id,
1974 )
1975 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1976 let handoff = ToolAttemptHandoff::new(
1977 context.attempt().fence().clone(),
1978 invocation,
1979 plan.attempt_id,
1980 events,
1981 context.cancellation().clone(),
1982 None,
1983 )
1984 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
1985 match self
1986 .shared
1987 .invocation_executor
1988 .start_tool_attempt(handoff)
1989 .await
1990 .map_err(|_| ProviderNativeAgentNodeError::InvocationExecutor)?
1991 {
1992 ToolAttemptStartOutcome::Started(dispatch) => {
1993 head = latest_journal_head(head, dispatch.invocation().journal_head().clone())?;
1994 let executor = self.shared.invocation_executor.clone();
1995 Ok((
1996 ProviderNativeToolWaveEntry::Dispatched {
1997 provider_call_id,
1998 task: AbortOnDropToolTask::new(tokio::spawn(async move {
1999 executor.dispatch_started_tool(*dispatch).await
2000 })),
2001 },
2002 head,
2003 ))
2004 }
2005 ToolAttemptStartOutcome::Recovered { invocation } => {
2006 let invocation = self
2007 .dispatch_or_recover_tool(context, plan, *invocation)
2008 .await?;
2009 head = latest_journal_head(head, invocation.journal_head().clone())?;
2010 Ok((
2011 ProviderNativeToolWaveEntry::Resolved {
2012 provider_call_id,
2013 invocation: Box::new(invocation),
2014 },
2015 head,
2016 ))
2017 }
2018 }
2019 }
2020
2021 async fn settle_read_only_tool_wave(
2022 &self,
2023 entries: Vec<ProviderNativeToolWaveEntry>,
2024 ) -> Result<Vec<ProviderNativeToolStep>, ProviderNativeAgentNodeError> {
2025 let mut steps = Vec::with_capacity(entries.len());
2026 let mut first_error = None;
2027 for entry in entries {
2028 let result = match entry {
2029 ProviderNativeToolWaveEntry::Resolved {
2030 provider_call_id,
2031 invocation,
2032 } => self.completed_tool_step(provider_call_id, invocation.as_ref()),
2033 ProviderNativeToolWaveEntry::Dispatched {
2034 provider_call_id,
2035 task,
2036 } => match task.join().await {
2037 Ok(terminal) => match self
2038 .shared
2039 .invocation_executor
2040 .commit_tool_terminal(terminal)
2041 .await
2042 {
2043 Ok(
2044 ToolAttemptOutcome::Dispatched { invocation, .. }
2045 | ToolAttemptOutcome::Recovered { invocation },
2046 ) => self.completed_tool_step(provider_call_id, &invocation),
2047 Err(_) => Err(ProviderNativeAgentNodeError::InvocationExecutor),
2048 },
2049 Err(_) => Err(ProviderNativeAgentNodeError::InvocationExecutor),
2050 },
2051 };
2052 match result {
2053 Ok(step) if first_error.is_none() => steps.push(step),
2054 Err(error) if first_error.is_none() => first_error = Some(error),
2055 Ok(_) | Err(_) => {}
2056 }
2057 }
2058 if let Some(error) = first_error {
2059 return Err(error);
2060 }
2061 Ok(steps)
2062 }
2063
2064 #[allow(clippy::too_many_arguments)]
2065 fn prepare_tool_intent(
2066 &self,
2067 context: &GraphNodeContext,
2068 stored: &stateknot_store_postgres::StoredAgentAdmission,
2069 model_invocation_id: InvocationId,
2070 index: usize,
2071 proposal: &ModelToolCallProposal,
2072 plan: &ProviderNativeToolPlan,
2073 ) -> Result<
2074 (
2075 stateknot_core::ModelProviderToolCallId,
2076 ToolInvocationIntent,
2077 ),
2078 ProviderNativeAgentNodeError,
2079 > {
2080 if usize::from(plan.proposal_index) != index {
2081 return Err(ProviderNativeAgentNodeError::Integrity);
2082 }
2083 let expected_action = action_digest(
2084 stored.admission().digest(),
2085 model_invocation_id,
2086 index,
2087 proposal,
2088 )?;
2089 if expected_action != plan.action_digest {
2090 return Err(ProviderNativeAgentNodeError::Integrity);
2091 }
2092 let provider_call_id = proposal
2093 .provider_call_id()
2094 .cloned()
2095 .ok_or(ProviderNativeAgentNodeError::InvalidModelOutput)?;
2096 let descriptor = self
2097 .shared
2098 .definition
2099 .descriptor
2100 .tools()
2101 .iter()
2102 .find(|tool| tool.metadata().identity() == proposal.tool())
2103 .cloned()
2104 .ok_or(ProviderNativeAgentNodeError::InvalidModelOutput)?;
2105 self.shared
2106 .schemas
2107 .validate_bounded(descriptor.input_schema(), proposal.arguments())
2108 .map_err(|_| ProviderNativeAgentNodeError::InvalidModelOutput)?;
2109 let input = ToolInput::new(
2110 descriptor.input_schema().clone(),
2111 proposal.arguments().clone(),
2112 )
2113 .map_err(|_| ProviderNativeAgentNodeError::InvalidModelOutput)?;
2114 let expected_intent = ToolInvocationIntent::new(
2115 context.attempt().activation().clone(),
2116 plan.invocation_id,
2117 descriptor.clone(),
2118 input,
2119 descriptor.limits().clone(),
2120 )
2121 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
2122 Ok((provider_call_id, expected_intent))
2123 }
2124
2125 async fn load_or_prepare_tool(
2126 &self,
2127 context: &GraphNodeContext,
2128 head: stateknot_core::JournalHead,
2129 plan: &ProviderNativeToolPlan,
2130 expected_intent: ToolInvocationIntent,
2131 ) -> Result<ToolInvocation, ProviderNativeAgentNodeError> {
2132 let invocation = match self
2133 .shared
2134 .store
2135 .load_tool_invocation(
2136 context.attempt().fence().tenant_id(),
2137 context.attempt().fence().run_id(),
2138 plan.invocation_id,
2139 )
2140 .await
2141 {
2142 Ok(invocation) => {
2143 if invocation.intent() != &expected_intent {
2144 return Err(ProviderNativeAgentNodeError::Integrity);
2145 }
2146 invocation
2147 }
2148 Err(StoreError::ToolInvocationNotFound) => {
2149 let payload = invocation_payload(
2150 &self.shared.schemas,
2151 &self.shared.invocation_event_schema,
2152 "tool-invocation-prepared",
2153 "tool_invocation_prepared",
2154 "tool",
2155 plan.invocation_id,
2156 plan.attempt_id,
2157 expected_intent.intent_digest(),
2158 )?;
2159 let append = worker_append(
2160 context.attempt().fence(),
2161 head,
2162 plan.prepared_event_id,
2163 payload,
2164 )?;
2165 self.shared
2166 .store
2167 .prepare_tool_invocation(append, expected_intent)
2168 .await
2169 .map_err(ProviderNativeAgentNodeError::Store)?
2170 .invocation()
2171 .clone()
2172 }
2173 Err(error) => return Err(ProviderNativeAgentNodeError::Store(error)),
2174 };
2175 Ok(invocation)
2176 }
2177
2178 async fn dispatch_or_recover_tool(
2179 &self,
2180 context: &GraphNodeContext,
2181 plan: &ProviderNativeToolPlan,
2182 invocation: ToolInvocation,
2183 ) -> Result<ToolInvocation, ProviderNativeAgentNodeError> {
2184 let invocation = match invocation.state() {
2185 ToolInvocationState::Prepared => {
2186 let events = InvocationAttemptEventIds::new(
2187 plan.attempt_start_event_id,
2188 plan.attempt_terminal_event_id,
2189 )
2190 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
2191 let handoff = ToolAttemptHandoff::new(
2192 context.attempt().fence().clone(),
2193 invocation,
2194 plan.attempt_id,
2195 events,
2196 context.cancellation().clone(),
2197 None,
2198 )
2199 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
2200 match self
2201 .shared
2202 .invocation_executor
2203 .execute_tool(handoff)
2204 .await
2205 .map_err(|_| ProviderNativeAgentNodeError::InvocationExecutor)?
2206 {
2207 ToolAttemptOutcome::Dispatched { invocation, .. }
2208 | ToolAttemptOutcome::Recovered { invocation } => invocation,
2209 }
2210 }
2211 ToolInvocationState::Executing { .. } => {
2212 return Err(ProviderNativeAgentNodeError::UncertainInvocation);
2213 }
2214 ToolInvocationState::Unknown { .. }
2215 | ToolInvocationState::Committed { .. }
2216 | ToolInvocationState::Failed { .. } => invocation,
2217 };
2218
2219 let ToolInvocationState::Unknown { .. } = invocation.state() else {
2220 return Ok(invocation);
2221 };
2222 if !invocation
2223 .intent()
2224 .descriptor()
2225 .semantics()
2226 .supports_status_query()
2227 {
2228 return Err(ProviderNativeAgentNodeError::UncertainInvocation);
2229 }
2230 let available = self
2231 .shared
2232 .invocation_executor
2233 .supports_tool_reconciliation(invocation.intent().descriptor())
2234 .map_err(|_| ProviderNativeAgentNodeError::InvocationExecutor)?;
2235 if !available {
2236 return Err(ProviderNativeAgentNodeError::UncertainInvocation);
2237 }
2238 let handoff = ToolReconciliationAttemptHandoff::new(
2239 context.attempt().fence().clone(),
2240 invocation,
2241 plan.reconciliation_event_id(),
2242 context.cancellation().clone(),
2243 )
2244 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
2245 let invocation = match self
2246 .shared
2247 .invocation_executor
2248 .reconcile_tool(handoff)
2249 .await
2250 .map_err(ProviderNativeAgentNodeError::Reconciliation)?
2251 {
2252 ToolReconciliationAttemptOutcome::Resolved { invocation, .. } => invocation,
2253 ToolReconciliationAttemptOutcome::Pending {
2254 retry_after,
2255 invocation: _,
2256 } => {
2257 return Err(ProviderNativeAgentNodeError::ReconciliationPending { retry_after });
2258 }
2259 };
2260 Ok(invocation)
2261 }
2262}
2263
2264struct ProviderNativeToolsObservation {
2265 state: ProviderNativeAgentState,
2266 stored: stateknot_store_postgres::StoredAgentAdmission,
2267 model_invocation_id: InvocationId,
2268 plans: Vec<ProviderNativeToolPlan>,
2269 response: stateknot_core::ModelResponse,
2270}
2271
2272struct ProviderNativeToolStep {
2273 outcome: ModelToolOutcome,
2274 binding: NodeInvocationBinding,
2275 journal_head: stateknot_core::JournalHead,
2276 usage: BudgetUsage,
2277}
2278
2279enum ProviderNativeToolWaveEntry {
2280 Resolved {
2281 provider_call_id: stateknot_core::ModelProviderToolCallId,
2282 invocation: Box<ToolInvocation>,
2283 },
2284 Dispatched {
2285 provider_call_id: stateknot_core::ModelProviderToolCallId,
2286 task: AbortOnDropToolTask,
2287 },
2288}
2289
2290struct AbortOnDropToolTask {
2294 task: tokio::task::JoinHandle<ToolTerminalCommitHandoff>,
2295}
2296
2297impl AbortOnDropToolTask {
2298 const fn new(task: tokio::task::JoinHandle<ToolTerminalCommitHandoff>) -> Self {
2299 Self { task }
2300 }
2301
2302 async fn join(mut self) -> Result<ToolTerminalCommitHandoff, tokio::task::JoinError> {
2303 (&mut self.task).await
2304 }
2305}
2306
2307impl Drop for AbortOnDropToolTask {
2308 fn drop(&mut self) {
2309 self.task.abort();
2310 }
2311}
2312
2313fn latest_journal_head(
2314 current: stateknot_core::JournalHead,
2315 candidate: stateknot_core::JournalHead,
2316) -> Result<stateknot_core::JournalHead, ProviderNativeAgentNodeError> {
2317 if current.tenant_id() != candidate.tenant_id() || current.run_id() != candidate.run_id() {
2318 return Err(ProviderNativeAgentNodeError::Integrity);
2319 }
2320 match current.sequence().cmp(&candidate.sequence()) {
2321 std::cmp::Ordering::Less => Ok(candidate),
2322 std::cmp::Ordering::Greater => Ok(current),
2323 std::cmp::Ordering::Equal if current == candidate => Ok(current),
2324 std::cmp::Ordering::Equal => Err(ProviderNativeAgentNodeError::Integrity),
2325 }
2326}
2327
2328fn validate_admission(
2329 stored: &stateknot_store_postgres::StoredAgentAdmission,
2330 definition: &ProviderNativeAgentGraph,
2331) -> Result<(), ProviderNativeAgentNodeError> {
2332 let intent = stored.admission().intent();
2333 if intent.descriptor() != &definition.descriptor
2334 || intent.graph() != &definition.graph.reference()
2335 {
2336 return Err(ProviderNativeAgentNodeError::Integrity);
2337 }
2338 Ok(())
2339}
2340
2341async fn reconstruct_transcript(
2342 store: &PostgresStore,
2343 tenant_id: &stateknot_core::TenantId,
2344 run_id: stateknot_core::RunId,
2345 state: &ProviderNativeAgentState,
2346 definition: &ProviderNativeAgentGraph,
2347) -> Result<(ModelTranscript, BudgetUsage, ExecutionCount), ProviderNativeAgentNodeError> {
2348 let mut turns = Vec::with_capacity(state.completed_turns.len());
2349 let mut usage = BudgetUsage::zero();
2350 let mut output_repair_turns = 0_u64;
2351 for turn in &state.completed_turns {
2352 let model = store
2353 .load_model_invocation(tenant_id, run_id, turn.model_invocation_id)
2354 .await
2355 .map_err(ProviderNativeAgentNodeError::Store)?;
2356 if turn.requires_output_repair() {
2357 let model_delta = match model.state() {
2358 ModelInvocationState::Committed { response } => {
2359 let valid_repair_marker = match response.finish_reason() {
2360 ModelFinishReason::Completed => true,
2361 ModelFinishReason::ToolCalls => output_repair_turns != 0,
2362 _ => false,
2363 };
2364 if !valid_repair_marker {
2365 return Err(ProviderNativeAgentNodeError::Integrity);
2366 }
2367 model_usage(&model, response, definition.accounting.as_ref())?
2368 }
2369 ModelInvocationState::Failed { error } if repairable_model_output_error(error) => {
2370 failed_model_usage(&model, error, definition.accounting.as_ref(), true)?
2371 }
2372 _ => return Err(ProviderNativeAgentNodeError::Integrity),
2373 };
2374 usage = usage
2375 .checked_accumulate(&model_delta)
2376 .map_err(|_| ProviderNativeAgentNodeError::Budget)?;
2377 output_repair_turns = output_repair_turns
2378 .checked_add(1)
2379 .ok_or(ProviderNativeAgentNodeError::Budget)?;
2380 continue;
2381 }
2382 let ModelInvocationState::Committed { response } = model.state() else {
2383 return Err(ProviderNativeAgentNodeError::Integrity);
2384 };
2385 if output_repair_turns != 0 || response.finish_reason() != ModelFinishReason::ToolCalls {
2386 return Err(ProviderNativeAgentNodeError::Integrity);
2387 }
2388 let model_delta = model_usage(&model, response, definition.accounting.as_ref())?;
2389 usage = usage
2390 .checked_accumulate(&model_delta)
2391 .map_err(|_| ProviderNativeAgentNodeError::Budget)?;
2392 if response.tool_call_count() != turn.tool_invocation_ids.len() {
2393 return Err(ProviderNativeAgentNodeError::Integrity);
2394 }
2395 let mut outcomes = Vec::with_capacity(turn.tool_invocation_ids.len());
2396 for (proposal, invocation_id) in response.tool_calls().zip(&turn.tool_invocation_ids) {
2397 let provider_call_id = proposal
2398 .provider_call_id()
2399 .cloned()
2400 .ok_or(ProviderNativeAgentNodeError::Integrity)?;
2401 let tool = store
2402 .load_tool_invocation(tenant_id, run_id, *invocation_id)
2403 .await
2404 .map_err(ProviderNativeAgentNodeError::Store)?;
2405 let outcome = match tool.state() {
2406 ToolInvocationState::Committed { result } => {
2407 ModelToolOutcome::succeeded(provider_call_id, result.clone())
2408 }
2409 ToolInvocationState::Failed { error } => {
2410 ModelToolOutcome::failed(provider_call_id, error)
2411 }
2412 ToolInvocationState::Prepared
2413 | ToolInvocationState::Executing { .. }
2414 | ToolInvocationState::Unknown { .. } => {
2415 return Err(ProviderNativeAgentNodeError::UncertainInvocation);
2416 }
2417 };
2418 let tool_delta = tool_usage(&tool, definition.accounting.as_ref())?;
2419 usage = usage
2420 .checked_accumulate(&tool_delta)
2421 .map_err(|_| ProviderNativeAgentNodeError::Budget)?;
2422 outcomes.push(outcome);
2423 }
2424 turns.push(
2425 ModelTranscriptTurn::new(response.clone(), outcomes)
2426 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?,
2427 );
2428 }
2429 let transcript =
2430 ModelTranscript::try_new(turns).map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
2431 Ok((transcript, usage, ExecutionCount::new(output_repair_turns)))
2432}
2433
2434fn validate_terminal_model_binding(
2435 invocation: &ModelInvocation,
2436 plan: &ProviderNativeModelPlan,
2437 checkpoint: &Checkpoint,
2438 model_node_id: &NodeId,
2439 expected_request: &ModelRequest,
2440) -> Result<(), GraphLifecycleEvidenceError> {
2441 let intent = invocation.intent();
2442 if intent.invocation_id() != plan.invocation_id()
2443 || invocation.attempt_id() != Some(plan.attempt_id())
2444 || intent.activation().base_checkpoint() != &checkpoint.head()
2445 || intent.activation().node_id() != model_node_id
2446 || intent.request() != expected_request
2447 {
2448 return Err(GraphLifecycleEvidenceError::Corrupt);
2449 }
2450 Ok(())
2451}
2452
2453async fn recover_failure_usage(
2454 store: &PostgresStore,
2455 definition: &ProviderNativeAgentGraph,
2456 checkpoint: &Checkpoint,
2457 state: &ProviderNativeAgentState,
2458) -> Result<BudgetUsage, GraphLifecycleEvidenceError> {
2459 let (_, usage, _) = reconstruct_transcript(
2460 store,
2461 checkpoint.tenant_id(),
2462 checkpoint.run_id(),
2463 state,
2464 definition,
2465 )
2466 .await
2467 .map_err(evidence_node_error)?;
2468 match state.phase() {
2469 ProviderNativeAgentPhase::Model { plan } => {
2470 recover_model_phase_usage(store, definition, checkpoint, plan, usage).await
2471 }
2472 ProviderNativeAgentPhase::Tools {
2473 model_invocation_id,
2474 plans,
2475 } => {
2476 recover_tools_phase_usage(
2477 store,
2478 definition,
2479 checkpoint,
2480 *model_invocation_id,
2481 plans,
2482 usage,
2483 )
2484 .await
2485 }
2486 }
2487}
2488
2489async fn recover_model_phase_usage(
2490 store: &PostgresStore,
2491 definition: &ProviderNativeAgentGraph,
2492 checkpoint: &Checkpoint,
2493 plan: &ProviderNativeModelPlan,
2494 usage: BudgetUsage,
2495) -> Result<BudgetUsage, GraphLifecycleEvidenceError> {
2496 let invocation = match store
2497 .load_model_invocation(
2498 checkpoint.tenant_id(),
2499 checkpoint.run_id(),
2500 plan.invocation_id(),
2501 )
2502 .await
2503 {
2504 Ok(invocation) => invocation,
2505 Err(StoreError::ModelInvocationNotFound) => return Ok(usage),
2506 Err(error) => return Err(evidence_store_error(&error)),
2507 };
2508 if invocation.intent().activation().base_checkpoint() != &checkpoint.head()
2509 || invocation.intent().activation().node_id() != &definition.model_node_id
2510 || invocation
2511 .attempt_id()
2512 .is_some_and(|id| id != plan.attempt_id())
2513 {
2514 return Err(GraphLifecycleEvidenceError::Corrupt);
2515 }
2516 let delta = match invocation.state() {
2517 ModelInvocationState::Prepared => return Ok(usage),
2518 ModelInvocationState::Executing { .. } => {
2519 return Err(GraphLifecycleEvidenceError::Unavailable);
2520 }
2521 ModelInvocationState::Committed { response } => {
2522 model_usage(&invocation, response, definition.accounting.as_ref())
2523 .map_err(evidence_node_error)?
2524 }
2525 ModelInvocationState::Failed { error } => {
2526 if error.usage().is_none() {
2527 return Err(GraphLifecycleEvidenceError::Unavailable);
2528 }
2529 failed_model_usage(
2530 &invocation,
2531 error,
2532 definition.accounting.as_ref(),
2533 definition.descriptor.execution().max_output_repair_turns() != ExecutionCount::ZERO
2534 && repairable_model_output_error(error),
2535 )
2536 .map_err(evidence_node_error)?
2537 }
2538 };
2539 usage
2540 .checked_accumulate(&delta)
2541 .map_err(|_| GraphLifecycleEvidenceError::Corrupt)
2542}
2543
2544async fn recover_tools_phase_usage(
2545 store: &PostgresStore,
2546 definition: &ProviderNativeAgentGraph,
2547 checkpoint: &Checkpoint,
2548 model_invocation_id: InvocationId,
2549 plans: &[ProviderNativeToolPlan],
2550 mut usage: BudgetUsage,
2551) -> Result<BudgetUsage, GraphLifecycleEvidenceError> {
2552 let model = store
2553 .load_model_invocation(
2554 checkpoint.tenant_id(),
2555 checkpoint.run_id(),
2556 model_invocation_id,
2557 )
2558 .await
2559 .map_err(|error| evidence_store_error(&error))?;
2560 let response = match model.state() {
2561 ModelInvocationState::Committed { response }
2562 if response.finish_reason() == ModelFinishReason::ToolCalls =>
2563 {
2564 response
2565 }
2566 _ => return Err(GraphLifecycleEvidenceError::Corrupt),
2567 };
2568 if response.tool_call_count() != plans.len()
2569 || checkpoint.parent().is_none_or(|parent| {
2570 model.intent().activation().base_checkpoint() != parent
2571 || model.intent().activation().node_id() != &definition.model_node_id
2572 })
2573 {
2574 return Err(GraphLifecycleEvidenceError::Corrupt);
2575 }
2576 let model_delta = model_usage(&model, response, definition.accounting.as_ref())
2577 .map_err(evidence_node_error)?;
2578 usage = usage
2579 .checked_accumulate(&model_delta)
2580 .map_err(|_| GraphLifecycleEvidenceError::Corrupt)?;
2581
2582 let mut undispatched = false;
2583 for plan in plans {
2584 let tool = match store
2585 .load_tool_invocation(
2586 checkpoint.tenant_id(),
2587 checkpoint.run_id(),
2588 plan.invocation_id(),
2589 )
2590 .await
2591 {
2592 Ok(tool) if !undispatched => tool,
2593 Ok(_) => return Err(GraphLifecycleEvidenceError::Corrupt),
2594 Err(StoreError::ToolInvocationNotFound) => {
2595 undispatched = true;
2596 continue;
2597 }
2598 Err(error) => return Err(evidence_store_error(&error)),
2599 };
2600 if tool.intent().activation().base_checkpoint() != &checkpoint.head()
2601 || tool.intent().activation().node_id() != &definition.tools_node_id
2602 || tool.attempt_id().is_some_and(|id| id != plan.attempt_id())
2603 {
2604 return Err(GraphLifecycleEvidenceError::Corrupt);
2605 }
2606 match tool.state() {
2607 ToolInvocationState::Prepared => undispatched = true,
2608 ToolInvocationState::Executing { .. } | ToolInvocationState::Unknown { .. } => {
2609 return Err(GraphLifecycleEvidenceError::Unavailable);
2610 }
2611 ToolInvocationState::Committed { .. } | ToolInvocationState::Failed { .. } => {
2612 let delta = tool_usage(&tool, definition.accounting.as_ref())
2613 .map_err(evidence_node_error)?;
2614 usage = usage
2615 .checked_accumulate(&delta)
2616 .map_err(|_| GraphLifecycleEvidenceError::Corrupt)?;
2617 }
2618 }
2619 }
2620 Ok(usage)
2621}
2622
2623async fn load_current_node_failure(
2624 store: &PostgresStore,
2625 checkpoint: &Checkpoint,
2626) -> Result<Failure, GraphLifecycleEvidenceError> {
2627 if checkpoint.ready_nodes().len() != 1 {
2628 return Err(GraphLifecycleEvidenceError::Corrupt);
2629 }
2630 let node_id = checkpoint
2631 .ready_nodes()
2632 .iter()
2633 .next()
2634 .cloned()
2635 .ok_or(GraphLifecycleEvidenceError::Corrupt)?;
2636 let activation = NodeActivation::for_ready_root(checkpoint, node_id)
2637 .map_err(|_| GraphLifecycleEvidenceError::Corrupt)?;
2638 let page_size = NodeAttemptHistoryPageSize::new(NodeAttemptHistoryPageSize::MAX)
2639 .map_err(|error| evidence_store_error(&error))?;
2640 let mut cursor = None;
2641 let mut latest = None;
2642 let mut observed = 0_usize;
2643 loop {
2644 let page = store
2645 .load_node_attempt_history_page(&activation, cursor.as_ref(), page_size)
2646 .await
2647 .map_err(|error| evidence_store_error(&error))?;
2648 observed = observed.saturating_add(page.records().len());
2649 if observed > 64 {
2650 return Err(GraphLifecycleEvidenceError::Corrupt);
2651 }
2652 for attempt in page.records() {
2653 if let Some(failure) = attempt
2654 .completion()
2655 .and_then(|completion| completion.outcome().failure())
2656 {
2657 latest = Some(failure.clone());
2658 }
2659 }
2660 if !page.has_more() {
2661 break;
2662 }
2663 cursor = page.next_cursor();
2664 if cursor.is_none() {
2665 return Err(GraphLifecycleEvidenceError::Corrupt);
2666 }
2667 }
2668 latest.ok_or(GraphLifecycleEvidenceError::Unavailable)
2669}
2670
2671fn evidence_node_error(error: ProviderNativeAgentNodeError) -> GraphLifecycleEvidenceError {
2672 match error {
2673 ProviderNativeAgentNodeError::Store(error) => evidence_store_error(&error),
2674 ProviderNativeAgentNodeError::UncertainInvocation => {
2675 GraphLifecycleEvidenceError::Unavailable
2676 }
2677 _ => GraphLifecycleEvidenceError::Corrupt,
2678 }
2679}
2680
2681fn evidence_store_error(error: &StoreError) -> GraphLifecycleEvidenceError {
2682 if error.corrupt_record().is_some() {
2683 GraphLifecycleEvidenceError::Corrupt
2684 } else if error.is_retryable() {
2685 GraphLifecycleEvidenceError::TemporarilyUnavailable
2686 } else {
2687 GraphLifecycleEvidenceError::Unavailable
2688 }
2689}
2690
2691fn build_model_request(
2692 stored: &stateknot_store_postgres::StoredAgentAdmission,
2693 definition: &ProviderNativeAgentGraph,
2694 input_message_id: MessageId,
2695 transcript: ModelTranscript,
2696 prior_usage: &BudgetUsage,
2697 output_repair_ordinal: ExecutionCount,
2698) -> Result<ModelRequest, ProviderNativeAgentNodeError> {
2699 let admission = stored.admission();
2700 let intent = admission.intent();
2701 let remaining = intent
2702 .budget()
2703 .remaining(prior_usage, admission.admitted_at())
2704 .map_err(|_| ProviderNativeAgentNodeError::Budget)?;
2705 let (max_input_tokens, max_output_tokens) = model_token_limits(
2706 definition.descriptor.model(),
2707 remaining.input_tokens(),
2708 remaining.output_tokens(),
2709 )?;
2710 let max_content_bytes = remaining
2711 .input_bytes()
2712 .min(ModelRequestLimits::HARD_MAX_CONTENT_BYTES);
2713 let limits = ModelRequestLimits::new(max_input_tokens, max_output_tokens, max_content_bytes)
2714 .map_err(|_| ProviderNativeAgentNodeError::Budget)?;
2715 let metadata =
2716 ContentMetadata::untrusted(ContentSource::User, definition.input_security_label.clone());
2717 let content = ContentPart::Json(JsonContent::new(
2718 intent.request().input().clone(),
2719 Some(intent.request().input_schema().clone()),
2720 metadata,
2721 ));
2722 let parts =
2723 MessageParts::try_new([content]).map_err(|_| ProviderNativeAgentNodeError::InvalidState)?;
2724 let message = Message::new(
2725 input_message_id,
2726 MessageRole::User,
2727 parts,
2728 MessageProvenance::new(
2729 intent.provenance().run_id(),
2730 stored.event().event_id(),
2731 MessageProducer::Principal {
2732 principal: intent.authority().principal().clone(),
2733 },
2734 ),
2735 )
2736 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
2737 let repairing_output = output_repair_ordinal != ExecutionCount::ZERO;
2738 let historical_tools = if repairing_output {
2739 transcript
2740 .iter()
2741 .flat_map(|turn| turn.response().tool_calls())
2742 .map(|proposal| proposal.tool().clone())
2743 .collect::<BTreeSet<_>>()
2744 } else {
2745 BTreeSet::new()
2746 };
2747 let mut builder = ModelRequest::builder(limits)
2748 .message(message)
2749 .transcript(transcript)
2750 .text_output_format(Some(ModelTextOutputFormat::json_schema(
2751 definition.descriptor.output_schema().clone(),
2752 )));
2753 for instruction in definition.descriptor.instructions() {
2754 builder = builder.instruction(instruction.clone());
2755 }
2756 if repairing_output {
2757 let instruction = definition
2758 .output_repair_instruction
2759 .as_ref()
2760 .ok_or(ProviderNativeAgentNodeError::Integrity)?;
2761 builder = builder.instruction(instruction.clone());
2762 }
2763 if repairing_output || definition.descriptor.tools().is_empty() {
2764 for tool in definition.descriptor.tools() {
2765 if historical_tools.contains(tool.metadata().identity()) {
2766 builder = builder.tool(tool.clone());
2767 }
2768 }
2769 builder = builder
2770 .tool_selection(ModelToolSelection::none())
2771 .max_tool_calls_per_response(ExecutionCount::ZERO)
2772 .strict_tool_arguments(false);
2773 } else {
2774 for tool in definition.descriptor.tools() {
2775 builder = builder.tool(tool.clone());
2776 }
2777 builder = builder
2778 .tool_selection(ModelToolSelection::auto())
2779 .max_tool_calls_per_response(
2780 definition.descriptor.execution().max_tool_calls_per_turn(),
2781 )
2782 .strict_tool_arguments(true);
2783 }
2784 builder
2785 .build()
2786 .map_err(|_| ProviderNativeAgentNodeError::InvalidState)
2787}
2788
2789fn model_token_limits(
2790 descriptor: &stateknot_core::ModelDescriptor,
2791 remaining_input: TokenCount,
2792 remaining_output: TokenCount,
2793) -> Result<(TokenCount, TokenCount), ProviderNativeAgentNodeError> {
2794 let published = descriptor.capabilities().token_limits();
2795 let mut input = published
2796 .max_input_tokens()
2797 .map_or(remaining_input, |maximum| maximum.min(remaining_input));
2798 let mut output = published
2799 .max_output_tokens()
2800 .map_or(remaining_output, |maximum| maximum.min(remaining_output));
2801 if input == TokenCount::ZERO || output == TokenCount::ZERO {
2802 return Err(ProviderNativeAgentNodeError::Budget);
2803 }
2804 if let Some(context) = published.max_context_tokens() {
2805 let sum = input
2806 .checked_add(output)
2807 .ok_or(ProviderNativeAgentNodeError::Budget)?;
2808 if sum > context {
2809 if context.get() < 2 {
2810 return Err(ProviderNativeAgentNodeError::Budget);
2811 }
2812 output = output.min(TokenCount::new(context.get() - 1));
2813 input = input.min(TokenCount::new(context.get() - output.get()));
2814 }
2815 }
2816 if input == TokenCount::ZERO || output == TokenCount::ZERO {
2817 return Err(ProviderNativeAgentNodeError::Budget);
2818 }
2819 Ok((input, output))
2820}
2821
2822fn completed_json_output(
2823 response: &stateknot_core::ModelResponse,
2824) -> Result<BoundedJson, ProviderNativeAgentNodeError> {
2825 let mut content = response
2826 .output()
2827 .iter()
2828 .filter_map(ModelOutputItem::as_content);
2829 let Some(ContentPart::Json(json)) = content.next() else {
2830 return Err(ProviderNativeAgentNodeError::InvalidModelOutput);
2831 };
2832 if content.next().is_some() {
2833 return Err(ProviderNativeAgentNodeError::InvalidModelOutput);
2834 }
2835 Ok(json.value().clone())
2836}
2837
2838fn output_repair_active(state: &ProviderNativeAgentState) -> bool {
2839 state
2840 .completed_turns
2841 .last()
2842 .is_some_and(ProviderNativeCompletedTurn::requires_output_repair)
2843}
2844
2845fn output_repair_ordinal(
2846 state: &ProviderNativeAgentState,
2847) -> Result<ExecutionCount, ProviderNativeAgentNodeError> {
2848 let repairs = state
2849 .completed_turns
2850 .iter()
2851 .filter(|turn| turn.requires_output_repair())
2852 .count();
2853 Ok(ExecutionCount::new(
2854 u64::try_from(repairs).map_err(|_| ProviderNativeAgentNodeError::Budget)?,
2855 ))
2856}
2857
2858fn repairable_model_output_error(error: &ModelError) -> bool {
2859 error.phase() == ModelErrorPhase::Response
2860 && error.failure().code().as_str() == "response.malformed"
2861 && error.usage().is_some()
2862}
2863
2864fn model_usage(
2865 invocation: &ModelInvocation,
2866 response: &stateknot_core::ModelResponse,
2867 accounting: &dyn AgentInvocationAccounting,
2868) -> Result<BudgetUsage, ProviderNativeAgentNodeError> {
2869 let observed = response.usage();
2870 let input_bytes = serde_json_canonicalizer::to_vec(invocation.intent().request())
2871 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?
2872 .len();
2873 let builder = BudgetUsage::builder()
2874 .model_attempts(ExecutionCount::new(1))
2875 .model_turns(ExecutionCount::new(1))
2876 .input_tokens(observed.input_tokens())
2877 .cached_input_tokens(
2878 observed
2879 .cached_input_tokens()
2880 .unwrap_or_else(|| observed.input_tokens()),
2881 )
2882 .output_tokens(observed.output_tokens())
2883 .reasoning_tokens(
2884 observed
2885 .reasoning_tokens()
2886 .unwrap_or_else(|| observed.output_tokens()),
2887 )
2888 .input_bytes(ByteCount::new(input_bytes as u64))
2889 .output_bytes(response.inline_payload_bytes());
2890 let builder = match accounting.model_charge(invocation) {
2891 AgentInvocationCharge::Known(costs) => builder.known_costs(costs),
2892 AgentInvocationCharge::Unpriced => builder.unpriced_cost_events(ExecutionCount::new(1)),
2893 };
2894 builder
2895 .build()
2896 .map_err(|_| ProviderNativeAgentNodeError::Budget)
2897}
2898
2899fn failed_model_usage(
2900 invocation: &ModelInvocation,
2901 error: &stateknot_core::ModelError,
2902 accounting: &dyn AgentInvocationAccounting,
2903 count_model_turn: bool,
2904) -> Result<BudgetUsage, ProviderNativeAgentNodeError> {
2905 let input_bytes = serde_json_canonicalizer::to_vec(invocation.intent().request())
2906 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?
2907 .len();
2908 let builder = BudgetUsage::builder()
2909 .model_attempts(ExecutionCount::new(1))
2910 .model_turns(if count_model_turn {
2911 ExecutionCount::new(1)
2912 } else {
2913 ExecutionCount::ZERO
2914 })
2915 .input_bytes(ByteCount::new(input_bytes as u64));
2916 let mut builder = match accounting.model_charge(invocation) {
2917 AgentInvocationCharge::Known(costs) => builder.known_costs(costs),
2918 AgentInvocationCharge::Unpriced => builder.unpriced_cost_events(ExecutionCount::new(1)),
2919 };
2920 if let Some(observed) = error.usage() {
2921 builder = builder
2922 .input_tokens(observed.input_tokens())
2923 .cached_input_tokens(
2924 observed
2925 .cached_input_tokens()
2926 .unwrap_or_else(|| observed.input_tokens()),
2927 )
2928 .output_tokens(observed.output_tokens())
2929 .reasoning_tokens(
2930 observed
2931 .reasoning_tokens()
2932 .unwrap_or_else(|| observed.output_tokens()),
2933 );
2934 }
2935 builder
2936 .build()
2937 .map_err(|_| ProviderNativeAgentNodeError::Budget)
2938}
2939
2940fn tool_usage(
2941 invocation: &ToolInvocation,
2942 accounting: &dyn AgentInvocationAccounting,
2943) -> Result<BudgetUsage, ProviderNativeAgentNodeError> {
2944 let descriptor = invocation.intent().descriptor();
2945 let input_bytes =
2946 ByteCount::new(invocation.intent().input().value().stats().compact_bytes() as u64);
2947 let (output_bytes, artifact_bytes) = match invocation.state() {
2948 ToolInvocationState::Committed { result } => (
2949 ByteCount::new(result.output().stats().compact_bytes() as u64),
2950 result.artifacts().total_bytes(),
2951 ),
2952 ToolInvocationState::Failed { .. } => (ByteCount::ZERO, ByteCount::ZERO),
2953 ToolInvocationState::Prepared
2954 | ToolInvocationState::Executing { .. }
2955 | ToolInvocationState::Unknown { .. } => {
2956 return Err(ProviderNativeAgentNodeError::UncertainInvocation);
2957 }
2958 };
2959 let builder = BudgetUsage::builder()
2960 .tool_calls(ExecutionCount::new(1))
2961 .write_calls(if descriptor.semantics().risk() == ToolRisk::ReadOnly {
2962 ExecutionCount::ZERO
2963 } else {
2964 ExecutionCount::new(1)
2965 })
2966 .input_bytes(input_bytes)
2967 .output_bytes(output_bytes)
2968 .artifact_bytes(artifact_bytes);
2969 let builder = match accounting.tool_charge(invocation) {
2970 AgentInvocationCharge::Known(costs) => builder.known_costs(costs),
2971 AgentInvocationCharge::Unpriced => builder.unpriced_cost_events(ExecutionCount::new(1)),
2972 };
2973 builder
2974 .build()
2975 .map_err(|_| ProviderNativeAgentNodeError::Budget)
2976}
2977
2978#[allow(clippy::too_many_arguments)]
2979fn invocation_payload(
2980 schemas: &JsonSchemaRegistry,
2981 schema: &SchemaReference,
2982 event_kind: &'static str,
2983 operation: &'static str,
2984 binding_kind: &'static str,
2985 invocation_id: InvocationId,
2986 attempt_id: AttemptId,
2987 intent_digest: Digest,
2988) -> Result<JournalPayload, ProviderNativeAgentNodeError> {
2989 let data = BoundedJson::try_from_value(json!({
2990 "operation": operation,
2991 "binding_kind": binding_kind,
2992 "invocation_id": invocation_id.to_string(),
2993 "attempt_id": attempt_id.to_string(),
2994 "intent_digest": digest_hex(intent_digest),
2995 }))
2996 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
2997 schemas
2998 .validate_bounded(schema, &data)
2999 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
3000 JournalPayload::new(
3001 schema.clone(),
3002 JournalEventKind::new(event_kind).map_err(|_| ProviderNativeAgentNodeError::Integrity)?,
3003 data,
3004 )
3005 .map_err(|_| ProviderNativeAgentNodeError::Integrity)
3006}
3007
3008fn digest_hex(digest: Digest) -> String {
3009 const HEX: &[u8; 16] = b"0123456789abcdef";
3010 let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
3011 for byte in digest.as_bytes() {
3012 value.push(char::from(HEX[usize::from(byte >> 4)]));
3013 value.push(char::from(HEX[usize::from(byte & 0x0f)]));
3014 }
3015 value
3016}
3017
3018fn worker_append(
3019 fence: &stateknot_core::RunFence,
3020 head: stateknot_core::JournalHead,
3021 event_id: EventId,
3022 payload: JournalPayload,
3023) -> Result<JournalAppend, ProviderNativeAgentNodeError> {
3024 let intent = JournalEventIntent::worker(
3025 fence.tenant_id().clone(),
3026 fence.run_id(),
3027 event_id,
3028 fence.clone(),
3029 payload,
3030 )
3031 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
3032 JournalAppend::new(JournalExpectation::exact(head), intent)
3033 .map_err(|_| ProviderNativeAgentNodeError::Integrity)
3034}
3035
3036#[allow(clippy::too_many_lines)]
3037fn node_execution_error(
3038 error: ProviderNativeAgentNodeError,
3039 usage: BudgetUsage,
3040) -> GraphNodeExecutionError {
3041 match &error {
3042 ProviderNativeAgentNodeError::ModelFailed(failure)
3043 | ProviderNativeAgentNodeError::PolicyDenied(failure) => {
3044 if let Ok(error) = GraphNodeExecutionError::new(failure.as_ref().clone(), usage.clone())
3045 {
3046 return error;
3047 }
3048 }
3049 _ => {}
3050 }
3051 let (category, code, message, advice) = match &error {
3052 ProviderNativeAgentNodeError::Store(source) if source.corrupt_record().is_some() => (
3053 FailureCategory::DataCorruption,
3054 "runtime.agent.corrupt_evidence",
3055 "Durable agent evidence failed integrity validation.",
3056 RetryAdvice::Never,
3057 ),
3058 ProviderNativeAgentNodeError::Store(source) if source.is_retryable() => (
3059 FailureCategory::DependencyUnavailable,
3060 "runtime.agent.store_unavailable",
3061 "Durable agent storage is temporarily unavailable.",
3062 RetryAdvice::SafeAfter {
3063 delay: DurationMillis::new(100).expect("static duration is valid"),
3064 },
3065 ),
3066 ProviderNativeAgentNodeError::UncertainInvocation => (
3067 FailureCategory::Conflict,
3068 "runtime.agent.invocation_uncertain",
3069 "An external invocation must be reconciled before the run can continue.",
3070 RetryAdvice::Never,
3071 ),
3072 ProviderNativeAgentNodeError::ReconciliationPending { retry_after } => (
3073 FailureCategory::DependencyUnavailable,
3074 "runtime.agent.reconciliation_pending",
3075 "The external invocation has no authoritative outcome yet.",
3076 RetryAdvice::SafeAfter {
3077 delay: *retry_after,
3078 },
3079 ),
3080 ProviderNativeAgentNodeError::Reconciliation(source) => {
3081 let advice = match source.retry_advice() {
3082 RetryAdvice::ReconcileFirst => RetryAdvice::Never,
3083 advice => advice,
3084 };
3085 (
3086 match advice {
3087 RetryAdvice::SafeAfter { .. } => FailureCategory::DependencyUnavailable,
3088 RetryAdvice::Never | RetryAdvice::ReconcileFirst => FailureCategory::Internal,
3089 },
3090 "runtime.agent.reconciliation_failed",
3091 "The external invocation could not be reconciled safely.",
3092 advice,
3093 )
3094 }
3095 ProviderNativeAgentNodeError::Budget => (
3096 FailureCategory::RateLimited,
3097 "runtime.agent.budget_exhausted",
3098 "The agent exhausted a finite execution budget.",
3099 RetryAdvice::Never,
3100 ),
3101 ProviderNativeAgentNodeError::InvalidModelOutput
3102 | ProviderNativeAgentNodeError::IncompleteModelOutput => (
3103 FailureCategory::InvalidInput,
3104 "runtime.agent.invalid_model_output",
3105 "The model did not produce a valid terminal output or tool turn.",
3106 RetryAdvice::Never,
3107 ),
3108 ProviderNativeAgentNodeError::OutputRepairExhausted => (
3109 FailureCategory::InvalidInput,
3110 "runtime.agent.output_repair_exhausted",
3111 "The model exhausted the configured structured-output repair turns.",
3112 RetryAdvice::Never,
3113 ),
3114 ProviderNativeAgentNodeError::Policy => (
3115 FailureCategory::DependencyUnavailable,
3116 "runtime.agent.policy_unavailable",
3117 "The pinned agent tool policy is unavailable.",
3118 RetryAdvice::SafeAfter {
3119 delay: DurationMillis::new(100).expect("static duration is valid"),
3120 },
3121 ),
3122 ProviderNativeAgentNodeError::InvocationExecutor => (
3123 FailureCategory::DependencyUnavailable,
3124 "runtime.agent.invocation_executor",
3125 "The durable invocation executor could not complete the attempt.",
3126 RetryAdvice::Never,
3127 ),
3128 ProviderNativeAgentNodeError::Store(_)
3129 | ProviderNativeAgentNodeError::Integrity
3130 | ProviderNativeAgentNodeError::InvalidState
3131 | ProviderNativeAgentNodeError::ModelFailed(_)
3132 | ProviderNativeAgentNodeError::PolicyDenied(_) => (
3133 FailureCategory::Internal,
3134 "runtime.agent.invalid_state",
3135 "The provider-native agent state could not be executed safely.",
3136 RetryAdvice::Never,
3137 ),
3138 };
3139 let failure = Failure::new(
3140 FailureId::generate(),
3141 category,
3142 FailureCode::new(code).expect("static failure code is valid"),
3143 FailureOrigin::new("stateknot.runtime.provider-native-agent")
3144 .expect("static failure origin is valid"),
3145 FailureMessage::new(message).expect("static failure message is valid"),
3146 advice,
3147 )
3148 .expect("static failure semantics are coherent")
3149 .with_private_source(error);
3150 GraphNodeExecutionError::new(failure, usage)
3151 .expect("provider-native node failures are uncaused and never reconcile-first")
3152}
3153
3154fn validate_supported_execution(
3155 descriptor: &AgentDescriptor,
3156) -> Result<(), ProviderNativeAgentGraphBuildError> {
3157 if descriptor.execution().structured_output() != AgentStructuredOutputStrategy::ModelNative {
3158 return Err(ProviderNativeAgentGraphBuildError::StructuredOutputUnsupported);
3159 }
3160 if descriptor.execution().max_output_repair_turns() != ExecutionCount::ZERO
3161 && !descriptor.tools().is_empty()
3162 && !descriptor
3163 .model()
3164 .capabilities()
3165 .tools()
3166 .choices()
3167 .contains(ModelToolChoice::None)
3168 {
3169 return Err(ProviderNativeAgentGraphBuildError::OutputRepairToolSelectionUnsupported);
3170 }
3171 if descriptor.execution().max_tool_calls_per_turn().get() > 256 {
3172 return Err(ProviderNativeAgentGraphBuildError::ToolCallsPerTurnLimit);
3173 }
3174 Ok(())
3175}
3176
3177fn build_output_repair_instruction(
3178 descriptor: &AgentDescriptor,
3179) -> Result<Option<Instruction>, ProviderNativeAgentGraphBuildError> {
3180 if descriptor.execution().max_output_repair_turns() == ExecutionCount::ZERO {
3181 return Ok(None);
3182 }
3183 if descriptor.instructions().len() == AgentInstructions::MAX_LEN {
3184 return Err(ProviderNativeAgentGraphBuildError::OutputRepairInstructionCapacity);
3185 }
3186 if descriptor
3187 .instructions()
3188 .iter()
3189 .any(|instruction| instruction.identity().name().as_str() == OUTPUT_REPAIR_INSTRUCTION_NAME)
3190 {
3191 return Err(ProviderNativeAgentGraphBuildError::OutputRepairInstructionConflict);
3192 }
3193 let metadata = ContentMetadata::new(
3194 ContentSource::Application,
3195 ContentTrust::ApplicationControlled,
3196 SecurityLabel::new("stateknot/internal/output-repair")
3197 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?,
3198 RedactionState::NotApplied,
3199 );
3200 let content = TextContent::new(OUTPUT_REPAIR_INSTRUCTION_TEXT, None, metadata)
3201 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?;
3202 let identity = InstructionIdentity::new(
3203 InstructionName::new(OUTPUT_REPAIR_INSTRUCTION_NAME)
3204 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)?,
3205 Version::new(1, 0, 0),
3206 );
3207 Instruction::new(
3208 identity,
3209 content.into(),
3210 InstructionProvenance::new(descriptor.metadata().identity().owner().clone()),
3211 )
3212 .map(Some)
3213 .map_err(|_| ProviderNativeAgentGraphBuildError::StaticDefinition)
3214}
3215
3216fn composition_digest(
3217 descriptor: &AgentDescriptor,
3218 policy: &AgentToolPolicyReference,
3219 accounting: &AgentInvocationAccountingReference,
3220 input_security_label: &SecurityLabel,
3221) -> Result<Digest, ProviderNativeAgentGraphBuildError> {
3222 #[derive(Serialize)]
3223 struct Wire<'a> {
3224 implementation: &'static str,
3225 descriptor: &'a AgentDescriptor,
3226 policy: &'a AgentToolPolicyReference,
3227 accounting: &'a AgentInvocationAccountingReference,
3228 input_security_label: &'a SecurityLabel,
3229 }
3230 let canonical = serde_json_canonicalizer::to_vec(&Wire {
3231 implementation: IMPLEMENTATION_VERSION,
3232 descriptor,
3233 policy,
3234 accounting,
3235 input_security_label,
3236 })
3237 .map_err(|_| ProviderNativeAgentGraphBuildError::Canonicalization)?;
3238 let mut preimage = b"stateknot.provider-native-agent.contract.v1\0".to_vec();
3239 preimage.extend_from_slice(&canonical);
3240 Ok(Digest::sha256(preimage))
3241}
3242
3243fn build_state_schema(
3244 schema_id: SchemaId,
3245 contract_digest: Digest,
3246 maximum_turns: u64,
3247 maximum_calls: u64,
3248 maximum_repairs: u64,
3249) -> Result<(SchemaReference, Value), ProviderNativeAgentGraphBuildError> {
3250 let mut document = serde_json::to_value(schemars::schema_for!(ProviderNativeAgentState))
3251 .map_err(|_| ProviderNativeAgentGraphBuildError::StateSchema)?;
3252 let object = document
3253 .as_object_mut()
3254 .ok_or(ProviderNativeAgentGraphBuildError::StateSchema)?;
3255 object.insert(
3256 "$schema".into(),
3257 Value::String("https://json-schema.org/draft/2020-12/schema".into()),
3258 );
3259 object.insert("$id".into(), Value::String(schema_id.as_str().into()));
3260 let description = if maximum_repairs == 0 {
3261 format!(
3262 "StateKnot provider-native agent state; contract={contract_digest:?}; max_turns={maximum_turns}; max_calls_per_turn={maximum_calls}"
3263 )
3264 } else {
3265 format!(
3266 "StateKnot provider-native agent state; contract={contract_digest:?}; max_turns={maximum_turns}; max_calls_per_turn={maximum_calls}; max_output_repair_turns={maximum_repairs}"
3267 )
3268 };
3269 object.insert("description".into(), Value::String(description));
3270 let canonical = serde_json_canonicalizer::to_vec(&document)
3271 .map_err(|_| ProviderNativeAgentGraphBuildError::Canonicalization)?;
3272 Ok((
3273 SchemaReference::new(schema_id, Version::new(1, 0, 0), Digest::sha256(canonical)),
3274 document,
3275 ))
3276}
3277
3278fn decode_state(
3279 value: &BoundedJson,
3280) -> Result<ProviderNativeAgentState, ProviderNativeAgentStateError> {
3281 serde_json::from_value(value.as_value().clone())
3282 .map_err(|_| ProviderNativeAgentStateError::Transition)
3283}
3284
3285fn validate_state(
3286 state: &ProviderNativeAgentState,
3287 graph: &ProviderNativeAgentGraph,
3288) -> Result<(), ProviderNativeAgentStateError> {
3289 if state.contract_digest != graph.contract_digest {
3290 return Err(ProviderNativeAgentStateError::ContractMismatch);
3291 }
3292 let execution = graph.descriptor.execution();
3293 let maximum_turns = usize::try_from(execution.max_model_turns().get())
3294 .map_err(|_| ProviderNativeAgentStateError::Bounds)?;
3295 let maximum_calls = usize::try_from(execution.max_tool_calls_per_turn().get())
3296 .map_err(|_| ProviderNativeAgentStateError::Bounds)?;
3297 let maximum_repairs = usize::try_from(execution.max_output_repair_turns().get())
3298 .map_err(|_| ProviderNativeAgentStateError::Bounds)?;
3299 if state.completed_turns.len() >= maximum_turns {
3300 return Err(ProviderNativeAgentStateError::Bounds);
3301 }
3302 let mut reference_count = 0_usize;
3303 let mut invocation_ids = BTreeSet::new();
3304 let mut output_repair_turns = 0_usize;
3305 let mut output_repair_active = false;
3306 for turn in &state.completed_turns {
3307 if !invocation_ids.insert(turn.model_invocation_id) {
3308 return Err(ProviderNativeAgentStateError::Transition);
3309 }
3310 if turn.requires_output_repair() {
3311 output_repair_active = true;
3312 output_repair_turns = output_repair_turns
3313 .checked_add(1)
3314 .ok_or(ProviderNativeAgentStateError::Bounds)?;
3315 continue;
3316 }
3317 if output_repair_active || turn.tool_invocation_ids.len() > maximum_calls {
3318 return Err(ProviderNativeAgentStateError::Bounds);
3319 }
3320 for invocation_id in &turn.tool_invocation_ids {
3321 if !invocation_ids.insert(*invocation_id) {
3322 return Err(ProviderNativeAgentStateError::Transition);
3323 }
3324 }
3325 reference_count = reference_count
3326 .checked_add(turn.tool_invocation_ids.len())
3327 .ok_or(ProviderNativeAgentStateError::Bounds)?;
3328 }
3329 if output_repair_turns > maximum_repairs {
3330 return Err(ProviderNativeAgentStateError::Bounds);
3331 }
3332 if u64::try_from(reference_count).map_err(|_| ProviderNativeAgentStateError::Bounds)?
3333 > MAX_CHECKPOINT_INVOCATION_REFERENCES
3334 {
3335 return Err(ProviderNativeAgentStateError::Bounds);
3336 }
3337 match &state.phase {
3338 ProviderNativeAgentPhase::Model { plan } => {
3339 if !invocation_ids.insert(plan.invocation_id) {
3340 return Err(ProviderNativeAgentStateError::Transition);
3341 }
3342 }
3343 ProviderNativeAgentPhase::Tools {
3344 model_invocation_id,
3345 plans,
3346 } => {
3347 let consumed_model_turns = state
3348 .completed_turns
3349 .len()
3350 .checked_add(1)
3351 .ok_or(ProviderNativeAgentStateError::Bounds)?;
3352 if output_repair_active
3353 || consumed_model_turns >= maximum_turns
3354 || plans.is_empty()
3355 || plans.len() > maximum_calls
3356 || !invocation_ids.insert(*model_invocation_id)
3357 {
3358 return Err(ProviderNativeAgentStateError::Bounds);
3359 }
3360 reference_count = reference_count
3361 .checked_add(plans.len())
3362 .ok_or(ProviderNativeAgentStateError::Bounds)?;
3363 if u64::try_from(reference_count).map_err(|_| ProviderNativeAgentStateError::Bounds)?
3364 > MAX_CHECKPOINT_INVOCATION_REFERENCES
3365 {
3366 return Err(ProviderNativeAgentStateError::Bounds);
3367 }
3368 for (index, plan) in plans.iter().enumerate() {
3369 if usize::from(plan.proposal_index) != index
3370 || !invocation_ids.insert(plan.invocation_id)
3371 {
3372 return Err(ProviderNativeAgentStateError::Transition);
3373 }
3374 }
3375 }
3376 }
3377 Ok(())
3378}
3379
3380fn validate_transition(
3381 node_id: &NodeId,
3382 current: &ProviderNativeAgentState,
3383 next: &ProviderNativeAgentState,
3384 graph: &ProviderNativeAgentGraph,
3385) -> Result<(), ProviderNativeAgentStateError> {
3386 if current.contract_digest != next.contract_digest
3387 || current.input_message_id != next.input_message_id
3388 {
3389 return Err(ProviderNativeAgentStateError::Transition);
3390 }
3391 if node_id == &graph.model_node_id {
3392 let ProviderNativeAgentPhase::Model { plan } = ¤t.phase else {
3393 return Err(ProviderNativeAgentStateError::Transition);
3394 };
3395 match &next.phase {
3396 ProviderNativeAgentPhase::Tools {
3397 model_invocation_id,
3398 ..
3399 } if !output_repair_active(current)
3400 && current.completed_turns == next.completed_turns
3401 && plan.invocation_id == *model_invocation_id =>
3402 {
3403 Ok(())
3404 }
3405 ProviderNativeAgentPhase::Model { plan: next_plan }
3406 if next.completed_turns.len() == current.completed_turns.len() + 1
3407 && next.completed_turns[..current.completed_turns.len()]
3408 == current.completed_turns
3409 && next.completed_turns.last().is_some_and(|turn| {
3410 turn.requires_output_repair()
3411 && turn.model_invocation_id == plan.invocation_id
3412 })
3413 && next_plan.invocation_id != plan.invocation_id
3414 && next_plan.attempt_id != plan.attempt_id =>
3415 {
3416 Ok(())
3417 }
3418 _ => Err(ProviderNativeAgentStateError::Transition),
3419 }
3420 } else if node_id == &graph.tools_node_id {
3421 let ProviderNativeAgentPhase::Tools {
3422 model_invocation_id,
3423 plans,
3424 } = ¤t.phase
3425 else {
3426 return Err(ProviderNativeAgentStateError::Transition);
3427 };
3428 if !matches!(next.phase, ProviderNativeAgentPhase::Model { .. })
3429 || next.completed_turns.len() != current.completed_turns.len() + 1
3430 || next.completed_turns[..current.completed_turns.len()] != current.completed_turns
3431 {
3432 return Err(ProviderNativeAgentStateError::Transition);
3433 }
3434 let appended = next
3435 .completed_turns
3436 .last()
3437 .ok_or(ProviderNativeAgentStateError::Transition)?;
3438 let expected_tools = plans
3439 .iter()
3440 .map(|plan| plan.invocation_id)
3441 .collect::<Vec<_>>();
3442 if appended.model_invocation_id != *model_invocation_id
3443 || appended.tool_invocation_ids != expected_tools
3444 {
3445 return Err(ProviderNativeAgentStateError::Transition);
3446 }
3447 Ok(())
3448 } else {
3449 Err(ProviderNativeAgentStateError::Transition)
3450 }
3451}
3452
3453fn action_digest(
3454 admission_digest: Digest,
3455 model_invocation_id: InvocationId,
3456 proposal_index: usize,
3457 proposal: &ModelToolCallProposal,
3458) -> Result<Digest, ProviderNativeAgentNodeError> {
3459 #[derive(Serialize)]
3460 struct Wire<'a> {
3461 admission_digest: Digest,
3462 model_invocation_id: InvocationId,
3463 proposal_index: usize,
3464 proposal: &'a ModelToolCallProposal,
3465 }
3466 let canonical = serde_json_canonicalizer::to_vec(&Wire {
3467 admission_digest,
3468 model_invocation_id,
3469 proposal_index,
3470 proposal,
3471 })
3472 .map_err(|_| ProviderNativeAgentNodeError::Integrity)?;
3473 let mut preimage = b"stateknot.provider-native-agent.action.v1\0".to_vec();
3474 preimage.extend_from_slice(&canonical);
3475 Ok(Digest::sha256(preimage))
3476}
3477
3478#[derive(Debug, Error)]
3479enum ProviderNativeAgentNodeError {
3480 #[error("provider-native agent durable evidence is unavailable")]
3481 Store(#[source] StoreError),
3482 #[error("provider-native agent durable evidence failed integrity validation")]
3483 Integrity,
3484 #[error("provider-native agent reached an unsupported or inconsistent state")]
3485 InvalidState,
3486 #[error("provider-native agent exhausted a finite execution budget")]
3487 Budget,
3488 #[error("provider-native agent observed an uncertain external invocation")]
3489 UncertainInvocation,
3490 #[error("provider-native agent reconciliation is still pending")]
3491 ReconciliationPending { retry_after: DurationMillis },
3492 #[error("provider-native agent reconciliation failed")]
3493 Reconciliation(#[source] ToolReconciliationAttemptExecutionError),
3494 #[error("provider-native agent policy dependency is unavailable")]
3495 Policy,
3496 #[error("provider-native agent invocation executor failed")]
3497 InvocationExecutor,
3498 #[error("provider-native agent model attempt failed")]
3499 ModelFailed(Box<Failure>),
3500 #[error("provider-native agent tool policy denied the action")]
3501 PolicyDenied(Box<Failure>),
3502 #[error("provider-native agent model output is invalid")]
3503 InvalidModelOutput,
3504 #[error("provider-native agent exhausted configured output-repair turns")]
3505 OutputRepairExhausted,
3506 #[error("provider-native agent model output is incomplete")]
3507 IncompleteModelOutput,
3508}
3509
3510#[derive(Debug)]
3511struct ProviderNativeAgentExecutionError {
3512 source: ProviderNativeAgentNodeError,
3513 usage: BudgetUsage,
3514}
3515
3516impl ProviderNativeAgentExecutionError {
3517 fn observed(source: ProviderNativeAgentNodeError, usage: BudgetUsage) -> Self {
3518 Self { source, usage }
3519 }
3520
3521 fn into_graph_error(self) -> GraphNodeExecutionError {
3522 node_execution_error(self.source, self.usage)
3523 }
3524}
3525
3526impl From<ProviderNativeAgentNodeError> for ProviderNativeAgentExecutionError {
3527 fn from(source: ProviderNativeAgentNodeError) -> Self {
3528 Self {
3529 source,
3530 usage: BudgetUsage::zero(),
3531 }
3532 }
3533}
3534
3535#[cfg(test)]
3536mod tests {
3537 use std::{future::pending, time::Duration};
3538
3539 use serde_json::Value;
3540 use stateknot_core::{
3541 CapabilityName, CapabilityReference, IssuerId, PrincipalIdentity, SubjectId,
3542 };
3543
3544 use super::*;
3545
3546 struct AllowAllPolicy {
3547 reference: AgentToolPolicyReference,
3548 }
3549
3550 impl AgentToolPolicy for AllowAllPolicy {
3551 fn reference(&self) -> &AgentToolPolicyReference {
3552 &self.reference
3553 }
3554
3555 fn evaluate(
3556 &self,
3557 context: AgentToolPolicyContext,
3558 ) -> BoxFuture<'_, Result<AgentToolPolicyDecision, AgentToolPolicyError>> {
3559 Box::pin(async move {
3560 Ok(AgentToolPolicyDecision::Allow {
3561 evidence_digest: context.action_digest(),
3562 })
3563 })
3564 }
3565 }
3566
3567 struct KnownFreeAccounting {
3568 reference: AgentInvocationAccountingReference,
3569 }
3570
3571 impl AgentInvocationAccounting for KnownFreeAccounting {
3572 fn reference(&self) -> &AgentInvocationAccountingReference {
3573 &self.reference
3574 }
3575
3576 fn model_charge(&self, _: &ModelInvocation) -> AgentInvocationCharge {
3577 AgentInvocationCharge::Known(KnownCosts::empty())
3578 }
3579
3580 fn tool_charge(&self, _: &ToolInvocation) -> AgentInvocationCharge {
3581 AgentInvocationCharge::Known(KnownCosts::empty())
3582 }
3583 }
3584
3585 fn accounting() -> Arc<dyn AgentInvocationAccounting> {
3586 Arc::new(KnownFreeAccounting {
3587 reference: AgentInvocationAccountingReference::new(
3588 identity("invocation-accounting"),
3589 Digest::sha256(b"known-free-test-accounting-v1"),
3590 ),
3591 })
3592 }
3593
3594 fn identity(name: &str) -> CapabilityIdentity {
3595 CapabilityIdentity::new(
3596 PrincipalIdentity::new(
3597 IssuerId::new("https://issuer.example.com").unwrap(),
3598 SubjectId::new("provider-native-tests").unwrap(),
3599 ),
3600 CapabilityReference::new(CapabilityName::new(name).unwrap(), Version::new(1, 0, 0)),
3601 )
3602 }
3603
3604 fn descriptor_with_repairs(repairs: u64) -> AgentDescriptor {
3605 let mut fixture: Value = serde_json::from_str(include_str!(
3606 "../../stateknot-core/tests/fixtures/core-agent-v1.json"
3607 ))
3608 .unwrap();
3609 let mut descriptor = fixture["descriptors"]["valid"][0].take();
3610 descriptor["execution"]["max_output_repair_turns"] = Value::String(repairs.to_string());
3611 if repairs != 0 {
3612 descriptor["model"]["capabilities"]["tools"]["choices"] = json!(["auto", "none"]);
3613 }
3614 serde_json::from_value(descriptor).unwrap()
3615 }
3616
3617 fn descriptor() -> AgentDescriptor {
3618 descriptor_with_repairs(0)
3619 }
3620
3621 fn definition_with_descriptor(descriptor: AgentDescriptor) -> ProviderNativeAgentGraph {
3622 let policy: Arc<dyn AgentToolPolicy> = Arc::new(AllowAllPolicy {
3623 reference: AgentToolPolicyReference::new(
3624 identity("tool-policy"),
3625 Digest::sha256(b"allow-all-v1"),
3626 ),
3627 });
3628 ProviderNativeAgentGraph::compile(
3629 descriptor,
3630 identity("provider-native-graph"),
3631 identity("provider-native-reducer"),
3632 "https://schemas.example.com/provider-native/state/1.0.0"
3633 .parse()
3634 .unwrap(),
3635 SecurityLabel::new("tenant/user-input").unwrap(),
3636 policy,
3637 accounting(),
3638 )
3639 .unwrap()
3640 }
3641
3642 fn definition() -> ProviderNativeAgentGraph {
3643 definition_with_descriptor(descriptor())
3644 }
3645
3646 #[test]
3647 fn definition_is_digest_pinned_bounded_and_schema_validated() {
3648 let definition = definition();
3649 assert_eq!(definition.graph().nodes().len(), 2);
3650 assert_eq!(definition.graph().limits().maximum_parallelism(), 1);
3651 assert_eq!(
3652 definition.graph().limits().maximum_supersteps().get(),
3653 definition.descriptor().execution().max_model_turns().get() * 2
3654 );
3655 let initial = definition.initial_state().unwrap();
3656 let state = definition.decode_checkpoint(&initial).unwrap();
3657 assert!(state.completed_turns().is_empty());
3658 assert!(matches!(
3659 state.phase(),
3660 ProviderNativeAgentPhase::Model { .. }
3661 ));
3662
3663 let mut schemas = JsonSchemaRegistryBuilder::default();
3664 definition.register_schema(&mut schemas).unwrap();
3665 let schemas = schemas.build().unwrap();
3666 schemas
3667 .validate_bounded(initial.schema(), initial.data())
3668 .unwrap();
3669 }
3670
3671 #[test]
3672 fn output_repair_compiles_as_a_bounded_model_self_loop_without_new_state_fields() {
3673 let definition = definition_with_descriptor(descriptor_with_repairs(2));
3674 let mut unsupported = serde_json::to_value(definition.descriptor()).unwrap();
3675 unsupported["model"]["capabilities"]["tools"]["choices"] = json!(["auto"]);
3676 let unsupported = serde_json::from_value::<AgentDescriptor>(unsupported).unwrap();
3677 assert!(matches!(
3678 validate_supported_execution(&unsupported),
3679 Err(ProviderNativeAgentGraphBuildError::OutputRepairToolSelectionUnsupported)
3680 ));
3681 let model = definition.graph().node(&definition.model_node_id).unwrap();
3682 assert_eq!(
3683 model.continue_to().unwrap().iter().collect::<Vec<_>>(),
3684 vec![&definition.model_node_id]
3685 );
3686 let instruction = definition.output_repair_instruction.as_ref().unwrap();
3687 assert_eq!(
3688 instruction.identity().name().as_str(),
3689 OUTPUT_REPAIR_INSTRUCTION_NAME
3690 );
3691 let initial = definition.initial_state().unwrap();
3692 assert_eq!(
3693 initial
3694 .data()
3695 .as_value()
3696 .as_object()
3697 .unwrap()
3698 .keys()
3699 .cloned()
3700 .collect::<BTreeSet<_>>(),
3701 [
3702 "completed_turns",
3703 "contract_digest",
3704 "input_message_id",
3705 "phase"
3706 ]
3707 .into_iter()
3708 .map(str::to_owned)
3709 .collect()
3710 );
3711 }
3712
3713 #[test]
3714 fn reducer_accepts_only_an_exact_distinct_output_repair_transition() {
3715 let definition = definition_with_descriptor(descriptor_with_repairs(2));
3716 let initial = definition.initial_state().unwrap();
3717 let current = definition.decode_checkpoint(&initial).unwrap();
3718 let current_plan = match current.phase() {
3719 ProviderNativeAgentPhase::Model { plan } => plan.clone(),
3720 ProviderNativeAgentPhase::Tools { .. } => panic!("initial phase must be model"),
3721 };
3722 let next_plan = ProviderNativeModelPlan::generate();
3723 let mut next = current.clone();
3724 next.completed_turns.push(ProviderNativeCompletedTurn {
3725 model_invocation_id: current_plan.invocation_id(),
3726 tool_invocation_ids: Vec::new(),
3727 });
3728 next.phase = ProviderNativeAgentPhase::Model {
3729 plan: next_plan.clone(),
3730 };
3731 validate_state(&next, &definition).unwrap();
3732 validate_transition(&definition.model_node_id, ¤t, &next, &definition).unwrap();
3733 assert!(next.completed_turns()[0].requires_output_repair());
3734 assert_ne!(current_plan.invocation_id(), next_plan.invocation_id());
3735 assert_ne!(current_plan.attempt_id(), next_plan.attempt_id());
3736
3737 let mut reused_attempt = next.clone();
3738 reused_attempt.phase = ProviderNativeAgentPhase::Model {
3739 plan: ProviderNativeModelPlan {
3740 invocation_id: next_plan.invocation_id(),
3741 attempt_id: current_plan.attempt_id(),
3742 ..next_plan.clone()
3743 },
3744 };
3745 assert_eq!(
3746 validate_transition(
3747 &definition.model_node_id,
3748 ¤t,
3749 &reused_attempt,
3750 &definition,
3751 ),
3752 Err(ProviderNativeAgentStateError::Transition)
3753 );
3754
3755 let mut reused = next.clone();
3756 reused.phase = ProviderNativeAgentPhase::Model { plan: current_plan };
3757 assert_eq!(
3758 validate_state(&reused, &definition),
3759 Err(ProviderNativeAgentStateError::Transition)
3760 );
3761 }
3762
3763 #[test]
3764 fn output_repair_reserves_one_framework_instruction_slot() {
3765 let mut fixture: Value = serde_json::from_str(include_str!(
3766 "../../stateknot-core/tests/fixtures/core-agent-v1.json"
3767 ))
3768 .unwrap();
3769 let descriptor = &mut fixture["descriptors"]["valid"][0];
3770 descriptor["execution"]["max_output_repair_turns"] = Value::String("1".into());
3771 descriptor["instructions"][0]["identity"]["name"] =
3772 Value::String(OUTPUT_REPAIR_INSTRUCTION_NAME.into());
3773 let conflict: AgentDescriptor = serde_json::from_value(descriptor.clone()).unwrap();
3774 assert!(matches!(
3775 build_output_repair_instruction(&conflict),
3776 Err(ProviderNativeAgentGraphBuildError::OutputRepairInstructionConflict)
3777 ));
3778
3779 let template = descriptor["instructions"][0].clone();
3780 descriptor["instructions"] = Value::Array(
3781 (0..AgentInstructions::MAX_LEN)
3782 .map(|index| {
3783 let mut instruction = template.clone();
3784 instruction["identity"]["name"] =
3785 Value::String(format!("application.instruction.{index}"));
3786 instruction
3787 })
3788 .collect(),
3789 );
3790 let full: AgentDescriptor = serde_json::from_value(descriptor.clone()).unwrap();
3791 assert!(matches!(
3792 build_output_repair_instruction(&full),
3793 Err(ProviderNativeAgentGraphBuildError::OutputRepairInstructionCapacity)
3794 ));
3795 }
3796
3797 #[test]
3798 fn reducer_accepts_only_the_exact_model_tools_model_transition() {
3799 let definition = Arc::new(definition());
3800 let initial = definition.initial_state().unwrap();
3801 let current = definition.decode_checkpoint(&initial).unwrap();
3802 let model_plan = match current.phase() {
3803 ProviderNativeAgentPhase::Model { plan } => plan.clone(),
3804 ProviderNativeAgentPhase::Tools { .. } => panic!("initial phase must be model"),
3805 };
3806 let mut next = current.clone();
3807 next.phase = ProviderNativeAgentPhase::Tools {
3808 model_invocation_id: model_plan.invocation_id,
3809 plans: vec![ProviderNativeToolPlan::generate(
3810 0,
3811 Digest::sha256(b"action"),
3812 Digest::sha256(b"policy"),
3813 )],
3814 };
3815 let model_node = NodeId::new(PROVIDER_NATIVE_MODEL_NODE_ID).unwrap();
3816 validate_transition(&model_node, ¤t, &next, &definition).unwrap();
3817
3818 let mut tampered = next;
3819 tampered.input_message_id = MessageId::generate();
3820 assert_eq!(
3821 validate_transition(&model_node, ¤t, &tampered, &definition),
3822 Err(ProviderNativeAgentStateError::Transition)
3823 );
3824 }
3825
3826 #[test]
3827 fn reconciliation_event_identity_is_stable_without_changing_checkpoint_wire_shape() {
3828 let plan = ProviderNativeToolPlan::generate(
3829 0,
3830 Digest::sha256(b"action"),
3831 Digest::sha256(b"policy"),
3832 );
3833 let event_id = plan.reconciliation_event_id();
3834 let wire = serde_json::to_value(&plan).unwrap();
3835 assert!(wire.get("reconciliation_event_id").is_none());
3836 let restored: ProviderNativeToolPlan = serde_json::from_value(wire).unwrap();
3837 assert_eq!(restored.reconciliation_event_id(), event_id);
3838 assert_ne!(event_id, plan.prepared_event_id());
3839 assert_ne!(event_id, plan.attempt_start_event_id());
3840 assert_ne!(event_id, plan.attempt_terminal_event_id());
3841 }
3842
3843 #[test]
3844 fn pending_reconciliation_becomes_a_durable_safe_after_node_retry() {
3845 let delay = DurationMillis::new(750).unwrap();
3846 let error = node_execution_error(
3847 ProviderNativeAgentNodeError::ReconciliationPending { retry_after: delay },
3848 BudgetUsage::zero(),
3849 );
3850 assert_eq!(
3851 error.failure().retry_advice(),
3852 RetryAdvice::SafeAfter { delay }
3853 );
3854 assert_eq!(
3855 error.failure().code().as_str(),
3856 "runtime.agent.reconciliation_pending"
3857 );
3858 }
3859
3860 #[test]
3861 fn unsupported_semantics_fail_closed_at_compile_time() {
3862 let mut fixture: Value = serde_json::from_str(include_str!(
3863 "../../stateknot-core/tests/fixtures/core-agent-v1.json"
3864 ))
3865 .unwrap();
3866 let mut value = fixture["descriptors"]["valid"][0].take();
3867 value["execution"]["structured_output"] = Value::String("tool_call".into());
3868 value["execution"]["max_output_repair_turns"] = Value::String("0".into());
3869 let unsupported: AgentDescriptor = serde_json::from_value(value).unwrap();
3870 let policy: Arc<dyn AgentToolPolicy> = Arc::new(AllowAllPolicy {
3871 reference: AgentToolPolicyReference::new(
3872 identity("tool-policy"),
3873 Digest::sha256(b"allow-all-v1"),
3874 ),
3875 });
3876 assert!(matches!(
3877 ProviderNativeAgentGraph::compile(
3878 unsupported,
3879 identity("provider-native-graph"),
3880 identity("provider-native-reducer"),
3881 "https://schemas.example.com/provider-native/state/1.0.0"
3882 .parse()
3883 .unwrap(),
3884 SecurityLabel::new("tenant/user-input").unwrap(),
3885 policy,
3886 accounting(),
3887 ),
3888 Err(ProviderNativeAgentGraphBuildError::StructuredOutputUnsupported)
3889 ));
3890 }
3891
3892 #[test]
3893 fn bounded_parallel_read_only_execution_compiles_without_changing_graph_wire_shape() {
3894 let mut fixture: Value = serde_json::from_str(include_str!(
3895 "../../stateknot-core/tests/fixtures/core-agent-v1.json"
3896 ))
3897 .unwrap();
3898 let mut value = fixture["descriptors"]["valid"][0].take();
3899 value["execution"]["max_output_repair_turns"] = Value::String("0".into());
3900 value["execution"]["tool_concurrency"] = serde_json::json!({
3901 "mode": "parallel_read_only",
3902 "max_concurrency": "2"
3903 });
3904 let descriptor: AgentDescriptor = serde_json::from_value(value).unwrap();
3905 let definition = ProviderNativeAgentGraph::compile(
3906 descriptor,
3907 identity("provider-native-parallel-graph"),
3908 identity("provider-native-parallel-reducer"),
3909 "https://schemas.example.com/provider-native/parallel-state/1.0.0"
3910 .parse()
3911 .unwrap(),
3912 SecurityLabel::new("tenant/user-input").unwrap(),
3913 Arc::new(AllowAllPolicy {
3914 reference: AgentToolPolicyReference::new(
3915 identity("parallel-tool-policy"),
3916 Digest::sha256(b"parallel-allow-all-v1"),
3917 ),
3918 }),
3919 accounting(),
3920 )
3921 .unwrap();
3922 assert_eq!(
3923 definition.descriptor().execution().tool_concurrency(),
3924 AgentToolConcurrency::parallel_read_only(ExecutionCount::new(2))
3925 );
3926 assert_eq!(definition.graph().limits().maximum_parallelism(), 1);
3927 }
3928
3929 #[tokio::test]
3930 async fn dropping_a_parallel_tool_wave_aborts_its_provider_task() {
3931 struct NotifyOnDrop(Arc<tokio::sync::Notify>);
3932
3933 impl Drop for NotifyOnDrop {
3934 fn drop(&mut self) {
3935 self.0.notify_one();
3936 }
3937 }
3938
3939 let dropped = Arc::new(tokio::sync::Notify::new());
3940 let task_guard = NotifyOnDrop(Arc::clone(&dropped));
3941 let task = tokio::spawn(async move {
3942 let _task_guard = task_guard;
3943 pending::<ToolTerminalCommitHandoff>().await
3944 });
3945 drop(AbortOnDropToolTask::new(task));
3946
3947 tokio::time::timeout(Duration::from_secs(1), dropped.notified())
3948 .await
3949 .expect("dropping a Tool wave must cancel its detached provider future");
3950 }
3951}