1mod artifact_observation;
7mod read_precondition;
8pub(crate) use artifact_observation::artifact_model_output;
9pub use artifact_observation::ArtifactReadObservation;
10use read_precondition::execution_invocation;
11pub use read_precondition::{
12 CompleteFileRead, FrozenToolObservations, ModelToolObservations, ObservedFileRead,
13};
14
15use std::collections::BTreeMap;
16use std::panic::AssertUnwindSafe;
17use std::sync::{Arc, Mutex as StdMutex, RwLock, Weak};
18use std::time::Duration;
19
20use async_trait::async_trait;
21use bytes::Bytes;
22use futures_util::FutureExt;
23use futures_util::StreamExt;
24use orchestral_core::agent_protocol::wire::{
25 ArtifactRef, ArtifactRefWithDigest, Digest, RunId, ToolActivityEvidence,
26};
27use orchestral_core::io::{BlobId, BlobIoError, BlobStore, BlobWriteRequest};
28use orchestral_core::spi::{HookRegistry, RuntimeHookContext, RuntimeHookEventEnvelope, SpiMeta};
29use orchestral_core::tool_effect::{
30 replay_tool_effect, InMemoryToolEffectJournalStore, PreparedToolEffect, ToolArgumentResolution,
31 ToolAuthorizationEvidence, ToolEffectAttemptId, ToolEffectError, ToolEffectEvent,
32 ToolEffectEventDraft, ToolEffectEventId, ToolEffectJournalStore, ToolEffectKey,
33 ToolEffectPhase, ToolEffectProjection,
34};
35use orchestral_core::tool_protocol::{
36 ApprovalBinding, ApprovalCapability, ApprovalCapabilityStore, ApprovalPolicy,
37 CapabilityRequest, CapabilitySelector, EffectScope, EffectiveToolPolicy, HostApprovalVerifier,
38 HostToolPolicy, ModelToolSchema, RunToolGrant, ToolArtifact, ToolCallId, ToolConcurrency,
39 ToolDescriptor, ToolId, ToolIdempotency, ToolInvocation, ToolOperationPlan, ToolOperationRisk,
40 ToolOutcome, ToolOutput, ToolProtocolError, ToolProtocolErrorCode, VerifiedApprovalCapability,
41};
42use tokio::sync::{Mutex as AsyncMutex, Notify, OwnedMutexGuard};
43use tokio_util::sync::CancellationToken;
44
45#[derive(Debug, Clone)]
51pub struct CapabilityLease {
52 operation_digest: Digest,
53 granted: CapabilityRequest,
54 approval: Option<VerifiedApprovalCapability>,
55}
56
57impl CapabilityLease {
58 fn policy(operation: &ToolOperationPlan) -> Result<Self, ToolProtocolError> {
59 Ok(Self {
60 operation_digest: operation.digest()?,
61 granted: operation.required_capabilities.clone(),
62 approval: None,
63 })
64 }
65
66 fn approved(
67 operation: &ToolOperationPlan,
68 approval: VerifiedApprovalCapability,
69 ) -> Result<Self, ToolProtocolError> {
70 Ok(Self {
71 operation_digest: operation.digest()?,
72 granted: operation.required_capabilities.clone(),
73 approval: Some(approval),
74 })
75 }
76
77 pub fn operation_digest(&self) -> &Digest {
78 &self.operation_digest
79 }
80
81 pub fn granted(&self) -> &CapabilityRequest {
82 &self.granted
83 }
84
85 pub fn was_approved(&self) -> bool {
86 self.approval.is_some()
87 }
88
89 pub fn approval(&self) -> Option<&VerifiedApprovalCapability> {
90 self.approval.as_ref()
91 }
92
93 pub fn validate_for(
97 &self,
98 invocation: &ToolInvocation,
99 operation: &ToolOperationPlan,
100 effective_policy: &EffectiveToolPolicy,
101 ) -> Result<(), ToolProtocolError> {
102 let operation_digest = operation.digest()?;
103 if self.operation_digest != operation_digest
104 || self.granted != operation.required_capabilities
105 {
106 return Err(ToolProtocolError::new(
107 ToolProtocolErrorCode::CapabilityBindingMismatch,
108 "capability lease does not match the dispatched Tool operation",
109 ));
110 }
111 if let Some(approval) = &self.approval {
112 let binding = approval.binding();
113 if binding.run_id != invocation.run_id
114 || binding.call_id != invocation.call_id
115 || binding.tool_id != invocation.tool_id
116 || binding.args_digest != invocation.args_digest()?
117 || binding.operation_digest != operation_digest
118 || binding.requested_capabilities != self.granted
119 || binding.policy_digest != effective_policy.digest()?
120 {
121 return Err(ToolProtocolError::new(
122 ToolProtocolErrorCode::CapabilityBindingMismatch,
123 "approved capability lease does not match the executor dispatch",
124 ));
125 }
126 }
127 Ok(())
128 }
129}
130
131#[derive(Debug, Clone)]
136pub struct GuardedToolExecution {
137 pub invocation: ToolInvocation,
138 pub operation: ToolOperationPlan,
141 pub effective_policy: EffectiveToolPolicy,
142 pub lease: CapabilityLease,
143 pub cancellation: CancellationToken,
144 pub run_cancellation: CancellationToken,
147 pub yield_requested: CancellationToken,
150 pub deadline: Option<tokio::time::Instant>,
152}
153
154#[async_trait]
156pub trait GuardedToolExecutor: Send + Sync {
157 fn project_model_output(
161 &self,
162 _invocation: &ToolInvocation,
163 output: &serde_json::Value,
164 ) -> serde_json::Value {
165 output.clone()
166 }
167
168 fn model_output_contract(&self) -> serde_json::Value {
170 serde_json::json!({ "contract": "orchestral.model-output/identity/v1" })
171 }
172
173 fn complete_file_read(
176 &self,
177 _invocation: &ToolInvocation,
178 _output: &serde_json::Value,
179 ) -> Option<CompleteFileRead> {
180 None
181 }
182
183 fn artifact_read_observation(
189 &self,
190 _invocation: &ToolInvocation,
191 _output: &serde_json::Value,
192 ) -> Option<ArtifactReadObservation> {
193 None
194 }
195
196 fn requires_observed_arguments(&self, _invocation: &ToolInvocation) -> bool {
197 false
198 }
199
200 fn resolve_arguments(
203 &self,
204 _invocation: &ToolInvocation,
205 _reads: &[ObservedFileRead],
206 ) -> Result<Option<ToolArgumentResolution>, ToolOutcome> {
207 Ok(None)
208 }
209
210 fn planning_contract(&self) -> serde_json::Value {
213 serde_json::json!({
214 "contract": "orchestral.tool-operation-planner/static-envelope/v1"
215 })
216 }
217
218 fn plan_operation(
223 &self,
224 invocation: &ToolInvocation,
225 descriptor: &ToolDescriptor,
226 _effective_policy: &EffectiveToolPolicy,
227 ) -> Result<ToolOperationPlan, ToolOutcome> {
228 let mut required_capabilities =
229 CapabilityRequest::from_effects(descriptor.effect_scopes.clone());
230 if required_capabilities.requires(EffectScope::Network) {
235 required_capabilities
236 .insert_resource(EffectScope::Network, CapabilitySelector::Unrestricted);
237 }
238 Ok(ToolOperationPlan {
239 required_capabilities,
240 risk: ToolOperationRisk::Routine,
241 session_approval_scope: None,
242 summary: sanitize_approval_summary(
243 &self.approval_summary(invocation),
244 &invocation.tool_id,
245 ),
246 })
247 }
248
249 fn approval_summary(&self, invocation: &ToolInvocation) -> String {
253 let args_digest = invocation
254 .args_digest()
255 .map(|digest| digest.to_string())
256 .unwrap_or_else(|_| "invalid-arguments".to_owned());
257 format!(
258 "Invoke Tool {} with arguments {}",
259 invocation.tool_id.as_str(),
260 args_digest
261 )
262 }
263
264 fn activity_evidence(
270 &self,
271 _invocation: &ToolInvocation,
272 _outcome: Option<&ToolOutcome>,
273 ) -> Vec<ToolActivityEvidence> {
274 Vec::new()
275 }
276
277 async fn execute(&self, execution: GuardedToolExecution) -> ToolOutcome;
278}
279
280#[derive(Debug, Clone, PartialEq, Eq)]
282#[non_exhaustive]
283pub enum ToolPermissionDecision {
284 Allow,
285 RequireApproval,
286 Deny { code: String, message: String },
287}
288
289pub trait ToolPermissionPolicy: Send + Sync {
293 fn contract_digest(&self) -> Digest;
294
295 fn decide(
296 &self,
297 descriptor: &ToolDescriptor,
298 operation: &ToolOperationPlan,
299 effective_policy: &EffectiveToolPolicy,
300 ) -> ToolPermissionDecision;
301}
302
303#[derive(Debug, Default)]
306pub struct DescriptorPermissionPolicy;
307
308impl ToolPermissionPolicy for DescriptorPermissionPolicy {
309 fn contract_digest(&self) -> Digest {
310 Digest::sha256("orchestral.permission-policy/descriptor/v1")
311 }
312
313 fn decide(
314 &self,
315 _descriptor: &ToolDescriptor,
316 _operation: &ToolOperationPlan,
317 effective_policy: &EffectiveToolPolicy,
318 ) -> ToolPermissionDecision {
319 match effective_policy.bounds().approval {
320 ApprovalPolicy::NotRequired => ToolPermissionDecision::Allow,
321 ApprovalPolicy::Required => ToolPermissionDecision::RequireApproval,
322 ApprovalPolicy::Deny => ToolPermissionDecision::Deny {
323 code: "approval_policy_denied".to_owned(),
324 message: "effective Host policy denies this Tool operation".to_owned(),
325 },
326 _ => ToolPermissionDecision::Deny {
327 code: "approval_policy_unknown".to_owned(),
328 message: "effective Host policy contains an unsupported approval mode".to_owned(),
329 },
330 }
331 }
332}
333
334#[derive(Debug, Default)]
341pub struct WorkspacePermissionPolicy;
342
343impl ToolPermissionPolicy for WorkspacePermissionPolicy {
344 fn contract_digest(&self) -> Digest {
345 Digest::sha256("orchestral.permission-policy/workspace/v2")
346 }
347
348 fn decide(
349 &self,
350 _descriptor: &ToolDescriptor,
351 operation: &ToolOperationPlan,
352 effective_policy: &EffectiveToolPolicy,
353 ) -> ToolPermissionDecision {
354 let bounds = effective_policy.bounds();
355 if bounds.approval == ApprovalPolicy::Deny {
356 return ToolPermissionDecision::Deny {
357 code: "approval_policy_denied".to_owned(),
358 message: "effective Host policy denies this Tool operation".to_owned(),
359 };
360 }
361 if bounds.approval == ApprovalPolicy::Required
362 || !matches!(
363 operation.risk,
364 ToolOperationRisk::Routine | ToolOperationRisk::Elevated
365 )
366 || operation
367 .required_capabilities
368 .effects
369 .iter()
370 .any(|scope| matches!(scope, EffectScope::SecretRead | EffectScope::HostExecution))
371 || (operation.risk != ToolOperationRisk::Routine
372 && operation.required_capabilities.effects.iter().any(|scope| {
373 matches!(
374 scope,
375 EffectScope::Network | EffectScope::ExternalSideEffect
376 )
377 }))
378 || (!bounds.sandbox.required
379 && operation.required_capabilities.effects.iter().any(|scope| {
380 matches!(scope, EffectScope::Process | EffectScope::FilesystemWrite)
381 }))
382 {
383 ToolPermissionDecision::RequireApproval
384 } else {
385 ToolPermissionDecision::Allow
386 }
387 }
388}
389
390fn constrain_permission_decision(
394 effective_policy: &EffectiveToolPolicy,
395 operation: &ToolOperationPlan,
396 proposed: ToolPermissionDecision,
397) -> ToolPermissionDecision {
398 let proposed = if operation
402 .required_capabilities
403 .requires(EffectScope::HostExecution)
404 {
405 match proposed {
406 ToolPermissionDecision::Deny { code, message } => {
407 ToolPermissionDecision::Deny { code, message }
408 }
409 ToolPermissionDecision::Allow | ToolPermissionDecision::RequireApproval => {
410 ToolPermissionDecision::RequireApproval
411 }
412 }
413 } else {
414 proposed
415 };
416 match effective_policy.bounds().approval {
417 ApprovalPolicy::Deny => ToolPermissionDecision::Deny {
418 code: "approval_policy_denied".to_owned(),
419 message: "effective Host policy denies this Tool operation".to_owned(),
420 },
421 ApprovalPolicy::Required => match proposed {
422 ToolPermissionDecision::Deny { code, message } => {
423 ToolPermissionDecision::Deny { code, message }
424 }
425 ToolPermissionDecision::Allow | ToolPermissionDecision::RequireApproval => {
426 ToolPermissionDecision::RequireApproval
427 }
428 },
429 ApprovalPolicy::NotRequired => proposed,
430 _ => ToolPermissionDecision::Deny {
431 code: "approval_policy_unknown".to_owned(),
432 message: "effective Host policy contains an unsupported approval mode".to_owned(),
433 },
434 }
435}
436
437pub fn tool_permission_decision_digest(
443 policy: &dyn ToolPermissionPolicy,
444 decision: &ToolPermissionDecision,
445) -> Result<Digest, ToolProtocolError> {
446 let decision = match decision {
447 ToolPermissionDecision::Allow => serde_json::json!({ "kind": "allow" }),
448 ToolPermissionDecision::RequireApproval => {
449 serde_json::json!({ "kind": "require_approval" })
450 }
451 ToolPermissionDecision::Deny { code, message } => serde_json::json!({
452 "kind": "deny",
453 "code": code,
454 "message": message,
455 }),
456 };
457 let binding = serde_json::json!({
458 "contract": "orchestral.tool-permission-decision/v1",
459 "policy_contract_digest": policy.contract_digest(),
460 "decision": decision,
461 });
462 let bytes = serde_jcs::to_vec(&binding).map_err(|error| {
463 ToolProtocolError::new(
464 ToolProtocolErrorCode::InvalidInvocation,
465 format!("canonicalize Tool permission decision failed: {error}"),
466 )
467 })?;
468 Ok(Digest::sha256(bytes))
469}
470
471#[async_trait]
474pub trait AgentToolRuntime: Send + Sync {
475 fn project_model_output(
476 &self,
477 _invocation: &ToolInvocation,
478 output: &serde_json::Value,
479 ) -> Result<serde_json::Value, ToolRuntimeError> {
480 Ok(output.clone())
481 }
482
483 async fn freeze_model_observations(
484 &self,
485 _run_id: &RunId,
486 _observations: &ModelToolObservations,
487 _pending_calls: &[ToolCallId],
488 ) -> Result<FrozenToolObservations, ToolOutcome> {
489 Ok(FrozenToolObservations::default())
490 }
491
492 fn execution_contract_digest(&self) -> Result<Digest, ToolRuntimeError>;
500
501 fn model_tool_schemas(&self) -> Result<Vec<ModelToolSchema>, ToolRuntimeError>;
502
503 fn resolve_tool_id(&self, model_name: &str) -> Result<Option<ToolId>, ToolRuntimeError>;
504
505 fn activity_evidence(
506 &self,
507 invocation: &ToolInvocation,
508 outcome: Option<&ToolOutcome>,
509 ) -> Result<Vec<ToolActivityEvidence>, ToolRuntimeError>;
510
511 async fn inspect_effect(
515 &self,
516 key: &ToolEffectKey,
517 ) -> Result<Option<ToolEffectProjection>, ToolOutcomeRecoveryError>;
518
519 async fn recover_outcome(
524 &self,
525 invocation: ToolInvocation,
526 run_grant: RunToolGrant,
527 ) -> Result<Option<ToolOutcome>, ToolOutcomeRecoveryError>;
528
529 async fn invoke(
530 &self,
531 invocation: ToolInvocation,
532 run_grant: RunToolGrant,
533 approval: Option<ApprovalCapability>,
534 run_cancellation: CancellationToken,
535 ) -> GuardedToolResult;
536
537 async fn invoke_with_yield(
540 &self,
541 invocation: ToolInvocation,
542 run_grant: RunToolGrant,
543 approval: Option<ApprovalCapability>,
544 run_cancellation: CancellationToken,
545 _yield_requested: CancellationToken,
546 ) -> GuardedToolResult {
547 self.invoke(invocation, run_grant, approval, run_cancellation)
548 .await
549 }
550
551 async fn invoke_with_observations(
554 &self,
555 invocation: ToolInvocation,
556 run_grant: RunToolGrant,
557 approval: Option<ApprovalCapability>,
558 run_cancellation: CancellationToken,
559 yield_requested: CancellationToken,
560 _observations: &FrozenToolObservations,
561 ) -> GuardedToolResult {
562 self.invoke_with_yield(
563 invocation,
564 run_grant,
565 approval,
566 run_cancellation,
567 yield_requested,
568 )
569 .await
570 }
571}
572
573#[derive(Debug, Clone, PartialEq)]
575#[non_exhaustive]
576pub enum GuardedToolResult {
577 ApprovalRequired {
580 binding: ApprovalBinding,
581 summary: String,
582 },
583 Outcome { outcome: ToolOutcome, cached: bool },
586}
587
588#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
592#[error("Tool outcome recovery failed ({code}): {message}")]
593pub struct ToolOutcomeRecoveryError {
594 pub code: String,
595 pub message: String,
596}
597
598#[derive(Debug, thiserror::Error)]
599#[non_exhaustive]
600pub enum ToolRuntimeError {
601 #[error("invalid Host Tool policy: {0}")]
602 InvalidHostPolicy(#[source] ToolProtocolError),
603 #[error("invalid Tool descriptor: {0}")]
604 InvalidDescriptor(#[source] ToolProtocolError),
605 #[error("tool id is already registered: {0}")]
606 DuplicateToolId(ToolId),
607 #[error("model tool name is already registered: {0}")]
608 DuplicateModelName(String),
609 #[error("Tool is not registered: {0}")]
610 UnknownTool(ToolId),
611 #[error("Tool Runtime execution contract cannot be encoded: {0}")]
612 InvalidExecutionContract(String),
613 #[error("Tool activity evidence is invalid: {0}")]
614 InvalidActivityEvidence(String),
615 #[error("Tool Runtime state is unavailable")]
616 StateUnavailable,
617}
618
619#[derive(Clone)]
624pub struct ToolArtifactStore {
625 store: Arc<dyn BlobStore>,
626 max_artifact_bytes: u64,
627 summary_max_chars: usize,
628 inline_output_limit: Option<std::num::NonZeroU64>,
629 hooks: Option<Arc<HookRegistry>>,
630}
631
632impl ToolArtifactStore {
633 pub fn new(
634 store: Arc<dyn BlobStore>,
635 max_artifact_bytes: u64,
636 summary_max_chars: usize,
637 ) -> Result<Self, ToolArtifactError> {
638 if max_artifact_bytes == 0 || summary_max_chars == 0 {
639 return Err(ToolArtifactError::InvalidConfig(
640 "artifact byte and summary limits must be positive".to_owned(),
641 ));
642 }
643 Ok(Self {
644 store,
645 max_artifact_bytes,
646 summary_max_chars,
647 inline_output_limit: None,
648 hooks: None,
649 })
650 }
651
652 pub fn with_inline_output_limit(mut self, max_bytes: std::num::NonZeroU64) -> Self {
655 self.inline_output_limit = Some(max_bytes);
656 self
657 }
658
659 pub fn inline_output_limit(&self) -> Option<u64> {
661 self.inline_output_limit.map(std::num::NonZeroU64::get)
662 }
663
664 pub fn with_hooks(mut self, hooks: Arc<HookRegistry>) -> Self {
669 self.hooks = Some(hooks);
670 self
671 }
672
673 pub fn max_artifact_bytes(&self) -> u64 {
674 self.max_artifact_bytes
675 }
676
677 pub async fn resolve(&self, artifact: &ToolArtifact) -> Result<Vec<u8>, ToolArtifactError> {
680 artifact
681 .validate()
682 .map_err(|error| ToolArtifactError::Integrity(error.message))?;
683 if artifact.byte_size > self.max_artifact_bytes {
684 return Err(ToolArtifactError::LimitExceeded {
685 observed: artifact.byte_size,
686 maximum: self.max_artifact_bytes,
687 });
688 }
689 let blob_id = BlobId::new(artifact.artifact.artifact_ref.as_str());
690 let mut read = self.store.read(&blob_id).await?;
691 if read.meta.id != blob_id
692 || read.meta.byte_size != artifact.byte_size
693 || read.meta.mime_type.as_deref() != Some(artifact.media_type.as_str())
694 {
695 return Err(ToolArtifactError::Integrity(
696 "artifact metadata does not match its durable reference".to_owned(),
697 ));
698 }
699 if let Some(checksum) = &read.meta.checksum_sha256 {
700 if checksum != artifact.artifact.digest.as_str() {
701 return Err(ToolArtifactError::Integrity(
702 "artifact store checksum does not match its durable digest".to_owned(),
703 ));
704 }
705 }
706 let mut bytes = Vec::with_capacity(usize::try_from(artifact.byte_size).unwrap_or(0));
707 while let Some(chunk) = read.body.next().await {
708 let chunk = chunk?;
709 let next_size = bytes.len().saturating_add(chunk.len()) as u64;
710 if next_size > artifact.byte_size || next_size > self.max_artifact_bytes {
711 return Err(ToolArtifactError::Integrity(
712 "artifact body exceeded its declared size".to_owned(),
713 ));
714 }
715 bytes.extend_from_slice(&chunk);
716 }
717 if bytes.len() as u64 != artifact.byte_size
718 || Digest::sha256(&bytes) != artifact.artifact.digest
719 {
720 return Err(ToolArtifactError::Integrity(
721 "artifact bytes do not match their declared size and digest".to_owned(),
722 ));
723 }
724 Ok(bytes)
725 }
726
727 async fn spill(
728 &self,
729 invocation: &ToolInvocation,
730 bytes: Vec<u8>,
731 summary: String,
732 inline_max_bytes: u64,
733 cancellation: &CancellationToken,
734 ) -> Result<ToolArtifact, ToolArtifactError> {
735 let byte_size = bytes.len() as u64;
736 let digest = Digest::sha256(&bytes);
737 let lifecycle_payload = serde_json::json!({
738 "protocol": "orchestral/tool-artifact/v1",
739 "run_id": invocation.run_id.as_str(),
740 "call_id": invocation.call_id.as_str(),
741 "tool_id": invocation.tool_id.as_str(),
742 "media_type": "application/json",
743 "byte_size": byte_size,
744 "digest": digest.as_str(),
745 });
746 if let Err(error) = self
747 .dispatch_artifact_hook("artifact.put", invocation, lifecycle_payload.clone())
748 .await
749 {
750 return Err(self
751 .report_artifact_failure(invocation, lifecycle_payload, error)
752 .await);
753 }
754
755 let result = if byte_size > self.max_artifact_bytes {
756 Err(ToolArtifactError::LimitExceeded {
757 observed: byte_size,
758 maximum: self.max_artifact_bytes,
759 })
760 } else if cancellation.is_cancelled() {
761 Err(ToolArtifactError::Cancelled)
762 } else {
763 self.write_artifact(invocation, bytes, byte_size, digest, summary, cancellation)
764 .await
765 };
766 let result = result.and_then(|mut artifact| {
767 if self.inline_output_limit.is_some() {
768 artifact_observation::fit_artifact_summary(&mut artifact, inline_max_bytes)?;
769 }
770 Ok(artifact)
771 });
772 match result {
773 Ok(artifact) => {
774 let mut payload = lifecycle_payload;
775 payload["artifact_ref"] =
776 serde_json::Value::String(artifact.artifact.artifact_ref.to_string());
777 if let Err(error) = self
778 .dispatch_artifact_hook("artifact.commit", invocation, payload.clone())
779 .await
780 {
781 return Err(self
782 .report_artifact_failure(invocation, payload, error)
783 .await);
784 }
785 Ok(artifact)
786 }
787 Err(error) => Err(self
788 .report_artifact_failure(invocation, lifecycle_payload, error)
789 .await),
790 }
791 }
792
793 async fn report_artifact_failure(
794 &self,
795 invocation: &ToolInvocation,
796 mut payload: serde_json::Value,
797 error: ToolArtifactError,
798 ) -> ToolArtifactError {
799 payload["error"] = serde_json::Value::String(error.to_string());
800 match self
801 .dispatch_artifact_hook("artifact.fail", invocation, payload)
802 .await
803 {
804 Ok(()) => error,
805 Err(fail_error) => ToolArtifactError::HookRejected {
806 event_type: "artifact.fail".to_owned(),
807 message: format!("{fail_error}; original error: {error}"),
808 },
809 }
810 }
811
812 async fn write_artifact(
813 &self,
814 invocation: &ToolInvocation,
815 bytes: Vec<u8>,
816 byte_size: u64,
817 digest: Digest,
818 summary: String,
819 cancellation: &CancellationToken,
820 ) -> Result<ToolArtifact, ToolArtifactError> {
821 let body = Box::pin(futures_util::stream::once(
822 async move { Ok(Bytes::from(bytes)) },
823 ));
824 let request = BlobWriteRequest::new(body)
825 .with_file_name(Some(format!(
826 "tool-{}-{}.json",
827 invocation.run_id.as_str(),
828 invocation.call_id.as_str()
829 )))
830 .with_mime_type(Some("application/json".to_owned()))
831 .with_metadata(serde_json::json!({
832 "protocol": "orchestral/tool-artifact/v1",
833 "run_id": invocation.run_id.as_str(),
834 "call_id": invocation.call_id.as_str(),
835 "tool_id": invocation.tool_id.as_str(),
836 "sha256": digest.as_str(),
837 }));
838 let write = self.store.write(request);
839 tokio::pin!(write);
840 let meta = tokio::select! {
841 _ = cancellation.cancelled() => return Err(ToolArtifactError::Cancelled),
842 result = &mut write => result?,
843 };
844 if meta.id.as_str().trim().is_empty()
845 || meta.byte_size != byte_size
846 || meta.mime_type.as_deref() != Some("application/json")
847 || meta
848 .checksum_sha256
849 .as_ref()
850 .is_some_and(|checksum| checksum != digest.as_str())
851 {
852 return Err(ToolArtifactError::Integrity(
853 "artifact store returned metadata inconsistent with the written bytes".to_owned(),
854 ));
855 }
856 let artifact = ToolArtifact {
857 artifact: ArtifactRefWithDigest {
858 artifact_ref: ArtifactRef::new(meta.id.as_str()),
859 digest,
860 },
861 media_type: "application/json".to_owned(),
862 byte_size,
863 summary,
864 };
865 artifact
866 .validate()
867 .map_err(|error| ToolArtifactError::Integrity(error.message))?;
868 Ok(artifact)
869 }
870
871 async fn dispatch_artifact_hook(
872 &self,
873 event_type: &str,
874 invocation: &ToolInvocation,
875 payload: serde_json::Value,
876 ) -> Result<(), ToolArtifactError> {
877 let Some(hooks) = &self.hooks else {
878 return Ok(());
879 };
880 let event = RuntimeHookEventEnvelope {
881 meta: SpiMeta::runtime_defaults(env!("CARGO_PKG_VERSION")),
882 event_type: event_type.to_owned(),
883 event_version: "1.0.0".to_owned(),
884 occurred_at_unix_ms: chrono::Utc::now().timestamp_millis(),
885 payload,
886 extensions: serde_json::Map::new(),
887 };
888 let context = RuntimeHookContext {
889 session_id: None,
890 run_id: Some(invocation.run_id.clone()),
891 workflow_id: None,
892 step_id: None,
893 tool_name: Some(invocation.tool_id.to_string()),
894 message: None,
895 metadata: serde_json::json!({
896 "run_id": invocation.run_id.as_str(),
897 "call_id": invocation.call_id.as_str(),
898 }),
899 extensions: serde_json::Map::new(),
900 };
901 hooks
902 .dispatch_checked(&event, &context)
903 .await
904 .map_err(|error| ToolArtifactError::HookRejected {
905 event_type: event_type.to_owned(),
906 message: error.to_string(),
907 })
908 }
909}
910
911#[derive(Debug, thiserror::Error)]
912#[non_exhaustive]
913pub enum ToolArtifactError {
914 #[error("invalid Tool Artifact configuration: {0}")]
915 InvalidConfig(String),
916 #[error("artifact size {observed} exceeds the Host ceiling {maximum}")]
917 LimitExceeded { observed: u64, maximum: u64 },
918 #[error("artifact storage failed: {0}")]
919 Store(#[from] BlobIoError),
920 #[error("artifact integrity check failed: {0}")]
921 Integrity(String),
922 #[error("artifact persistence was cancelled")]
923 Cancelled,
924 #[error("artifact lifecycle hook rejected {event_type}: {message}")]
925 HookRejected { event_type: String, message: String },
926}
927
928struct RegisteredTool {
929 descriptor: ToolDescriptor,
930 executor: Arc<dyn GuardedToolExecutor>,
931 global_gate: Arc<AsyncMutex<()>>,
932}
933
934#[derive(Debug, Clone, PartialEq, Eq)]
935struct InvocationIdentity {
936 tool_id: ToolId,
937 args_digest: Digest,
938 operation_digest: Digest,
939 permission_digest: Digest,
940 policy_digest: Digest,
941 descriptor_digest: Digest,
942 argument_resolution_digest: Option<Digest>,
943}
944
945struct InvocationEntry {
946 identity: InvocationIdentity,
947 state: AsyncMutex<InvocationState>,
948 changed: Notify,
949}
950
951enum InvocationState {
952 Ready,
953 Running,
954 Completed(ToolOutcome),
955}
956
957enum DurableInvocationStart {
958 Execute { lease: Box<CapabilityLease> },
959 Replay { outcome: ToolOutcome },
960}
961
962struct PlannedInvocation {
963 operation: ToolOperationPlan,
964 effective_policy: EffectiveToolPolicy,
965 permission: ToolPermissionDecision,
966 permission_digest: Digest,
967 approval_binding: ApprovalBinding,
968 argument_resolution: Option<ToolArgumentResolution>,
969}
970
971type InvocationKey = (RunId, ToolCallId);
972type PerRunGateKey = (ToolId, RunId);
973
974pub struct GuardedToolRuntime<S> {
980 host_ceiling: HostToolPolicy,
981 permission_policy: Arc<dyn ToolPermissionPolicy>,
982 approval_verifier: HostApprovalVerifier<S>,
983 effect_journal: Arc<dyn ToolEffectJournalStore>,
984 artifact_store: Option<ToolArtifactStore>,
985 registry: RwLock<BTreeMap<ToolId, Arc<RegisteredTool>>>,
986 invocations: StdMutex<BTreeMap<InvocationKey, Arc<InvocationEntry>>>,
987 per_run_gates: StdMutex<BTreeMap<PerRunGateKey, Weak<AsyncMutex<()>>>>,
988}
989
990impl<S: ApprovalCapabilityStore> GuardedToolRuntime<S> {
991 pub fn new(
992 host_ceiling: HostToolPolicy,
993 approval_verifier: HostApprovalVerifier<S>,
994 ) -> Result<Self, ToolRuntimeError> {
995 Self::new_with_effect_journal(
996 host_ceiling,
997 approval_verifier,
998 Arc::new(InMemoryToolEffectJournalStore::default()),
999 )
1000 }
1001
1002 pub fn new_with_effect_journal(
1003 host_ceiling: HostToolPolicy,
1004 approval_verifier: HostApprovalVerifier<S>,
1005 effect_journal: Arc<dyn ToolEffectJournalStore>,
1006 ) -> Result<Self, ToolRuntimeError> {
1007 Self::new_with_services(host_ceiling, approval_verifier, effect_journal, None)
1008 }
1009
1010 pub fn new_with_effect_journal_and_artifacts(
1011 host_ceiling: HostToolPolicy,
1012 approval_verifier: HostApprovalVerifier<S>,
1013 effect_journal: Arc<dyn ToolEffectJournalStore>,
1014 artifact_store: ToolArtifactStore,
1015 ) -> Result<Self, ToolRuntimeError> {
1016 Self::new_with_services(
1017 host_ceiling,
1018 approval_verifier,
1019 effect_journal,
1020 Some(artifact_store),
1021 )
1022 }
1023
1024 fn new_with_services(
1025 host_ceiling: HostToolPolicy,
1026 approval_verifier: HostApprovalVerifier<S>,
1027 effect_journal: Arc<dyn ToolEffectJournalStore>,
1028 artifact_store: Option<ToolArtifactStore>,
1029 ) -> Result<Self, ToolRuntimeError> {
1030 host_ceiling
1031 .bounds
1032 .validate()
1033 .map_err(ToolRuntimeError::InvalidHostPolicy)?;
1034 Ok(Self {
1035 host_ceiling,
1036 permission_policy: Arc::new(DescriptorPermissionPolicy),
1037 approval_verifier,
1038 effect_journal,
1039 artifact_store,
1040 registry: RwLock::new(BTreeMap::new()),
1041 invocations: StdMutex::new(BTreeMap::new()),
1042 per_run_gates: StdMutex::new(BTreeMap::new()),
1043 })
1044 }
1045
1046 pub fn with_permission_policy(mut self, policy: Arc<dyn ToolPermissionPolicy>) -> Self {
1049 self.permission_policy = policy;
1050 self
1051 }
1052
1053 pub fn register(
1055 &self,
1056 descriptor: ToolDescriptor,
1057 executor: Arc<dyn GuardedToolExecutor>,
1058 ) -> Result<(), ToolRuntimeError> {
1059 descriptor
1060 .validate()
1061 .map_err(ToolRuntimeError::InvalidDescriptor)?;
1062 let mut registry = self
1063 .registry
1064 .write()
1065 .map_err(|_| ToolRuntimeError::StateUnavailable)?;
1066 if registry.contains_key(&descriptor.tool_id) {
1067 return Err(ToolRuntimeError::DuplicateToolId(
1068 descriptor.tool_id.clone(),
1069 ));
1070 }
1071 if registry.values().any(|registered| {
1072 registered.descriptor.model_schema.name == descriptor.model_schema.name
1073 }) {
1074 return Err(ToolRuntimeError::DuplicateModelName(
1075 descriptor.model_schema.name.clone(),
1076 ));
1077 }
1078 registry.insert(
1079 descriptor.tool_id.clone(),
1080 Arc::new(RegisteredTool {
1081 descriptor,
1082 executor,
1083 global_gate: Arc::new(AsyncMutex::new(())),
1084 }),
1085 );
1086 Ok(())
1087 }
1088
1089 pub fn execution_contract_digest(&self) -> Result<Digest, ToolRuntimeError> {
1093 let registry = self
1094 .registry
1095 .read()
1096 .map_err(|_| ToolRuntimeError::StateUnavailable)?;
1097 let registrations = registry
1098 .values()
1099 .map(|registered| {
1100 serde_json::json!({
1101 "descriptor": ®istered.descriptor,
1102 "planning_contract": registered.executor.planning_contract(),
1103 "model_output_contract": registered.executor.model_output_contract(),
1104 })
1105 })
1106 .collect::<Vec<_>>();
1107 let artifact_contract = self.artifact_store.as_ref().map(|store| {
1108 let mut contract = serde_json::json!({
1109 "max_artifact_bytes": store.max_artifact_bytes,
1110 "summary_max_chars": store.summary_max_chars,
1111 "hooks_enabled": store.hooks.is_some(),
1112 });
1113 if let Some(limit) = store.inline_output_limit() {
1114 contract["inline_output_limit"] = serde_json::json!(limit);
1115 }
1116 contract
1117 });
1118 let contract = serde_json::json!({
1119 "contract": "orchestral.guarded-tool-runtime/v1",
1120 "host_ceiling": &self.host_ceiling,
1121 "permission_policy": self.permission_policy.contract_digest(),
1122 "registered_tools": registrations,
1123 "artifact_store": artifact_contract,
1124 });
1125 let bytes = serde_jcs::to_vec(&contract)
1126 .map_err(|error| ToolRuntimeError::InvalidExecutionContract(error.to_string()))?;
1127 Ok(Digest::sha256(bytes))
1128 }
1129
1130 pub fn project_model_output(
1131 &self,
1132 invocation: &ToolInvocation,
1133 output: &serde_json::Value,
1134 ) -> Result<serde_json::Value, ToolRuntimeError> {
1135 let producer = self
1136 .registered_tool(&invocation.tool_id)?
1137 .ok_or_else(|| ToolRuntimeError::UnknownTool(invocation.tool_id.clone()))?;
1138 Ok(producer.executor.project_model_output(invocation, output))
1139 }
1140
1141 pub fn model_tool_schemas(&self) -> Result<Vec<ModelToolSchema>, ToolRuntimeError> {
1144 let registry = self
1145 .registry
1146 .read()
1147 .map_err(|_| ToolRuntimeError::StateUnavailable)?;
1148 Ok(registry
1149 .values()
1150 .map(|registered| registered.descriptor.model_schema().clone())
1151 .collect())
1152 }
1153
1154 pub fn resolve_tool_id(&self, model_name: &str) -> Result<Option<ToolId>, ToolRuntimeError> {
1155 let registry = self
1156 .registry
1157 .read()
1158 .map_err(|_| ToolRuntimeError::StateUnavailable)?;
1159 Ok(registry
1160 .values()
1161 .find(|registered| registered.descriptor.model_schema.name == model_name)
1162 .map(|registered| registered.descriptor.tool_id.clone()))
1163 }
1164
1165 pub fn activity_evidence(
1166 &self,
1167 invocation: &ToolInvocation,
1168 outcome: Option<&ToolOutcome>,
1169 ) -> Result<Vec<ToolActivityEvidence>, ToolRuntimeError> {
1170 invocation
1171 .validate()
1172 .map_err(|error| ToolRuntimeError::InvalidActivityEvidence(error.message))?;
1173 let Some(registered) = self.registered_tool(&invocation.tool_id)? else {
1174 return Ok(Vec::new());
1175 };
1176 registered
1177 .descriptor
1178 .model_schema
1179 .validate_arguments(&invocation.arguments)
1180 .map_err(|error| ToolRuntimeError::InvalidActivityEvidence(error.message))?;
1181 let evidence = registered.executor.activity_evidence(invocation, outcome);
1182 if evidence.len() > 16 {
1183 return Err(ToolRuntimeError::InvalidActivityEvidence(
1184 "a Tool adapter emitted more than sixteen evidence items".to_owned(),
1185 ));
1186 }
1187 for item in &evidence {
1188 item.validate_integrity()
1189 .map_err(|error| ToolRuntimeError::InvalidActivityEvidence(error.message))?;
1190 }
1191 Ok(evidence)
1192 }
1193
1194 pub async fn recover_outcome(
1198 &self,
1199 invocation: ToolInvocation,
1200 run_grant: RunToolGrant,
1201 ) -> Result<Option<ToolOutcome>, ToolOutcomeRecoveryError> {
1202 if let Err(error) = invocation.validate() {
1203 return Err(tool_outcome_recovery_error(
1204 "invalid_invocation",
1205 error.message,
1206 ));
1207 }
1208 let registered = match self.registered_tool(&invocation.tool_id) {
1209 Ok(Some(registered)) => registered,
1210 Ok(None) => {
1211 return Err(tool_outcome_recovery_error(
1212 "tool_not_found",
1213 format!("tool is not registered: {}", invocation.tool_id),
1214 ))
1215 }
1216 Err(error) => {
1217 return Err(tool_outcome_recovery_error(
1218 "runtime_unavailable",
1219 error.to_string(),
1220 ))
1221 }
1222 };
1223 if let Err(error) = registered
1224 .descriptor
1225 .model_schema
1226 .validate_arguments(&invocation.arguments)
1227 {
1228 return Err(tool_outcome_recovery_error(
1229 "input_schema_violation",
1230 error.message,
1231 ));
1232 }
1233 let recovery_key =
1236 ToolEffectKey::new(invocation.run_id.clone(), invocation.call_id.clone());
1237 if self
1238 .effect_journal
1239 .load_effect(&recovery_key)
1240 .await
1241 .map_err(effect_journal_recovery_error)?
1242 .is_empty()
1243 {
1244 return Ok(None);
1245 }
1246 let effective_policy = EffectiveToolPolicy::resolve(
1247 &self.host_ceiling,
1248 &run_grant,
1249 ®istered.descriptor.restriction,
1250 )
1251 .map_err(|error| tool_outcome_recovery_error("invalid_effective_policy", error.message))?;
1252 let argument_resolution = self
1253 .resolve_invocation_arguments(
1254 &invocation,
1255 ®istered,
1256 &FrozenToolObservations::default(),
1257 )
1258 .await
1259 .map_err(|outcome| {
1260 tool_outcome_recovery_error(
1261 "operation_planning_failed",
1262 format!("Tool argument resolution failed: {outcome:?}"),
1263 )
1264 })?;
1265 let resolved_invocation = execution_invocation(&invocation, argument_resolution.as_ref());
1266 let operation = registered
1267 .executor
1268 .plan_operation(
1269 &resolved_invocation,
1270 ®istered.descriptor,
1271 &effective_policy,
1272 )
1273 .map_err(|outcome| {
1274 tool_outcome_recovery_error(
1275 "operation_planning_failed",
1276 format!("Tool operation planning failed: {outcome:?}"),
1277 )
1278 })?;
1279 operation
1280 .validate_envelope(®istered.descriptor.effect_scopes)
1281 .map_err(|error| {
1282 tool_outcome_recovery_error("invalid_operation_plan", error.message)
1283 })?;
1284 if !effective_policy.authorizes_request(&operation.required_capabilities) {
1285 return Err(tool_outcome_recovery_error(
1286 "policy_denied",
1287 "tool effects are outside the effective Host policy",
1288 ));
1289 }
1290 let permission = constrain_permission_decision(
1291 &effective_policy,
1292 &operation,
1293 self.permission_policy
1294 .decide(®istered.descriptor, &operation, &effective_policy),
1295 );
1296 let permission_digest =
1297 tool_permission_decision_digest(self.permission_policy.as_ref(), &permission).map_err(
1298 |error| tool_outcome_recovery_error("invalid_permission_decision", error.message),
1299 )?;
1300 let prepared = PreparedToolEffect {
1301 invocation: invocation.clone(),
1302 argument_resolution: argument_resolution.map(Box::new),
1303 args_digest: invocation.args_digest().map_err(|error| {
1304 tool_outcome_recovery_error("invalid_invocation", error.message)
1305 })?,
1306 operation_digest: operation.digest().map_err(|error| {
1307 tool_outcome_recovery_error("invalid_operation_plan", error.message)
1308 })?,
1309 permission_digest,
1310 policy_digest: effective_policy.digest().map_err(|error| {
1311 tool_outcome_recovery_error("invalid_effective_policy", error.message)
1312 })?,
1313 descriptor_digest: registered.descriptor.digest().map_err(|error| {
1314 tool_outcome_recovery_error("invalid_descriptor", error.message)
1315 })?,
1316 idempotency: registered.descriptor.idempotency,
1317 effect_scopes: operation.required_capabilities.effects.clone(),
1318 };
1319 let key = prepared.key();
1320
1321 for _ in 0..4 {
1322 let records = self
1323 .effect_journal
1324 .load_effect(&key)
1325 .await
1326 .map_err(effect_journal_recovery_error)?;
1327 let Some(projection) =
1328 replay_tool_effect(&key, &records).map_err(effect_journal_recovery_error)?
1329 else {
1330 return Ok(None);
1331 };
1332 if projection.prepared != prepared {
1333 return Err(tool_outcome_recovery_error(
1334 "call_identity_conflict",
1335 "durable Tool effect identity differs for the same run_id/call_id",
1336 ));
1337 }
1338 match projection.phase {
1339 ToolEffectPhase::Prepared => return Ok(None),
1340 ToolEffectPhase::Observed { outcome, .. } => {
1341 let outcome_digest = outcome.digest().map_err(|error| {
1342 tool_outcome_recovery_error("invalid_tool_outcome", error.message)
1343 })?;
1344 match self
1345 .effect_journal
1346 .append(
1347 projection.last_effect_seq,
1348 ToolEffectEventDraft {
1349 event_id: effect_event_id(&key, "committed"),
1350 key: key.clone(),
1351 payload: ToolEffectEvent::Committed { outcome_digest },
1352 },
1353 )
1354 .await
1355 {
1356 Ok(_) | Err(ToolEffectError::SequenceConflict { .. }) => continue,
1357 Err(error) => return Err(effect_journal_recovery_error(error)),
1358 }
1359 }
1360 ToolEffectPhase::Committed { outcome, .. } => return Ok(Some(outcome)),
1361 ToolEffectPhase::Invoked { .. } => {
1362 let reason = "durable invocation has no observation after runtime recovery";
1363 match self
1364 .effect_journal
1365 .append(
1366 projection.last_effect_seq,
1367 ToolEffectEventDraft {
1368 event_id: effect_event_id(&key, "unknown"),
1369 key: key.clone(),
1370 payload: ToolEffectEvent::EffectUnknown {
1371 reason: reason.to_owned(),
1372 },
1373 },
1374 )
1375 .await
1376 {
1377 Ok(_) | Err(ToolEffectError::SequenceConflict { .. }) => continue,
1378 Err(error) => return Err(effect_journal_recovery_error(error)),
1379 }
1380 }
1381 ToolEffectPhase::UnknownEffect { reason, .. } => {
1382 return Ok(Some(unknown_effect(reason)))
1383 }
1384 }
1385 }
1386 Err(tool_outcome_recovery_error(
1387 "effect_journal_contention",
1388 "Tool effect journal did not converge during recovery",
1389 ))
1390 }
1391
1392 pub async fn invoke(
1398 &self,
1399 invocation: ToolInvocation,
1400 run_grant: RunToolGrant,
1401 approval: Option<ApprovalCapability>,
1402 run_cancellation: CancellationToken,
1403 ) -> GuardedToolResult {
1404 self.invoke_with_yield(
1405 invocation,
1406 run_grant,
1407 approval,
1408 run_cancellation,
1409 CancellationToken::new(),
1410 )
1411 .await
1412 }
1413
1414 pub async fn invoke_with_yield(
1417 &self,
1418 invocation: ToolInvocation,
1419 run_grant: RunToolGrant,
1420 approval: Option<ApprovalCapability>,
1421 run_cancellation: CancellationToken,
1422 yield_requested: CancellationToken,
1423 ) -> GuardedToolResult {
1424 self.invoke_with_observations(
1425 invocation,
1426 run_grant,
1427 approval,
1428 run_cancellation,
1429 yield_requested,
1430 &FrozenToolObservations::default(),
1431 )
1432 .await
1433 }
1434
1435 pub async fn invoke_with_observations(
1436 &self,
1437 invocation: ToolInvocation,
1438 run_grant: RunToolGrant,
1439 approval: Option<ApprovalCapability>,
1440 run_cancellation: CancellationToken,
1441 yield_requested: CancellationToken,
1442 observations: &FrozenToolObservations,
1443 ) -> GuardedToolResult {
1444 if let Err(error) = invocation.validate() {
1445 return rejected("invalid_invocation", error.message);
1446 }
1447 let registered = match self.registered_tool(&invocation.tool_id) {
1448 Ok(Some(registered)) => registered,
1449 Ok(None) => {
1450 return rejected(
1451 "tool_not_found",
1452 format!("tool is not registered: {}", invocation.tool_id),
1453 )
1454 }
1455 Err(error) => return rejected("runtime_unavailable", error.to_string()),
1456 };
1457 if let Err(error) = registered
1458 .descriptor
1459 .model_schema
1460 .validate_arguments(&invocation.arguments)
1461 {
1462 return rejected("input_schema_violation", error.message);
1463 }
1464
1465 let effective_policy = match EffectiveToolPolicy::resolve(
1466 &self.host_ceiling,
1467 &run_grant,
1468 ®istered.descriptor.restriction,
1469 ) {
1470 Ok(policy) => policy,
1471 Err(error) => return rejected("invalid_effective_policy", error.message),
1472 };
1473 let argument_resolution = match self
1474 .resolve_invocation_arguments(&invocation, ®istered, observations)
1475 .await
1476 {
1477 Ok(resolution) => resolution,
1478 Err(outcome) => {
1479 return GuardedToolResult::Outcome {
1480 outcome,
1481 cached: false,
1482 }
1483 }
1484 };
1485 let resolved_invocation = execution_invocation(&invocation, argument_resolution.as_ref());
1486 let operation = match registered.executor.plan_operation(
1487 &resolved_invocation,
1488 ®istered.descriptor,
1489 &effective_policy,
1490 ) {
1491 Ok(operation) => operation,
1492 Err(outcome) => {
1493 return GuardedToolResult::Outcome {
1494 outcome,
1495 cached: false,
1496 }
1497 }
1498 };
1499 if let Err(error) = operation.validate_envelope(®istered.descriptor.effect_scopes) {
1500 return rejected("invalid_operation_plan", error.message);
1501 }
1502 if !effective_policy.authorizes_request(&operation.required_capabilities) {
1503 return rejected(
1504 "policy_denied",
1505 "tool effects are outside the effective Host policy",
1506 );
1507 }
1508 let permission = constrain_permission_decision(
1509 &effective_policy,
1510 &operation,
1511 self.permission_policy
1512 .decide(®istered.descriptor, &operation, &effective_policy),
1513 );
1514 if let ToolPermissionDecision::Deny { code, message } = &permission {
1515 return rejected(code.clone(), message.clone());
1516 }
1517 let permission_digest =
1518 match tool_permission_decision_digest(self.permission_policy.as_ref(), &permission) {
1519 Ok(digest) => digest,
1520 Err(error) => return rejected("invalid_permission_decision", error.message),
1521 };
1522 let approval_binding = match ApprovalBinding::for_operation(
1523 &resolved_invocation,
1524 &operation,
1525 &effective_policy,
1526 permission_digest.clone(),
1527 ) {
1528 Ok(binding) => binding,
1529 Err(error) => return rejected("policy_denied", error.message),
1530 };
1531 let identity = match invocation_identity(
1532 &invocation,
1533 &operation,
1534 &effective_policy,
1535 &permission_digest,
1536 ®istered.descriptor,
1537 argument_resolution.as_ref(),
1538 ) {
1539 Ok(identity) => identity,
1540 Err(error) => return rejected("invalid_invocation", error.message),
1541 };
1542 let planned = PlannedInvocation {
1543 operation,
1544 effective_policy,
1545 permission,
1546 permission_digest,
1547 approval_binding,
1548 argument_resolution,
1549 };
1550 let effect_key = ToolEffectKey::new(invocation.run_id.clone(), invocation.call_id.clone());
1551 let entry = match self.invocation_entry(&invocation, identity) {
1552 Ok(entry) => entry,
1553 Err(result) => return *result,
1554 };
1555
1556 let lease = loop {
1557 let changed = entry.changed.notified();
1560 let mut state = entry.state.lock().await;
1561 match &*state {
1562 InvocationState::Completed(outcome) => {
1563 return GuardedToolResult::Outcome {
1564 outcome: outcome.clone(),
1565 cached: true,
1566 };
1567 }
1568 InvocationState::Running => {
1569 drop(state);
1570 changed.await;
1571 }
1572 InvocationState::Ready => {
1573 match self
1574 .prepare_durable_invocation(
1575 ®istered,
1576 &invocation,
1577 &planned,
1578 approval.as_ref(),
1579 &run_cancellation,
1580 )
1581 .await
1582 {
1583 Ok(DurableInvocationStart::Execute { lease }) => {
1584 *state = InvocationState::Running;
1585 break *lease;
1586 }
1587 Ok(DurableInvocationStart::Replay { outcome }) => {
1588 *state = InvocationState::Completed(outcome.clone());
1589 drop(state);
1590 entry.changed.notify_waiters();
1591 return GuardedToolResult::Outcome {
1592 outcome,
1593 cached: true,
1594 };
1595 }
1596 Err(result) => return result,
1597 }
1598 }
1599 }
1600 };
1601
1602 let execution_cancellation = run_cancellation.child_token();
1603 let PlannedInvocation {
1604 operation,
1605 effective_policy,
1606 ..
1607 } = planned;
1608 let outcome = match self
1609 .concurrency_gate(®istered, &invocation, &execution_cancellation)
1610 .await
1611 {
1612 Ok(gate_guard) => {
1613 let _gate_guard = gate_guard;
1614 let deadline = effective_policy
1615 .bounds()
1616 .max_timeout_ms
1617 .map(|ms| tokio::time::Instant::now() + Duration::from_millis(ms));
1618 self.execute(
1619 registered,
1620 GuardedToolExecution {
1621 invocation: resolved_invocation,
1622 operation,
1623 effective_policy,
1624 lease,
1625 cancellation: execution_cancellation,
1626 run_cancellation,
1627 yield_requested,
1628 deadline,
1629 },
1630 )
1631 .await
1632 }
1633 Err(outcome) => outcome,
1634 };
1635
1636 let outcome = self.commit_durable_outcome(&effect_key, outcome).await;
1637 let mut state = entry.state.lock().await;
1638 *state = InvocationState::Completed(outcome.clone());
1639 drop(state);
1640 entry.changed.notify_waiters();
1641 GuardedToolResult::Outcome {
1642 outcome,
1643 cached: false,
1644 }
1645 }
1646
1647 pub async fn inspect_effect(
1651 &self,
1652 key: &ToolEffectKey,
1653 ) -> Result<Option<ToolEffectProjection>, ToolOutcomeRecoveryError> {
1654 key.validate().map_err(effect_journal_recovery_error)?;
1655 let records = self
1656 .effect_journal
1657 .load_effect(key)
1658 .await
1659 .map_err(effect_journal_recovery_error)?;
1660 replay_tool_effect(key, &records).map_err(effect_journal_recovery_error)
1661 }
1662
1663 async fn prepare_durable_invocation(
1664 &self,
1665 registered: &Arc<RegisteredTool>,
1666 invocation: &ToolInvocation,
1667 planned: &PlannedInvocation,
1668 approval: Option<&ApprovalCapability>,
1669 run_cancellation: &CancellationToken,
1670 ) -> Result<DurableInvocationStart, GuardedToolResult> {
1671 let PlannedInvocation {
1672 operation,
1673 effective_policy,
1674 permission,
1675 permission_digest,
1676 approval_binding,
1677 argument_resolution,
1678 } = planned;
1679 let prepared = PreparedToolEffect {
1680 invocation: invocation.clone(),
1681 argument_resolution: argument_resolution.clone().map(Box::new),
1682 args_digest: invocation
1683 .args_digest()
1684 .map_err(|error| rejected("invalid_invocation", error.message))?,
1685 operation_digest: operation
1686 .digest()
1687 .map_err(|error| rejected("invalid_operation_plan", error.message))?,
1688 permission_digest: permission_digest.clone(),
1689 policy_digest: effective_policy
1690 .digest()
1691 .map_err(|error| rejected("invalid_effective_policy", error.message))?,
1692 descriptor_digest: registered
1693 .descriptor
1694 .digest()
1695 .map_err(|error| rejected("invalid_descriptor", error.message))?,
1696 idempotency: registered.descriptor.idempotency,
1697 effect_scopes: operation.required_capabilities.effects.clone(),
1698 };
1699 let key = prepared.key();
1700
1701 for _ in 0..4 {
1702 let records = self
1703 .effect_journal
1704 .load_effect(&key)
1705 .await
1706 .map_err(effect_journal_rejected)?;
1707 let projection = replay_tool_effect(&key, &records).map_err(effect_journal_rejected)?;
1708 let Some(projection) = projection else {
1709 match self
1710 .effect_journal
1711 .append(
1712 0,
1713 ToolEffectEventDraft {
1714 event_id: effect_event_id(&key, "prepared"),
1715 key: key.clone(),
1716 payload: ToolEffectEvent::Prepared {
1717 effect: prepared.clone(),
1718 },
1719 },
1720 )
1721 .await
1722 {
1723 Ok(_) | Err(ToolEffectError::SequenceConflict { .. }) => continue,
1724 Err(error) => return Err(effect_journal_rejected(error)),
1725 }
1726 };
1727 if projection.prepared != prepared {
1728 return Err(rejected(
1729 "call_identity_conflict",
1730 "durable Tool effect identity differs for the same run_id/call_id",
1731 ));
1732 }
1733 match projection.phase {
1734 ToolEffectPhase::Prepared => {
1735 if run_cancellation.is_cancelled() {
1740 return Err(GuardedToolResult::Outcome {
1741 outcome: ToolOutcome::Cancelled,
1742 cached: false,
1743 });
1744 }
1745 let (lease, authorization) =
1746 if matches!(permission, ToolPermissionDecision::RequireApproval) {
1747 let Some(capability) = approval else {
1748 return Err(GuardedToolResult::ApprovalRequired {
1749 binding: approval_binding.clone(),
1750 summary: sanitize_approval_summary(
1751 &operation.summary,
1752 &invocation.tool_id,
1753 ),
1754 });
1755 };
1756 let verified = self
1757 .approval_verifier
1758 .verify_and_consume(
1759 capability,
1760 approval_binding,
1761 chrono::Utc::now().timestamp_millis(),
1762 )
1763 .map_err(|error| {
1764 rejected(approval_error_code(error.code), error.message)
1765 })?;
1766 let evidence = ToolAuthorizationEvidence::Approval {
1767 nonce: verified.nonce().clone(),
1768 };
1769 let lease = CapabilityLease::approved(operation, verified).map_err(
1770 |error| rejected("invalid_capability_lease", error.message),
1771 )?;
1772 (lease, evidence)
1773 } else {
1774 let lease = CapabilityLease::policy(operation).map_err(|error| {
1775 rejected("invalid_capability_lease", error.message)
1776 })?;
1777 (lease, ToolAuthorizationEvidence::Policy)
1778 };
1779 let appended = self
1780 .effect_journal
1781 .append(
1782 projection.last_effect_seq,
1783 ToolEffectEventDraft {
1784 event_id: effect_event_id(&key, "invoked"),
1785 key: key.clone(),
1786 payload: ToolEffectEvent::Invoked {
1787 attempt_id: ToolEffectAttemptId::new(format!(
1788 "attempt:{}:{}",
1789 key.run_id.as_str(),
1790 key.call_id.as_str()
1791 )),
1792 authorization,
1793 },
1794 },
1795 )
1796 .await;
1797 match appended {
1798 Ok(_) => {
1799 return Ok(DurableInvocationStart::Execute {
1800 lease: Box::new(lease),
1801 })
1802 }
1803 Err(ToolEffectError::SequenceConflict { .. }) => continue,
1804 Err(error) => return Err(effect_journal_rejected(error)),
1805 }
1806 }
1807 ToolEffectPhase::Observed { outcome, .. } => {
1808 let outcome_digest = outcome
1809 .digest()
1810 .map_err(|error| rejected("invalid_tool_outcome", error.message))?;
1811 match self
1812 .effect_journal
1813 .append(
1814 projection.last_effect_seq,
1815 ToolEffectEventDraft {
1816 event_id: effect_event_id(&key, "committed"),
1817 key: key.clone(),
1818 payload: ToolEffectEvent::Committed { outcome_digest },
1819 },
1820 )
1821 .await
1822 {
1823 Ok(_) => return Ok(DurableInvocationStart::Replay { outcome }),
1824 Err(ToolEffectError::SequenceConflict { .. }) => continue,
1825 Err(error) => return Err(effect_journal_rejected(error)),
1826 }
1827 }
1828 ToolEffectPhase::Committed { outcome, .. } => {
1829 return Ok(DurableInvocationStart::Replay { outcome })
1830 }
1831 ToolEffectPhase::Invoked { .. } => {
1832 let reason = "durable invocation has no observation after runtime recovery";
1833 match self
1834 .effect_journal
1835 .append(
1836 projection.last_effect_seq,
1837 ToolEffectEventDraft {
1838 event_id: effect_event_id(&key, "unknown"),
1839 key: key.clone(),
1840 payload: ToolEffectEvent::EffectUnknown {
1841 reason: reason.to_owned(),
1842 },
1843 },
1844 )
1845 .await
1846 {
1847 Ok(_) => {
1848 return Ok(DurableInvocationStart::Replay {
1849 outcome: unknown_effect(reason),
1850 })
1851 }
1852 Err(ToolEffectError::SequenceConflict { .. }) => continue,
1853 Err(error) => {
1854 return Ok(DurableInvocationStart::Replay {
1855 outcome: unknown_effect(format!(
1856 "{reason}; journal update failed: {error}"
1857 )),
1858 })
1859 }
1860 }
1861 }
1862 ToolEffectPhase::UnknownEffect { reason, .. } => {
1863 return Ok(DurableInvocationStart::Replay {
1864 outcome: unknown_effect(reason),
1865 })
1866 }
1867 }
1868 }
1869 Err(rejected(
1870 "effect_journal_contention",
1871 "Tool effect journal did not converge after concurrent updates",
1872 ))
1873 }
1874
1875 async fn commit_durable_outcome(
1876 &self,
1877 key: &ToolEffectKey,
1878 outcome: ToolOutcome,
1879 ) -> ToolOutcome {
1880 for _ in 0..5 {
1881 let records = match self.effect_journal.load_effect(key).await {
1882 Ok(records) => records,
1883 Err(error) => {
1884 return unknown_effect(format!(
1885 "Tool effect completed but its journal is unavailable: {error}"
1886 ))
1887 }
1888 };
1889 let projection = match replay_tool_effect(key, &records) {
1890 Ok(Some(projection)) => projection,
1891 Ok(None) => {
1892 return unknown_effect(
1893 "Tool effect completed without a durable Prepared record",
1894 )
1895 }
1896 Err(error) => {
1897 return unknown_effect(format!(
1898 "Tool effect completed but its journal is corrupt: {error}"
1899 ))
1900 }
1901 };
1902 match (&projection.phase, &outcome) {
1903 (ToolEffectPhase::Invoked { .. }, ToolOutcome::UnknownEffect { message }) => {
1904 match self
1905 .effect_journal
1906 .append(
1907 projection.last_effect_seq,
1908 ToolEffectEventDraft {
1909 event_id: effect_event_id(key, "unknown"),
1910 key: key.clone(),
1911 payload: ToolEffectEvent::EffectUnknown {
1912 reason: message.clone(),
1913 },
1914 },
1915 )
1916 .await
1917 {
1918 Ok(_) => return outcome,
1919 Err(ToolEffectError::SequenceConflict { .. }) => continue,
1920 Err(error) => {
1921 return unknown_effect(format!(
1922 "{message}; journal update failed: {error}"
1923 ))
1924 }
1925 }
1926 }
1927 (ToolEffectPhase::Invoked { .. }, _) => {
1928 match self
1929 .effect_journal
1930 .append(
1931 projection.last_effect_seq,
1932 ToolEffectEventDraft {
1933 event_id: effect_event_id(key, "observed"),
1934 key: key.clone(),
1935 payload: ToolEffectEvent::Observed {
1936 outcome: outcome.clone(),
1937 },
1938 },
1939 )
1940 .await
1941 {
1942 Ok(_) | Err(ToolEffectError::SequenceConflict { .. }) => continue,
1943 Err(error) => {
1944 return unknown_effect(format!(
1945 "Tool effect outcome could not be observed durably: {error}"
1946 ))
1947 }
1948 }
1949 }
1950 (
1951 ToolEffectPhase::Observed {
1952 outcome: observed, ..
1953 },
1954 _,
1955 ) => {
1956 if observed != &outcome {
1957 return unknown_effect(
1958 "durable Tool observation differs from the live outcome",
1959 );
1960 }
1961 let outcome_digest = match outcome.digest() {
1962 Ok(digest) => digest,
1963 Err(error) => {
1964 return unknown_effect(format!(
1965 "Tool effect produced an invalid outcome: {}",
1966 error.message
1967 ))
1968 }
1969 };
1970 match self
1971 .effect_journal
1972 .append(
1973 projection.last_effect_seq,
1974 ToolEffectEventDraft {
1975 event_id: effect_event_id(key, "committed"),
1976 key: key.clone(),
1977 payload: ToolEffectEvent::Committed { outcome_digest },
1978 },
1979 )
1980 .await
1981 {
1982 Ok(_) => return outcome,
1983 Err(ToolEffectError::SequenceConflict { .. }) => continue,
1984 Err(error) => {
1985 return unknown_effect(format!(
1986 "Tool effect observation was durable but commit failed: {error}"
1987 ))
1988 }
1989 }
1990 }
1991 (
1992 ToolEffectPhase::Committed {
1993 outcome: committed, ..
1994 },
1995 _,
1996 ) => {
1997 return if committed == &outcome {
1998 committed.clone()
1999 } else {
2000 unknown_effect("committed Tool outcome differs from the live outcome")
2001 }
2002 }
2003 (ToolEffectPhase::UnknownEffect { reason, .. }, _) => {
2004 return unknown_effect(reason.clone())
2005 }
2006 (ToolEffectPhase::Prepared, _) => {
2007 return unknown_effect(
2008 "Tool executor was entered without a durable Invoked boundary",
2009 )
2010 }
2011 }
2012 }
2013 unknown_effect("Tool effect journal did not converge while committing the outcome")
2014 }
2015
2016 pub fn forget_run(&self, run_id: &RunId) -> Result<(), ToolRuntimeError> {
2019 self.invocations
2020 .lock()
2021 .map_err(|_| ToolRuntimeError::StateUnavailable)?
2022 .retain(|(entry_run_id, _), _| entry_run_id != run_id);
2023 self.per_run_gates
2024 .lock()
2025 .map_err(|_| ToolRuntimeError::StateUnavailable)?
2026 .retain(|(_, entry_run_id), _| entry_run_id != run_id);
2027 Ok(())
2028 }
2029
2030 fn registered_tool(
2031 &self,
2032 tool_id: &ToolId,
2033 ) -> Result<Option<Arc<RegisteredTool>>, ToolRuntimeError> {
2034 Ok(self
2035 .registry
2036 .read()
2037 .map_err(|_| ToolRuntimeError::StateUnavailable)?
2038 .get(tool_id)
2039 .cloned())
2040 }
2041
2042 fn invocation_entry(
2043 &self,
2044 invocation: &ToolInvocation,
2045 identity: InvocationIdentity,
2046 ) -> Result<Arc<InvocationEntry>, Box<GuardedToolResult>> {
2047 let key = (invocation.run_id.clone(), invocation.call_id.clone());
2048 let mut invocations = self.invocations.lock().map_err(|_| {
2049 Box::new(rejected(
2050 "runtime_unavailable",
2051 "Tool call ledger is unavailable",
2052 ))
2053 })?;
2054 if let Some(entry) = invocations.get(&key) {
2055 if entry.identity != identity {
2056 return Err(Box::new(rejected(
2057 "call_identity_conflict",
2058 "the same run_id/call_id was reused with different content or policy",
2059 )));
2060 }
2061 return Ok(entry.clone());
2062 }
2063 let entry = Arc::new(InvocationEntry {
2064 identity,
2065 state: AsyncMutex::new(InvocationState::Ready),
2066 changed: Notify::new(),
2067 });
2068 invocations.insert(key, entry.clone());
2069 Ok(entry)
2070 }
2071
2072 async fn concurrency_gate(
2073 &self,
2074 registered: &Arc<RegisteredTool>,
2075 invocation: &ToolInvocation,
2076 cancellation: &CancellationToken,
2077 ) -> Result<Option<OwnedMutexGuard<()>>, ToolOutcome> {
2078 let gate = match registered.descriptor.concurrency {
2079 ToolConcurrency::ParallelSafe => return Ok(None),
2080 ToolConcurrency::PerRunSerial => {
2081 match self.per_run_gate(&invocation.tool_id, &invocation.run_id) {
2082 Ok(gate) => gate,
2083 Err(error) => {
2084 return Err(ToolOutcome::Failed {
2085 code: "runtime_unavailable".to_owned(),
2086 message: error.to_string(),
2087 retryable: true,
2088 })
2089 }
2090 }
2091 }
2092 ToolConcurrency::GlobalSerial => registered.global_gate.clone(),
2093 _ => registered.global_gate.clone(),
2095 };
2096 tokio::select! {
2097 _ = cancellation.cancelled() => Err(ToolOutcome::Cancelled),
2098 guard = gate.lock_owned() => Ok(Some(guard)),
2099 }
2100 }
2101
2102 fn per_run_gate(
2103 &self,
2104 tool_id: &ToolId,
2105 run_id: &RunId,
2106 ) -> Result<Arc<AsyncMutex<()>>, ToolRuntimeError> {
2107 let key = (tool_id.clone(), run_id.clone());
2108 let mut gates = self
2109 .per_run_gates
2110 .lock()
2111 .map_err(|_| ToolRuntimeError::StateUnavailable)?;
2112 if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
2113 return Ok(gate);
2114 }
2115 let gate = Arc::new(AsyncMutex::new(()));
2116 gates.insert(key, Arc::downgrade(&gate));
2117 Ok(gate)
2118 }
2119
2120 async fn execute(
2121 &self,
2122 registered: Arc<RegisteredTool>,
2123 execution: GuardedToolExecution,
2124 ) -> ToolOutcome {
2125 if let Err(error) = execution.lease.validate_for(
2126 &execution.invocation,
2127 &execution.operation,
2128 &execution.effective_policy,
2129 ) {
2130 return ToolOutcome::Rejected {
2131 code: "invalid_capability_lease".to_owned(),
2132 message: error.message,
2133 };
2134 }
2135 let effective_policy = execution.effective_policy.clone();
2136 let deadline = execution.deadline;
2137 let output_invocation = execution.invocation.clone();
2138 let cancellation = execution.cancellation.clone();
2139 let execution = registered.executor.execute(execution);
2140 let execution = AssertUnwindSafe(execution).catch_unwind();
2141 tokio::pin!(execution);
2142
2143 let outcome = match deadline {
2144 Some(deadline) => {
2145 let timeout = tokio::time::sleep_until(deadline);
2146 tokio::pin!(timeout);
2147 tokio::select! {
2148 _ = cancellation.cancelled() => {
2149 let fallback = cancellation_outcome(®istered.descriptor);
2150 settle_cancelled_executor(&mut execution, fallback).await
2151 },
2152 _ = &mut timeout => {
2153 cancellation.cancel();
2154 let fallback = timeout_outcome(®istered.descriptor);
2155 settle_cancelled_executor(&mut execution, fallback).await
2156 }
2157 result = &mut execution => map_execution_result(result),
2158 }
2159 }
2160 None => {
2161 tokio::select! {
2162 _ = cancellation.cancelled() => {
2163 let fallback = cancellation_outcome(®istered.descriptor);
2164 settle_cancelled_executor(&mut execution, fallback).await
2165 },
2166 result = &mut execution => map_execution_result(result),
2167 }
2168 }
2169 };
2170 let outcome = normalize_post_dispatch_outcome(®istered.descriptor, outcome);
2176 normalize_completed_outcome(
2177 ®istered.descriptor,
2178 registered.executor.as_ref(),
2179 &effective_policy,
2180 &output_invocation,
2181 self.artifact_store.as_ref(),
2182 &cancellation,
2183 outcome,
2184 )
2185 .await
2186 }
2187}
2188
2189#[async_trait]
2190impl<S> AgentToolRuntime for GuardedToolRuntime<S>
2191where
2192 S: ApprovalCapabilityStore + 'static,
2193{
2194 fn project_model_output(
2195 &self,
2196 invocation: &ToolInvocation,
2197 output: &serde_json::Value,
2198 ) -> Result<serde_json::Value, ToolRuntimeError> {
2199 GuardedToolRuntime::project_model_output(self, invocation, output)
2200 }
2201
2202 async fn freeze_model_observations(
2203 &self,
2204 run_id: &RunId,
2205 observations: &ModelToolObservations,
2206 pending_calls: &[ToolCallId],
2207 ) -> Result<FrozenToolObservations, ToolOutcome> {
2208 GuardedToolRuntime::freeze_model_observations(self, run_id, observations, pending_calls)
2209 .await
2210 }
2211
2212 fn execution_contract_digest(&self) -> Result<Digest, ToolRuntimeError> {
2213 GuardedToolRuntime::execution_contract_digest(self)
2214 }
2215
2216 fn model_tool_schemas(&self) -> Result<Vec<ModelToolSchema>, ToolRuntimeError> {
2217 GuardedToolRuntime::model_tool_schemas(self)
2218 }
2219
2220 fn resolve_tool_id(&self, model_name: &str) -> Result<Option<ToolId>, ToolRuntimeError> {
2221 GuardedToolRuntime::resolve_tool_id(self, model_name)
2222 }
2223
2224 fn activity_evidence(
2225 &self,
2226 invocation: &ToolInvocation,
2227 outcome: Option<&ToolOutcome>,
2228 ) -> Result<Vec<ToolActivityEvidence>, ToolRuntimeError> {
2229 GuardedToolRuntime::activity_evidence(self, invocation, outcome)
2230 }
2231
2232 async fn inspect_effect(
2233 &self,
2234 key: &ToolEffectKey,
2235 ) -> Result<Option<ToolEffectProjection>, ToolOutcomeRecoveryError> {
2236 GuardedToolRuntime::inspect_effect(self, key).await
2237 }
2238
2239 async fn recover_outcome(
2240 &self,
2241 invocation: ToolInvocation,
2242 run_grant: RunToolGrant,
2243 ) -> Result<Option<ToolOutcome>, ToolOutcomeRecoveryError> {
2244 GuardedToolRuntime::recover_outcome(self, invocation, run_grant).await
2245 }
2246
2247 async fn invoke(
2248 &self,
2249 invocation: ToolInvocation,
2250 run_grant: RunToolGrant,
2251 approval: Option<ApprovalCapability>,
2252 run_cancellation: CancellationToken,
2253 ) -> GuardedToolResult {
2254 GuardedToolRuntime::invoke(self, invocation, run_grant, approval, run_cancellation).await
2255 }
2256
2257 async fn invoke_with_yield(
2258 &self,
2259 invocation: ToolInvocation,
2260 run_grant: RunToolGrant,
2261 approval: Option<ApprovalCapability>,
2262 run_cancellation: CancellationToken,
2263 yield_requested: CancellationToken,
2264 ) -> GuardedToolResult {
2265 GuardedToolRuntime::invoke_with_yield(
2266 self,
2267 invocation,
2268 run_grant,
2269 approval,
2270 run_cancellation,
2271 yield_requested,
2272 )
2273 .await
2274 }
2275
2276 async fn invoke_with_observations(
2277 &self,
2278 invocation: ToolInvocation,
2279 run_grant: RunToolGrant,
2280 approval: Option<ApprovalCapability>,
2281 run_cancellation: CancellationToken,
2282 yield_requested: CancellationToken,
2283 observations: &FrozenToolObservations,
2284 ) -> GuardedToolResult {
2285 GuardedToolRuntime::invoke_with_observations(
2286 self,
2287 invocation,
2288 run_grant,
2289 approval,
2290 run_cancellation,
2291 yield_requested,
2292 observations,
2293 )
2294 .await
2295 }
2296}
2297
2298fn invocation_identity(
2299 invocation: &ToolInvocation,
2300 operation: &ToolOperationPlan,
2301 effective_policy: &EffectiveToolPolicy,
2302 permission_digest: &Digest,
2303 descriptor: &ToolDescriptor,
2304 argument_resolution: Option<&ToolArgumentResolution>,
2305) -> Result<InvocationIdentity, ToolProtocolError> {
2306 Ok(InvocationIdentity {
2307 tool_id: invocation.tool_id.clone(),
2308 args_digest: invocation.args_digest()?,
2309 operation_digest: operation.digest()?,
2310 permission_digest: permission_digest.clone(),
2311 policy_digest: effective_policy.digest()?,
2312 descriptor_digest: descriptor.digest()?,
2313 argument_resolution_digest: argument_resolution
2314 .map(|resolution| {
2315 serde_jcs::to_vec(resolution)
2316 .map(Digest::sha256)
2317 .map_err(|error| {
2318 ToolProtocolError::new(
2319 ToolProtocolErrorCode::InvalidInvocation,
2320 error.to_string(),
2321 )
2322 })
2323 })
2324 .transpose()?,
2325 })
2326}
2327
2328fn effect_event_id(key: &ToolEffectKey, phase: &str) -> ToolEffectEventId {
2329 ToolEffectEventId::new(format!(
2330 "effect:{}:{}:{phase}",
2331 key.run_id.as_str(),
2332 key.call_id.as_str()
2333 ))
2334}
2335
2336fn effect_journal_rejected(error: ToolEffectError) -> GuardedToolResult {
2337 rejected("effect_journal_unavailable", error.to_string())
2338}
2339
2340fn tool_outcome_recovery_error(
2341 code: impl Into<String>,
2342 message: impl Into<String>,
2343) -> ToolOutcomeRecoveryError {
2344 ToolOutcomeRecoveryError {
2345 code: code.into(),
2346 message: message.into(),
2347 }
2348}
2349
2350fn effect_journal_recovery_error(error: ToolEffectError) -> ToolOutcomeRecoveryError {
2351 tool_outcome_recovery_error("effect_journal_unavailable", error.to_string())
2352}
2353
2354fn unknown_effect(message: impl Into<String>) -> ToolOutcome {
2355 ToolOutcome::UnknownEffect {
2356 message: message.into(),
2357 }
2358}
2359
2360fn cancellation_outcome(descriptor: &ToolDescriptor) -> ToolOutcome {
2361 if matches!(descriptor.idempotency, ToolIdempotency::NonIdempotent) {
2362 unknown_effect(
2363 "non-idempotent Tool was cancelled after its durable invocation boundary; effect completion is unknown",
2364 )
2365 } else {
2366 ToolOutcome::Cancelled
2367 }
2368}
2369
2370fn normalize_post_dispatch_outcome(
2371 descriptor: &ToolDescriptor,
2372 outcome: ToolOutcome,
2373) -> ToolOutcome {
2374 match outcome {
2375 ToolOutcome::Cancelled => cancellation_outcome(descriptor),
2376 outcome => outcome,
2377 }
2378}
2379
2380fn timeout_outcome(descriptor: &ToolDescriptor) -> ToolOutcome {
2381 if matches!(descriptor.idempotency, ToolIdempotency::NonIdempotent) {
2382 unknown_effect(
2383 "non-idempotent Tool timed out after its durable invocation boundary; effect completion is unknown",
2384 )
2385 } else {
2386 ToolOutcome::Failed {
2387 code: "timeout".to_owned(),
2388 message: "tool execution exceeded its Host timeout".to_owned(),
2389 retryable: false,
2390 }
2391 }
2392}
2393
2394async fn settle_cancelled_executor<F>(
2395 execution: &mut std::pin::Pin<&mut F>,
2396 fallback: ToolOutcome,
2397) -> ToolOutcome
2398where
2399 F: std::future::Future<Output = Result<ToolOutcome, Box<dyn std::any::Any + Send>>>,
2400{
2401 match tokio::time::timeout(Duration::from_millis(250), execution).await {
2402 Ok(result) => match map_execution_result(result) {
2403 outcome @ ToolOutcome::UnknownEffect { .. } => outcome,
2404 _ => fallback,
2405 },
2406 Err(_) => fallback,
2407 }
2408}
2409
2410fn map_execution_result(result: Result<ToolOutcome, Box<dyn std::any::Any + Send>>) -> ToolOutcome {
2411 match result {
2412 Ok(outcome) => outcome,
2413 Err(_) => ToolOutcome::UnknownEffect {
2414 message: "tool executor panicked; effect completion is unknown".to_owned(),
2415 },
2416 }
2417}
2418
2419async fn normalize_completed_outcome(
2420 descriptor: &ToolDescriptor,
2421 executor: &dyn GuardedToolExecutor,
2422 effective_policy: &EffectiveToolPolicy,
2423 invocation: &ToolInvocation,
2424 artifact_store: Option<&ToolArtifactStore>,
2425 cancellation: &CancellationToken,
2426 outcome: ToolOutcome,
2427) -> ToolOutcome {
2428 let ToolOutcome::Completed { output } = outcome else {
2429 return outcome;
2430 };
2431 let ToolOutput::Inline(output) = output else {
2432 return ToolOutcome::Failed {
2433 code: "executor_artifact_forbidden".to_owned(),
2434 message: "Tool executors cannot mint Artifact references; only the Host may spill validated output"
2435 .to_owned(),
2436 retryable: false,
2437 };
2438 };
2439 if let Err(error) = descriptor.validate_output(&output) {
2440 return ToolOutcome::Failed {
2441 code: "output_schema_violation".to_owned(),
2442 message: error.message,
2443 retryable: false,
2444 };
2445 }
2446 let bytes = match serde_jcs::to_vec(&output) {
2447 Ok(bytes) => bytes,
2448 Err(error) => {
2449 return ToolOutcome::Failed {
2450 code: "output_serialization_failed".to_owned(),
2451 message: error.to_string(),
2452 retryable: false,
2453 }
2454 }
2455 };
2456 let policy_max = effective_policy.bounds().max_output_bytes;
2457 let model_max = artifact_store.and_then(ToolArtifactStore::inline_output_limit);
2458 let model_fits = match model_max {
2459 Some(maximum) => {
2460 match serde_jcs::to_vec(&executor.project_model_output(invocation, &output)) {
2461 Ok(bytes) => bytes.len() as u64 <= maximum,
2462 Err(error) => {
2463 return ToolOutcome::Failed {
2464 code: "model_output_serialization_failed".to_owned(),
2465 message: error.to_string(),
2466 retryable: false,
2467 }
2468 }
2469 }
2470 }
2471 None => true,
2472 };
2473 if model_fits && policy_max.is_none_or(|maximum| bytes.len() as u64 <= maximum) {
2474 return ToolOutcome::Completed {
2475 output: ToolOutput::Inline(output),
2476 };
2477 }
2478 let inline_max_bytes = match (policy_max, model_max) {
2479 (Some(policy), Some(model)) => Some(policy.min(model)),
2480 (policy, model) => policy.or(model),
2481 };
2482 let Some(inline_max_bytes) = inline_max_bytes else {
2483 return ToolOutcome::Completed {
2484 output: ToolOutput::Inline(output),
2485 };
2486 };
2487 let Some(artifact_store) = artifact_store else {
2488 return ToolOutcome::Failed {
2489 code: "output_limit_exceeded".to_owned(),
2490 message: "Tool output exceeded its Host inline byte limit and no Artifact store is configured"
2491 .to_owned(),
2492 retryable: false,
2493 };
2494 };
2495 let summary = summarize_tool_output(
2496 &output,
2497 bytes.len() as u64,
2498 artifact_store.summary_max_chars,
2499 );
2500 match artifact_store
2501 .spill(invocation, bytes, summary, inline_max_bytes, cancellation)
2502 .await
2503 {
2504 Ok(artifact) => ToolOutcome::Completed {
2505 output: ToolOutput::Artifact(artifact),
2506 },
2507 Err(ToolArtifactError::Cancelled) => cancellation_outcome(descriptor),
2508 Err(error) if matches!(descriptor.idempotency, ToolIdempotency::NonIdempotent) => {
2509 unknown_effect(format!(
2510 "non-idempotent Tool completed but its result could not be persisted: {error}"
2511 ))
2512 }
2513 Err(error) => ToolOutcome::Failed {
2514 code: "artifact_persistence_failed".to_owned(),
2515 message: error.to_string(),
2516 retryable: true,
2517 },
2518 }
2519}
2520
2521fn summarize_tool_output(output: &serde_json::Value, byte_size: u64, max_chars: usize) -> String {
2522 let shape = match output {
2523 serde_json::Value::Object(values) => {
2524 format!("JSON object with {} top-level fields", values.len())
2525 }
2526 serde_json::Value::Array(values) => format!("JSON array with {} items", values.len()),
2527 serde_json::Value::String(_) => "JSON string".to_owned(),
2528 serde_json::Value::Number(_) => "JSON number".to_owned(),
2529 serde_json::Value::Bool(_) => "JSON boolean".to_owned(),
2530 serde_json::Value::Null => "JSON null".to_owned(),
2531 };
2532 let preview = serde_json::to_string(output).unwrap_or_else(|_| "<unavailable>".to_owned());
2533 let mut chars = preview.chars();
2534 let mut preview = chars.by_ref().take(max_chars).collect::<String>();
2535 if chars.next().is_some() {
2536 preview.push('…');
2537 }
2538 format!("{shape}; {byte_size} bytes. Preview: {preview}")
2539}
2540
2541fn sanitize_approval_summary(summary: &str, tool_id: &ToolId) -> String {
2542 const MAX_CHARS: usize = 512;
2543 let normalized = summary
2544 .chars()
2545 .map(|character| {
2546 if character.is_control() {
2547 ' '
2548 } else {
2549 character
2550 }
2551 })
2552 .collect::<String>()
2553 .split_whitespace()
2554 .collect::<Vec<_>>()
2555 .join(" ");
2556 let normalized = if normalized.is_empty() {
2557 format!("Invoke Tool {}", tool_id.as_str())
2558 } else {
2559 normalized
2560 };
2561 let mut chars = normalized.chars();
2562 let mut bounded = chars.by_ref().take(MAX_CHARS).collect::<String>();
2563 if chars.next().is_some() {
2564 bounded.push('…');
2565 }
2566 bounded
2567}
2568
2569fn rejected(code: impl Into<String>, message: impl Into<String>) -> GuardedToolResult {
2570 GuardedToolResult::Outcome {
2571 outcome: ToolOutcome::Rejected {
2572 code: code.into(),
2573 message: message.into(),
2574 },
2575 cached: false,
2576 }
2577}
2578
2579fn approval_error_code(code: ToolProtocolErrorCode) -> &'static str {
2580 match code {
2581 ToolProtocolErrorCode::CapabilityExpired => "approval_expired",
2582 ToolProtocolErrorCode::CapabilityBindingMismatch => "approval_binding_mismatch",
2583 ToolProtocolErrorCode::CapabilityReplayed => "approval_replayed",
2584 ToolProtocolErrorCode::StoreFailure => "approval_store_failure",
2585 ToolProtocolErrorCode::InvalidCapability => "invalid_approval_capability",
2586 _ => "approval_validation_failed",
2587 }
2588}