Skip to main content

orchestral_runtime/
tool_runtime.rs

1//! Guarded, provider-neutral Tool execution boundary.
2//!
3//! An executor must opt in to the Host-owned effective policy
4//! and cancellation contract by implementing [`GuardedToolExecutor`].
5
6mod artifact_observation;
7mod read_precondition;
8pub(crate) use artifact_observation::artifact_model_output;
9pub use artifact_observation::ArtifactReadObservation;
10use read_precondition::execution_invocation;
11pub use read_precondition::{
12    CompleteFileRead, FrozenToolObservations, ModelToolObservations, ObservedFileRead,
13};
14
15use std::collections::BTreeMap;
16use std::panic::AssertUnwindSafe;
17use std::sync::{Arc, Mutex as StdMutex, RwLock, Weak};
18use std::time::Duration;
19
20use async_trait::async_trait;
21use bytes::Bytes;
22use futures_util::FutureExt;
23use futures_util::StreamExt;
24use orchestral_core::agent_protocol::wire::{
25    ArtifactRef, ArtifactRefWithDigest, Digest, RunId, ToolActivityEvidence,
26};
27use orchestral_core::io::{BlobId, BlobIoError, BlobStore, BlobWriteRequest};
28use orchestral_core::spi::{HookRegistry, RuntimeHookContext, RuntimeHookEventEnvelope, SpiMeta};
29use orchestral_core::tool_effect::{
30    replay_tool_effect, InMemoryToolEffectJournalStore, PreparedToolEffect, ToolArgumentResolution,
31    ToolAuthorizationEvidence, ToolEffectAttemptId, ToolEffectError, ToolEffectEvent,
32    ToolEffectEventDraft, ToolEffectEventId, ToolEffectJournalStore, ToolEffectKey,
33    ToolEffectPhase, ToolEffectProjection,
34};
35use orchestral_core::tool_protocol::{
36    ApprovalBinding, ApprovalCapability, ApprovalCapabilityStore, ApprovalPolicy,
37    CapabilityRequest, CapabilitySelector, EffectScope, EffectiveToolPolicy, HostApprovalVerifier,
38    HostToolPolicy, ModelToolSchema, RunToolGrant, ToolArtifact, ToolCallId, ToolConcurrency,
39    ToolDescriptor, ToolId, ToolIdempotency, ToolInvocation, ToolOperationPlan, ToolOperationRisk,
40    ToolOutcome, ToolOutput, ToolProtocolError, ToolProtocolErrorCode, VerifiedApprovalCapability,
41};
42use tokio::sync::{Mutex as AsyncMutex, Notify, OwnedMutexGuard};
43use tokio_util::sync::CancellationToken;
44
45/// Host-issued, operation-bound authority consumed by one executor dispatch.
46///
47/// Automatic policy and interactive approval produce the same executor-facing
48/// object. Executors therefore consume granted authority instead of inferring
49/// it from configuration or from the presence of a user prompt.
50#[derive(Debug, Clone)]
51pub struct CapabilityLease {
52    operation_digest: Digest,
53    granted: CapabilityRequest,
54    approval: Option<VerifiedApprovalCapability>,
55}
56
57impl CapabilityLease {
58    fn policy(operation: &ToolOperationPlan) -> Result<Self, ToolProtocolError> {
59        Ok(Self {
60            operation_digest: operation.digest()?,
61            granted: operation.required_capabilities.clone(),
62            approval: None,
63        })
64    }
65
66    fn approved(
67        operation: &ToolOperationPlan,
68        approval: VerifiedApprovalCapability,
69    ) -> Result<Self, ToolProtocolError> {
70        Ok(Self {
71            operation_digest: operation.digest()?,
72            granted: operation.required_capabilities.clone(),
73            approval: Some(approval),
74        })
75    }
76
77    pub fn operation_digest(&self) -> &Digest {
78        &self.operation_digest
79    }
80
81    pub fn granted(&self) -> &CapabilityRequest {
82        &self.granted
83    }
84
85    pub fn was_approved(&self) -> bool {
86        self.approval.is_some()
87    }
88
89    pub fn approval(&self) -> Option<&VerifiedApprovalCapability> {
90        self.approval.as_ref()
91    }
92
93    /// Revalidates the non-serializable lease at the final executor boundary.
94    /// This keeps a cloned lease from being reused with a different
95    /// invocation, operation, or effective Host policy by an adapter.
96    pub fn validate_for(
97        &self,
98        invocation: &ToolInvocation,
99        operation: &ToolOperationPlan,
100        effective_policy: &EffectiveToolPolicy,
101    ) -> Result<(), ToolProtocolError> {
102        let operation_digest = operation.digest()?;
103        if self.operation_digest != operation_digest
104            || self.granted != operation.required_capabilities
105        {
106            return Err(ToolProtocolError::new(
107                ToolProtocolErrorCode::CapabilityBindingMismatch,
108                "capability lease does not match the dispatched Tool operation",
109            ));
110        }
111        if let Some(approval) = &self.approval {
112            let binding = approval.binding();
113            if binding.run_id != invocation.run_id
114                || binding.call_id != invocation.call_id
115                || binding.tool_id != invocation.tool_id
116                || binding.args_digest != invocation.args_digest()?
117                || binding.operation_digest != operation_digest
118                || binding.requested_capabilities != self.granted
119                || binding.policy_digest != effective_policy.digest()?
120            {
121                return Err(ToolProtocolError::new(
122                    ToolProtocolErrorCode::CapabilityBindingMismatch,
123                    "approved capability lease does not match the executor dispatch",
124                ));
125            }
126        }
127        Ok(())
128    }
129}
130
131/// The only context passed to a production Tool executor.
132///
133/// Policy and cancellation are Host-derived. `approval` is a non-serializable
134/// proof produced by the Host verifier, never a model-provided boolean.
135#[derive(Debug, Clone)]
136pub struct GuardedToolExecution {
137    pub invocation: ToolInvocation,
138    /// Host-inspected, invocation-specific operation. Executors must stay
139    /// within this plan as well as the effective authority ceiling.
140    pub operation: ToolOperationPlan,
141    pub effective_policy: EffectiveToolPolicy,
142    pub lease: CapabilityLease,
143    pub cancellation: CancellationToken,
144    /// Run lifetime signal. Unlike dispatch cancellation, a Tool timeout does
145    /// not cancel this token or release other Run-owned resources.
146    pub run_cancellation: CancellationToken,
147    /// Cooperative request to return an observation at a safe point. This is
148    /// not cancellation and must never stop or replay an external effect.
149    pub yield_requested: CancellationToken,
150    /// Absolute Host deadline for this dispatch, including executor setup.
151    pub deadline: Option<tokio::time::Instant>,
152}
153
154/// Explicit opt-in SPI for implementations that enforce Host Tool policy.
155#[async_trait]
156pub trait GuardedToolExecutor: Send + Sync {
157    /// Deterministic model view of this producer's own successful output.
158    /// The canonical output remains in the Effect Journal. A complete file
159    /// read must retain its original content bytes in this view.
160    fn project_model_output(
161        &self,
162        _invocation: &ToolInvocation,
163        output: &serde_json::Value,
164    ) -> serde_json::Value {
165        output.clone()
166    }
167
168    /// Version the model view separately from execution and output schemas.
169    fn model_output_contract(&self) -> serde_json::Value {
170        serde_json::json!({ "contract": "orchestral.model-output/identity/v1" })
171    }
172
173    /// Declares a complete read from this executor's own validated result
174    /// contract. Other executors' JSON fields are never guessed as evidence.
175    fn complete_file_read(
176        &self,
177        _invocation: &ToolInvocation,
178        _output: &serde_json::Value,
179    ) -> Option<CompleteFileRead> {
180        None
181    }
182
183    /// Declares Artifact bytes retained in this executor's model view. Opt-in
184    /// readers must version this behavior in their planning contract. The
185    /// runtime verifies committed visible pages and the original result digest
186    /// before recognizing a complete read; matching JSON field names alone is
187    /// never evidence.
188    fn artifact_read_observation(
189        &self,
190        _invocation: &ToolInvocation,
191        _output: &serde_json::Value,
192    ) -> Option<ArtifactReadObservation> {
193        None
194    }
195
196    fn requires_observed_arguments(&self, _invocation: &ToolInvocation) -> bool {
197        false
198    }
199
200    /// Resolve omitted arguments from committed observations already shown to
201    /// the model. The runtime journals this result before issuing authority.
202    fn resolve_arguments(
203        &self,
204        _invocation: &ToolInvocation,
205        _reads: &[ObservedFileRead],
206    ) -> Result<Option<ToolArgumentResolution>, ToolOutcome> {
207        Ok(None)
208    }
209
210    /// Stable identity of the pre-execution planner implemented by this Tool.
211    /// It becomes part of the runtime execution contract used by recovery.
212    fn planning_contract(&self) -> serde_json::Value {
213        serde_json::json!({
214            "contract": "orchestral.tool-operation-planner/static-envelope/v1"
215        })
216    }
217
218    /// Inspects one invocation without producing an externally observable
219    /// effect. The default is conservative: it requests the Tool's entire
220    /// registered effect envelope. Built-ins should narrow this plan whenever
221    /// their arguments provide stronger information.
222    fn plan_operation(
223        &self,
224        invocation: &ToolInvocation,
225        descriptor: &ToolDescriptor,
226        _effective_policy: &EffectiveToolPolicy,
227    ) -> Result<ToolOperationPlan, ToolOutcome> {
228        let mut required_capabilities =
229            CapabilityRequest::from_effects(descriptor.effect_scopes.clone());
230        // A generic executor cannot claim an enforceable target boundary for
231        // open-world network access. It must request the wider capability and
232        // let Host policy decide; silently omitting Network would bypass the
233        // approval control plane.
234        if required_capabilities.requires(EffectScope::Network) {
235            required_capabilities
236                .insert_resource(EffectScope::Network, CapabilitySelector::Unrestricted);
237        }
238        Ok(ToolOperationPlan {
239            required_capabilities,
240            risk: ToolOperationRisk::Routine,
241            session_approval_scope: None,
242            summary: sanitize_approval_summary(
243                &self.approval_summary(invocation),
244                &invocation.tool_id,
245            ),
246        })
247    }
248
249    /// Host-owned, human-facing description for an approval prompt. It is not
250    /// authority: the signed [`ApprovalBinding`] remains the exact operation.
251    /// Implementations should redact credential-bearing fields.
252    fn approval_summary(&self, invocation: &ToolInvocation) -> String {
253        let args_digest = invocation
254            .args_digest()
255            .map(|digest| digest.to_string())
256            .unwrap_or_else(|_| "invalid-arguments".to_owned());
257        format!(
258            "Invoke Tool {} with arguments {}",
259            invocation.tool_id.as_str(),
260            args_digest
261        )
262    }
263
264    /// Projects bounded, presentation-safe evidence for Agent clients.
265    ///
266    /// The Tool adapter owns this projection because it understands its own
267    /// argument and result contracts. Generic Agent loops and UIs must not
268    /// reverse-engineer arbitrary Tool JSON or dispatch on Tool names.
269    fn activity_evidence(
270        &self,
271        _invocation: &ToolInvocation,
272        _outcome: Option<&ToolOutcome>,
273    ) -> Vec<ToolActivityEvidence> {
274        Vec::new()
275    }
276
277    async fn execute(&self, execution: GuardedToolExecution) -> ToolOutcome;
278}
279
280/// Host decision for one already-inspected Tool operation.
281#[derive(Debug, Clone, PartialEq, Eq)]
282#[non_exhaustive]
283pub enum ToolPermissionDecision {
284    Allow,
285    RequireApproval,
286    Deny { code: String, message: String },
287}
288
289/// Policy SPI kept separate from Tool planning and capability issuance.
290/// Implementations decide; only the Host approval broker can issue an exact
291/// capability for a reviewed operation.
292pub trait ToolPermissionPolicy: Send + Sync {
293    fn contract_digest(&self) -> Digest;
294
295    fn decide(
296        &self,
297        descriptor: &ToolDescriptor,
298        operation: &ToolOperationPlan,
299        effective_policy: &EffectiveToolPolicy,
300    ) -> ToolPermissionDecision;
301}
302
303/// Compatibility policy used by SDK-created runtimes: the composed static
304/// approval bound remains authoritative.
305#[derive(Debug, Default)]
306pub struct DescriptorPermissionPolicy;
307
308impl ToolPermissionPolicy for DescriptorPermissionPolicy {
309    fn contract_digest(&self) -> Digest {
310        Digest::sha256("orchestral.permission-policy/descriptor/v1")
311    }
312
313    fn decide(
314        &self,
315        _descriptor: &ToolDescriptor,
316        _operation: &ToolOperationPlan,
317        effective_policy: &EffectiveToolPolicy,
318    ) -> ToolPermissionDecision {
319        match effective_policy.bounds().approval {
320            ApprovalPolicy::NotRequired => ToolPermissionDecision::Allow,
321            ApprovalPolicy::Required => ToolPermissionDecision::RequireApproval,
322            ApprovalPolicy::Deny => ToolPermissionDecision::Deny {
323                code: "approval_policy_denied".to_owned(),
324                message: "effective Host policy denies this Tool operation".to_owned(),
325            },
326            _ => ToolPermissionDecision::Deny {
327                code: "approval_policy_unknown".to_owned(),
328                message: "effective Host policy contains an unsupported approval mode".to_owned(),
329            },
330        }
331    }
332}
333
334/// Default interactive workspace policy used by the CLI.
335///
336/// Routine operations asserted by Host-owned planners and non-destructive
337/// workspace mutation stay inside the configured sandbox and run
338/// automatically. Destructive or ambiguous open-world operations, secrets,
339/// and any Tool that statically requires approval still route to the reviewer.
340#[derive(Debug, Default)]
341pub struct WorkspacePermissionPolicy;
342
343impl ToolPermissionPolicy for WorkspacePermissionPolicy {
344    fn contract_digest(&self) -> Digest {
345        Digest::sha256("orchestral.permission-policy/workspace/v2")
346    }
347
348    fn decide(
349        &self,
350        _descriptor: &ToolDescriptor,
351        operation: &ToolOperationPlan,
352        effective_policy: &EffectiveToolPolicy,
353    ) -> ToolPermissionDecision {
354        let bounds = effective_policy.bounds();
355        if bounds.approval == ApprovalPolicy::Deny {
356            return ToolPermissionDecision::Deny {
357                code: "approval_policy_denied".to_owned(),
358                message: "effective Host policy denies this Tool operation".to_owned(),
359            };
360        }
361        if bounds.approval == ApprovalPolicy::Required
362            || !matches!(
363                operation.risk,
364                ToolOperationRisk::Routine | ToolOperationRisk::Elevated
365            )
366            || operation
367                .required_capabilities
368                .effects
369                .iter()
370                .any(|scope| matches!(scope, EffectScope::SecretRead | EffectScope::HostExecution))
371            || (operation.risk != ToolOperationRisk::Routine
372                && operation.required_capabilities.effects.iter().any(|scope| {
373                    matches!(
374                        scope,
375                        EffectScope::Network | EffectScope::ExternalSideEffect
376                    )
377                }))
378            || (!bounds.sandbox.required
379                && operation.required_capabilities.effects.iter().any(|scope| {
380                    matches!(scope, EffectScope::Process | EffectScope::FilesystemWrite)
381                }))
382        {
383            ToolPermissionDecision::RequireApproval
384        } else {
385            ToolPermissionDecision::Allow
386        }
387    }
388}
389
390/// The pluggable policy may only tighten the statically composed Host bound.
391/// `Required` and `Deny` are ceilings, never suggestions that an application
392/// policy can relax.
393fn constrain_permission_decision(
394    effective_policy: &EffectiveToolPolicy,
395    operation: &ToolOperationPlan,
396    proposed: ToolPermissionDecision,
397) -> ToolPermissionDecision {
398    // Leaving the configured OS sandbox is never an automatic policy path.
399    // Even a pluggable policy that would otherwise allow the operation must
400    // produce an exact, verified Host approval capability for this effect.
401    let proposed = if operation
402        .required_capabilities
403        .requires(EffectScope::HostExecution)
404    {
405        match proposed {
406            ToolPermissionDecision::Deny { code, message } => {
407                ToolPermissionDecision::Deny { code, message }
408            }
409            ToolPermissionDecision::Allow | ToolPermissionDecision::RequireApproval => {
410                ToolPermissionDecision::RequireApproval
411            }
412        }
413    } else {
414        proposed
415    };
416    match effective_policy.bounds().approval {
417        ApprovalPolicy::Deny => ToolPermissionDecision::Deny {
418            code: "approval_policy_denied".to_owned(),
419            message: "effective Host policy denies this Tool operation".to_owned(),
420        },
421        ApprovalPolicy::Required => match proposed {
422            ToolPermissionDecision::Deny { code, message } => {
423                ToolPermissionDecision::Deny { code, message }
424            }
425            ToolPermissionDecision::Allow | ToolPermissionDecision::RequireApproval => {
426                ToolPermissionDecision::RequireApproval
427            }
428        },
429        ApprovalPolicy::NotRequired => proposed,
430        _ => ToolPermissionDecision::Deny {
431            code: "approval_policy_unknown".to_owned(),
432            message: "effective Host policy contains an unsupported approval mode".to_owned(),
433        },
434    }
435}
436
437/// Produces the durable identity of one normalized permission decision.
438///
439/// Journal builders and recovery adapters use the same function so a change
440/// from reviewed to automatic execution (or the reverse) is detected before
441/// an executor can run.
442pub fn tool_permission_decision_digest(
443    policy: &dyn ToolPermissionPolicy,
444    decision: &ToolPermissionDecision,
445) -> Result<Digest, ToolProtocolError> {
446    let decision = match decision {
447        ToolPermissionDecision::Allow => serde_json::json!({ "kind": "allow" }),
448        ToolPermissionDecision::RequireApproval => {
449            serde_json::json!({ "kind": "require_approval" })
450        }
451        ToolPermissionDecision::Deny { code, message } => serde_json::json!({
452            "kind": "deny",
453            "code": code,
454            "message": message,
455        }),
456    };
457    let binding = serde_json::json!({
458        "contract": "orchestral.tool-permission-decision/v1",
459        "policy_contract_digest": policy.contract_digest(),
460        "decision": decision,
461    });
462    let bytes = serde_jcs::to_vec(&binding).map_err(|error| {
463        ToolProtocolError::new(
464            ToolProtocolErrorCode::InvalidInvocation,
465            format!("canonicalize Tool permission decision failed: {error}"),
466        )
467    })?;
468    Ok(Digest::sha256(bytes))
469}
470
471/// Object-safe surface consumed by an Agent loop. Concrete approval stores and
472/// reference-monitor state stay behind this Host-owned boundary.
473#[async_trait]
474pub trait AgentToolRuntime: Send + Sync {
475    fn project_model_output(
476        &self,
477        _invocation: &ToolInvocation,
478        output: &serde_json::Value,
479    ) -> Result<serde_json::Value, ToolRuntimeError> {
480        Ok(output.clone())
481    }
482
483    async fn freeze_model_observations(
484        &self,
485        _run_id: &RunId,
486        _observations: &ModelToolObservations,
487        _pending_calls: &[ToolCallId],
488    ) -> Result<FrozenToolObservations, ToolOutcome> {
489        Ok(FrozenToolObservations::default())
490    }
491
492    /// Stable identity of the Host-side execution contract used to decide
493    /// whether a private Agent checkpoint may continue after restart.
494    ///
495    /// Implementations must cover authority ceilings, registered Tool
496    /// descriptors, and other durable policy that can change whether an
497    /// invocation is accepted or how its result is represented. Credentials,
498    /// live ledgers, and other ephemeral state must not enter this digest.
499    fn execution_contract_digest(&self) -> Result<Digest, ToolRuntimeError>;
500
501    fn model_tool_schemas(&self) -> Result<Vec<ModelToolSchema>, ToolRuntimeError>;
502
503    fn resolve_tool_id(&self, model_name: &str) -> Result<Option<ToolId>, ToolRuntimeError>;
504
505    fn activity_evidence(
506        &self,
507        invocation: &ToolInvocation,
508        outcome: Option<&ToolOutcome>,
509    ) -> Result<Vec<ToolActivityEvidence>, ToolRuntimeError>;
510
511    /// Reads one durable effect projection without changing its phase.
512    /// Workflow recovery uses this to reject an entire replay before any new
513    /// sibling Tool is dispatched when one prior invocation is unresolved.
514    async fn inspect_effect(
515        &self,
516        key: &ToolEffectKey,
517    ) -> Result<Option<ToolEffectProjection>, ToolOutcomeRecoveryError>;
518
519    /// Recovers an already-started invocation from the durable Effect Journal
520    /// without ever calling its executor or creating a fresh effect record.
521    /// `Ok(None)` means no durable outcome exists. Callers must establish
522    /// exclusive recovery ownership before using this operation.
523    async fn recover_outcome(
524        &self,
525        invocation: ToolInvocation,
526        run_grant: RunToolGrant,
527    ) -> Result<Option<ToolOutcome>, ToolOutcomeRecoveryError>;
528
529    async fn invoke(
530        &self,
531        invocation: ToolInvocation,
532        run_grant: RunToolGrant,
533        approval: Option<ApprovalCapability>,
534        run_cancellation: CancellationToken,
535    ) -> GuardedToolResult;
536
537    /// Invoke with a Host signal for cooperative waits. Implementations that
538    /// do not support yielding retain their normal execution semantics.
539    async fn invoke_with_yield(
540        &self,
541        invocation: ToolInvocation,
542        run_grant: RunToolGrant,
543        approval: Option<ApprovalCapability>,
544        run_cancellation: CancellationToken,
545        _yield_requested: CancellationToken,
546    ) -> GuardedToolResult {
547        self.invoke(invocation, run_grant, approval, run_cancellation)
548            .await
549    }
550
551    /// Uses only observations from the Host's already-dispatched model request.
552    /// Runtimes without observation resolution preserve their existing path.
553    async fn invoke_with_observations(
554        &self,
555        invocation: ToolInvocation,
556        run_grant: RunToolGrant,
557        approval: Option<ApprovalCapability>,
558        run_cancellation: CancellationToken,
559        yield_requested: CancellationToken,
560        _observations: &FrozenToolObservations,
561    ) -> GuardedToolResult {
562        self.invoke_with_yield(
563            invocation,
564            run_grant,
565            approval,
566            run_cancellation,
567            yield_requested,
568        )
569        .await
570    }
571}
572
573/// Structured result returned to the Agent loop.
574#[derive(Debug, Clone, PartialEq)]
575#[non_exhaustive]
576pub enum GuardedToolResult {
577    /// No executor was called. The Host may issue a capability for this exact
578    /// binding and retry the same `(run_id, call_id)`.
579    ApprovalRequired {
580        binding: ApprovalBinding,
581        summary: String,
582    },
583    /// Semantic Tool result. `cached=true` means this call joined or replayed
584    /// an invocation that another caller already executed.
585    Outcome { outcome: ToolOutcome, cached: bool },
586}
587
588/// Structured failure from replay-only Tool outcome recovery. This is kept
589/// separate from a semantic [`ToolOutcome`] so callers cannot confuse a
590/// recovery-contract violation with a result produced by the Tool.
591#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
592#[error("Tool outcome recovery failed ({code}): {message}")]
593pub struct ToolOutcomeRecoveryError {
594    pub code: String,
595    pub message: String,
596}
597
598#[derive(Debug, thiserror::Error)]
599#[non_exhaustive]
600pub enum ToolRuntimeError {
601    #[error("invalid Host Tool policy: {0}")]
602    InvalidHostPolicy(#[source] ToolProtocolError),
603    #[error("invalid Tool descriptor: {0}")]
604    InvalidDescriptor(#[source] ToolProtocolError),
605    #[error("tool id is already registered: {0}")]
606    DuplicateToolId(ToolId),
607    #[error("model tool name is already registered: {0}")]
608    DuplicateModelName(String),
609    #[error("Tool is not registered: {0}")]
610    UnknownTool(ToolId),
611    #[error("Tool Runtime execution contract cannot be encoded: {0}")]
612    InvalidExecutionContract(String),
613    #[error("Tool activity evidence is invalid: {0}")]
614    InvalidActivityEvidence(String),
615    #[error("Tool Runtime state is unavailable")]
616    StateUnavailable,
617}
618
619/// Host-owned Artifact service used by the Tool Runtime for large results.
620///
621/// The byte ceiling is independent from `max_output_bytes`: the latter is the
622/// maximum inline context payload, while this is the hard storage ceiling.
623#[derive(Clone)]
624pub struct ToolArtifactStore {
625    store: Arc<dyn BlobStore>,
626    max_artifact_bytes: u64,
627    summary_max_chars: usize,
628    inline_output_limit: Option<std::num::NonZeroU64>,
629    hooks: Option<Arc<HookRegistry>>,
630}
631
632impl ToolArtifactStore {
633    pub fn new(
634        store: Arc<dyn BlobStore>,
635        max_artifact_bytes: u64,
636        summary_max_chars: usize,
637    ) -> Result<Self, ToolArtifactError> {
638        if max_artifact_bytes == 0 || summary_max_chars == 0 {
639            return Err(ToolArtifactError::InvalidConfig(
640                "artifact byte and summary limits must be positive".to_owned(),
641            ));
642        }
643        Ok(Self {
644            store,
645            max_artifact_bytes,
646            summary_max_chars,
647            inline_output_limit: None,
648            hooks: None,
649        })
650    }
651
652    /// Spill validated results above this model-inline ceiling without reducing
653    /// executor collection limits or the durable Artifact storage ceiling.
654    pub fn with_inline_output_limit(mut self, max_bytes: std::num::NonZeroU64) -> Self {
655        self.inline_output_limit = Some(max_bytes);
656        self
657    }
658
659    /// Maximum serialized inline result size, when configured by the Host.
660    pub fn inline_output_limit(&self) -> Option<u64> {
661        self.inline_output_limit.map(std::num::NonZeroU64::get)
662    }
663
664    /// Attaches the Host runtime hook registry to artifact lifecycle events.
665    /// The registry's failure policy controls whether a hook rejection is
666    /// observational (`FailOpen`) or aborts the artifact operation
667    /// (`FailClosed`).
668    pub fn with_hooks(mut self, hooks: Arc<HookRegistry>) -> Self {
669        self.hooks = Some(hooks);
670        self
671    }
672
673    pub fn max_artifact_bytes(&self) -> u64 {
674        self.max_artifact_bytes
675    }
676
677    /// Resolves and verifies an immutable Artifact reference. A store cannot
678    /// make corrupt or substituted bytes valid merely by returning metadata.
679    pub async fn resolve(&self, artifact: &ToolArtifact) -> Result<Vec<u8>, ToolArtifactError> {
680        artifact
681            .validate()
682            .map_err(|error| ToolArtifactError::Integrity(error.message))?;
683        if artifact.byte_size > self.max_artifact_bytes {
684            return Err(ToolArtifactError::LimitExceeded {
685                observed: artifact.byte_size,
686                maximum: self.max_artifact_bytes,
687            });
688        }
689        let blob_id = BlobId::new(artifact.artifact.artifact_ref.as_str());
690        let mut read = self.store.read(&blob_id).await?;
691        if read.meta.id != blob_id
692            || read.meta.byte_size != artifact.byte_size
693            || read.meta.mime_type.as_deref() != Some(artifact.media_type.as_str())
694        {
695            return Err(ToolArtifactError::Integrity(
696                "artifact metadata does not match its durable reference".to_owned(),
697            ));
698        }
699        if let Some(checksum) = &read.meta.checksum_sha256 {
700            if checksum != artifact.artifact.digest.as_str() {
701                return Err(ToolArtifactError::Integrity(
702                    "artifact store checksum does not match its durable digest".to_owned(),
703                ));
704            }
705        }
706        let mut bytes = Vec::with_capacity(usize::try_from(artifact.byte_size).unwrap_or(0));
707        while let Some(chunk) = read.body.next().await {
708            let chunk = chunk?;
709            let next_size = bytes.len().saturating_add(chunk.len()) as u64;
710            if next_size > artifact.byte_size || next_size > self.max_artifact_bytes {
711                return Err(ToolArtifactError::Integrity(
712                    "artifact body exceeded its declared size".to_owned(),
713                ));
714            }
715            bytes.extend_from_slice(&chunk);
716        }
717        if bytes.len() as u64 != artifact.byte_size
718            || Digest::sha256(&bytes) != artifact.artifact.digest
719        {
720            return Err(ToolArtifactError::Integrity(
721                "artifact bytes do not match their declared size and digest".to_owned(),
722            ));
723        }
724        Ok(bytes)
725    }
726
727    async fn spill(
728        &self,
729        invocation: &ToolInvocation,
730        bytes: Vec<u8>,
731        summary: String,
732        inline_max_bytes: u64,
733        cancellation: &CancellationToken,
734    ) -> Result<ToolArtifact, ToolArtifactError> {
735        let byte_size = bytes.len() as u64;
736        let digest = Digest::sha256(&bytes);
737        let lifecycle_payload = serde_json::json!({
738            "protocol": "orchestral/tool-artifact/v1",
739            "run_id": invocation.run_id.as_str(),
740            "call_id": invocation.call_id.as_str(),
741            "tool_id": invocation.tool_id.as_str(),
742            "media_type": "application/json",
743            "byte_size": byte_size,
744            "digest": digest.as_str(),
745        });
746        if let Err(error) = self
747            .dispatch_artifact_hook("artifact.put", invocation, lifecycle_payload.clone())
748            .await
749        {
750            return Err(self
751                .report_artifact_failure(invocation, lifecycle_payload, error)
752                .await);
753        }
754
755        let result = if byte_size > self.max_artifact_bytes {
756            Err(ToolArtifactError::LimitExceeded {
757                observed: byte_size,
758                maximum: self.max_artifact_bytes,
759            })
760        } else if cancellation.is_cancelled() {
761            Err(ToolArtifactError::Cancelled)
762        } else {
763            self.write_artifact(invocation, bytes, byte_size, digest, summary, cancellation)
764                .await
765        };
766        let result = result.and_then(|mut artifact| {
767            if self.inline_output_limit.is_some() {
768                artifact_observation::fit_artifact_summary(&mut artifact, inline_max_bytes)?;
769            }
770            Ok(artifact)
771        });
772        match result {
773            Ok(artifact) => {
774                let mut payload = lifecycle_payload;
775                payload["artifact_ref"] =
776                    serde_json::Value::String(artifact.artifact.artifact_ref.to_string());
777                if let Err(error) = self
778                    .dispatch_artifact_hook("artifact.commit", invocation, payload.clone())
779                    .await
780                {
781                    return Err(self
782                        .report_artifact_failure(invocation, payload, error)
783                        .await);
784                }
785                Ok(artifact)
786            }
787            Err(error) => Err(self
788                .report_artifact_failure(invocation, lifecycle_payload, error)
789                .await),
790        }
791    }
792
793    async fn report_artifact_failure(
794        &self,
795        invocation: &ToolInvocation,
796        mut payload: serde_json::Value,
797        error: ToolArtifactError,
798    ) -> ToolArtifactError {
799        payload["error"] = serde_json::Value::String(error.to_string());
800        match self
801            .dispatch_artifact_hook("artifact.fail", invocation, payload)
802            .await
803        {
804            Ok(()) => error,
805            Err(fail_error) => ToolArtifactError::HookRejected {
806                event_type: "artifact.fail".to_owned(),
807                message: format!("{fail_error}; original error: {error}"),
808            },
809        }
810    }
811
812    async fn write_artifact(
813        &self,
814        invocation: &ToolInvocation,
815        bytes: Vec<u8>,
816        byte_size: u64,
817        digest: Digest,
818        summary: String,
819        cancellation: &CancellationToken,
820    ) -> Result<ToolArtifact, ToolArtifactError> {
821        let body = Box::pin(futures_util::stream::once(
822            async move { Ok(Bytes::from(bytes)) },
823        ));
824        let request = BlobWriteRequest::new(body)
825            .with_file_name(Some(format!(
826                "tool-{}-{}.json",
827                invocation.run_id.as_str(),
828                invocation.call_id.as_str()
829            )))
830            .with_mime_type(Some("application/json".to_owned()))
831            .with_metadata(serde_json::json!({
832                "protocol": "orchestral/tool-artifact/v1",
833                "run_id": invocation.run_id.as_str(),
834                "call_id": invocation.call_id.as_str(),
835                "tool_id": invocation.tool_id.as_str(),
836                "sha256": digest.as_str(),
837            }));
838        let write = self.store.write(request);
839        tokio::pin!(write);
840        let meta = tokio::select! {
841            _ = cancellation.cancelled() => return Err(ToolArtifactError::Cancelled),
842            result = &mut write => result?,
843        };
844        if meta.id.as_str().trim().is_empty()
845            || meta.byte_size != byte_size
846            || meta.mime_type.as_deref() != Some("application/json")
847            || meta
848                .checksum_sha256
849                .as_ref()
850                .is_some_and(|checksum| checksum != digest.as_str())
851        {
852            return Err(ToolArtifactError::Integrity(
853                "artifact store returned metadata inconsistent with the written bytes".to_owned(),
854            ));
855        }
856        let artifact = ToolArtifact {
857            artifact: ArtifactRefWithDigest {
858                artifact_ref: ArtifactRef::new(meta.id.as_str()),
859                digest,
860            },
861            media_type: "application/json".to_owned(),
862            byte_size,
863            summary,
864        };
865        artifact
866            .validate()
867            .map_err(|error| ToolArtifactError::Integrity(error.message))?;
868        Ok(artifact)
869    }
870
871    async fn dispatch_artifact_hook(
872        &self,
873        event_type: &str,
874        invocation: &ToolInvocation,
875        payload: serde_json::Value,
876    ) -> Result<(), ToolArtifactError> {
877        let Some(hooks) = &self.hooks else {
878            return Ok(());
879        };
880        let event = RuntimeHookEventEnvelope {
881            meta: SpiMeta::runtime_defaults(env!("CARGO_PKG_VERSION")),
882            event_type: event_type.to_owned(),
883            event_version: "1.0.0".to_owned(),
884            occurred_at_unix_ms: chrono::Utc::now().timestamp_millis(),
885            payload,
886            extensions: serde_json::Map::new(),
887        };
888        let context = RuntimeHookContext {
889            session_id: None,
890            run_id: Some(invocation.run_id.clone()),
891            workflow_id: None,
892            step_id: None,
893            tool_name: Some(invocation.tool_id.to_string()),
894            message: None,
895            metadata: serde_json::json!({
896                "run_id": invocation.run_id.as_str(),
897                "call_id": invocation.call_id.as_str(),
898            }),
899            extensions: serde_json::Map::new(),
900        };
901        hooks
902            .dispatch_checked(&event, &context)
903            .await
904            .map_err(|error| ToolArtifactError::HookRejected {
905                event_type: event_type.to_owned(),
906                message: error.to_string(),
907            })
908    }
909}
910
911#[derive(Debug, thiserror::Error)]
912#[non_exhaustive]
913pub enum ToolArtifactError {
914    #[error("invalid Tool Artifact configuration: {0}")]
915    InvalidConfig(String),
916    #[error("artifact size {observed} exceeds the Host ceiling {maximum}")]
917    LimitExceeded { observed: u64, maximum: u64 },
918    #[error("artifact storage failed: {0}")]
919    Store(#[from] BlobIoError),
920    #[error("artifact integrity check failed: {0}")]
921    Integrity(String),
922    #[error("artifact persistence was cancelled")]
923    Cancelled,
924    #[error("artifact lifecycle hook rejected {event_type}: {message}")]
925    HookRejected { event_type: String, message: String },
926}
927
928struct RegisteredTool {
929    descriptor: ToolDescriptor,
930    executor: Arc<dyn GuardedToolExecutor>,
931    global_gate: Arc<AsyncMutex<()>>,
932}
933
934#[derive(Debug, Clone, PartialEq, Eq)]
935struct InvocationIdentity {
936    tool_id: ToolId,
937    args_digest: Digest,
938    operation_digest: Digest,
939    permission_digest: Digest,
940    policy_digest: Digest,
941    descriptor_digest: Digest,
942    argument_resolution_digest: Option<Digest>,
943}
944
945struct InvocationEntry {
946    identity: InvocationIdentity,
947    state: AsyncMutex<InvocationState>,
948    changed: Notify,
949}
950
951enum InvocationState {
952    Ready,
953    Running,
954    Completed(ToolOutcome),
955}
956
957enum DurableInvocationStart {
958    Execute { lease: Box<CapabilityLease> },
959    Replay { outcome: ToolOutcome },
960}
961
962struct PlannedInvocation {
963    operation: ToolOperationPlan,
964    effective_policy: EffectiveToolPolicy,
965    permission: ToolPermissionDecision,
966    permission_digest: Digest,
967    approval_binding: ApprovalBinding,
968    argument_resolution: Option<ToolArgumentResolution>,
969}
970
971type InvocationKey = (RunId, ToolCallId);
972type PerRunGateKey = (ToolId, RunId);
973
974/// In-process Host reference monitor and execution gate.
975///
976/// The policy ceiling, registry, call ledger, and approval verifier are all
977/// Host-owned. Callers can grant less authority per Run but cannot replace the
978/// ceiling or a registered descriptor.
979pub struct GuardedToolRuntime<S> {
980    host_ceiling: HostToolPolicy,
981    permission_policy: Arc<dyn ToolPermissionPolicy>,
982    approval_verifier: HostApprovalVerifier<S>,
983    effect_journal: Arc<dyn ToolEffectJournalStore>,
984    artifact_store: Option<ToolArtifactStore>,
985    registry: RwLock<BTreeMap<ToolId, Arc<RegisteredTool>>>,
986    invocations: StdMutex<BTreeMap<InvocationKey, Arc<InvocationEntry>>>,
987    per_run_gates: StdMutex<BTreeMap<PerRunGateKey, Weak<AsyncMutex<()>>>>,
988}
989
990impl<S: ApprovalCapabilityStore> GuardedToolRuntime<S> {
991    pub fn new(
992        host_ceiling: HostToolPolicy,
993        approval_verifier: HostApprovalVerifier<S>,
994    ) -> Result<Self, ToolRuntimeError> {
995        Self::new_with_effect_journal(
996            host_ceiling,
997            approval_verifier,
998            Arc::new(InMemoryToolEffectJournalStore::default()),
999        )
1000    }
1001
1002    pub fn new_with_effect_journal(
1003        host_ceiling: HostToolPolicy,
1004        approval_verifier: HostApprovalVerifier<S>,
1005        effect_journal: Arc<dyn ToolEffectJournalStore>,
1006    ) -> Result<Self, ToolRuntimeError> {
1007        Self::new_with_services(host_ceiling, approval_verifier, effect_journal, None)
1008    }
1009
1010    pub fn new_with_effect_journal_and_artifacts(
1011        host_ceiling: HostToolPolicy,
1012        approval_verifier: HostApprovalVerifier<S>,
1013        effect_journal: Arc<dyn ToolEffectJournalStore>,
1014        artifact_store: ToolArtifactStore,
1015    ) -> Result<Self, ToolRuntimeError> {
1016        Self::new_with_services(
1017            host_ceiling,
1018            approval_verifier,
1019            effect_journal,
1020            Some(artifact_store),
1021        )
1022    }
1023
1024    fn new_with_services(
1025        host_ceiling: HostToolPolicy,
1026        approval_verifier: HostApprovalVerifier<S>,
1027        effect_journal: Arc<dyn ToolEffectJournalStore>,
1028        artifact_store: Option<ToolArtifactStore>,
1029    ) -> Result<Self, ToolRuntimeError> {
1030        host_ceiling
1031            .bounds
1032            .validate()
1033            .map_err(ToolRuntimeError::InvalidHostPolicy)?;
1034        Ok(Self {
1035            host_ceiling,
1036            permission_policy: Arc::new(DescriptorPermissionPolicy),
1037            approval_verifier,
1038            effect_journal,
1039            artifact_store,
1040            registry: RwLock::new(BTreeMap::new()),
1041            invocations: StdMutex::new(BTreeMap::new()),
1042            per_run_gates: StdMutex::new(BTreeMap::new()),
1043        })
1044    }
1045
1046    /// Replaces the immutable invocation permission policy before the runtime
1047    /// is shared or registered with an Agent composition root.
1048    pub fn with_permission_policy(mut self, policy: Arc<dyn ToolPermissionPolicy>) -> Self {
1049        self.permission_policy = policy;
1050        self
1051    }
1052
1053    /// Registers an immutable descriptor and policy-aware executor.
1054    pub fn register(
1055        &self,
1056        descriptor: ToolDescriptor,
1057        executor: Arc<dyn GuardedToolExecutor>,
1058    ) -> Result<(), ToolRuntimeError> {
1059        descriptor
1060            .validate()
1061            .map_err(ToolRuntimeError::InvalidDescriptor)?;
1062        let mut registry = self
1063            .registry
1064            .write()
1065            .map_err(|_| ToolRuntimeError::StateUnavailable)?;
1066        if registry.contains_key(&descriptor.tool_id) {
1067            return Err(ToolRuntimeError::DuplicateToolId(
1068                descriptor.tool_id.clone(),
1069            ));
1070        }
1071        if registry.values().any(|registered| {
1072            registered.descriptor.model_schema.name == descriptor.model_schema.name
1073        }) {
1074            return Err(ToolRuntimeError::DuplicateModelName(
1075                descriptor.model_schema.name.clone(),
1076            ));
1077        }
1078        registry.insert(
1079            descriptor.tool_id.clone(),
1080            Arc::new(RegisteredTool {
1081                descriptor,
1082                executor,
1083                global_gate: Arc::new(AsyncMutex::new(())),
1084            }),
1085        );
1086        Ok(())
1087    }
1088
1089    /// Digests only the declared execution boundary. Executor pointers,
1090    /// approval signing material, and mutable invocation state are
1091    /// intentionally excluded.
1092    pub fn execution_contract_digest(&self) -> Result<Digest, ToolRuntimeError> {
1093        let registry = self
1094            .registry
1095            .read()
1096            .map_err(|_| ToolRuntimeError::StateUnavailable)?;
1097        let registrations = registry
1098            .values()
1099            .map(|registered| {
1100                serde_json::json!({
1101                    "descriptor": &registered.descriptor,
1102                    "planning_contract": registered.executor.planning_contract(),
1103                    "model_output_contract": registered.executor.model_output_contract(),
1104                })
1105            })
1106            .collect::<Vec<_>>();
1107        let artifact_contract = self.artifact_store.as_ref().map(|store| {
1108            let mut contract = serde_json::json!({
1109                "max_artifact_bytes": store.max_artifact_bytes,
1110                "summary_max_chars": store.summary_max_chars,
1111                "hooks_enabled": store.hooks.is_some(),
1112            });
1113            if let Some(limit) = store.inline_output_limit() {
1114                contract["inline_output_limit"] = serde_json::json!(limit);
1115            }
1116            contract
1117        });
1118        let contract = serde_json::json!({
1119            "contract": "orchestral.guarded-tool-runtime/v1",
1120            "host_ceiling": &self.host_ceiling,
1121            "permission_policy": self.permission_policy.contract_digest(),
1122            "registered_tools": registrations,
1123            "artifact_store": artifact_contract,
1124        });
1125        let bytes = serde_jcs::to_vec(&contract)
1126            .map_err(|error| ToolRuntimeError::InvalidExecutionContract(error.to_string()))?;
1127        Ok(Digest::sha256(bytes))
1128    }
1129
1130    pub fn project_model_output(
1131        &self,
1132        invocation: &ToolInvocation,
1133        output: &serde_json::Value,
1134    ) -> Result<serde_json::Value, ToolRuntimeError> {
1135        let producer = self
1136            .registered_tool(&invocation.tool_id)?
1137            .ok_or_else(|| ToolRuntimeError::UnknownTool(invocation.tool_id.clone()))?;
1138        Ok(producer.executor.project_model_output(invocation, output))
1139    }
1140
1141    /// Projects only the model-facing schema. Host policy, effect declarations,
1142    /// approval state, and executor details cannot enter this return type.
1143    pub fn model_tool_schemas(&self) -> Result<Vec<ModelToolSchema>, ToolRuntimeError> {
1144        let registry = self
1145            .registry
1146            .read()
1147            .map_err(|_| ToolRuntimeError::StateUnavailable)?;
1148        Ok(registry
1149            .values()
1150            .map(|registered| registered.descriptor.model_schema().clone())
1151            .collect())
1152    }
1153
1154    pub fn resolve_tool_id(&self, model_name: &str) -> Result<Option<ToolId>, ToolRuntimeError> {
1155        let registry = self
1156            .registry
1157            .read()
1158            .map_err(|_| ToolRuntimeError::StateUnavailable)?;
1159        Ok(registry
1160            .values()
1161            .find(|registered| registered.descriptor.model_schema.name == model_name)
1162            .map(|registered| registered.descriptor.tool_id.clone()))
1163    }
1164
1165    pub fn activity_evidence(
1166        &self,
1167        invocation: &ToolInvocation,
1168        outcome: Option<&ToolOutcome>,
1169    ) -> Result<Vec<ToolActivityEvidence>, ToolRuntimeError> {
1170        invocation
1171            .validate()
1172            .map_err(|error| ToolRuntimeError::InvalidActivityEvidence(error.message))?;
1173        let Some(registered) = self.registered_tool(&invocation.tool_id)? else {
1174            return Ok(Vec::new());
1175        };
1176        registered
1177            .descriptor
1178            .model_schema
1179            .validate_arguments(&invocation.arguments)
1180            .map_err(|error| ToolRuntimeError::InvalidActivityEvidence(error.message))?;
1181        let evidence = registered.executor.activity_evidence(invocation, outcome);
1182        if evidence.len() > 16 {
1183            return Err(ToolRuntimeError::InvalidActivityEvidence(
1184                "a Tool adapter emitted more than sixteen evidence items".to_owned(),
1185            ));
1186        }
1187        for item in &evidence {
1188            item.validate_integrity()
1189                .map_err(|error| ToolRuntimeError::InvalidActivityEvidence(error.message))?;
1190        }
1191        Ok(evidence)
1192    }
1193
1194    /// Replays only durable Tool state. This path can close an Observed result
1195    /// or classify an orphaned Invoked effect as unknown, but it never creates
1196    /// Prepared/Invoked records and never enters an executor.
1197    pub async fn recover_outcome(
1198        &self,
1199        invocation: ToolInvocation,
1200        run_grant: RunToolGrant,
1201    ) -> Result<Option<ToolOutcome>, ToolOutcomeRecoveryError> {
1202        if let Err(error) = invocation.validate() {
1203            return Err(tool_outcome_recovery_error(
1204                "invalid_invocation",
1205                error.message,
1206            ));
1207        }
1208        let registered = match self.registered_tool(&invocation.tool_id) {
1209            Ok(Some(registered)) => registered,
1210            Ok(None) => {
1211                return Err(tool_outcome_recovery_error(
1212                    "tool_not_found",
1213                    format!("tool is not registered: {}", invocation.tool_id),
1214                ))
1215            }
1216            Err(error) => {
1217                return Err(tool_outcome_recovery_error(
1218                    "runtime_unavailable",
1219                    error.to_string(),
1220                ))
1221            }
1222        };
1223        if let Err(error) = registered
1224            .descriptor
1225            .model_schema
1226            .validate_arguments(&invocation.arguments)
1227        {
1228            return Err(tool_outcome_recovery_error(
1229                "input_schema_violation",
1230                error.message,
1231            ));
1232        }
1233        // Recovery never creates a missing preparation. In particular an
1234        // omitted precondition needs no new read when no durable effect exists.
1235        let recovery_key =
1236            ToolEffectKey::new(invocation.run_id.clone(), invocation.call_id.clone());
1237        if self
1238            .effect_journal
1239            .load_effect(&recovery_key)
1240            .await
1241            .map_err(effect_journal_recovery_error)?
1242            .is_empty()
1243        {
1244            return Ok(None);
1245        }
1246        let effective_policy = EffectiveToolPolicy::resolve(
1247            &self.host_ceiling,
1248            &run_grant,
1249            &registered.descriptor.restriction,
1250        )
1251        .map_err(|error| tool_outcome_recovery_error("invalid_effective_policy", error.message))?;
1252        let argument_resolution = self
1253            .resolve_invocation_arguments(
1254                &invocation,
1255                &registered,
1256                &FrozenToolObservations::default(),
1257            )
1258            .await
1259            .map_err(|outcome| {
1260                tool_outcome_recovery_error(
1261                    "operation_planning_failed",
1262                    format!("Tool argument resolution failed: {outcome:?}"),
1263                )
1264            })?;
1265        let resolved_invocation = execution_invocation(&invocation, argument_resolution.as_ref());
1266        let operation = registered
1267            .executor
1268            .plan_operation(
1269                &resolved_invocation,
1270                &registered.descriptor,
1271                &effective_policy,
1272            )
1273            .map_err(|outcome| {
1274                tool_outcome_recovery_error(
1275                    "operation_planning_failed",
1276                    format!("Tool operation planning failed: {outcome:?}"),
1277                )
1278            })?;
1279        operation
1280            .validate_envelope(&registered.descriptor.effect_scopes)
1281            .map_err(|error| {
1282                tool_outcome_recovery_error("invalid_operation_plan", error.message)
1283            })?;
1284        if !effective_policy.authorizes_request(&operation.required_capabilities) {
1285            return Err(tool_outcome_recovery_error(
1286                "policy_denied",
1287                "tool effects are outside the effective Host policy",
1288            ));
1289        }
1290        let permission = constrain_permission_decision(
1291            &effective_policy,
1292            &operation,
1293            self.permission_policy
1294                .decide(&registered.descriptor, &operation, &effective_policy),
1295        );
1296        let permission_digest =
1297            tool_permission_decision_digest(self.permission_policy.as_ref(), &permission).map_err(
1298                |error| tool_outcome_recovery_error("invalid_permission_decision", error.message),
1299            )?;
1300        let prepared = PreparedToolEffect {
1301            invocation: invocation.clone(),
1302            argument_resolution: argument_resolution.map(Box::new),
1303            args_digest: invocation.args_digest().map_err(|error| {
1304                tool_outcome_recovery_error("invalid_invocation", error.message)
1305            })?,
1306            operation_digest: operation.digest().map_err(|error| {
1307                tool_outcome_recovery_error("invalid_operation_plan", error.message)
1308            })?,
1309            permission_digest,
1310            policy_digest: effective_policy.digest().map_err(|error| {
1311                tool_outcome_recovery_error("invalid_effective_policy", error.message)
1312            })?,
1313            descriptor_digest: registered.descriptor.digest().map_err(|error| {
1314                tool_outcome_recovery_error("invalid_descriptor", error.message)
1315            })?,
1316            idempotency: registered.descriptor.idempotency,
1317            effect_scopes: operation.required_capabilities.effects.clone(),
1318        };
1319        let key = prepared.key();
1320
1321        for _ in 0..4 {
1322            let records = self
1323                .effect_journal
1324                .load_effect(&key)
1325                .await
1326                .map_err(effect_journal_recovery_error)?;
1327            let Some(projection) =
1328                replay_tool_effect(&key, &records).map_err(effect_journal_recovery_error)?
1329            else {
1330                return Ok(None);
1331            };
1332            if projection.prepared != prepared {
1333                return Err(tool_outcome_recovery_error(
1334                    "call_identity_conflict",
1335                    "durable Tool effect identity differs for the same run_id/call_id",
1336                ));
1337            }
1338            match projection.phase {
1339                ToolEffectPhase::Prepared => return Ok(None),
1340                ToolEffectPhase::Observed { outcome, .. } => {
1341                    let outcome_digest = outcome.digest().map_err(|error| {
1342                        tool_outcome_recovery_error("invalid_tool_outcome", error.message)
1343                    })?;
1344                    match self
1345                        .effect_journal
1346                        .append(
1347                            projection.last_effect_seq,
1348                            ToolEffectEventDraft {
1349                                event_id: effect_event_id(&key, "committed"),
1350                                key: key.clone(),
1351                                payload: ToolEffectEvent::Committed { outcome_digest },
1352                            },
1353                        )
1354                        .await
1355                    {
1356                        Ok(_) | Err(ToolEffectError::SequenceConflict { .. }) => continue,
1357                        Err(error) => return Err(effect_journal_recovery_error(error)),
1358                    }
1359                }
1360                ToolEffectPhase::Committed { outcome, .. } => return Ok(Some(outcome)),
1361                ToolEffectPhase::Invoked { .. } => {
1362                    let reason = "durable invocation has no observation after runtime recovery";
1363                    match self
1364                        .effect_journal
1365                        .append(
1366                            projection.last_effect_seq,
1367                            ToolEffectEventDraft {
1368                                event_id: effect_event_id(&key, "unknown"),
1369                                key: key.clone(),
1370                                payload: ToolEffectEvent::EffectUnknown {
1371                                    reason: reason.to_owned(),
1372                                },
1373                            },
1374                        )
1375                        .await
1376                    {
1377                        Ok(_) | Err(ToolEffectError::SequenceConflict { .. }) => continue,
1378                        Err(error) => return Err(effect_journal_recovery_error(error)),
1379                    }
1380                }
1381                ToolEffectPhase::UnknownEffect { reason, .. } => {
1382                    return Ok(Some(unknown_effect(reason)))
1383                }
1384            }
1385        }
1386        Err(tool_outcome_recovery_error(
1387            "effect_journal_contention",
1388            "Tool effect journal did not converge during recovery",
1389        ))
1390    }
1391
1392    /// Executes the fixed guarded pipeline:
1393    ///
1394    /// invocation/input schema → effective policy → operation planning →
1395    /// permission decision/approval → concurrency gate/executor → output
1396    /// schema and output limit.
1397    pub async fn invoke(
1398        &self,
1399        invocation: ToolInvocation,
1400        run_grant: RunToolGrant,
1401        approval: Option<ApprovalCapability>,
1402        run_cancellation: CancellationToken,
1403    ) -> GuardedToolResult {
1404        self.invoke_with_yield(
1405            invocation,
1406            run_grant,
1407            approval,
1408            run_cancellation,
1409            CancellationToken::new(),
1410        )
1411        .await
1412    }
1413
1414    /// Runs the same guarded, journaled invocation while allowing an executor
1415    /// to yield an observation when new Host input arrives.
1416    pub async fn invoke_with_yield(
1417        &self,
1418        invocation: ToolInvocation,
1419        run_grant: RunToolGrant,
1420        approval: Option<ApprovalCapability>,
1421        run_cancellation: CancellationToken,
1422        yield_requested: CancellationToken,
1423    ) -> GuardedToolResult {
1424        self.invoke_with_observations(
1425            invocation,
1426            run_grant,
1427            approval,
1428            run_cancellation,
1429            yield_requested,
1430            &FrozenToolObservations::default(),
1431        )
1432        .await
1433    }
1434
1435    pub async fn invoke_with_observations(
1436        &self,
1437        invocation: ToolInvocation,
1438        run_grant: RunToolGrant,
1439        approval: Option<ApprovalCapability>,
1440        run_cancellation: CancellationToken,
1441        yield_requested: CancellationToken,
1442        observations: &FrozenToolObservations,
1443    ) -> GuardedToolResult {
1444        if let Err(error) = invocation.validate() {
1445            return rejected("invalid_invocation", error.message);
1446        }
1447        let registered = match self.registered_tool(&invocation.tool_id) {
1448            Ok(Some(registered)) => registered,
1449            Ok(None) => {
1450                return rejected(
1451                    "tool_not_found",
1452                    format!("tool is not registered: {}", invocation.tool_id),
1453                )
1454            }
1455            Err(error) => return rejected("runtime_unavailable", error.to_string()),
1456        };
1457        if let Err(error) = registered
1458            .descriptor
1459            .model_schema
1460            .validate_arguments(&invocation.arguments)
1461        {
1462            return rejected("input_schema_violation", error.message);
1463        }
1464
1465        let effective_policy = match EffectiveToolPolicy::resolve(
1466            &self.host_ceiling,
1467            &run_grant,
1468            &registered.descriptor.restriction,
1469        ) {
1470            Ok(policy) => policy,
1471            Err(error) => return rejected("invalid_effective_policy", error.message),
1472        };
1473        let argument_resolution = match self
1474            .resolve_invocation_arguments(&invocation, &registered, observations)
1475            .await
1476        {
1477            Ok(resolution) => resolution,
1478            Err(outcome) => {
1479                return GuardedToolResult::Outcome {
1480                    outcome,
1481                    cached: false,
1482                }
1483            }
1484        };
1485        let resolved_invocation = execution_invocation(&invocation, argument_resolution.as_ref());
1486        let operation = match registered.executor.plan_operation(
1487            &resolved_invocation,
1488            &registered.descriptor,
1489            &effective_policy,
1490        ) {
1491            Ok(operation) => operation,
1492            Err(outcome) => {
1493                return GuardedToolResult::Outcome {
1494                    outcome,
1495                    cached: false,
1496                }
1497            }
1498        };
1499        if let Err(error) = operation.validate_envelope(&registered.descriptor.effect_scopes) {
1500            return rejected("invalid_operation_plan", error.message);
1501        }
1502        if !effective_policy.authorizes_request(&operation.required_capabilities) {
1503            return rejected(
1504                "policy_denied",
1505                "tool effects are outside the effective Host policy",
1506            );
1507        }
1508        let permission = constrain_permission_decision(
1509            &effective_policy,
1510            &operation,
1511            self.permission_policy
1512                .decide(&registered.descriptor, &operation, &effective_policy),
1513        );
1514        if let ToolPermissionDecision::Deny { code, message } = &permission {
1515            return rejected(code.clone(), message.clone());
1516        }
1517        let permission_digest =
1518            match tool_permission_decision_digest(self.permission_policy.as_ref(), &permission) {
1519                Ok(digest) => digest,
1520                Err(error) => return rejected("invalid_permission_decision", error.message),
1521            };
1522        let approval_binding = match ApprovalBinding::for_operation(
1523            &resolved_invocation,
1524            &operation,
1525            &effective_policy,
1526            permission_digest.clone(),
1527        ) {
1528            Ok(binding) => binding,
1529            Err(error) => return rejected("policy_denied", error.message),
1530        };
1531        let identity = match invocation_identity(
1532            &invocation,
1533            &operation,
1534            &effective_policy,
1535            &permission_digest,
1536            &registered.descriptor,
1537            argument_resolution.as_ref(),
1538        ) {
1539            Ok(identity) => identity,
1540            Err(error) => return rejected("invalid_invocation", error.message),
1541        };
1542        let planned = PlannedInvocation {
1543            operation,
1544            effective_policy,
1545            permission,
1546            permission_digest,
1547            approval_binding,
1548            argument_resolution,
1549        };
1550        let effect_key = ToolEffectKey::new(invocation.run_id.clone(), invocation.call_id.clone());
1551        let entry = match self.invocation_entry(&invocation, identity) {
1552            Ok(entry) => entry,
1553            Err(result) => return *result,
1554        };
1555
1556        let lease = loop {
1557            // Register the waiter before observing the state to avoid a missed
1558            // notification between unlocking and awaiting.
1559            let changed = entry.changed.notified();
1560            let mut state = entry.state.lock().await;
1561            match &*state {
1562                InvocationState::Completed(outcome) => {
1563                    return GuardedToolResult::Outcome {
1564                        outcome: outcome.clone(),
1565                        cached: true,
1566                    };
1567                }
1568                InvocationState::Running => {
1569                    drop(state);
1570                    changed.await;
1571                }
1572                InvocationState::Ready => {
1573                    match self
1574                        .prepare_durable_invocation(
1575                            &registered,
1576                            &invocation,
1577                            &planned,
1578                            approval.as_ref(),
1579                            &run_cancellation,
1580                        )
1581                        .await
1582                    {
1583                        Ok(DurableInvocationStart::Execute { lease }) => {
1584                            *state = InvocationState::Running;
1585                            break *lease;
1586                        }
1587                        Ok(DurableInvocationStart::Replay { outcome }) => {
1588                            *state = InvocationState::Completed(outcome.clone());
1589                            drop(state);
1590                            entry.changed.notify_waiters();
1591                            return GuardedToolResult::Outcome {
1592                                outcome,
1593                                cached: true,
1594                            };
1595                        }
1596                        Err(result) => return result,
1597                    }
1598                }
1599            }
1600        };
1601
1602        let execution_cancellation = run_cancellation.child_token();
1603        let PlannedInvocation {
1604            operation,
1605            effective_policy,
1606            ..
1607        } = planned;
1608        let outcome = match self
1609            .concurrency_gate(&registered, &invocation, &execution_cancellation)
1610            .await
1611        {
1612            Ok(gate_guard) => {
1613                let _gate_guard = gate_guard;
1614                let deadline = effective_policy
1615                    .bounds()
1616                    .max_timeout_ms
1617                    .map(|ms| tokio::time::Instant::now() + Duration::from_millis(ms));
1618                self.execute(
1619                    registered,
1620                    GuardedToolExecution {
1621                        invocation: resolved_invocation,
1622                        operation,
1623                        effective_policy,
1624                        lease,
1625                        cancellation: execution_cancellation,
1626                        run_cancellation,
1627                        yield_requested,
1628                        deadline,
1629                    },
1630                )
1631                .await
1632            }
1633            Err(outcome) => outcome,
1634        };
1635
1636        let outcome = self.commit_durable_outcome(&effect_key, outcome).await;
1637        let mut state = entry.state.lock().await;
1638        *state = InvocationState::Completed(outcome.clone());
1639        drop(state);
1640        entry.changed.notify_waiters();
1641        GuardedToolResult::Outcome {
1642            outcome,
1643            cached: false,
1644        }
1645    }
1646
1647    /// Loads a durable Tool effect without closing `Observed` or classifying
1648    /// `Invoked`. This is deliberately read-only so a workflow can perform a
1649    /// global recovery preflight before it dispatches any new work.
1650    pub async fn inspect_effect(
1651        &self,
1652        key: &ToolEffectKey,
1653    ) -> Result<Option<ToolEffectProjection>, ToolOutcomeRecoveryError> {
1654        key.validate().map_err(effect_journal_recovery_error)?;
1655        let records = self
1656            .effect_journal
1657            .load_effect(key)
1658            .await
1659            .map_err(effect_journal_recovery_error)?;
1660        replay_tool_effect(key, &records).map_err(effect_journal_recovery_error)
1661    }
1662
1663    async fn prepare_durable_invocation(
1664        &self,
1665        registered: &Arc<RegisteredTool>,
1666        invocation: &ToolInvocation,
1667        planned: &PlannedInvocation,
1668        approval: Option<&ApprovalCapability>,
1669        run_cancellation: &CancellationToken,
1670    ) -> Result<DurableInvocationStart, GuardedToolResult> {
1671        let PlannedInvocation {
1672            operation,
1673            effective_policy,
1674            permission,
1675            permission_digest,
1676            approval_binding,
1677            argument_resolution,
1678        } = planned;
1679        let prepared = PreparedToolEffect {
1680            invocation: invocation.clone(),
1681            argument_resolution: argument_resolution.clone().map(Box::new),
1682            args_digest: invocation
1683                .args_digest()
1684                .map_err(|error| rejected("invalid_invocation", error.message))?,
1685            operation_digest: operation
1686                .digest()
1687                .map_err(|error| rejected("invalid_operation_plan", error.message))?,
1688            permission_digest: permission_digest.clone(),
1689            policy_digest: effective_policy
1690                .digest()
1691                .map_err(|error| rejected("invalid_effective_policy", error.message))?,
1692            descriptor_digest: registered
1693                .descriptor
1694                .digest()
1695                .map_err(|error| rejected("invalid_descriptor", error.message))?,
1696            idempotency: registered.descriptor.idempotency,
1697            effect_scopes: operation.required_capabilities.effects.clone(),
1698        };
1699        let key = prepared.key();
1700
1701        for _ in 0..4 {
1702            let records = self
1703                .effect_journal
1704                .load_effect(&key)
1705                .await
1706                .map_err(effect_journal_rejected)?;
1707            let projection = replay_tool_effect(&key, &records).map_err(effect_journal_rejected)?;
1708            let Some(projection) = projection else {
1709                match self
1710                    .effect_journal
1711                    .append(
1712                        0,
1713                        ToolEffectEventDraft {
1714                            event_id: effect_event_id(&key, "prepared"),
1715                            key: key.clone(),
1716                            payload: ToolEffectEvent::Prepared {
1717                                effect: prepared.clone(),
1718                            },
1719                        },
1720                    )
1721                    .await
1722                {
1723                    Ok(_) | Err(ToolEffectError::SequenceConflict { .. }) => continue,
1724                    Err(error) => return Err(effect_journal_rejected(error)),
1725                }
1726            };
1727            if projection.prepared != prepared {
1728                return Err(rejected(
1729                    "call_identity_conflict",
1730                    "durable Tool effect identity differs for the same run_id/call_id",
1731                ));
1732            }
1733            match projection.phase {
1734                ToolEffectPhase::Prepared => {
1735                    // Prepared records contain intent only. Cancellation here
1736                    // proves the executor never crossed the durable Invoked
1737                    // boundary, while prior Invoked/Observed/Committed phases
1738                    // below still retain their conservative replay semantics.
1739                    if run_cancellation.is_cancelled() {
1740                        return Err(GuardedToolResult::Outcome {
1741                            outcome: ToolOutcome::Cancelled,
1742                            cached: false,
1743                        });
1744                    }
1745                    let (lease, authorization) =
1746                        if matches!(permission, ToolPermissionDecision::RequireApproval) {
1747                            let Some(capability) = approval else {
1748                                return Err(GuardedToolResult::ApprovalRequired {
1749                                    binding: approval_binding.clone(),
1750                                    summary: sanitize_approval_summary(
1751                                        &operation.summary,
1752                                        &invocation.tool_id,
1753                                    ),
1754                                });
1755                            };
1756                            let verified = self
1757                                .approval_verifier
1758                                .verify_and_consume(
1759                                    capability,
1760                                    approval_binding,
1761                                    chrono::Utc::now().timestamp_millis(),
1762                                )
1763                                .map_err(|error| {
1764                                    rejected(approval_error_code(error.code), error.message)
1765                                })?;
1766                            let evidence = ToolAuthorizationEvidence::Approval {
1767                                nonce: verified.nonce().clone(),
1768                            };
1769                            let lease = CapabilityLease::approved(operation, verified).map_err(
1770                                |error| rejected("invalid_capability_lease", error.message),
1771                            )?;
1772                            (lease, evidence)
1773                        } else {
1774                            let lease = CapabilityLease::policy(operation).map_err(|error| {
1775                                rejected("invalid_capability_lease", error.message)
1776                            })?;
1777                            (lease, ToolAuthorizationEvidence::Policy)
1778                        };
1779                    let appended = self
1780                        .effect_journal
1781                        .append(
1782                            projection.last_effect_seq,
1783                            ToolEffectEventDraft {
1784                                event_id: effect_event_id(&key, "invoked"),
1785                                key: key.clone(),
1786                                payload: ToolEffectEvent::Invoked {
1787                                    attempt_id: ToolEffectAttemptId::new(format!(
1788                                        "attempt:{}:{}",
1789                                        key.run_id.as_str(),
1790                                        key.call_id.as_str()
1791                                    )),
1792                                    authorization,
1793                                },
1794                            },
1795                        )
1796                        .await;
1797                    match appended {
1798                        Ok(_) => {
1799                            return Ok(DurableInvocationStart::Execute {
1800                                lease: Box::new(lease),
1801                            })
1802                        }
1803                        Err(ToolEffectError::SequenceConflict { .. }) => continue,
1804                        Err(error) => return Err(effect_journal_rejected(error)),
1805                    }
1806                }
1807                ToolEffectPhase::Observed { outcome, .. } => {
1808                    let outcome_digest = outcome
1809                        .digest()
1810                        .map_err(|error| rejected("invalid_tool_outcome", error.message))?;
1811                    match self
1812                        .effect_journal
1813                        .append(
1814                            projection.last_effect_seq,
1815                            ToolEffectEventDraft {
1816                                event_id: effect_event_id(&key, "committed"),
1817                                key: key.clone(),
1818                                payload: ToolEffectEvent::Committed { outcome_digest },
1819                            },
1820                        )
1821                        .await
1822                    {
1823                        Ok(_) => return Ok(DurableInvocationStart::Replay { outcome }),
1824                        Err(ToolEffectError::SequenceConflict { .. }) => continue,
1825                        Err(error) => return Err(effect_journal_rejected(error)),
1826                    }
1827                }
1828                ToolEffectPhase::Committed { outcome, .. } => {
1829                    return Ok(DurableInvocationStart::Replay { outcome })
1830                }
1831                ToolEffectPhase::Invoked { .. } => {
1832                    let reason = "durable invocation has no observation after runtime recovery";
1833                    match self
1834                        .effect_journal
1835                        .append(
1836                            projection.last_effect_seq,
1837                            ToolEffectEventDraft {
1838                                event_id: effect_event_id(&key, "unknown"),
1839                                key: key.clone(),
1840                                payload: ToolEffectEvent::EffectUnknown {
1841                                    reason: reason.to_owned(),
1842                                },
1843                            },
1844                        )
1845                        .await
1846                    {
1847                        Ok(_) => {
1848                            return Ok(DurableInvocationStart::Replay {
1849                                outcome: unknown_effect(reason),
1850                            })
1851                        }
1852                        Err(ToolEffectError::SequenceConflict { .. }) => continue,
1853                        Err(error) => {
1854                            return Ok(DurableInvocationStart::Replay {
1855                                outcome: unknown_effect(format!(
1856                                    "{reason}; journal update failed: {error}"
1857                                )),
1858                            })
1859                        }
1860                    }
1861                }
1862                ToolEffectPhase::UnknownEffect { reason, .. } => {
1863                    return Ok(DurableInvocationStart::Replay {
1864                        outcome: unknown_effect(reason),
1865                    })
1866                }
1867            }
1868        }
1869        Err(rejected(
1870            "effect_journal_contention",
1871            "Tool effect journal did not converge after concurrent updates",
1872        ))
1873    }
1874
1875    async fn commit_durable_outcome(
1876        &self,
1877        key: &ToolEffectKey,
1878        outcome: ToolOutcome,
1879    ) -> ToolOutcome {
1880        for _ in 0..5 {
1881            let records = match self.effect_journal.load_effect(key).await {
1882                Ok(records) => records,
1883                Err(error) => {
1884                    return unknown_effect(format!(
1885                        "Tool effect completed but its journal is unavailable: {error}"
1886                    ))
1887                }
1888            };
1889            let projection = match replay_tool_effect(key, &records) {
1890                Ok(Some(projection)) => projection,
1891                Ok(None) => {
1892                    return unknown_effect(
1893                        "Tool effect completed without a durable Prepared record",
1894                    )
1895                }
1896                Err(error) => {
1897                    return unknown_effect(format!(
1898                        "Tool effect completed but its journal is corrupt: {error}"
1899                    ))
1900                }
1901            };
1902            match (&projection.phase, &outcome) {
1903                (ToolEffectPhase::Invoked { .. }, ToolOutcome::UnknownEffect { message }) => {
1904                    match self
1905                        .effect_journal
1906                        .append(
1907                            projection.last_effect_seq,
1908                            ToolEffectEventDraft {
1909                                event_id: effect_event_id(key, "unknown"),
1910                                key: key.clone(),
1911                                payload: ToolEffectEvent::EffectUnknown {
1912                                    reason: message.clone(),
1913                                },
1914                            },
1915                        )
1916                        .await
1917                    {
1918                        Ok(_) => return outcome,
1919                        Err(ToolEffectError::SequenceConflict { .. }) => continue,
1920                        Err(error) => {
1921                            return unknown_effect(format!(
1922                                "{message}; journal update failed: {error}"
1923                            ))
1924                        }
1925                    }
1926                }
1927                (ToolEffectPhase::Invoked { .. }, _) => {
1928                    match self
1929                        .effect_journal
1930                        .append(
1931                            projection.last_effect_seq,
1932                            ToolEffectEventDraft {
1933                                event_id: effect_event_id(key, "observed"),
1934                                key: key.clone(),
1935                                payload: ToolEffectEvent::Observed {
1936                                    outcome: outcome.clone(),
1937                                },
1938                            },
1939                        )
1940                        .await
1941                    {
1942                        Ok(_) | Err(ToolEffectError::SequenceConflict { .. }) => continue,
1943                        Err(error) => {
1944                            return unknown_effect(format!(
1945                                "Tool effect outcome could not be observed durably: {error}"
1946                            ))
1947                        }
1948                    }
1949                }
1950                (
1951                    ToolEffectPhase::Observed {
1952                        outcome: observed, ..
1953                    },
1954                    _,
1955                ) => {
1956                    if observed != &outcome {
1957                        return unknown_effect(
1958                            "durable Tool observation differs from the live outcome",
1959                        );
1960                    }
1961                    let outcome_digest = match outcome.digest() {
1962                        Ok(digest) => digest,
1963                        Err(error) => {
1964                            return unknown_effect(format!(
1965                                "Tool effect produced an invalid outcome: {}",
1966                                error.message
1967                            ))
1968                        }
1969                    };
1970                    match self
1971                        .effect_journal
1972                        .append(
1973                            projection.last_effect_seq,
1974                            ToolEffectEventDraft {
1975                                event_id: effect_event_id(key, "committed"),
1976                                key: key.clone(),
1977                                payload: ToolEffectEvent::Committed { outcome_digest },
1978                            },
1979                        )
1980                        .await
1981                    {
1982                        Ok(_) => return outcome,
1983                        Err(ToolEffectError::SequenceConflict { .. }) => continue,
1984                        Err(error) => {
1985                            return unknown_effect(format!(
1986                                "Tool effect observation was durable but commit failed: {error}"
1987                            ))
1988                        }
1989                    }
1990                }
1991                (
1992                    ToolEffectPhase::Committed {
1993                        outcome: committed, ..
1994                    },
1995                    _,
1996                ) => {
1997                    return if committed == &outcome {
1998                        committed.clone()
1999                    } else {
2000                        unknown_effect("committed Tool outcome differs from the live outcome")
2001                    }
2002                }
2003                (ToolEffectPhase::UnknownEffect { reason, .. }, _) => {
2004                    return unknown_effect(reason.clone())
2005                }
2006                (ToolEffectPhase::Prepared, _) => {
2007                    return unknown_effect(
2008                        "Tool executor was entered without a durable Invoked boundary",
2009                    )
2010                }
2011            }
2012        }
2013        unknown_effect("Tool effect journal did not converge while committing the outcome")
2014    }
2015
2016    /// Releases replay and per-Run gate state once the owning Agent Run is no
2017    /// longer resumable in this process.
2018    pub fn forget_run(&self, run_id: &RunId) -> Result<(), ToolRuntimeError> {
2019        self.invocations
2020            .lock()
2021            .map_err(|_| ToolRuntimeError::StateUnavailable)?
2022            .retain(|(entry_run_id, _), _| entry_run_id != run_id);
2023        self.per_run_gates
2024            .lock()
2025            .map_err(|_| ToolRuntimeError::StateUnavailable)?
2026            .retain(|(_, entry_run_id), _| entry_run_id != run_id);
2027        Ok(())
2028    }
2029
2030    fn registered_tool(
2031        &self,
2032        tool_id: &ToolId,
2033    ) -> Result<Option<Arc<RegisteredTool>>, ToolRuntimeError> {
2034        Ok(self
2035            .registry
2036            .read()
2037            .map_err(|_| ToolRuntimeError::StateUnavailable)?
2038            .get(tool_id)
2039            .cloned())
2040    }
2041
2042    fn invocation_entry(
2043        &self,
2044        invocation: &ToolInvocation,
2045        identity: InvocationIdentity,
2046    ) -> Result<Arc<InvocationEntry>, Box<GuardedToolResult>> {
2047        let key = (invocation.run_id.clone(), invocation.call_id.clone());
2048        let mut invocations = self.invocations.lock().map_err(|_| {
2049            Box::new(rejected(
2050                "runtime_unavailable",
2051                "Tool call ledger is unavailable",
2052            ))
2053        })?;
2054        if let Some(entry) = invocations.get(&key) {
2055            if entry.identity != identity {
2056                return Err(Box::new(rejected(
2057                    "call_identity_conflict",
2058                    "the same run_id/call_id was reused with different content or policy",
2059                )));
2060            }
2061            return Ok(entry.clone());
2062        }
2063        let entry = Arc::new(InvocationEntry {
2064            identity,
2065            state: AsyncMutex::new(InvocationState::Ready),
2066            changed: Notify::new(),
2067        });
2068        invocations.insert(key, entry.clone());
2069        Ok(entry)
2070    }
2071
2072    async fn concurrency_gate(
2073        &self,
2074        registered: &Arc<RegisteredTool>,
2075        invocation: &ToolInvocation,
2076        cancellation: &CancellationToken,
2077    ) -> Result<Option<OwnedMutexGuard<()>>, ToolOutcome> {
2078        let gate = match registered.descriptor.concurrency {
2079            ToolConcurrency::ParallelSafe => return Ok(None),
2080            ToolConcurrency::PerRunSerial => {
2081                match self.per_run_gate(&invocation.tool_id, &invocation.run_id) {
2082                    Ok(gate) => gate,
2083                    Err(error) => {
2084                        return Err(ToolOutcome::Failed {
2085                            code: "runtime_unavailable".to_owned(),
2086                            message: error.to_string(),
2087                            retryable: true,
2088                        })
2089                    }
2090                }
2091            }
2092            ToolConcurrency::GlobalSerial => registered.global_gate.clone(),
2093            // Unknown future modes are conservatively serialized globally.
2094            _ => registered.global_gate.clone(),
2095        };
2096        tokio::select! {
2097            _ = cancellation.cancelled() => Err(ToolOutcome::Cancelled),
2098            guard = gate.lock_owned() => Ok(Some(guard)),
2099        }
2100    }
2101
2102    fn per_run_gate(
2103        &self,
2104        tool_id: &ToolId,
2105        run_id: &RunId,
2106    ) -> Result<Arc<AsyncMutex<()>>, ToolRuntimeError> {
2107        let key = (tool_id.clone(), run_id.clone());
2108        let mut gates = self
2109            .per_run_gates
2110            .lock()
2111            .map_err(|_| ToolRuntimeError::StateUnavailable)?;
2112        if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
2113            return Ok(gate);
2114        }
2115        let gate = Arc::new(AsyncMutex::new(()));
2116        gates.insert(key, Arc::downgrade(&gate));
2117        Ok(gate)
2118    }
2119
2120    async fn execute(
2121        &self,
2122        registered: Arc<RegisteredTool>,
2123        execution: GuardedToolExecution,
2124    ) -> ToolOutcome {
2125        if let Err(error) = execution.lease.validate_for(
2126            &execution.invocation,
2127            &execution.operation,
2128            &execution.effective_policy,
2129        ) {
2130            return ToolOutcome::Rejected {
2131                code: "invalid_capability_lease".to_owned(),
2132                message: error.message,
2133            };
2134        }
2135        let effective_policy = execution.effective_policy.clone();
2136        let deadline = execution.deadline;
2137        let output_invocation = execution.invocation.clone();
2138        let cancellation = execution.cancellation.clone();
2139        let execution = registered.executor.execute(execution);
2140        let execution = AssertUnwindSafe(execution).catch_unwind();
2141        tokio::pin!(execution);
2142
2143        let outcome = match deadline {
2144            Some(deadline) => {
2145                let timeout = tokio::time::sleep_until(deadline);
2146                tokio::pin!(timeout);
2147                tokio::select! {
2148                    _ = cancellation.cancelled() => {
2149                        let fallback = cancellation_outcome(&registered.descriptor);
2150                        settle_cancelled_executor(&mut execution, fallback).await
2151                    },
2152                    _ = &mut timeout => {
2153                        cancellation.cancel();
2154                        let fallback = timeout_outcome(&registered.descriptor);
2155                        settle_cancelled_executor(&mut execution, fallback).await
2156                    }
2157                    result = &mut execution => map_execution_result(result),
2158                }
2159            }
2160            None => {
2161                tokio::select! {
2162                    _ = cancellation.cancelled() => {
2163                        let fallback = cancellation_outcome(&registered.descriptor);
2164                        settle_cancelled_executor(&mut execution, fallback).await
2165                    },
2166                    result = &mut execution => map_execution_result(result),
2167                }
2168            }
2169        };
2170        // The executor future has crossed the Host's dispatch boundary. A
2171        // non-idempotent executor may observe the same cancellation as this
2172        // outer select and return `Cancelled` first; that race cannot prove
2173        // whether its external effect happened, so preserve the conservative
2174        // `UnknownEffect` contract.
2175        let outcome = normalize_post_dispatch_outcome(&registered.descriptor, outcome);
2176        normalize_completed_outcome(
2177            &registered.descriptor,
2178            registered.executor.as_ref(),
2179            &effective_policy,
2180            &output_invocation,
2181            self.artifact_store.as_ref(),
2182            &cancellation,
2183            outcome,
2184        )
2185        .await
2186    }
2187}
2188
2189#[async_trait]
2190impl<S> AgentToolRuntime for GuardedToolRuntime<S>
2191where
2192    S: ApprovalCapabilityStore + 'static,
2193{
2194    fn project_model_output(
2195        &self,
2196        invocation: &ToolInvocation,
2197        output: &serde_json::Value,
2198    ) -> Result<serde_json::Value, ToolRuntimeError> {
2199        GuardedToolRuntime::project_model_output(self, invocation, output)
2200    }
2201
2202    async fn freeze_model_observations(
2203        &self,
2204        run_id: &RunId,
2205        observations: &ModelToolObservations,
2206        pending_calls: &[ToolCallId],
2207    ) -> Result<FrozenToolObservations, ToolOutcome> {
2208        GuardedToolRuntime::freeze_model_observations(self, run_id, observations, pending_calls)
2209            .await
2210    }
2211
2212    fn execution_contract_digest(&self) -> Result<Digest, ToolRuntimeError> {
2213        GuardedToolRuntime::execution_contract_digest(self)
2214    }
2215
2216    fn model_tool_schemas(&self) -> Result<Vec<ModelToolSchema>, ToolRuntimeError> {
2217        GuardedToolRuntime::model_tool_schemas(self)
2218    }
2219
2220    fn resolve_tool_id(&self, model_name: &str) -> Result<Option<ToolId>, ToolRuntimeError> {
2221        GuardedToolRuntime::resolve_tool_id(self, model_name)
2222    }
2223
2224    fn activity_evidence(
2225        &self,
2226        invocation: &ToolInvocation,
2227        outcome: Option<&ToolOutcome>,
2228    ) -> Result<Vec<ToolActivityEvidence>, ToolRuntimeError> {
2229        GuardedToolRuntime::activity_evidence(self, invocation, outcome)
2230    }
2231
2232    async fn inspect_effect(
2233        &self,
2234        key: &ToolEffectKey,
2235    ) -> Result<Option<ToolEffectProjection>, ToolOutcomeRecoveryError> {
2236        GuardedToolRuntime::inspect_effect(self, key).await
2237    }
2238
2239    async fn recover_outcome(
2240        &self,
2241        invocation: ToolInvocation,
2242        run_grant: RunToolGrant,
2243    ) -> Result<Option<ToolOutcome>, ToolOutcomeRecoveryError> {
2244        GuardedToolRuntime::recover_outcome(self, invocation, run_grant).await
2245    }
2246
2247    async fn invoke(
2248        &self,
2249        invocation: ToolInvocation,
2250        run_grant: RunToolGrant,
2251        approval: Option<ApprovalCapability>,
2252        run_cancellation: CancellationToken,
2253    ) -> GuardedToolResult {
2254        GuardedToolRuntime::invoke(self, invocation, run_grant, approval, run_cancellation).await
2255    }
2256
2257    async fn invoke_with_yield(
2258        &self,
2259        invocation: ToolInvocation,
2260        run_grant: RunToolGrant,
2261        approval: Option<ApprovalCapability>,
2262        run_cancellation: CancellationToken,
2263        yield_requested: CancellationToken,
2264    ) -> GuardedToolResult {
2265        GuardedToolRuntime::invoke_with_yield(
2266            self,
2267            invocation,
2268            run_grant,
2269            approval,
2270            run_cancellation,
2271            yield_requested,
2272        )
2273        .await
2274    }
2275
2276    async fn invoke_with_observations(
2277        &self,
2278        invocation: ToolInvocation,
2279        run_grant: RunToolGrant,
2280        approval: Option<ApprovalCapability>,
2281        run_cancellation: CancellationToken,
2282        yield_requested: CancellationToken,
2283        observations: &FrozenToolObservations,
2284    ) -> GuardedToolResult {
2285        GuardedToolRuntime::invoke_with_observations(
2286            self,
2287            invocation,
2288            run_grant,
2289            approval,
2290            run_cancellation,
2291            yield_requested,
2292            observations,
2293        )
2294        .await
2295    }
2296}
2297
2298fn invocation_identity(
2299    invocation: &ToolInvocation,
2300    operation: &ToolOperationPlan,
2301    effective_policy: &EffectiveToolPolicy,
2302    permission_digest: &Digest,
2303    descriptor: &ToolDescriptor,
2304    argument_resolution: Option<&ToolArgumentResolution>,
2305) -> Result<InvocationIdentity, ToolProtocolError> {
2306    Ok(InvocationIdentity {
2307        tool_id: invocation.tool_id.clone(),
2308        args_digest: invocation.args_digest()?,
2309        operation_digest: operation.digest()?,
2310        permission_digest: permission_digest.clone(),
2311        policy_digest: effective_policy.digest()?,
2312        descriptor_digest: descriptor.digest()?,
2313        argument_resolution_digest: argument_resolution
2314            .map(|resolution| {
2315                serde_jcs::to_vec(resolution)
2316                    .map(Digest::sha256)
2317                    .map_err(|error| {
2318                        ToolProtocolError::new(
2319                            ToolProtocolErrorCode::InvalidInvocation,
2320                            error.to_string(),
2321                        )
2322                    })
2323            })
2324            .transpose()?,
2325    })
2326}
2327
2328fn effect_event_id(key: &ToolEffectKey, phase: &str) -> ToolEffectEventId {
2329    ToolEffectEventId::new(format!(
2330        "effect:{}:{}:{phase}",
2331        key.run_id.as_str(),
2332        key.call_id.as_str()
2333    ))
2334}
2335
2336fn effect_journal_rejected(error: ToolEffectError) -> GuardedToolResult {
2337    rejected("effect_journal_unavailable", error.to_string())
2338}
2339
2340fn tool_outcome_recovery_error(
2341    code: impl Into<String>,
2342    message: impl Into<String>,
2343) -> ToolOutcomeRecoveryError {
2344    ToolOutcomeRecoveryError {
2345        code: code.into(),
2346        message: message.into(),
2347    }
2348}
2349
2350fn effect_journal_recovery_error(error: ToolEffectError) -> ToolOutcomeRecoveryError {
2351    tool_outcome_recovery_error("effect_journal_unavailable", error.to_string())
2352}
2353
2354fn unknown_effect(message: impl Into<String>) -> ToolOutcome {
2355    ToolOutcome::UnknownEffect {
2356        message: message.into(),
2357    }
2358}
2359
2360fn cancellation_outcome(descriptor: &ToolDescriptor) -> ToolOutcome {
2361    if matches!(descriptor.idempotency, ToolIdempotency::NonIdempotent) {
2362        unknown_effect(
2363            "non-idempotent Tool was cancelled after its durable invocation boundary; effect completion is unknown",
2364        )
2365    } else {
2366        ToolOutcome::Cancelled
2367    }
2368}
2369
2370fn normalize_post_dispatch_outcome(
2371    descriptor: &ToolDescriptor,
2372    outcome: ToolOutcome,
2373) -> ToolOutcome {
2374    match outcome {
2375        ToolOutcome::Cancelled => cancellation_outcome(descriptor),
2376        outcome => outcome,
2377    }
2378}
2379
2380fn timeout_outcome(descriptor: &ToolDescriptor) -> ToolOutcome {
2381    if matches!(descriptor.idempotency, ToolIdempotency::NonIdempotent) {
2382        unknown_effect(
2383            "non-idempotent Tool timed out after its durable invocation boundary; effect completion is unknown",
2384        )
2385    } else {
2386        ToolOutcome::Failed {
2387            code: "timeout".to_owned(),
2388            message: "tool execution exceeded its Host timeout".to_owned(),
2389            retryable: false,
2390        }
2391    }
2392}
2393
2394async fn settle_cancelled_executor<F>(
2395    execution: &mut std::pin::Pin<&mut F>,
2396    fallback: ToolOutcome,
2397) -> ToolOutcome
2398where
2399    F: std::future::Future<Output = Result<ToolOutcome, Box<dyn std::any::Any + Send>>>,
2400{
2401    match tokio::time::timeout(Duration::from_millis(250), execution).await {
2402        Ok(result) => match map_execution_result(result) {
2403            outcome @ ToolOutcome::UnknownEffect { .. } => outcome,
2404            _ => fallback,
2405        },
2406        Err(_) => fallback,
2407    }
2408}
2409
2410fn map_execution_result(result: Result<ToolOutcome, Box<dyn std::any::Any + Send>>) -> ToolOutcome {
2411    match result {
2412        Ok(outcome) => outcome,
2413        Err(_) => ToolOutcome::UnknownEffect {
2414            message: "tool executor panicked; effect completion is unknown".to_owned(),
2415        },
2416    }
2417}
2418
2419async fn normalize_completed_outcome(
2420    descriptor: &ToolDescriptor,
2421    executor: &dyn GuardedToolExecutor,
2422    effective_policy: &EffectiveToolPolicy,
2423    invocation: &ToolInvocation,
2424    artifact_store: Option<&ToolArtifactStore>,
2425    cancellation: &CancellationToken,
2426    outcome: ToolOutcome,
2427) -> ToolOutcome {
2428    let ToolOutcome::Completed { output } = outcome else {
2429        return outcome;
2430    };
2431    let ToolOutput::Inline(output) = output else {
2432        return ToolOutcome::Failed {
2433            code: "executor_artifact_forbidden".to_owned(),
2434            message: "Tool executors cannot mint Artifact references; only the Host may spill validated output"
2435                .to_owned(),
2436            retryable: false,
2437        };
2438    };
2439    if let Err(error) = descriptor.validate_output(&output) {
2440        return ToolOutcome::Failed {
2441            code: "output_schema_violation".to_owned(),
2442            message: error.message,
2443            retryable: false,
2444        };
2445    }
2446    let bytes = match serde_jcs::to_vec(&output) {
2447        Ok(bytes) => bytes,
2448        Err(error) => {
2449            return ToolOutcome::Failed {
2450                code: "output_serialization_failed".to_owned(),
2451                message: error.to_string(),
2452                retryable: false,
2453            }
2454        }
2455    };
2456    let policy_max = effective_policy.bounds().max_output_bytes;
2457    let model_max = artifact_store.and_then(ToolArtifactStore::inline_output_limit);
2458    let model_fits = match model_max {
2459        Some(maximum) => {
2460            match serde_jcs::to_vec(&executor.project_model_output(invocation, &output)) {
2461                Ok(bytes) => bytes.len() as u64 <= maximum,
2462                Err(error) => {
2463                    return ToolOutcome::Failed {
2464                        code: "model_output_serialization_failed".to_owned(),
2465                        message: error.to_string(),
2466                        retryable: false,
2467                    }
2468                }
2469            }
2470        }
2471        None => true,
2472    };
2473    if model_fits && policy_max.is_none_or(|maximum| bytes.len() as u64 <= maximum) {
2474        return ToolOutcome::Completed {
2475            output: ToolOutput::Inline(output),
2476        };
2477    }
2478    let inline_max_bytes = match (policy_max, model_max) {
2479        (Some(policy), Some(model)) => Some(policy.min(model)),
2480        (policy, model) => policy.or(model),
2481    };
2482    let Some(inline_max_bytes) = inline_max_bytes else {
2483        return ToolOutcome::Completed {
2484            output: ToolOutput::Inline(output),
2485        };
2486    };
2487    let Some(artifact_store) = artifact_store else {
2488        return ToolOutcome::Failed {
2489            code: "output_limit_exceeded".to_owned(),
2490            message: "Tool output exceeded its Host inline byte limit and no Artifact store is configured"
2491                .to_owned(),
2492            retryable: false,
2493        };
2494    };
2495    let summary = summarize_tool_output(
2496        &output,
2497        bytes.len() as u64,
2498        artifact_store.summary_max_chars,
2499    );
2500    match artifact_store
2501        .spill(invocation, bytes, summary, inline_max_bytes, cancellation)
2502        .await
2503    {
2504        Ok(artifact) => ToolOutcome::Completed {
2505            output: ToolOutput::Artifact(artifact),
2506        },
2507        Err(ToolArtifactError::Cancelled) => cancellation_outcome(descriptor),
2508        Err(error) if matches!(descriptor.idempotency, ToolIdempotency::NonIdempotent) => {
2509            unknown_effect(format!(
2510                "non-idempotent Tool completed but its result could not be persisted: {error}"
2511            ))
2512        }
2513        Err(error) => ToolOutcome::Failed {
2514            code: "artifact_persistence_failed".to_owned(),
2515            message: error.to_string(),
2516            retryable: true,
2517        },
2518    }
2519}
2520
2521fn summarize_tool_output(output: &serde_json::Value, byte_size: u64, max_chars: usize) -> String {
2522    let shape = match output {
2523        serde_json::Value::Object(values) => {
2524            format!("JSON object with {} top-level fields", values.len())
2525        }
2526        serde_json::Value::Array(values) => format!("JSON array with {} items", values.len()),
2527        serde_json::Value::String(_) => "JSON string".to_owned(),
2528        serde_json::Value::Number(_) => "JSON number".to_owned(),
2529        serde_json::Value::Bool(_) => "JSON boolean".to_owned(),
2530        serde_json::Value::Null => "JSON null".to_owned(),
2531    };
2532    let preview = serde_json::to_string(output).unwrap_or_else(|_| "<unavailable>".to_owned());
2533    let mut chars = preview.chars();
2534    let mut preview = chars.by_ref().take(max_chars).collect::<String>();
2535    if chars.next().is_some() {
2536        preview.push('…');
2537    }
2538    format!("{shape}; {byte_size} bytes. Preview: {preview}")
2539}
2540
2541fn sanitize_approval_summary(summary: &str, tool_id: &ToolId) -> String {
2542    const MAX_CHARS: usize = 512;
2543    let normalized = summary
2544        .chars()
2545        .map(|character| {
2546            if character.is_control() {
2547                ' '
2548            } else {
2549                character
2550            }
2551        })
2552        .collect::<String>()
2553        .split_whitespace()
2554        .collect::<Vec<_>>()
2555        .join(" ");
2556    let normalized = if normalized.is_empty() {
2557        format!("Invoke Tool {}", tool_id.as_str())
2558    } else {
2559        normalized
2560    };
2561    let mut chars = normalized.chars();
2562    let mut bounded = chars.by_ref().take(MAX_CHARS).collect::<String>();
2563    if chars.next().is_some() {
2564        bounded.push('…');
2565    }
2566    bounded
2567}
2568
2569fn rejected(code: impl Into<String>, message: impl Into<String>) -> GuardedToolResult {
2570    GuardedToolResult::Outcome {
2571        outcome: ToolOutcome::Rejected {
2572            code: code.into(),
2573            message: message.into(),
2574        },
2575        cached: false,
2576    }
2577}
2578
2579fn approval_error_code(code: ToolProtocolErrorCode) -> &'static str {
2580    match code {
2581        ToolProtocolErrorCode::CapabilityExpired => "approval_expired",
2582        ToolProtocolErrorCode::CapabilityBindingMismatch => "approval_binding_mismatch",
2583        ToolProtocolErrorCode::CapabilityReplayed => "approval_replayed",
2584        ToolProtocolErrorCode::StoreFailure => "approval_store_failure",
2585        ToolProtocolErrorCode::InvalidCapability => "invalid_approval_capability",
2586        _ => "approval_validation_failed",
2587    }
2588}