conversation_api/execution/
llm.rs

1//! Agent model port and execution policy, using the shared model contract.
2use crate::execution::{ExternalError, InvocationContext};
3use async_trait::async_trait;
4pub use llm_api::{
5    Completion, CompletionRequest, ContentPart, Continuation, FinishReason, INPUT_AUDIO,
6    INPUT_FILE, INPUT_IMAGE, INPUT_VIDEO, Message, MessageRole, ModelCapabilities,
7    ModelConstraints, ModelMode, ModelProfile, TokenUsage, ToolCall, UseCase,
8    input_modality_for_kind, payload_input_modalities,
9};
10use serde::{Deserialize, Serialize};
11use serde_json::Value;
12use thiserror::Error;
13
14/// Tool schema exposed to an LLM.
15#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
16pub struct ToolDefinition {
17    /// Stable tool name.
18    pub name: String,
19    /// Human-readable behavior description.
20    pub description: String,
21    /// JSON Schema for tool input.
22    pub input_schema: Value,
23    /// Runtime policy for executing the tool.
24    pub policy: ToolPolicy,
25}
26
27/// Side-effect class used by Runtime access policy.
28#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
29#[serde(rename_all = "snake_case")]
30pub enum ToolEffect {
31    /// Tool is safe to execute as a query.
32    ReadOnly,
33    /// Tool can change application state; its implementation selects the execution flow.
34    Mutating,
35}
36
37/// Runtime policy declared by a tool implementation.
38#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
39pub struct ToolPolicy {
40    /// Side-effect class of the tool.
41    pub effect: ToolEffect,
42    /// Whether the Agent declares independent calls safe for parallel execution.
43    pub parallel_safe: bool,
44}
45
46/// Aggregated model and tool usage for one Agent run.
47#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
48#[serde(deny_unknown_fields)]
49pub struct UsageSummary {
50    /// Model requests attempted by Runtime.
51    pub model_requests: u32,
52    /// Requests whose provider returned token accounting.
53    pub reported_model_requests: u32,
54    /// Aggregated input tokens.
55    pub input_tokens: u64,
56    /// Aggregated output tokens.
57    pub output_tokens: u64,
58    /// Aggregated cached input tokens when any provider reported them.
59    pub cached_input_tokens: Option<u64>,
60    /// Aggregated reasoning output tokens when any provider reported them.
61    pub reasoning_output_tokens: Option<u64>,
62    /// Largest single-request input observed or estimated during the run.
63    pub peak_input_tokens: u64,
64    /// Aggregated provider-normalized billable credits when reported.
65    pub credits: Option<u64>,
66    /// Tool calls requested by models.
67    pub tool_calls: u32,
68    /// Number of prompt-window truncations across the main loop and derived workflows.
69    pub window_truncations: u32,
70    /// Number of prompt messages removed by window truncation.
71    pub trimmed_messages: u64,
72}
73
74impl UsageSummary {
75    /// Records that Runtime attempted one provider request.
76    pub fn record_model_attempt(&mut self) {
77        self.model_requests = self.model_requests.saturating_add(1);
78    }
79
80    /// Records normalized usage without incrementing the already-recorded attempt count.
81    pub fn record_usage(&mut self, usage: TokenUsage, reported: bool) {
82        if reported {
83            self.reported_model_requests = self.reported_model_requests.saturating_add(1);
84        }
85        self.input_tokens = self.input_tokens.saturating_add(usage.input_tokens);
86        self.output_tokens = self.output_tokens.saturating_add(usage.output_tokens);
87        self.peak_input_tokens = self.peak_input_tokens.max(usage.input_tokens);
88        add_optional(&mut self.cached_input_tokens, usage.cached_input_tokens);
89        add_optional(
90            &mut self.reasoning_output_tokens,
91            usage.reasoning_output_tokens,
92        );
93        add_optional(&mut self.credits, usage.credits);
94    }
95
96    /// Records one completed model request while preserving missing telemetry.
97    pub fn record_model_request(&mut self, usage: Option<TokenUsage>) {
98        self.record_model_attempt();
99        let Some(usage) = usage else {
100            return;
101        };
102        self.record_usage(usage, true);
103    }
104
105    /// Records one deterministic prompt-history truncation.
106    pub fn record_window_truncation(&mut self, removed_messages: usize) {
107        self.window_truncations = self.window_truncations.saturating_add(1);
108        self.trimmed_messages = self
109            .trimmed_messages
110            .saturating_add(u64::try_from(removed_messages).unwrap_or(u64::MAX));
111    }
112}
113
114fn add_optional(total: &mut Option<u64>, value: Option<u64>) {
115    if let Some(value) = value {
116        *total = Some(total.unwrap_or(0).saturating_add(value));
117    }
118}
119
120/// Model-specific failure that can retain usage returned before output validation failed.
121#[derive(Clone, Debug, Error, Eq, PartialEq)]
122#[error("{error}")]
123pub struct LlmFailure {
124    /// Stable provider-neutral failure.
125    pub error: ExternalError,
126    /// Usage already reported by the provider, when available.
127    pub usage: Option<TokenUsage>,
128}
129
130impl From<ExternalError> for LlmFailure {
131    fn from(error: ExternalError) -> Self {
132        Self { error, usage: None }
133    }
134}
135
136/// LLM capability consumed by Runtime.
137///
138/// Provider discovery, credentials, model mappings, and provider-specific retries belong to the
139/// implementation and are intentionally absent from this interface.
140#[async_trait]
141pub trait Llm: Send + Sync {
142    /// Returns limits for the exact route frozen for one logical selector.
143    async fn model_profile(
144        &self,
145        context: &InvocationContext,
146        use_case: &UseCase,
147        model_mode: &ModelMode,
148    ) -> Result<ModelProfile, ExternalError>;
149
150    /// Completes one logical model invocation.
151    async fn complete(
152        &self,
153        context: &InvocationContext,
154        request: CompletionRequest,
155    ) -> Result<Completion, LlmFailure>;
156}
157
158impl ToolDefinition {
159    pub fn model_definition(&self) -> llm_api::ToolDefinition {
160        llm_api::ToolDefinition {
161            name: self.name.clone(),
162            description: self.description.clone(),
163            input_schema: self.input_schema.clone(),
164        }
165    }
166}
conversation_api/execution/llm.rs

conversation_api/execution/
llm.rs
