vtcode_llm/provider/
request.rs

1use serde::{Deserialize, Serialize};
2use serde_json::{Value, json};
3use std::sync::Arc;
4use vtcode_config::types::{ReasoningEffortLevel, VerbosityLevel};
5
6use super::{Message, ToolDefinition};
7
8/// Fallback model configuration for Anthropic server-side fallback.
9/// The explicit-list form of `fallbacks`, sent with the
10/// `server-side-fallback-2026-06-01` beta header (the `"default"` keyword form
11/// uses `server-side-fallback-2026-07-01`; pairing a header with the other form
12/// is rejected).
13#[derive(Debug, Clone, Serialize, Deserialize, Default)]
14pub struct FallbackModel {
15    /// The model identifier to fall back to (e.g., "claude-opus-5")
16    pub(crate) model: String,
17    /// Optional max_tokens override for this fallback attempt
18    #[serde(skip_serializing_if = "Option::is_none")]
19    pub(crate) max_tokens: Option<u32>,
20    /// Optional thinking configuration override for this fallback attempt
21    #[serde(skip_serializing_if = "Option::is_none")]
22    pub(crate) thinking: Option<AnthropicThinkingConfig>,
23}
24
25/// Anthropic thinking configuration for fallback models.
26#[derive(Debug, Clone, Serialize, Deserialize, Default)]
27#[serde(tag = "type", rename_all = "lowercase")]
28pub enum AnthropicThinkingConfig {
29    #[default]
30    Disabled,
31    Enabled {
32        budget_tokens: u32,
33        #[serde(skip_serializing_if = "Option::is_none")]
34        display: Option<String>,
35    },
36    Adaptive {
37        #[serde(skip_serializing_if = "Option::is_none")]
38        display: Option<String>,
39    },
40}
41
42#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
43#[serde(rename_all = "snake_case")]
44pub enum PromptCacheProfile {
45    BudgetContinuation,
46}
47
48#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
49#[serde(rename_all = "snake_case")]
50pub enum AnthropicThinkingModeOverride {
51    #[default]
52    Inherit,
53    Disabled,
54    Adaptive,
55    ManualBudget(u32),
56}
57
58#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
59#[serde(rename_all = "snake_case")]
60pub enum AnthropicThinkingDisplayOverride {
61    #[default]
62    Inherit,
63    Summarized,
64    Omitted,
65    Updates,
66}
67
68#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
69#[serde(rename_all = "snake_case")]
70pub enum AnthropicOptionalStringOverride {
71    #[default]
72    Inherit,
73    Omit,
74    Explicit(String),
75}
76
77#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
78#[serde(rename_all = "snake_case")]
79pub enum AnthropicOptionalU32Override {
80    #[default]
81    Inherit,
82    Omit,
83    Explicit(u32),
84}
85
86#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
87pub struct AnthropicRequestOverrides {
88    /// Explicit request speed takes precedence over provider configuration.
89    #[serde(default, skip_serializing_if = "Option::is_none")]
90    pub speed: Option<vtcode_config::AnthropicSpeed>,
91    #[serde(default)]
92    pub(crate) thinking_mode: AnthropicThinkingModeOverride,
93    #[serde(default)]
94    pub(crate) thinking_display: AnthropicThinkingDisplayOverride,
95    #[serde(default)]
96    pub(crate) effort: AnthropicOptionalStringOverride,
97    #[serde(default)]
98    pub(crate) task_budget_tokens: AnthropicOptionalU32Override,
99}
100
101/// Universal LLM request structure
102#[derive(Debug, Clone, Serialize, Deserialize, Default)]
103
104pub struct LLMRequest {
105    /// Conversation history shared via `Arc` so per-turn request construction
106    /// and continuation bookkeeping are O(1) instead of deep-copying the
107    /// entire history (which grows unbounded over a session).
108    pub messages: Arc<Vec<Message>>,
109    /// Shared via `Arc<str>` (single allocation) so per-turn request
110    /// construction can reuse the prompt envelope's `Arc<str>` via a cheap
111    /// refcount bump instead of cloning the full system prompt string.
112    pub system_prompt: Option<Arc<str>>,
113    pub tools: Option<Arc<Vec<ToolDefinition>>>,
114    pub model: String,
115    pub max_tokens: Option<u32>,
116    pub temperature: Option<f32>,
117    pub stream: bool,
118
119    /// Optional structured output JSON schema to request from providers that support it
120    /// For Anthropic this will be sent as `output_config.format = { type: "json_schema", schema: ... }`
121    pub output_format: Option<Value>,
122
123    /// Tool choice configuration based on official API docs
124    /// Supports: "auto" (default), "none", "any", or specific tool selection
125    pub tool_choice: Option<ToolChoice>,
126
127    /// Whether to enable parallel tool calls (OpenAI specific)
128    pub parallel_tool_calls: Option<bool>,
129
130    /// Parallel tool use configuration following Anthropic best practices
131    pub parallel_tool_config: Option<Box<ParallelToolConfig>>,
132
133    /// Reasoning effort level for models that support it (none, minimal, low, medium, high, xhigh)
134    /// Applies to: Claude, GPT-5 family, Gemini, Qwen3, DeepSeek with reasoning capability
135    pub reasoning_effort: Option<ReasoningEffortLevel>,
136
137    /// Effort level for overall token usage (low, medium, high, xhigh, max)
138    /// Applies to: Anthropic adaptive-thinking models such as Claude Opus 4.7
139    /// Controls how many tokens Claude uses when responding, trading off between
140    /// response thoroughness and token efficiency.
141    pub effort: Option<String>,
142
143    /// Verbosity level for output text (low, medium, high)
144    /// Applies to: GPT-5.4-family Responses workflows and other models that support verbosity control
145    pub verbosity: Option<VerbosityLevel>,
146
147    /// Advanced generation parameters
148    pub do_sample: Option<bool>,
149    pub top_p: Option<f32>,
150    pub top_k: Option<i32>,
151    pub presence_penalty: Option<f32>,
152    pub frequency_penalty: Option<f32>,
153    pub stop_sequences: Option<Vec<String>>,
154    /// Optional budget for extended thinking (Anthropic specific)
155    /// Minimum value: 1024
156    pub thinking_budget: Option<u32>,
157
158    /// Optional beta headers for Anthropic (and potentially others)
159    pub betas: Option<Vec<String>>,
160
161    /// Optional provider-specific context management configuration (Anthropic compaction/editing).
162    pub context_management: Option<Value>,
163
164    /// Optional coding agent specific settings
165    pub coding_agent_settings: Option<Box<CodingAgentSettings>>,
166
167    /// Optional turn metadata for git context (remote URLs, commit hash, etc.)
168    /// This is sent as X-Turn-Metadata header to providers that support it
169    pub metadata: Option<Value>,
170
171    /// Optional Responses API continuity pointer for server-side context chaining.
172    /// Used by providers that support stateful response continuation.
173    pub previous_response_id: Option<String>,
174
175    /// Optional Responses API storage flag.
176    /// When set, providers that support `store` pass this through directly.
177    pub response_store: Option<bool>,
178
179    /// Optional Responses API include fields (e.g. reasoning encrypted content).
180    /// Passed through only for providers/APIs that support include selectors.
181    pub responses_include: Option<Vec<String>>,
182
183    /// Optional native OpenAI `service_tier` request parameter.
184    /// Passed through only for native OpenAI endpoints that support service tiers.
185    pub service_tier: Option<String>,
186
187    /// Optional provider routing hint for prompt cache stickiness.
188    /// OpenAI uses this value to improve routing locality for repeated prefixes.
189    pub prompt_cache_key: Option<String>,
190
191    /// Optional request-scoped prompt cache profile for provider-specific TTL overrides.
192    pub prompt_cache_profile: Option<PromptCacheProfile>,
193
194    /// Optional explicit fallback models for Anthropic server-side fallback.
195    /// Overrides `provider.anthropic.fallbacks`; sent with the
196    /// `server-side-fallback-2026-06-01` beta header. Subject to the same
197    /// gates as the config list (first-party endpoint, a model that supports
198    /// server-side fallbacks, no credit-token retry) and the same entry
199    /// validation; an empty list sends nothing.
200    pub fallbacks: Option<Vec<FallbackModel>>,
201
202    /// Optional opaque credit token from a refused request's `stop_details.fallback_credit_token`.
203    /// Echoed on the retry to avoid paying the prompt-cache cost twice.
204    /// Sent with the `fallback-credit-2026-07-01` beta header. The refused request must have
205    /// carried that header or `server-side-fallback-2026-07-01`, which grants the same
206    /// `stop_details` fields.
207    pub fallback_credit_token: Option<String>,
208
209    /// Optional Anthropic-specific request overrides used when request semantics must
210    /// not inherit VT Code's provider defaults, such as the Anthropic compatibility server.
211    pub anthropic_request_overrides: Option<AnthropicRequestOverrides>,
212}
213
214/// Per-model sampling parameter overrides resolved from a custom provider
215/// profile. Every field defaults to `None`, meaning the agent loop's global
216/// configuration value applies unchanged.
217#[derive(Debug, Clone, Copy, Default, PartialEq)]
218pub struct SamplingOverrides {
219    /// Sampling temperature (0.0-2.0).
220    pub temperature: Option<f32>,
221    /// Nucleus sampling mass (0.0-1.0).
222    pub top_p: Option<f32>,
223    /// Top-k token cutoff (>= 0).
224    pub top_k: Option<i32>,
225    /// Presence penalty (-2.0-2.0).
226    pub presence_penalty: Option<f32>,
227    /// Frequency penalty (-2.0-2.0).
228    pub frequency_penalty: Option<f32>,
229    /// Maximum output tokens (> 0).
230    pub max_tokens: Option<u32>,
231    /// Reasoning effort for models that accept it.
232    pub reasoning_effort: Option<ReasoningEffortLevel>,
233    /// Whether this model's wire format rejects sampling parameters while
234    /// reasoning is active (e.g. custom endpoints speaking the Anthropic
235    /// Messages shape).
236    pub suppresses_sampling_with_reasoning: bool,
237    /// Set by providers that resolve overrides from an explicit user profile;
238    /// callers use it to trust these values over name-based heuristics.
239    pub profile_aware: bool,
240}
241
242impl SamplingOverrides {
243    /// Whether sampling parameters must be dropped for this request: either
244    /// the model's wire format rejects sampling while reasoning is active, or
245    /// a built-in Anthropic/MiniMax-shaped backend (`native_match`) enforces
246    /// the same rule outside of profile-aware routing.
247    pub fn suppresses_sampling(&self, native_match: bool, reasoning_active: bool) -> bool {
248        reasoning_active && (self.suppresses_sampling_with_reasoning || (!self.profile_aware && native_match))
249    }
250}
251
252/// Optional overrides for standalone Responses compaction requests.
253#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
254pub struct ResponsesCompactionOptions {
255    /// Optional custom instructions appended to the derived replay instructions.
256    pub instructions: Option<String>,
257    /// Optional output token limit for the compaction response.
258    pub max_output_tokens: Option<u32>,
259    /// Optional reasoning effort override for the compaction pass.
260    pub reasoning_effort: Option<ReasoningEffortLevel>,
261    /// Optional verbosity override for the compaction output text settings.
262    pub verbosity: Option<VerbosityLevel>,
263    /// Optional include selectors override.
264    pub responses_include: Option<Vec<String>>,
265    /// Optional storage override.
266    pub response_store: Option<bool>,
267    /// Optional native OpenAI service tier override.
268    pub service_tier: Option<String>,
269    /// Optional prompt cache routing override.
270    pub prompt_cache_key: Option<String>,
271}
272
273/// Settings to refine model behavior for coding agent tasks
274#[derive(Debug, Clone, Serialize, Deserialize, Default)]
275pub struct CodingAgentSettings {
276    /// Optimize for long context by hoisting the largest user message.
277    /// Hoisting reorders earlier turns, so the Anthropic builder skips it on
278    /// preserved-thinking models (Claude Sonnet 5.5, Claude Opus 5.5, Claude
279    /// Fable 5.1).
280    pub(crate) long_context_optimization: bool,
281}
282
283/// Tool choice configuration that works across different providers
284/// Based on OpenAI, Anthropic, and Gemini API specifications
285/// Follows Anthropic's tool use best practices for optimal performance
286#[derive(Debug, Clone, Serialize, Deserialize)]
287#[serde(untagged)]
288#[derive(Default)]
289pub enum ToolChoice {
290    /// Let the model decide whether to call tools ("auto")
291    /// Default behavior - allows model to use tools when appropriate
292    #[default]
293    Auto,
294
295    /// Force the model to not call any tools ("none")
296    /// Useful for pure conversational responses without tool usage
297    None,
298
299    /// Force the model to call at least one tool ("any")
300    /// Ensures tool usage even when model might prefer direct response
301    Any,
302
303    /// Force the model to call a specific tool
304    /// Useful for directing model to use particular functionality
305    Specific(SpecificToolChoice),
306
307    /// OpenAI Responses native allowed-tools filtering.
308    ///
309    /// This is advisory provider-side filtering only; runtime dispatch remains
310    /// authoritative. Providers without native support should degrade this to
311    /// the equivalent auto/required tool choice while keeping the full
312    /// catalogue visible.
313    AllowedTools(AllowedToolsChoice),
314}
315
316#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
317#[serde(rename_all = "lowercase")]
318pub enum AllowedToolsMode {
319    Auto,
320    Required,
321}
322
323impl AllowedToolsMode {
324    pub fn as_str(self) -> &'static str {
325        match self {
326            Self::Auto => "auto",
327            Self::Required => "required",
328        }
329    }
330
331    fn anthropic_mode(self) -> &'static str {
332        match self {
333            Self::Auto => "auto",
334            Self::Required => "any",
335        }
336    }
337
338    fn gemini_mode(self) -> &'static str {
339        match self {
340            Self::Auto => "auto",
341            Self::Required => "any",
342        }
343    }
344}
345
346#[derive(Debug, Clone, Serialize, Deserialize)]
347pub struct AllowedToolsChoice {
348    pub mode: AllowedToolsMode,
349    pub tools: Vec<String>,
350}
351
352/// Specific tool choice for forcing a particular function call
353#[derive(Debug, Clone, Serialize, Deserialize)]
354pub struct SpecificToolChoice {
355    #[serde(rename = "type")]
356    pub(crate) tool_type: String, // "function"
357
358    pub(crate) function: SpecificFunctionChoice,
359}
360
361/// Specific function choice details
362#[derive(Debug, Clone, Serialize, Deserialize)]
363pub struct SpecificFunctionChoice {
364    pub(crate) name: String,
365}
366
367impl ToolChoice {
368    /// Create auto tool choice (default behavior)
369    pub fn auto() -> Self {
370        Self::Auto
371    }
372
373    /// Create none tool choice (disable tool calling)
374    pub fn none() -> Self {
375        Self::None
376    }
377
378    /// Create any tool choice (force at least one tool call)
379    pub(crate) fn any() -> Self {
380        Self::Any
381    }
382
383    /// Create specific function tool choice
384    pub(crate) fn function(name: String) -> Self {
385        Self::Specific(SpecificToolChoice {
386            tool_type: "function".to_owned(),
387            function: SpecificFunctionChoice { name },
388        })
389    }
390
391    pub fn allowed_tools_auto(tools: Vec<String>) -> Self {
392        Self::AllowedTools(AllowedToolsChoice { mode: AllowedToolsMode::Auto, tools })
393    }
394
395    /// Check if this tool choice allows parallel tool use
396    /// Based on Anthropic's parallel tool use guidelines
397    pub fn allows_parallel_tools(&self) -> bool {
398        match self {
399            // Auto allows parallel tools by default
400            Self::Auto => true,
401            // Any forces at least one tool, may allow parallel
402            Self::Any => true,
403            Self::AllowedTools(choice) => matches!(choice.mode, AllowedToolsMode::Auto),
404            // Specific forces one particular tool, typically no parallel
405            Self::Specific(_) => false,
406            // None disables tools entirely
407            Self::None => false,
408        }
409    }
410
411    /// Get human-readable description of tool choice behavior
412    pub fn description(&self) -> &'static str {
413        match self {
414            Self::Auto => "Model decides when to use tools (allows parallel)",
415            Self::None => "No tools will be used",
416            Self::Any => "At least one tool must be used (allows parallel)",
417            Self::AllowedTools(_) => "Model decides when to use the currently allowed tool subset",
418            Self::Specific(_) => "Specific tool must be used (no parallel)",
419        }
420    }
421
422    /// OpenAI-compatible providers that share the same tool_choice format
423    const OPENAI_STYLE_PROVIDERS: &'static [&'static str] = &[
424        "openai",
425        "deepseek",
426        "huggingface",
427        "mistral",
428        "openrouter",
429        "zai",
430        "moonshot",
431        "stepfun",
432        "evolink",
433        "lmstudio",
434        "llamacpp",
435        "meta",
436    ];
437
438    /// Convert to provider-specific format
439    #[inline]
440    pub(crate) fn to_provider_format(&self, provider: &str) -> Value {
441        if Self::OPENAI_STYLE_PROVIDERS.contains(&provider) {
442            return self.to_openai_format();
443        }
444
445        match provider {
446            "anthropic" => self.to_anthropic_format(),
447            "gemini" => self.to_gemini_format(),
448            _ => self.to_openai_format(), // Default to OpenAI format
449        }
450    }
451
452    #[inline]
453    fn to_openai_format(&self) -> Value {
454        match self {
455            Self::Auto => json!("auto"),
456            Self::None => json!("none"),
457            Self::Any => json!("required"),
458            Self::Specific(choice) => json!(choice),
459            Self::AllowedTools(choice) => json!(choice.mode.as_str()),
460        }
461    }
462
463    #[inline]
464    fn to_anthropic_format(&self) -> Value {
465        match self {
466            Self::Auto => json!({"type": "auto"}),
467            Self::None => json!({"type": "none"}),
468            Self::Any => json!({"type": "any"}),
469            Self::Specific(choice) => json!({"type": "tool", "name": &choice.function.name}),
470            Self::AllowedTools(choice) => json!({"type": choice.mode.anthropic_mode()}),
471        }
472    }
473
474    #[inline]
475    fn to_gemini_format(&self) -> Value {
476        match self {
477            Self::Auto => json!({"mode": "auto"}),
478            Self::None => json!({"mode": "none"}),
479            Self::Any => json!({"mode": "any"}),
480            Self::AllowedTools(choice) => json!({"mode": choice.mode.gemini_mode()}),
481            Self::Specific(choice) => {
482                json!({"mode": "any", "allowed_function_names": [&choice.function.name]})
483            }
484        }
485    }
486}
487
488/// Configuration for parallel tool use behavior
489/// Based on Anthropic's parallel tool use guidelines
490#[derive(Debug, Clone, Serialize, Deserialize)]
491pub struct ParallelToolConfig {
492    /// Whether to disable parallel tool use
493    /// When true, forces sequential tool execution
494    pub(crate) disable_parallel_tool_use: bool,
495
496    /// Maximum number of tools to execute in parallel
497    /// None means no limit (provider default)
498    pub(crate) max_parallel_tools: Option<usize>,
499
500    /// Whether to encourage parallel tool use in prompts
501    pub(crate) encourage_parallel: bool,
502}
503
504impl Default for ParallelToolConfig {
505    fn default() -> Self {
506        Self {
507            disable_parallel_tool_use: false,
508            max_parallel_tools: Some(5), // Reasonable default
509            encourage_parallel: true,
510        }
511    }
512}
513
514impl ParallelToolConfig {
515    /// Create configuration optimized for Anthropic models
516    pub fn anthropic_optimized() -> Self {
517        Self {
518            disable_parallel_tool_use: false,
519            max_parallel_tools: None, // Let Anthropic decide
520            encourage_parallel: true,
521        }
522    }
523
524    /// Create configuration for sequential tool use
525    pub(crate) fn sequential_only() -> Self {
526        Self {
527            disable_parallel_tool_use: true,
528            max_parallel_tools: Some(1),
529            encourage_parallel: false,
530        }
531    }
532}
vtcode_llm/provider/request.rs

vtcode_llm/provider/
request.rs
