vtcode_llm/provider/
request.rs

1use serde::{Deserialize, Serialize};
2use serde_json::{Value, json};
3use std::sync::Arc;
4use vtcode_config::types::{ReasoningEffortLevel, VerbosityLevel};
5
6use super::{Message, ToolDefinition};
7
8/// Fallback model configuration for Anthropic server-side fallback.
9/// The explicit-list form of `fallbacks`, sent with the
10/// `server-side-fallback-2026-06-01` beta header (the `"default"` keyword form
11/// uses `server-side-fallback-2026-07-01`; pairing a header with the other form
12/// is rejected).
13#[derive(Debug, Clone, Serialize, Deserialize, Default)]
14pub struct FallbackModel {
15    /// The model identifier to fall back to (e.g., "claude-opus-5")
16    pub(crate) model: String,
17    /// Optional max_tokens override for this fallback attempt
18    #[serde(skip_serializing_if = "Option::is_none")]
19    pub(crate) max_tokens: Option<u32>,
20    /// Optional thinking configuration override for this fallback attempt
21    #[serde(skip_serializing_if = "Option::is_none")]
22    pub(crate) thinking: Option<AnthropicThinkingConfig>,
23}
24
25/// Anthropic thinking configuration for fallback models.
26#[derive(Debug, Clone, Serialize, Deserialize, Default)]
27#[serde(tag = "type", rename_all = "lowercase")]
28pub enum AnthropicThinkingConfig {
29    #[default]
30    Disabled,
31    Enabled {
32        budget_tokens: u32,
33        #[serde(skip_serializing_if = "Option::is_none")]
34        display: Option<String>,
35    },
36    Adaptive {
37        #[serde(skip_serializing_if = "Option::is_none")]
38        display: Option<String>,
39    },
40}
41
42#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
43#[serde(rename_all = "snake_case")]
44pub enum PromptCacheProfile {
45    BudgetContinuation,
46    /// Cache only stable instructions for an independent auxiliary request.
47    /// Native GPT-5.6+ uses explicit-only caching; other routes ignore this profile.
48    StablePrefix,
49}
50
51#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
52#[serde(rename_all = "snake_case")]
53pub enum AnthropicThinkingModeOverride {
54    #[default]
55    Inherit,
56    Disabled,
57    Adaptive,
58    ManualBudget(u32),
59}
60
61#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
62#[serde(rename_all = "snake_case")]
63pub enum AnthropicThinkingDisplayOverride {
64    #[default]
65    Inherit,
66    Summarized,
67    Omitted,
68    Updates,
69}
70
71#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
72#[serde(rename_all = "snake_case")]
73pub enum AnthropicOptionalStringOverride {
74    #[default]
75    Inherit,
76    Omit,
77    Explicit(String),
78}
79
80#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
81#[serde(rename_all = "snake_case")]
82pub enum AnthropicOptionalU32Override {
83    #[default]
84    Inherit,
85    Omit,
86    Explicit(u32),
87}
88
89#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
90pub struct AnthropicRequestOverrides {
91    /// Explicit request speed takes precedence over provider configuration.
92    #[serde(default, skip_serializing_if = "Option::is_none")]
93    pub speed: Option<vtcode_config::AnthropicSpeed>,
94    #[serde(default)]
95    pub(crate) thinking_mode: AnthropicThinkingModeOverride,
96    #[serde(default)]
97    pub(crate) thinking_display: AnthropicThinkingDisplayOverride,
98    #[serde(default)]
99    pub(crate) effort: AnthropicOptionalStringOverride,
100    #[serde(default)]
101    pub(crate) task_budget_tokens: AnthropicOptionalU32Override,
102}
103
104/// Universal LLM request structure
105#[derive(Debug, Clone, Serialize, Deserialize, Default)]
106
107pub struct LLMRequest {
108    /// Conversation history shared via `Arc` so per-turn request construction
109    /// and continuation bookkeeping are O(1) instead of deep-copying the
110    /// entire history (which grows unbounded over a session).
111    pub messages: Arc<Vec<Message>>,
112    /// Shared via `Arc<str>` (single allocation) so per-turn request
113    /// construction can reuse the prompt envelope's `Arc<str>` via a cheap
114    /// refcount bump instead of cloning the full system prompt string.
115    pub system_prompt: Option<Arc<str>>,
116    pub tools: Option<Arc<Vec<ToolDefinition>>>,
117    pub model: String,
118    pub max_tokens: Option<u32>,
119    pub temperature: Option<f32>,
120    pub stream: bool,
121
122    /// Optional structured output JSON schema to request from providers that support it
123    /// For Anthropic this will be sent as `output_config.format = { type: "json_schema", schema: ... }`
124    pub output_format: Option<Value>,
125
126    /// Tool choice configuration based on official API docs
127    /// Supports: "auto" (default), "none", "any", or specific tool selection
128    pub tool_choice: Option<ToolChoice>,
129
130    /// Whether to enable parallel tool calls (OpenAI specific)
131    pub parallel_tool_calls: Option<bool>,
132
133    /// Parallel tool use configuration following Anthropic best practices
134    pub parallel_tool_config: Option<Box<ParallelToolConfig>>,
135
136    /// Reasoning effort level for models that support it (none, minimal, low, medium, high, xhigh)
137    /// Applies to: Claude, GPT-5 family, Gemini, Qwen3, DeepSeek with reasoning capability
138    pub reasoning_effort: Option<ReasoningEffortLevel>,
139
140    /// Effort level for overall token usage (low, medium, high, xhigh, max)
141    /// Applies to: Anthropic adaptive-thinking models such as Claude Opus 4.7
142    /// Controls how many tokens Claude uses when responding, trading off between
143    /// response thoroughness and token efficiency.
144    pub effort: Option<String>,
145
146    /// Verbosity level for output text (low, medium, high)
147    /// Applies to: GPT-5.4-family Responses workflows and other models that support verbosity control
148    pub verbosity: Option<VerbosityLevel>,
149
150    /// Advanced generation parameters
151    pub do_sample: Option<bool>,
152    pub top_p: Option<f32>,
153    pub top_k: Option<i32>,
154    pub presence_penalty: Option<f32>,
155    pub frequency_penalty: Option<f32>,
156    pub stop_sequences: Option<Vec<String>>,
157    /// Optional budget for extended thinking (Anthropic specific)
158    /// Minimum value: 1024
159    pub thinking_budget: Option<u32>,
160
161    /// Optional beta headers for Anthropic (and potentially others)
162    pub betas: Option<Vec<String>>,
163
164    /// Optional provider-specific context management configuration (Anthropic compaction/editing).
165    pub context_management: Option<Value>,
166
167    /// Optional coding agent specific settings
168    pub coding_agent_settings: Option<Box<CodingAgentSettings>>,
169
170    /// Optional turn metadata for git context (remote URLs, commit hash, etc.)
171    /// This is sent as X-Turn-Metadata header to providers that support it
172    pub metadata: Option<Value>,
173
174    /// Optional Responses API continuity pointer for server-side context chaining.
175    /// Used by providers that support stateful response continuation.
176    pub previous_response_id: Option<String>,
177
178    /// Optional Responses API storage flag.
179    /// When set, providers that support `store` pass this through directly.
180    pub response_store: Option<bool>,
181
182    /// Optional Responses API include fields (e.g. reasoning encrypted content).
183    /// Passed through only for providers/APIs that support include selectors.
184    pub responses_include: Option<Vec<String>>,
185
186    /// Optional native OpenAI `service_tier` request parameter.
187    /// Passed through only for native OpenAI endpoints that support service tiers.
188    pub service_tier: Option<String>,
189
190    /// Optional provider routing hint for prompt cache stickiness.
191    /// OpenAI uses this value to improve routing locality for repeated prefixes.
192    pub prompt_cache_key: Option<String>,
193
194    /// Optional request-scoped prompt cache profile for provider-specific TTL overrides.
195    pub prompt_cache_profile: Option<PromptCacheProfile>,
196
197    /// Optional explicit fallback models for Anthropic server-side fallback.
198    /// Overrides `provider.anthropic.fallbacks`; sent with the
199    /// `server-side-fallback-2026-06-01` beta header. Subject to the same
200    /// gates as the config list (first-party endpoint, a model that supports
201    /// server-side fallbacks, no credit-token retry) and the same entry
202    /// validation; an empty list sends nothing.
203    pub fallbacks: Option<Vec<FallbackModel>>,
204
205    /// Optional opaque credit token from a refused request's `stop_details.fallback_credit_token`.
206    /// Echoed on the retry to avoid paying the prompt-cache cost twice.
207    /// Sent with the `fallback-credit-2026-07-01` beta header. The refused request must have
208    /// carried that header or `server-side-fallback-2026-07-01`, which grants the same
209    /// `stop_details` fields.
210    pub fallback_credit_token: Option<String>,
211
212    /// Optional Anthropic-specific request overrides used when request semantics must
213    /// not inherit VT Code's provider defaults, such as the Anthropic compatibility server.
214    pub anthropic_request_overrides: Option<AnthropicRequestOverrides>,
215}
216
217impl LLMRequest {
218    /// Budget a short auxiliary answer separately from any hidden reasoning.
219    ///
220    /// Select low effort only when advertised; otherwise retain the provider's
221    /// default. Reasoning routes receive 2,048 additional output tokens and no
222    /// sampling controls. This is a bounded allowance, not a completion guarantee.
223    /// Reapply after changing `model`, including on a fallback route.
224    pub fn set_auxiliary_output_budget(
225        &mut self,
226        provider: &(impl super::LLMProvider + ?Sized),
227        visible_output_tokens: u32,
228    ) {
229        let levels = provider.supported_reasoning_efforts(&self.model);
230        let reasoning = provider.supports_reasoning(&self.model) || !levels.is_empty();
231        self.reasoning_effort = levels.contains(&"low").then_some(ReasoningEffortLevel::Low);
232        self.max_tokens = Some(visible_output_tokens.max(1).saturating_add(if reasoning { 2048 } else { 0 }));
233        self.verbosity = Some(VerbosityLevel::Low);
234        if reasoning {
235            self.temperature = None;
236            self.top_p = None;
237            self.top_k = None;
238            self.do_sample = None;
239            self.presence_penalty = None;
240            self.frequency_penalty = None;
241        }
242    }
243}
244
245/// Per-model sampling parameter overrides resolved from a custom provider
246/// profile. Every field defaults to `None`, meaning the agent loop's global
247/// configuration value applies unchanged.
248#[derive(Debug, Clone, Copy, Default, PartialEq)]
249pub struct SamplingOverrides {
250    /// Sampling temperature (0.0-2.0).
251    pub temperature: Option<f32>,
252    /// Nucleus sampling mass (0.0-1.0).
253    pub top_p: Option<f32>,
254    /// Top-k token cutoff (>= 0).
255    pub top_k: Option<i32>,
256    /// Presence penalty (-2.0-2.0).
257    pub presence_penalty: Option<f32>,
258    /// Frequency penalty (-2.0-2.0).
259    pub frequency_penalty: Option<f32>,
260    /// Maximum output tokens (> 0).
261    pub max_tokens: Option<u32>,
262    /// Reasoning effort for models that accept it.
263    pub reasoning_effort: Option<ReasoningEffortLevel>,
264    /// Whether this model's wire format rejects sampling parameters while
265    /// reasoning is active (e.g. custom endpoints speaking the Anthropic
266    /// Messages shape).
267    pub suppresses_sampling_with_reasoning: bool,
268    /// Set by providers that resolve overrides from an explicit user profile;
269    /// callers use it to trust these values over name-based heuristics.
270    pub profile_aware: bool,
271}
272
273impl SamplingOverrides {
274    /// Whether sampling parameters must be dropped for this request: either
275    /// the model's wire format rejects sampling while reasoning is active, or
276    /// a built-in Anthropic/MiniMax-shaped backend (`native_match`) enforces
277    /// the same rule outside of profile-aware routing.
278    pub fn suppresses_sampling(&self, native_match: bool, reasoning_active: bool) -> bool {
279        reasoning_active && (self.suppresses_sampling_with_reasoning || (!self.profile_aware && native_match))
280    }
281}
282
283/// Optional overrides for standalone Responses compaction requests.
284#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
285pub struct ResponsesCompactionOptions {
286    /// Optional custom instructions appended to the derived replay instructions.
287    pub instructions: Option<String>,
288    /// Optional output token limit for the compaction response.
289    pub max_output_tokens: Option<u32>,
290    /// Optional reasoning effort override for the compaction pass.
291    pub reasoning_effort: Option<ReasoningEffortLevel>,
292    /// Optional verbosity override for the compaction output text settings.
293    pub verbosity: Option<VerbosityLevel>,
294    /// Optional include selectors override.
295    pub responses_include: Option<Vec<String>>,
296    /// Optional storage override.
297    pub response_store: Option<bool>,
298    /// Optional native OpenAI service tier override.
299    pub service_tier: Option<String>,
300    /// Optional prompt cache routing override.
301    pub prompt_cache_key: Option<String>,
302}
303
304/// Settings to refine model behavior for coding agent tasks
305#[derive(Debug, Clone, Serialize, Deserialize, Default)]
306pub struct CodingAgentSettings {
307    /// Optimize for long context by hoisting the largest user message.
308    /// Hoisting reorders earlier turns, so the Anthropic builder skips it on
309    /// preserved-thinking models (Claude Sonnet 5.5, Claude Opus 5.5, Claude
310    /// Fable 5.1).
311    pub(crate) long_context_optimization: bool,
312}
313
314/// Tool choice configuration that works across different providers
315/// Based on OpenAI, Anthropic, and Gemini API specifications
316/// Follows Anthropic's tool use best practices for optimal performance
317#[derive(Debug, Clone, Serialize, Deserialize)]
318#[serde(untagged)]
319#[derive(Default)]
320pub enum ToolChoice {
321    /// Let the model decide whether to call tools ("auto")
322    /// Default behavior - allows model to use tools when appropriate
323    #[default]
324    Auto,
325
326    /// Force the model to not call any tools ("none")
327    /// Useful for pure conversational responses without tool usage
328    None,
329
330    /// Force the model to call at least one tool ("any")
331    /// Ensures tool usage even when model might prefer direct response
332    Any,
333
334    /// Force the model to call a specific tool
335    /// Useful for directing model to use particular functionality
336    Specific(SpecificToolChoice),
337
338    /// OpenAI Responses native allowed-tools filtering.
339    ///
340    /// This is advisory provider-side filtering only; runtime dispatch remains
341    /// authoritative. Providers without native support should degrade this to
342    /// the equivalent auto/required tool choice while keeping the full
343    /// catalogue visible.
344    AllowedTools(AllowedToolsChoice),
345}
346
347#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
348#[serde(rename_all = "lowercase")]
349pub enum AllowedToolsMode {
350    Auto,
351    Required,
352}
353
354impl AllowedToolsMode {
355    pub fn as_str(self) -> &'static str {
356        match self {
357            Self::Auto => "auto",
358            Self::Required => "required",
359        }
360    }
361
362    fn anthropic_mode(self) -> &'static str {
363        match self {
364            Self::Auto => "auto",
365            Self::Required => "any",
366        }
367    }
368
369    fn gemini_mode(self) -> &'static str {
370        match self {
371            Self::Auto => "auto",
372            Self::Required => "any",
373        }
374    }
375}
376
377#[derive(Debug, Clone, Serialize, Deserialize)]
378pub struct AllowedToolsChoice {
379    pub mode: AllowedToolsMode,
380    pub tools: Vec<String>,
381}
382
383/// Specific tool choice for forcing a particular function call
384#[derive(Debug, Clone, Serialize, Deserialize)]
385pub struct SpecificToolChoice {
386    #[serde(rename = "type")]
387    pub(crate) tool_type: String, // "function"
388
389    pub(crate) function: SpecificFunctionChoice,
390}
391
392/// Specific function choice details
393#[derive(Debug, Clone, Serialize, Deserialize)]
394pub struct SpecificFunctionChoice {
395    pub(crate) name: String,
396}
397
398impl ToolChoice {
399    /// Create auto tool choice (default behavior)
400    pub fn auto() -> Self {
401        Self::Auto
402    }
403
404    /// Create none tool choice (disable tool calling)
405    pub fn none() -> Self {
406        Self::None
407    }
408
409    /// Create any tool choice (force at least one tool call)
410    pub(crate) fn any() -> Self {
411        Self::Any
412    }
413
414    /// Create specific function tool choice
415    pub(crate) fn function(name: String) -> Self {
416        Self::Specific(SpecificToolChoice {
417            tool_type: "function".to_owned(),
418            function: SpecificFunctionChoice { name },
419        })
420    }
421
422    pub fn allowed_tools_auto(tools: Vec<String>) -> Self {
423        Self::AllowedTools(AllowedToolsChoice { mode: AllowedToolsMode::Auto, tools })
424    }
425
426    /// Check if this tool choice allows parallel tool use
427    /// Based on Anthropic's parallel tool use guidelines
428    pub fn allows_parallel_tools(&self) -> bool {
429        match self {
430            // Auto allows parallel tools by default
431            Self::Auto => true,
432            // Any forces at least one tool, may allow parallel
433            Self::Any => true,
434            Self::AllowedTools(choice) => matches!(choice.mode, AllowedToolsMode::Auto),
435            // Specific forces one particular tool, typically no parallel
436            Self::Specific(_) => false,
437            // None disables tools entirely
438            Self::None => false,
439        }
440    }
441
442    /// Get human-readable description of tool choice behavior
443    pub fn description(&self) -> &'static str {
444        match self {
445            Self::Auto => "Model decides when to use tools (allows parallel)",
446            Self::None => "No tools will be used",
447            Self::Any => "At least one tool must be used (allows parallel)",
448            Self::AllowedTools(_) => "Model decides when to use the currently allowed tool subset",
449            Self::Specific(_) => "Specific tool must be used (no parallel)",
450        }
451    }
452
453    /// OpenAI-compatible providers that share the same tool_choice format
454    const OPENAI_STYLE_PROVIDERS: &'static [&'static str] = &[
455        "openai",
456        "deepseek",
457        "huggingface",
458        "mistral",
459        "openrouter",
460        "zai",
461        "moonshot",
462        "stepfun",
463        "evolink",
464        "lmstudio",
465        "llamacpp",
466        "meta",
467    ];
468
469    /// Convert to provider-specific format
470    #[inline]
471    pub(crate) fn to_provider_format(&self, provider: &str) -> Value {
472        if Self::OPENAI_STYLE_PROVIDERS.contains(&provider) {
473            return self.to_openai_format();
474        }
475
476        match provider {
477            "anthropic" => self.to_anthropic_format(),
478            "gemini" => self.to_gemini_format(),
479            _ => self.to_openai_format(), // Default to OpenAI format
480        }
481    }
482
483    #[inline]
484    pub(crate) fn to_responses_format(&self) -> Value {
485        match self {
486            Self::Specific(choice) => json!({"type": "function", "name": &choice.function.name}),
487            _ => self.to_openai_format(),
488        }
489    }
490
491    #[inline]
492    fn to_openai_format(&self) -> Value {
493        match self {
494            Self::Auto => json!("auto"),
495            Self::None => json!("none"),
496            Self::Any => json!("required"),
497            Self::Specific(choice) => json!(choice),
498            Self::AllowedTools(choice) => json!(choice.mode.as_str()),
499        }
500    }
501
502    #[inline]
503    fn to_anthropic_format(&self) -> Value {
504        match self {
505            Self::Auto => json!({"type": "auto"}),
506            Self::None => json!({"type": "none"}),
507            Self::Any => json!({"type": "any"}),
508            Self::Specific(choice) => json!({"type": "tool", "name": &choice.function.name}),
509            Self::AllowedTools(choice) => json!({"type": choice.mode.anthropic_mode()}),
510        }
511    }
512
513    #[inline]
514    fn to_gemini_format(&self) -> Value {
515        match self {
516            Self::Auto => json!({"mode": "auto"}),
517            Self::None => json!({"mode": "none"}),
518            Self::Any => json!({"mode": "any"}),
519            Self::AllowedTools(choice) => json!({"mode": choice.mode.gemini_mode()}),
520            Self::Specific(choice) => {
521                json!({"mode": "any", "allowed_function_names": [&choice.function.name]})
522            }
523        }
524    }
525}
526
527/// Configuration for parallel tool use behavior
528/// Based on Anthropic's parallel tool use guidelines
529#[derive(Debug, Clone, Serialize, Deserialize)]
530pub struct ParallelToolConfig {
531    /// Whether to disable parallel tool use
532    /// When true, forces sequential tool execution
533    pub(crate) disable_parallel_tool_use: bool,
534
535    /// Maximum number of tools to execute in parallel
536    /// None means no limit (provider default)
537    pub(crate) max_parallel_tools: Option<usize>,
538
539    /// Whether to encourage parallel tool use in prompts
540    pub(crate) encourage_parallel: bool,
541}
542
543impl Default for ParallelToolConfig {
544    fn default() -> Self {
545        Self {
546            disable_parallel_tool_use: false,
547            max_parallel_tools: Some(5), // Reasonable default
548            encourage_parallel: true,
549        }
550    }
551}
552
553impl ParallelToolConfig {
554    /// Create configuration optimized for Anthropic models
555    pub fn anthropic_optimized() -> Self {
556        Self {
557            disable_parallel_tool_use: false,
558            max_parallel_tools: None, // Let Anthropic decide
559            encourage_parallel: true,
560        }
561    }
562
563    /// Create configuration for sequential tool use
564    pub(crate) fn sequential_only() -> Self {
565        Self {
566            disable_parallel_tool_use: true,
567            max_parallel_tools: Some(1),
568            encourage_parallel: false,
569        }
570    }
571}
vtcode_llm/provider/request.rs

vtcode_llm/provider/
request.rs
