vtcode_llm/provider/
request.rs

1use serde::{Deserialize, Serialize};
2use serde_json::{Value, json};
3use std::sync::Arc;
4use vtcode_config::types::{ReasoningEffortLevel, VerbosityLevel};
5
6use super::{Message, ToolDefinition};
7
8/// Fallback model configuration for Anthropic server-side fallback.
9/// The explicit-list form of `fallbacks`, sent with the
10/// `server-side-fallback-2026-06-01` beta header (the `"default"` keyword form
11/// uses `server-side-fallback-2026-07-01`; pairing a header with the other form
12/// is rejected).
13#[derive(Debug, Clone, Serialize, Deserialize, Default)]
14pub struct FallbackModel {
15    /// The model identifier to fall back to (e.g., "claude-opus-5")
16    pub(crate) model: String,
17    /// Optional max_tokens override for this fallback attempt
18    #[serde(skip_serializing_if = "Option::is_none")]
19    pub(crate) max_tokens: Option<u32>,
20    /// Optional thinking configuration override for this fallback attempt
21    #[serde(skip_serializing_if = "Option::is_none")]
22    pub(crate) thinking: Option<AnthropicThinkingConfig>,
23}
24
25/// Anthropic thinking configuration for fallback models.
26#[derive(Debug, Clone, Serialize, Deserialize, Default)]
27#[serde(tag = "type", rename_all = "lowercase")]
28pub enum AnthropicThinkingConfig {
29    #[default]
30    Disabled,
31    Enabled {
32        budget_tokens: u32,
33        #[serde(skip_serializing_if = "Option::is_none")]
34        display: Option<String>,
35    },
36    Adaptive {
37        #[serde(skip_serializing_if = "Option::is_none")]
38        display: Option<String>,
39    },
40}
41
42#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
43#[serde(rename_all = "snake_case")]
44pub enum PromptCacheProfile {
45    BudgetContinuation,
46    /// Cache only stable instructions for an independent auxiliary request.
47    /// Native GPT-5.6+ uses explicit-only caching; other routes ignore this profile.
48    StablePrefix,
49}
50
51#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
52#[serde(rename_all = "snake_case")]
53pub enum AnthropicThinkingModeOverride {
54    #[default]
55    Inherit,
56    Disabled,
57    Adaptive,
58    ManualBudget(u32),
59}
60
61#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
62#[serde(rename_all = "snake_case")]
63pub enum AnthropicThinkingDisplayOverride {
64    #[default]
65    Inherit,
66    Summarized,
67    Omitted,
68    Updates,
69}
70
71#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
72#[serde(rename_all = "snake_case")]
73pub enum AnthropicOptionalStringOverride {
74    #[default]
75    Inherit,
76    Omit,
77    Explicit(String),
78}
79
80#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
81#[serde(rename_all = "snake_case")]
82pub enum AnthropicOptionalU32Override {
83    #[default]
84    Inherit,
85    Omit,
86    Explicit(u32),
87}
88
89#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
90pub struct AnthropicRequestOverrides {
91    /// Explicit request speed takes precedence over provider configuration.
92    #[serde(default, skip_serializing_if = "Option::is_none")]
93    pub speed: Option<vtcode_config::AnthropicSpeed>,
94    #[serde(default)]
95    pub(crate) thinking_mode: AnthropicThinkingModeOverride,
96    #[serde(default)]
97    pub(crate) thinking_display: AnthropicThinkingDisplayOverride,
98    #[serde(default)]
99    pub(crate) effort: AnthropicOptionalStringOverride,
100    #[serde(default)]
101    pub(crate) task_budget_tokens: AnthropicOptionalU32Override,
102}
103
104/// Universal LLM request structure
105#[derive(Debug, Clone, Serialize, Deserialize, Default)]
106
107pub struct LLMRequest {
108    /// Conversation history shared via `Arc` so per-turn request construction
109    /// and continuation bookkeeping are O(1) instead of deep-copying the
110    /// entire history (which grows unbounded over a session).
111    pub messages: Arc<Vec<Message>>,
112    /// Shared via `Arc<str>` (single allocation) so per-turn request
113    /// construction can reuse the prompt envelope's `Arc<str>` via a cheap
114    /// refcount bump instead of cloning the full system prompt string.
115    pub system_prompt: Option<Arc<str>>,
116    pub tools: Option<Arc<Vec<ToolDefinition>>>,
117    pub model: String,
118    pub max_tokens: Option<u32>,
119    pub temperature: Option<f32>,
120    pub stream: bool,
121
122    /// Optional structured output JSON schema to request from providers that support it
123    /// For Anthropic this will be sent as `output_config.format = { type: "json_schema", schema: ... }`
124    pub output_format: Option<Value>,
125
126    /// Tool choice configuration based on official API docs
127    /// Supports: "auto" (default), "none", "any", or specific tool selection
128    pub tool_choice: Option<ToolChoice>,
129
130    /// Whether to enable parallel tool calls (OpenAI specific)
131    pub parallel_tool_calls: Option<bool>,
132
133    /// Parallel tool use configuration following Anthropic best practices
134    pub parallel_tool_config: Option<Box<ParallelToolConfig>>,
135
136    /// Reasoning effort level for models that support it (none, minimal, low, medium, high, xhigh)
137    /// Applies to: Claude, GPT-5 family, Gemini, Qwen3, DeepSeek with reasoning capability
138    pub reasoning_effort: Option<ReasoningEffortLevel>,
139
140    /// Effort level for overall token usage (low, medium, high, xhigh, max)
141    /// Applies to: Anthropic adaptive-thinking models such as Claude Opus 4.7
142    /// Controls how many tokens Claude uses when responding, trading off between
143    /// response thoroughness and token efficiency.
144    pub effort: Option<String>,
145
146    /// Verbosity level for output text (low, medium, high)
147    /// Applies to: GPT-5.4-family Responses workflows and other models that support verbosity control
148    pub verbosity: Option<VerbosityLevel>,
149
150    /// Advanced generation parameters
151    pub do_sample: Option<bool>,
152    pub top_p: Option<f32>,
153    pub top_k: Option<i32>,
154    pub presence_penalty: Option<f32>,
155    pub frequency_penalty: Option<f32>,
156    pub stop_sequences: Option<Vec<String>>,
157    /// Optional budget for extended thinking (Anthropic specific)
158    /// Minimum value: 1024
159    pub thinking_budget: Option<u32>,
160
161    /// Optional beta headers for Anthropic (and potentially others)
162    pub betas: Option<Vec<String>>,
163
164    /// Optional provider-specific context management configuration (Anthropic compaction/editing).
165    pub context_management: Option<Value>,
166
167    /// Optional coding agent specific settings
168    pub coding_agent_settings: Option<Box<CodingAgentSettings>>,
169
170    /// Optional turn metadata for git context (remote URLs, commit hash, etc.)
171    /// This is sent as X-Turn-Metadata header to providers that support it
172    pub metadata: Option<Value>,
173
174    /// Optional Responses API continuity pointer for server-side context chaining.
175    /// Used by providers that support stateful response continuation.
176    pub previous_response_id: Option<String>,
177
178    /// Optional Responses API storage flag.
179    /// When set, providers that support `store` pass this through directly.
180    pub response_store: Option<bool>,
181
182    /// Optional Responses API include fields (e.g. reasoning encrypted content).
183    /// Passed through only for providers/APIs that support include selectors.
184    pub responses_include: Option<Vec<String>>,
185
186    /// Optional native OpenAI `service_tier` request parameter.
187    /// Passed through only for native OpenAI endpoints that support service tiers.
188    pub service_tier: Option<String>,
189
190    /// Optional provider routing hint for prompt cache stickiness.
191    /// OpenAI uses this value to improve routing locality for repeated prefixes.
192    pub prompt_cache_key: Option<String>,
193
194    /// Optional request-scoped prompt cache profile for provider-specific TTL overrides.
195    pub prompt_cache_profile: Option<PromptCacheProfile>,
196
197    /// Optional explicit fallback models for Anthropic server-side fallback.
198    /// Overrides `provider.anthropic.fallbacks`; sent with the
199    /// `server-side-fallback-2026-06-01` beta header. Subject to the same
200    /// gates as the config list (first-party endpoint, a model that supports
201    /// server-side fallbacks, no credit-token retry) and the same entry
202    /// validation; an empty list sends nothing.
203    pub fallbacks: Option<Vec<FallbackModel>>,
204
205    /// Optional opaque credit token from a refused request's `stop_details.fallback_credit_token`.
206    /// Echoed on the retry to avoid paying the prompt-cache cost twice.
207    /// Sent with the `fallback-credit-2026-07-01` beta header. The refused request must have
208    /// carried that header or `server-side-fallback-2026-07-01`, which grants the same
209    /// `stop_details` fields.
210    pub fallback_credit_token: Option<String>,
211
212    /// Optional Anthropic-specific request overrides used when request semantics must
213    /// not inherit VT Code's provider defaults, such as the Anthropic compatibility server.
214    pub anthropic_request_overrides: Option<AnthropicRequestOverrides>,
215}
216
217impl LLMRequest {
218    /// Budget a short auxiliary answer separately from any hidden reasoning.
219    ///
220    /// Select low effort only when advertised; otherwise retain the provider's
221    /// default. Reasoning routes receive 2,048 additional output tokens and no
222    /// sampling controls. This is a bounded allowance, not a completion guarantee.
223    /// Reapply after changing `model`, including on a fallback route.
224    pub fn set_auxiliary_output_budget(
225        &mut self,
226        provider: &(impl super::LLMProvider + ?Sized),
227        visible_output_tokens: u32,
228    ) {
229        let levels = provider.supported_reasoning_efforts(&self.model);
230        let reasoning = provider.supports_reasoning(&self.model) || !levels.is_empty();
231        self.reasoning_effort = levels.contains(&"low").then_some(ReasoningEffortLevel::Low);
232        self.max_tokens = Some(visible_output_tokens.max(1).saturating_add(if reasoning { 2048 } else { 0 }));
233        self.verbosity = Some(VerbosityLevel::Low);
234        // Routes that always reject sampling must not carry it even without
235        // reasoning: Anthropic Messages (API 2026-10-21) fails the whole
236        // request instead of ignoring the value.
237        if reasoning || provider.sampling_overrides(&self.model).drops_sampling_params(false, false) {
238            self.temperature = None;
239            self.top_p = None;
240            self.top_k = None;
241            self.do_sample = None;
242            self.presence_penalty = None;
243            self.frequency_penalty = None;
244        }
245    }
246}
247
248/// Per-model sampling parameter overrides resolved from a custom provider
249/// profile. Every field defaults to `None`, meaning the agent loop's global
250/// configuration value applies unchanged.
251#[derive(Debug, Clone, Copy, Default, PartialEq)]
252pub struct SamplingOverrides {
253    /// Sampling temperature (0.0-2.0).
254    pub temperature: Option<f32>,
255    /// Nucleus sampling mass (0.0-1.0).
256    pub top_p: Option<f32>,
257    /// Top-k token cutoff (>= 0).
258    pub top_k: Option<i32>,
259    /// Presence penalty (-2.0-2.0).
260    pub presence_penalty: Option<f32>,
261    /// Frequency penalty (-2.0-2.0).
262    pub frequency_penalty: Option<f32>,
263    /// Maximum output tokens (> 0).
264    pub max_tokens: Option<u32>,
265    /// Reasoning effort for models that accept it.
266    pub reasoning_effort: Option<ReasoningEffortLevel>,
267    /// Whether this wire format rejects sampling parameters, as Anthropic
268    /// Messages does under API version `2026-10-21`, independent of reasoning.
269    pub suppresses_sampling: bool,
270    /// Set by providers that resolve overrides from an explicit user profile;
271    /// callers use it to trust these values over name-based heuristics.
272    pub profile_aware: bool,
273}
274
275impl SamplingOverrides {
276    /// Whether sampling parameters must be dropped for this request.
277    ///
278    /// Anthropic Messages routes reject `temperature`, `top_p`, and `top_k`
279    /// under API version `2026-10-21` even at their default values, so the
280    /// drop is unconditional for them: either the provider declares that its
281    /// wire rejects sampling (`suppresses_sampling`), or it is a built-in
282    /// Anthropic/MiniMax backend (`native_match`) that resolves no user
283    /// profile. Reasoning state does not change the answer.
284    pub fn drops_sampling_params(&self, native_match: bool, reasoning_active: bool) -> bool {
285        let _ = reasoning_active;
286        self.suppresses_sampling || (!self.profile_aware && native_match)
287    }
288}
289
290/// Optional overrides for standalone Responses compaction requests.
291#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
292pub struct ResponsesCompactionOptions {
293    /// Optional custom instructions appended to the derived replay instructions.
294    pub instructions: Option<String>,
295    /// Optional output token limit for the compaction response.
296    pub max_output_tokens: Option<u32>,
297    /// Optional reasoning effort override for the compaction pass.
298    pub reasoning_effort: Option<ReasoningEffortLevel>,
299    /// Optional verbosity override for the compaction output text settings.
300    pub verbosity: Option<VerbosityLevel>,
301    /// Optional include selectors override.
302    pub responses_include: Option<Vec<String>>,
303    /// Optional storage override.
304    pub response_store: Option<bool>,
305    /// Optional native OpenAI service tier override.
306    pub service_tier: Option<String>,
307    /// Optional prompt cache routing override.
308    pub prompt_cache_key: Option<String>,
309}
310
311/// Settings to refine model behavior for coding agent tasks
312#[derive(Debug, Clone, Serialize, Deserialize, Default)]
313pub struct CodingAgentSettings {
314    /// Optimize for long context by hoisting the largest user message.
315    /// Hoisting reorders earlier turns, so the Anthropic builder skips it on
316    /// preserved-thinking models (Claude Sonnet 5.5, Claude Opus 5.5, Claude
317    /// Fable 5.1).
318    pub(crate) long_context_optimization: bool,
319}
320
321/// Tool choice configuration that works across different providers
322/// Based on OpenAI, Anthropic, and Gemini API specifications
323/// Follows Anthropic's tool use best practices for optimal performance
324#[derive(Debug, Clone, Serialize, Deserialize)]
325#[serde(untagged)]
326#[derive(Default)]
327pub enum ToolChoice {
328    /// Let the model decide whether to call tools ("auto")
329    /// Default behavior - allows model to use tools when appropriate
330    #[default]
331    Auto,
332
333    /// Force the model to not call any tools ("none")
334    /// Useful for pure conversational responses without tool usage
335    None,
336
337    /// Force the model to call at least one tool ("any")
338    /// Ensures tool usage even when model might prefer direct response
339    Any,
340
341    /// Force the model to call a specific tool
342    /// Useful for directing model to use particular functionality
343    Specific(SpecificToolChoice),
344
345    /// OpenAI Responses native allowed-tools filtering.
346    ///
347    /// This is advisory provider-side filtering only; runtime dispatch remains
348    /// authoritative. Providers without native support should degrade this to
349    /// the equivalent auto/required tool choice while keeping the full
350    /// catalogue visible.
351    AllowedTools(AllowedToolsChoice),
352}
353
354#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
355#[serde(rename_all = "lowercase")]
356pub enum AllowedToolsMode {
357    Auto,
358    Required,
359}
360
361impl AllowedToolsMode {
362    pub fn as_str(self) -> &'static str {
363        match self {
364            Self::Auto => "auto",
365            Self::Required => "required",
366        }
367    }
368
369    fn anthropic_mode(self) -> &'static str {
370        match self {
371            Self::Auto => "auto",
372            Self::Required => "any",
373        }
374    }
375
376    fn gemini_mode(self) -> &'static str {
377        match self {
378            Self::Auto => "auto",
379            Self::Required => "any",
380        }
381    }
382}
383
384#[derive(Debug, Clone, Serialize, Deserialize)]
385pub struct AllowedToolsChoice {
386    pub mode: AllowedToolsMode,
387    pub tools: Vec<String>,
388}
389
390/// Specific tool choice for forcing a particular function call
391#[derive(Debug, Clone, Serialize, Deserialize)]
392pub struct SpecificToolChoice {
393    #[serde(rename = "type")]
394    pub(crate) tool_type: String, // "function"
395
396    pub(crate) function: SpecificFunctionChoice,
397}
398
399/// Specific function choice details
400#[derive(Debug, Clone, Serialize, Deserialize)]
401pub struct SpecificFunctionChoice {
402    pub(crate) name: String,
403}
404
405impl ToolChoice {
406    /// Create auto tool choice (default behavior)
407    pub fn auto() -> Self {
408        Self::Auto
409    }
410
411    /// Create none tool choice (disable tool calling)
412    pub fn none() -> Self {
413        Self::None
414    }
415
416    /// Create any tool choice (force at least one tool call)
417    pub(crate) fn any() -> Self {
418        Self::Any
419    }
420
421    /// Create specific function tool choice
422    pub(crate) fn function(name: String) -> Self {
423        Self::Specific(SpecificToolChoice {
424            tool_type: "function".to_owned(),
425            function: SpecificFunctionChoice { name },
426        })
427    }
428
429    pub fn allowed_tools_auto(tools: Vec<String>) -> Self {
430        Self::AllowedTools(AllowedToolsChoice { mode: AllowedToolsMode::Auto, tools })
431    }
432
433    /// Check if this tool choice allows parallel tool use
434    /// Based on Anthropic's parallel tool use guidelines
435    pub fn allows_parallel_tools(&self) -> bool {
436        match self {
437            // Auto allows parallel tools by default
438            Self::Auto => true,
439            // Any forces at least one tool, may allow parallel
440            Self::Any => true,
441            Self::AllowedTools(choice) => matches!(choice.mode, AllowedToolsMode::Auto),
442            // Specific forces one particular tool, typically no parallel
443            Self::Specific(_) => false,
444            // None disables tools entirely
445            Self::None => false,
446        }
447    }
448
449    /// Get human-readable description of tool choice behavior
450    pub fn description(&self) -> &'static str {
451        match self {
452            Self::Auto => "Model decides when to use tools (allows parallel)",
453            Self::None => "No tools will be used",
454            Self::Any => "At least one tool must be used (allows parallel)",
455            Self::AllowedTools(_) => "Model decides when to use the currently allowed tool subset",
456            Self::Specific(_) => "Specific tool must be used (no parallel)",
457        }
458    }
459
460    /// OpenAI-compatible providers that share the same tool_choice format
461    const OPENAI_STYLE_PROVIDERS: &'static [&'static str] = &[
462        "openai",
463        "deepseek",
464        "huggingface",
465        "mistral",
466        "openrouter",
467        "zai",
468        "moonshot",
469        "stepfun",
470        "evolink",
471        "lmstudio",
472        "llamacpp",
473        "meta",
474    ];
475
476    /// Convert to provider-specific format
477    #[inline]
478    pub(crate) fn to_provider_format(&self, provider: &str) -> Value {
479        if Self::OPENAI_STYLE_PROVIDERS.contains(&provider) {
480            return self.to_openai_format();
481        }
482
483        match provider {
484            "anthropic" => self.to_anthropic_format(),
485            "gemini" => self.to_gemini_format(),
486            _ => self.to_openai_format(), // Default to OpenAI format
487        }
488    }
489
490    #[inline]
491    pub(crate) fn to_responses_format(&self) -> Value {
492        match self {
493            Self::Specific(choice) => json!({"type": "function", "name": &choice.function.name}),
494            _ => self.to_openai_format(),
495        }
496    }
497
498    #[inline]
499    fn to_openai_format(&self) -> Value {
500        match self {
501            Self::Auto => json!("auto"),
502            Self::None => json!("none"),
503            Self::Any => json!("required"),
504            Self::Specific(choice) => json!(choice),
505            Self::AllowedTools(choice) => json!(choice.mode.as_str()),
506        }
507    }
508
509    #[inline]
510    fn to_anthropic_format(&self) -> Value {
511        match self {
512            Self::Auto => json!({"type": "auto"}),
513            Self::None => json!({"type": "none"}),
514            Self::Any => json!({"type": "any"}),
515            Self::Specific(choice) => json!({"type": "tool", "name": &choice.function.name}),
516            Self::AllowedTools(choice) => json!({"type": choice.mode.anthropic_mode()}),
517        }
518    }
519
520    #[inline]
521    fn to_gemini_format(&self) -> Value {
522        match self {
523            Self::Auto => json!({"mode": "auto"}),
524            Self::None => json!({"mode": "none"}),
525            Self::Any => json!({"mode": "any"}),
526            Self::AllowedTools(choice) => json!({"mode": choice.mode.gemini_mode()}),
527            Self::Specific(choice) => {
528                json!({"mode": "any", "allowed_function_names": [&choice.function.name]})
529            }
530        }
531    }
532}
533
534/// Configuration for parallel tool use behavior
535/// Based on Anthropic's parallel tool use guidelines
536#[derive(Debug, Clone, Serialize, Deserialize)]
537pub struct ParallelToolConfig {
538    /// Whether to disable parallel tool use
539    /// When true, forces sequential tool execution
540    pub(crate) disable_parallel_tool_use: bool,
541
542    /// Maximum number of tools to execute in parallel
543    /// None means no limit (provider default)
544    pub(crate) max_parallel_tools: Option<usize>,
545
546    /// Whether to encourage parallel tool use in prompts
547    pub(crate) encourage_parallel: bool,
548}
549
550impl Default for ParallelToolConfig {
551    fn default() -> Self {
552        Self {
553            disable_parallel_tool_use: false,
554            max_parallel_tools: Some(5), // Reasonable default
555            encourage_parallel: true,
556        }
557    }
558}
559
560impl ParallelToolConfig {
561    /// Create configuration optimized for Anthropic models
562    pub fn anthropic_optimized() -> Self {
563        Self {
564            disable_parallel_tool_use: false,
565            max_parallel_tools: None, // Let Anthropic decide
566            encourage_parallel: true,
567        }
568    }
569
570    /// Create configuration for sequential tool use
571    pub(crate) fn sequential_only() -> Self {
572        Self {
573            disable_parallel_tool_use: true,
574            max_parallel_tools: Some(1),
575            encourage_parallel: false,
576        }
577    }
578}
vtcode_llm/provider/request.rs

vtcode_llm/provider/
request.rs
