vtcode_llm/providers/llamacpp/
mod.rs

1//! llama.cpp provider: a local, OpenAI-compatible server managed by vtcode.
2//!
3//! Module layout (per crate-local AGENTS.md: `providers/<name>/mod.rs`):
4//! - `mod.rs` (this file): `LlamaCppProvider`, provider trait impls, request
5//!   model selection, public API surface.
6//! - `probe.rs`: HTTP model discovery (`fetch_llamacpp_models`), `ServerProbe`,
7//!   `probe_server`.
8//! - `startup.rs`: model-path policy, binary/args resolution, process spawning,
9//!   readiness polling.
10//! - `managed.rs`: shared state registry, phase tracking, readiness
11//!   orchestration (`ensure_server_ready`).
12
13mod managed;
14mod probe;
15mod startup;
16
17pub use probe::fetch_llamacpp_models;
18
19use std::path::Path;
20use std::sync::Arc;
21
22use anyhow::Result;
23use async_trait::async_trait;
24
25use super::OpenAIProvider;
26use super::common::resolve_model;
27use crate::client::LLMClient;
28use crate::error_display;
29use crate::provider::{LLMError, LLMProvider, LLMRequest, LLMResponse, LLMStream, Message};
30use crate::providers::common::override_base_url;
31
32use vtcode_config::TimeoutsConfig;
33use vtcode_config::constants::{env_vars, models, urls};
34use vtcode_config::core::{AnthropicConfig, ModelConfig, PromptCachingConfig};
35
36const LLAMACPP_CONNECTION_ERROR: &str = "llama.cpp is not responding. Install from https://llama.app and either start `llama-server -m /path/to/model.gguf --port 8080` yourself or set LLAMACPP_MODEL_PATH so VT Code can manage startup.";
37
38pub struct LlamaCppProvider {
39    /// Concrete OpenAI-compatible inner provider.
40    ///
41    /// Stored without a `Box<dyn LLMProvider>` layer on purpose: the inner
42    /// type is always `OpenAIProvider`, so keeping it concrete lets the
43    /// compiler use static dispatch (monomorphization) for every delegated
44    /// capability call instead of a vtable lookup. Only the outer
45    /// `LlamaCppProvider` itself is boxed as `Box<dyn LLMProvider>` by the
46    /// factory, where dynamic dispatch is genuinely required for the
47    /// heterogeneous provider collection.
48    inner: OpenAIProvider,
49    api_key: Option<String>,
50    configured_model: Option<String>,
51    model_id: String,
52    base_url: String,
53    prompt_cache: Option<PromptCachingConfig>,
54    timeouts: Option<TimeoutsConfig>,
55    anthropic: Option<AnthropicConfig>,
56    model_behavior: Option<ModelConfig>,
57}
58
59impl LlamaCppProvider {
60    fn resolve_base_url(base_url: Option<String>) -> String {
61        override_base_url(urls::LLAMACPP_API_BASE, base_url, Some(env_vars::LLAMACPP_BASE_URL))
62    }
63
64    fn build_inner(
65        api_key: Option<String>,
66        model: Option<String>,
67        base_url: Option<String>,
68        prompt_cache: Option<PromptCachingConfig>,
69        timeouts: Option<TimeoutsConfig>,
70        anthropic: Option<AnthropicConfig>,
71        model_behavior: Option<ModelConfig>,
72    ) -> (OpenAIProvider, String) {
73        let resolved_model = resolve_model(model, models::llamacpp::DEFAULT_MODEL);
74        let resolved_base = Self::resolve_base_url(base_url);
75        let inner = OpenAIProvider::from_config(
76            api_key,
77            None,
78            Some(resolved_model.clone()),
79            Some(resolved_base),
80            prompt_cache,
81            timeouts,
82            anthropic,
83            None,
84            model_behavior,
85        );
86        (inner, resolved_model)
87    }
88
89    fn should_replace_request_model(&self, request_model: &str, discovered_models: &[String]) -> bool {
90        let trimmed = request_model.trim();
91        if trimmed.is_empty() || looks_like_local_model_path(trimmed) {
92            return true;
93        }
94
95        if discovered_models.len() == 1 {
96            let configured = self.configured_model.as_deref().map(str::trim).unwrap_or_default();
97            if trimmed == models::llamacpp::DEFAULT_MODEL || trimmed == configured {
98                return true;
99            }
100        }
101
102        !discovered_models.iter().any(|model| model == trimmed) && discovered_models.len() == 1
103    }
104
105    fn request_model_or_default(&self, request_model: &str) -> String {
106        let trimmed = request_model.trim();
107        if trimmed.is_empty() {
108            resolve_model(self.configured_model.clone(), models::llamacpp::DEFAULT_MODEL)
109        } else {
110            trimmed.to_string()
111        }
112    }
113
114    fn build_request_provider(&self, model: String) -> OpenAIProvider {
115        Self::build_inner(
116            self.api_key.clone(),
117            Some(model),
118            Some(self.base_url.clone()),
119            self.prompt_cache.clone(),
120            self.timeouts.clone(),
121            self.anthropic.clone(),
122            self.model_behavior.clone(),
123        )
124        .0
125    }
126
127    async fn prepare_request(&self, mut request: LLMRequest) -> Result<(OpenAIProvider, LLMRequest), LLMError> {
128        let discovered_model = managed::ensure_server_ready(&self.base_url, self.configured_model.as_deref()).await?;
129        let discovered_models = vec![discovered_model.clone()];
130
131        if self.should_replace_request_model(&request.model, &discovered_models) || request.model.trim().is_empty() {
132            request.model = discovered_model.clone();
133        } else {
134            request.model = self.request_model_or_default(&request.model);
135        }
136
137        Ok((self.build_request_provider(request.model.clone()), request))
138    }
139
140    pub fn from_config(
141        api_key: Option<String>,
142        model: Option<String>,
143        base_url: Option<String>,
144        prompt_cache: Option<PromptCachingConfig>,
145        timeouts: Option<TimeoutsConfig>,
146        anthropic: Option<AnthropicConfig>,
147        model_behavior: Option<ModelConfig>,
148    ) -> Self {
149        let resolved_base_url = Self::resolve_base_url(base_url.clone());
150        let (inner, model_id) = Self::build_inner(
151            api_key.clone(),
152            model.clone(),
153            base_url,
154            prompt_cache.clone(),
155            timeouts.clone(),
156            anthropic.clone(),
157            model_behavior.clone(),
158        );
159        Self {
160            inner,
161            api_key,
162            configured_model: model,
163            model_id,
164            base_url: resolved_base_url,
165            prompt_cache,
166            timeouts,
167            anthropic,
168            model_behavior,
169        }
170    }
171}
172
173#[async_trait]
174impl LLMProvider for LlamaCppProvider {
175    fn name(&self) -> &str {
176        "llamacpp"
177    }
178
179    fn supports_streaming(&self) -> bool {
180        self.inner.supports_streaming()
181    }
182
183    fn supports_non_streaming(&self, model: &str) -> bool {
184        // Delegated (fail-safe): the stream-timeout fallback keys on this.
185        self.inner.supports_non_streaming(model)
186    }
187
188    fn supports_reasoning(&self, model: &str) -> bool {
189        self.inner.supports_reasoning(model)
190    }
191
192    fn supports_reasoning_effort(&self, model: &str) -> bool {
193        self.inner.supports_reasoning_effort(model)
194    }
195
196    fn supports_tools(&self, model: &str) -> bool {
197        self.inner.supports_tools(model)
198    }
199
200    fn supports_parallel_tool_config(&self, model: &str) -> bool {
201        self.inner.supports_parallel_tool_config(model)
202    }
203
204    async fn generate(&self, request: LLMRequest) -> Result<LLMResponse, LLMError> {
205        let (provider, request) = self.prepare_request(request).await?;
206        provider.generate(request).await
207    }
208
209    async fn stream(&self, request: LLMRequest) -> Result<LLMStream, LLMError> {
210        let (provider, request) = self.prepare_request(request).await?;
211        provider.stream(request).await
212    }
213
214    fn supported_models(&self) -> Vec<String> {
215        models::llamacpp::SUPPORTED_MODELS
216            .iter()
217            .map(|model| model.to_string())
218            .collect()
219    }
220
221    fn validate_request(&self, request: &LLMRequest) -> Result<(), LLMError> {
222        if request.messages.is_empty() {
223            let formatted_error = error_display::format_llm_error("llama.cpp", "Messages cannot be empty");
224            return Err(LLMError::InvalidRequest { message: formatted_error, metadata: None });
225        }
226
227        for message in request.messages.iter() {
228            if let Err(err) = message.validate_for_provider("openai") {
229                let formatted = error_display::format_llm_error("llama.cpp", &err);
230                return Err(LLMError::InvalidRequest { message: formatted, metadata: None });
231            }
232        }
233
234        Ok(())
235    }
236}
237
238#[async_trait]
239impl LLMClient for LlamaCppProvider {
240    async fn generate(&mut self, prompt: &str) -> Result<LLMResponse, LLMError> {
241        LLMProvider::generate(
242            self,
243            LLMRequest {
244                messages: Arc::new(vec![Message::user(prompt.to_string())]),
245                model: self
246                    .configured_model
247                    .clone()
248                    .unwrap_or_else(|| models::llamacpp::DEFAULT_MODEL.to_string()),
249                ..Default::default()
250            },
251        )
252        .await
253    }
254
255    fn model_id(&self) -> &str {
256        &self.model_id
257    }
258}
259
260// Re-export the local-model-path heuristic at module scope so `startup.rs`
261// can call it as `super::looks_like_local_model_path` without going through
262// the provider type. This keeps the policy decision owned by the provider
263// module while letting the startup seam stay stateless.
264pub(super) fn looks_like_local_model_path(value: &str) -> bool {
265    let trimmed = value.trim();
266    if trimmed.is_empty() {
267        return false;
268    }
269
270    trimmed.ends_with(".gguf")
271        || trimmed.contains(std::path::MAIN_SEPARATOR)
272        || trimmed.contains('/')
273        || trimmed.starts_with('.')
274        || Path::new(trimmed).exists()
275}
276
277#[cfg(test)]
278mod tests {
279    use super::looks_like_local_model_path;
280
281    #[test]
282    fn empty_input_is_not_a_path() {
283        assert!(!looks_like_local_model_path(""));
284        assert!(!looks_like_local_model_path("   "));
285    }
286
287    #[test]
288    fn gguf_extension_is_local() {
289        assert!(looks_like_local_model_path("qwen.gguf"));
290        // The check is case-sensitive: uppercase .GGUF is not recognized as a
291        // model path on its own (it has no separator and likely does not exist
292        // on disk). This documents the existing behavior, not a desired design.
293        assert!(!looks_like_local_model_path("QWEN.GGUF"));
294    }
295
296    #[test]
297    fn paths_with_separator_are_local() {
298        assert!(looks_like_local_model_path("/abs/path/model.gguf"));
299        assert!(looks_like_local_model_path("models/qwen"));
300        assert!(looks_like_local_model_path("./relative/model"));
301    }
302
303    #[test]
304    fn leading_dot_is_local() {
305        assert!(looks_like_local_model_path("./model"));
306        assert!(looks_like_local_model_path(".hidden"));
307    }
308
309    #[test]
310    fn bare_model_ids_are_not_local() {
311        assert!(!looks_like_local_model_path("qwen2.5-7b"));
312        assert!(!looks_like_local_model_path("gpt-4o-mini"));
313    }
314
315    #[test]
316    fn existing_filesystem_path_is_local() {
317        // Cargo.toml always exists relative to the manifest dir.
318        let path = env!("CARGO_MANIFEST_DIR").to_string() + "/Cargo.toml";
319        assert!(looks_like_local_model_path(&path));
320    }
321}
vtcode_llm/providers/llamacpp/mod.rs

vtcode_llm/providers/llamacpp/
mod.rs
