llm/providers/local/
llama_cpp.rs

1#![doc = include_str!(concat!(env!("OUT_DIR"), "/docs/llamacpp.md"))]
2
3use super::util::get_local_config;
4use crate::providers::openai::OpenAiChatProvider;
5use crate::{ProviderConnectionConfig, ProviderFactory, Result};
6use async_openai::{Client, config::OpenAIConfig};
7
8pub struct LlamaCppProvider {
9    client: Client<OpenAIConfig>,
10}
11
12impl LlamaCppProvider {
13    pub fn new(base_url: &str) -> Self {
14        Self { client: Client::with_config(get_local_config(base_url)) }
15    }
16}
17
18impl Default for LlamaCppProvider {
19    fn default() -> Self {
20        Self { client: Client::with_config(get_local_config("http://localhost:8080/v1")) }
21    }
22}
23
24impl ProviderFactory for LlamaCppProvider {
25    async fn from_env() -> Result<Self> {
26        Self::from_env_with_connection(ProviderConnectionConfig::default()).await
27    }
28
29    async fn from_env_with_connection(connection: ProviderConnectionConfig) -> Result<Self> {
30        let base_url = connection.base_url.as_deref().unwrap_or("http://localhost:8080/v1");
31        Ok(Self { client: Client::with_config(get_local_config(base_url)) })
32    }
33
34    fn with_model(self, _model: &str) -> Self {
35        // LlamaCpp doesn't support model selection - it serves a single model
36        self
37    }
38}
39
40impl OpenAiChatProvider for LlamaCppProvider {
41    type Config = OpenAIConfig;
42
43    fn client(&self) -> &Client<Self::Config> {
44        &self.client
45    }
46
47    fn model(&self) -> &'static str {
48        "" // llama.cpp server serves a single model on boot and does not allow swapping models
49    }
50
51    fn provider_name(&self) -> &'static str {
52        "LlamaCpp"
53    }
54}
llm/providers/local/llama_cpp.rs

llm/providers/local/
llama_cpp.rs
