llm/providers/local/
llama_cpp.rs

1#![doc = include_str!(concat!(env!("OUT_DIR"), "/docs/llamacpp.md"))]
2
3use super::util::get_local_config;
4use crate::providers::http::openai_client;
5use crate::providers::openai::OpenAiChatProvider;
6use crate::{ProviderConnectionConfig, ProviderFactory, Result};
7use async_openai::{Client, config::OpenAIConfig};
8use std::future::ready;
9
10pub struct LlamaCppProvider {
11    client: Client<OpenAIConfig>,
12}
13
14impl LlamaCppProvider {
15    pub fn new(base_url: &str) -> Self {
16        Self { client: openai_client(get_local_config(base_url), reqwest::Client::new()) }
17    }
18}
19
20impl Default for LlamaCppProvider {
21    fn default() -> Self {
22        Self::new("http://localhost:8080/v1")
23    }
24}
25
26impl ProviderFactory for LlamaCppProvider {
27    async fn from_env() -> Result<Self> {
28        Self::from_env_with_connection(ProviderConnectionConfig::default()).await
29    }
30
31    fn from_env_with_connection(connection: ProviderConnectionConfig) -> impl Future<Output = Result<Self>> + Send {
32        let base_url = connection.base_url.as_deref().unwrap_or("http://localhost:8080/v1");
33        ready(Ok(Self::new(base_url)))
34    }
35
36    fn with_model(self, _model: &str) -> Self {
37        // LlamaCpp doesn't support model selection - it serves a single model
38        self
39    }
40}
41
42impl OpenAiChatProvider for LlamaCppProvider {
43    type Config = OpenAIConfig;
44
45    fn client(&self) -> &Client<Self::Config> {
46        &self.client
47    }
48
49    fn model(&self) -> &'static str {
50        "" // llama.cpp server serves a single model on boot and does not allow swapping models
51    }
52
53    fn provider_name(&self) -> &'static str {
54        "LlamaCpp"
55    }
56}
llm/providers/local/llama_cpp.rs

llm/providers/local/
llama_cpp.rs
