llm/providers/local/
llama_cpp.rs

1#![doc = include_str!(concat!(env!("OUT_DIR"), "/docs/llamacpp.md"))]
2
3use super::util::get_local_config;
4use crate::provider_connection::DEFAULT_STREAM_IDLE_TIMEOUT;
5use crate::providers::http::{http_client, openai_client};
6use crate::providers::openai::OpenAiChatProvider;
7use crate::{ProviderConnectionConfig, ProviderFactory, Result};
8use async_openai::{Client, config::OpenAIConfig};
9use std::future::ready;
10use std::time::Duration;
11
12pub struct LlamaCppProvider {
13    client: Client<OpenAIConfig>,
14    idle_timeout: Duration,
15}
16
17impl LlamaCppProvider {
18    pub fn new(base_url: &str) -> Self {
19        Self {
20            client: openai_client(get_local_config(base_url), http_client()),
21            idle_timeout: DEFAULT_STREAM_IDLE_TIMEOUT,
22        }
23    }
24}
25
26impl Default for LlamaCppProvider {
27    fn default() -> Self {
28        Self::new("http://localhost:8080/v1")
29    }
30}
31
32impl ProviderFactory for LlamaCppProvider {
33    async fn from_env() -> Result<Self> {
34        Self::from_env_with_connection(ProviderConnectionConfig::default()).await
35    }
36
37    fn from_env_with_connection(connection: ProviderConnectionConfig) -> impl Future<Output = Result<Self>> + Send {
38        let base_url = connection.base_url.as_deref().unwrap_or("http://localhost:8080/v1");
39        ready(Ok(Self { idle_timeout: connection.idle_timeout, ..Self::new(base_url) }))
40    }
41
42    fn with_model(self, _model: &str) -> Self {
43        // LlamaCpp doesn't support model selection - it serves a single model
44        self
45    }
46}
47
48impl OpenAiChatProvider for LlamaCppProvider {
49    type Config = OpenAIConfig;
50
51    fn client(&self) -> &Client<Self::Config> {
52        &self.client
53    }
54
55    fn model(&self) -> &'static str {
56        "" // llama.cpp server serves a single model on boot and does not allow swapping models
57    }
58
59    fn provider_name(&self) -> &'static str {
60        "LlamaCpp"
61    }
62
63    fn idle_timeout(&self) -> Duration {
64        self.idle_timeout
65    }
66}
llm/providers/local/llama_cpp.rs

llm/providers/local/
llama_cpp.rs
