llm/providers/local/
llama_cpp.rs

1#![doc = include_str!(concat!(env!("OUT_DIR"), "/docs/llamacpp.md"))]
2
3use super::util::get_local_config;
4use crate::providers::openai::OpenAiChatProvider;
5use crate::{ProviderConnectionConfig, ProviderFactory, Result};
6use async_openai::{Client, config::OpenAIConfig};
7use std::future::ready;
8
9pub struct LlamaCppProvider {
10    client: Client<OpenAIConfig>,
11}
12
13impl LlamaCppProvider {
14    pub fn new(base_url: &str) -> Self {
15        Self { client: Client::with_config(get_local_config(base_url)) }
16    }
17}
18
19impl Default for LlamaCppProvider {
20    fn default() -> Self {
21        Self { client: Client::with_config(get_local_config("http://localhost:8080/v1")) }
22    }
23}
24
25impl ProviderFactory for LlamaCppProvider {
26    async fn from_env() -> Result<Self> {
27        Self::from_env_with_connection(ProviderConnectionConfig::default()).await
28    }
29
30    fn from_env_with_connection(connection: ProviderConnectionConfig) -> impl Future<Output = Result<Self>> + Send {
31        let base_url = connection.base_url.as_deref().unwrap_or("http://localhost:8080/v1");
32        ready(Ok(Self { client: Client::with_config(get_local_config(base_url)) }))
33    }
34
35    fn with_model(self, _model: &str) -> Self {
36        // LlamaCpp doesn't support model selection - it serves a single model
37        self
38    }
39}
40
41impl OpenAiChatProvider for LlamaCppProvider {
42    type Config = OpenAIConfig;
43
44    fn client(&self) -> &Client<Self::Config> {
45        &self.client
46    }
47
48    fn model(&self) -> &'static str {
49        "" // llama.cpp server serves a single model on boot and does not allow swapping models
50    }
51
52    fn provider_name(&self) -> &'static str {
53        "LlamaCpp"
54    }
55}
llm/providers/local/llama_cpp.rs

llm/providers/local/
llama_cpp.rs
