use super::types::*;
use async_trait::async_trait;
use futures::Stream;
use std::pin::Pin;
pub type ChatStream = Pin<Box<dyn Stream<Item = LLMResult<ChatCompletionChunk>> + Send>>;
#[async_trait]
pub trait LLMProvider: Send + Sync {
fn name(&self) -> &str;
fn default_model(&self) -> &str {
""
}
fn supported_models(&self) -> Vec<&str> {
vec![]
}
fn supports_model(&self, model: &str) -> bool {
self.supported_models().contains(&model)
}
fn supports_streaming(&self) -> bool {
true
}
fn supports_tools(&self) -> bool {
true
}
fn supports_vision(&self) -> bool {
false
}
fn supports_embedding(&self) -> bool {
false
}
async fn chat(&self, request: ChatCompletionRequest) -> LLMResult<ChatCompletionResponse>;
async fn chat_stream(&self, _request: ChatCompletionRequest) -> LLMResult<ChatStream> {
Err(LLMError::ProviderNotSupported(format!(
"Provider {} does not support streaming",
self.name()
)))
}
async fn embedding(&self, _request: EmbeddingRequest) -> LLMResult<EmbeddingResponse> {
Err(LLMError::ProviderNotSupported(format!(
"Provider {} does not support embedding",
self.name()
)))
}
async fn health_check(&self) -> LLMResult<bool> {
Ok(true)
}
async fn get_model_info(&self, _model: &str) -> LLMResult<ModelInfo> {
Err(LLMError::ProviderNotSupported(format!(
"Provider {} does not support model info",
self.name()
)))
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ModelInfo {
pub id: String,
pub name: String,
pub description: Option<String>,
pub context_window: Option<u32>,
pub max_output_tokens: Option<u32>,
pub training_cutoff: Option<String>,
pub capabilities: ModelCapabilities,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
pub struct ModelCapabilities {
pub streaming: bool,
pub tools: bool,
pub vision: bool,
pub json_mode: bool,
pub json_schema: bool,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct LLMConfig {
pub provider: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub api_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub base_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub default_model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub default_temperature: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub default_max_tokens: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub timeout_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_retries: Option<u32>,
#[serde(flatten)]
pub extra: HashMap<String, serde_json::Value>,
}
impl Default for LLMConfig {
fn default() -> Self {
Self {
provider: "openai".to_string(),
api_key: None,
base_url: None,
default_model: None,
default_temperature: Some(0.7),
default_max_tokens: Some(4096),
timeout_secs: Some(60),
max_retries: Some(3),
extra: std::collections::HashMap::new(),
}
}
}
impl LLMConfig {
pub fn openai(api_key: impl Into<String>) -> Self {
Self {
provider: "openai".to_string(),
api_key: Some(api_key.into()),
base_url: Some("https://api.openai.com/v1".to_string()),
default_model: Some("gpt-4".to_string()),
..Default::default()
}
}
pub fn anthropic(api_key: impl Into<String>) -> Self {
Self {
provider: "anthropic".to_string(),
api_key: Some(api_key.into()),
base_url: Some("https://api.anthropic.com".to_string()),
default_model: Some("claude-3-sonnet-20240229".to_string()),
..Default::default()
}
}
pub fn ollama(model: impl Into<String>) -> Self {
Self {
provider: "ollama".to_string(),
api_key: None,
base_url: Some("http://localhost:11434".to_string()),
default_model: Some(model.into()),
..Default::default()
}
}
pub fn openai_compatible(
base_url: impl Into<String>,
api_key: impl Into<String>,
model: impl Into<String>,
) -> Self {
Self {
provider: "openai-compatible".to_string(),
api_key: Some(api_key.into()),
base_url: Some(base_url.into()),
default_model: Some(model.into()),
..Default::default()
}
}
pub fn model(mut self, model: impl Into<String>) -> Self {
self.default_model = Some(model.into());
self
}
pub fn temperature(mut self, temp: f32) -> Self {
self.default_temperature = Some(temp);
self
}
pub fn max_tokens(mut self, tokens: u32) -> Self {
self.default_max_tokens = Some(tokens);
self
}
}
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
pub type ProviderFactory = Box<dyn Fn(LLMConfig) -> LLMResult<Box<dyn LLMProvider>> + Send + Sync>;
pub struct LLMRegistry {
factories: RwLock<HashMap<String, ProviderFactory>>,
providers: RwLock<HashMap<String, Arc<dyn LLMProvider>>>,
}
impl LLMRegistry {
pub fn new() -> Self {
Self {
factories: RwLock::new(HashMap::new()),
providers: RwLock::new(HashMap::new()),
}
}
pub async fn register_factory<F>(&self, name: &str, factory: F)
where
F: Fn(LLMConfig) -> LLMResult<Box<dyn LLMProvider>> + Send + Sync + 'static,
{
let mut factories = self.factories.write().await;
factories.insert(name.to_string(), Box::new(factory));
}
pub async fn create(&self, config: LLMConfig) -> LLMResult<Arc<dyn LLMProvider>> {
let factories = self.factories.read().await;
let factory = factories
.get(&config.provider)
.ok_or_else(|| LLMError::ProviderNotSupported(config.provider.clone()))?;
let provider = factory(config)?;
Ok(Arc::from(provider))
}
pub async fn register(&self, name: &str, provider: Arc<dyn LLMProvider>) {
let mut providers = self.providers.write().await;
providers.insert(name.to_string(), provider);
}
pub async fn get(&self, name: &str) -> Option<Arc<dyn LLMProvider>> {
let providers = self.providers.read().await;
providers.get(name).cloned()
}
pub async fn list_providers(&self) -> Vec<String> {
let providers = self.providers.read().await;
providers.keys().cloned().collect()
}
pub async fn list_factories(&self) -> Vec<String> {
let factories = self.factories.read().await;
factories.keys().cloned().collect()
}
}
impl Default for LLMRegistry {
fn default() -> Self {
Self::new()
}
}
use std::sync::OnceLock;
static GLOBAL_REGISTRY: OnceLock<LLMRegistry> = OnceLock::new();
pub fn global_registry() -> &'static LLMRegistry {
GLOBAL_REGISTRY.get_or_init(LLMRegistry::new)
}