#[cfg(feature = "test-util")]
pub mod mock;
pub mod model_variants;
pub mod openai_compat;
pub mod openai_compat_provider;
pub mod sockudo;
pub mod tool_pairs;
use std::fmt::Display;
use std::future::Future;
use serde::{Deserialize, Serialize};
use crate::SecretString;
use crate::config::Settings;
use crate::image::ImageAttachment;
use crate::provider::openai_compat_provider::OpenAiCompatProvider;
use crate::provider::sockudo::SockudoProvider;
pub const STREAM_CHANNEL_CAPACITY: usize = 64;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolDefinition {
pub name: String,
pub description: String,
pub parameters: schemars::Schema,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolCall {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
pub function: ToolCallFunction,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolCallFunction {
pub name: String,
pub arguments: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub thought_signature: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatMessage {
pub content: String,
#[serde(default)]
pub tool_calls: Vec<ToolCall>,
#[serde(default)]
pub thinking: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct ChatMessageResponse {
pub message: ChatMessage,
pub done: bool,
#[serde(default)]
pub is_error: bool,
#[serde(default)]
pub usage: Option<TokenUsage>,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct TokenUsage {
pub prompt_tokens: u32,
pub completion_tokens: u32,
pub total_tokens: u32,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub enum Role {
System,
User,
Assistant,
Tool,
}
impl Display for Role {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Role::System => f.write_str("System"),
Role::User => f.write_str("User"),
Role::Assistant => f.write_str("Assistant"),
Role::Tool => f.write_str("Tool"),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Message {
pub role: Role,
pub content: String,
pub tool_calls: Vec<ToolCall>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tool_call_id: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub images: Vec<ImageAttachment>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thinking: Option<String>,
}
impl Default for Message {
fn default() -> Self {
Message::simple(Role::User, "")
}
}
impl Message {
pub fn simple(role: Role, content: impl Into<String>) -> Self {
Message {
role,
content: content.into(),
tool_calls: vec![],
tool_call_id: None,
images: vec![],
thinking: None,
}
}
}
pub trait Provider: Send + Sync {
fn health_check(&self) -> impl Future<Output = Result<(), String>> + Send;
fn list_models(&self) -> impl Future<Output = Vec<String>> + Send;
fn select_model(&mut self, name: String);
fn current_model(&self) -> Option<String>;
fn chat(
&mut self,
messages: Vec<Message>,
tools: Vec<ToolDefinition>,
) -> impl Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>> + Send;
fn set_timeout(&mut self, _timeout_secs: u64) {}
fn set_retries(&mut self, _max_retries: u32) {}
}
pub enum AnyProvider {
OpenAiCompat(crate::provider::openai_compat_provider::OpenAiCompatProvider),
Sockudo(crate::provider::sockudo::SockudoProvider),
#[cfg(feature = "test-util")]
Mock(crate::provider::mock::MockProviderHandle),
}
impl AnyProvider {
#[allow(clippy::too_many_arguments)]
pub fn build(
kind: crate::config::ProviderKind,
base_url: String,
api_key: Option<SecretString>,
timeout_secs: u64,
max_retries: u32,
sockudo: SockudoCredentials,
) -> Result<Self, String> {
Ok(match kind {
crate::config::ProviderKind::Ollama => AnyProvider::OpenAiCompat(
OpenAiCompatProvider::with_options(base_url, None, timeout_secs, max_retries),
),
crate::config::ProviderKind::LlamaCpp => AnyProvider::OpenAiCompat(
OpenAiCompatProvider::with_options(base_url, None, timeout_secs, max_retries)
.with_static_models(vec!["llama-cpp".to_string()]),
),
crate::config::ProviderKind::Vllm => AnyProvider::OpenAiCompat(
OpenAiCompatProvider::with_options(base_url, None, timeout_secs, max_retries),
),
crate::config::ProviderKind::OpenAiCompat => {
let key = api_key.filter(|k| !k.is_empty());
AnyProvider::OpenAiCompat(OpenAiCompatProvider::with_options(
base_url,
key,
timeout_secs,
max_retries,
))
}
crate::config::ProviderKind::Sockudo => AnyProvider::Sockudo(SockudoProvider::new(
base_url,
sockudo.app_id,
sockudo.app_key,
sockudo.app_secret.unwrap_or_default(),
)),
})
}
}
#[derive(Debug, Clone, Default)]
pub struct SockudoCredentials {
pub app_id: String,
pub app_key: String,
pub app_secret: Option<SecretString>,
}
impl From<&Settings> for SockudoCredentials {
fn from(settings: &Settings) -> Self {
SockudoCredentials {
app_id: settings.sockudo_app_id.clone().unwrap_or_default(),
app_key: settings.sockudo_app_key.clone().unwrap_or_default(),
app_secret: settings.sockudo_app_secret.clone(),
}
}
}
impl Provider for AnyProvider {
async fn health_check(&self) -> Result<(), String> {
match self {
AnyProvider::OpenAiCompat(p) => p.health_check().await,
AnyProvider::Sockudo(p) => p.health_check().await,
#[cfg(feature = "test-util")]
AnyProvider::Mock(p) => p.health_check().await,
}
}
async fn list_models(&self) -> Vec<String> {
match self {
AnyProvider::OpenAiCompat(p) => p.list_models().await,
AnyProvider::Sockudo(p) => p.list_models().await,
#[cfg(feature = "test-util")]
AnyProvider::Mock(p) => p.list_models().await,
}
}
fn select_model(&mut self, name: String) {
match self {
AnyProvider::OpenAiCompat(p) => p.select_model(name),
AnyProvider::Sockudo(p) => p.select_model(name),
#[cfg(feature = "test-util")]
AnyProvider::Mock(p) => p.select_model(name),
}
}
fn current_model(&self) -> Option<String> {
match self {
AnyProvider::OpenAiCompat(p) => p.current_model(),
AnyProvider::Sockudo(p) => p.current_model(),
#[cfg(feature = "test-util")]
AnyProvider::Mock(p) => p.current_model(),
}
}
async fn chat(
&mut self,
messages: Vec<Message>,
tools: Vec<ToolDefinition>,
) -> Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String> {
match self {
AnyProvider::OpenAiCompat(p) => p.chat(messages, tools).await,
AnyProvider::Sockudo(p) => p.chat(messages, tools).await,
#[cfg(feature = "test-util")]
AnyProvider::Mock(p) => p.chat(messages, tools).await,
}
}
fn set_timeout(&mut self, timeout_secs: u64) {
match self {
AnyProvider::OpenAiCompat(p) => p.set_timeout(timeout_secs),
AnyProvider::Sockudo(p) => p.set_timeout(timeout_secs),
#[cfg(feature = "test-util")]
AnyProvider::Mock(p) => p.set_timeout(timeout_secs),
}
}
fn set_retries(&mut self, max_retries: u32) {
match self {
AnyProvider::OpenAiCompat(p) => p.set_retries(max_retries),
AnyProvider::Sockudo(p) => p.set_retries(max_retries),
#[cfg(feature = "test-util")]
AnyProvider::Mock(p) => p.set_retries(max_retries),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::provider::mock::{MockProvider, MockProviderHandle};
use std::sync::{Arc, Mutex as StdMutex};
#[test]
fn build_returns_provider_for_each_kind() {
let creds = SockudoCredentials::default();
for kind in [
crate::config::ProviderKind::Ollama,
crate::config::ProviderKind::LlamaCpp,
crate::config::ProviderKind::Vllm,
crate::config::ProviderKind::OpenAiCompat,
crate::config::ProviderKind::Sockudo,
] {
let provider = AnyProvider::build(
kind,
"http://localhost:1234".to_string(),
Some(SecretString::new("k")),
30,
0,
creds.clone(),
)
.unwrap_or_else(|e| panic!("build({kind}) failed: {e}"));
let _ = provider.current_model();
}
}
#[tokio::test]
async fn any_provider_delegates_to_mock() {
let mock = Arc::new(StdMutex::new(MockProvider::new()));
mock.lock().unwrap().enqueue_text("via any");
let mut provider = AnyProvider::Mock(MockProviderHandle::new(mock));
assert_eq!(provider.current_model().as_deref(), Some("mock-model"));
provider.select_model("m2".to_string());
assert_eq!(provider.current_model().as_deref(), Some("m2"));
let mut rx = provider
.chat(vec![Message::simple(Role::User, "hi")], vec![])
.await
.expect("chat via AnyProvider should work");
let chunk = rx.recv().await.expect("chunk");
assert!(chunk.done);
assert_eq!(chunk.message.content, "via any");
}
#[tokio::test]
async fn any_provider_health_check_delegates() {
let mut mock = MockProvider::new();
assert!(mock.health_check().await.is_ok());
mock.set_health_ok(false);
assert!(mock.health_check().await.is_err());
}
}