use super::provider::{LLMConfig, LLMProvider};
use super::tool_executor::ToolExecutor;
use super::types::*;
use std::sync::Arc;
pub struct LLMClient {
provider: Arc<dyn LLMProvider>,
config: LLMConfig,
}
impl LLMClient {
pub fn new(provider: Arc<dyn LLMProvider>) -> Self {
Self {
provider,
config: LLMConfig::default(),
}
}
pub fn with_config(provider: Arc<dyn LLMProvider>, config: LLMConfig) -> Self {
Self { provider, config }
}
pub fn provider(&self) -> &Arc<dyn LLMProvider> {
&self.provider
}
pub fn config(&self) -> &LLMConfig {
&self.config
}
pub fn chat(&self) -> ChatRequestBuilder {
let model = self
.config
.default_model
.clone()
.unwrap_or_else(|| self.provider.default_model().to_string());
let mut builder = ChatRequestBuilder::new(self.provider.clone(), model);
if let Some(temp) = self.config.default_temperature {
builder = builder.temperature(temp);
}
if let Some(tokens) = self.config.default_max_tokens {
builder = builder.max_tokens(tokens);
}
builder
}
pub async fn embed(&self, input: impl Into<String>) -> LLMResult<Vec<f32>> {
let model = self
.config
.default_model
.clone()
.unwrap_or_else(|| "text-embedding-ada-002".to_string());
let request = EmbeddingRequest {
model,
input: EmbeddingInput::Single(input.into()),
encoding_format: None,
dimensions: None,
user: None,
};
let response = self.provider.embedding(request).await?;
response
.data
.into_iter()
.next()
.map(|d| d.embedding)
.ok_or_else(|| LLMError::Other("No embedding data returned".to_string()))
}
pub async fn embed_batch(&self, inputs: Vec<String>) -> LLMResult<Vec<Vec<f32>>> {
let model = self
.config
.default_model
.clone()
.unwrap_or_else(|| "text-embedding-ada-002".to_string());
let request = EmbeddingRequest {
model,
input: EmbeddingInput::Multiple(inputs),
encoding_format: None,
dimensions: None,
user: None,
};
let response = self.provider.embedding(request).await?;
Ok(response.data.into_iter().map(|d| d.embedding).collect())
}
pub async fn ask(&self, question: impl Into<String>) -> LLMResult<String> {
let response = self.chat().user(question).send().await?;
response
.content()
.map(|s| s.to_string())
.ok_or_else(|| LLMError::Other("No content in response".to_string()))
}
pub async fn ask_with_system(
&self,
system: impl Into<String>,
question: impl Into<String>,
) -> LLMResult<String> {
let response = self.chat().system(system).user(question).send().await?;
response
.content()
.map(|s| s.to_string())
.ok_or_else(|| LLMError::Other("No content in response".to_string()))
}
}
pub struct ChatRequestBuilder {
provider: Arc<dyn LLMProvider>,
request: ChatCompletionRequest,
tool_executor: Option<Arc<dyn ToolExecutor>>,
max_tool_rounds: u32,
retry_policy: Option<LLMRetryPolicy>,
retry_enabled: bool,
}
impl ChatRequestBuilder {
pub fn new(provider: Arc<dyn LLMProvider>, model: impl Into<String>) -> Self {
Self {
provider,
request: ChatCompletionRequest::new(model),
tool_executor: None,
max_tool_rounds: 10,
retry_policy: None,
retry_enabled: false,
}
}
pub fn system(mut self, content: impl Into<String>) -> Self {
self.request.messages.push(ChatMessage::system(content));
self
}
pub fn user(mut self, content: impl Into<String>) -> Self {
self.request.messages.push(ChatMessage::user(content));
self
}
pub fn user_with_content(mut self, content: MessageContent) -> Self {
self.request
.messages
.push(ChatMessage::user_with_content(content));
self
}
pub fn user_with_parts(mut self, parts: Vec<ContentPart>) -> Self {
self.request
.messages
.push(ChatMessage::user_with_parts(parts));
self
}
pub fn assistant(mut self, content: impl Into<String>) -> Self {
self.request.messages.push(ChatMessage::assistant(content));
self
}
pub fn message(mut self, message: ChatMessage) -> Self {
self.request.messages.push(message);
self
}
pub fn messages(mut self, messages: Vec<ChatMessage>) -> Self {
self.request.messages.extend(messages);
self
}
pub fn temperature(mut self, temp: f32) -> Self {
self.request.temperature = Some(temp);
self
}
pub fn max_tokens(mut self, tokens: u32) -> Self {
self.request.max_tokens = Some(tokens);
self
}
pub fn tool(mut self, tool: Tool) -> Self {
self.request.tools.get_or_insert_with(Vec::new).push(tool);
self
}
pub fn tools(mut self, tools: Vec<Tool>) -> Self {
self.request.tools = Some(tools);
self
}
pub fn with_tool_executor(mut self, executor: Arc<dyn ToolExecutor>) -> Self {
self.tool_executor = Some(executor);
self
}
pub fn max_tool_rounds(mut self, rounds: u32) -> Self {
self.max_tool_rounds = rounds;
self
}
pub fn json_mode(mut self) -> Self {
self.request.response_format = Some(ResponseFormat::json());
self
}
pub fn stop(mut self, sequences: Vec<String>) -> Self {
self.request.stop = Some(sequences);
self
}
pub fn with_retry(mut self) -> Self {
self.retry_enabled = true;
self.retry_policy = Some(LLMRetryPolicy::default());
self
}
pub fn with_retry_policy(mut self, policy: LLMRetryPolicy) -> Self {
self.retry_enabled = true;
self.retry_policy = Some(policy);
self
}
pub fn without_retry(mut self) -> Self {
self.retry_enabled = false;
self.retry_policy = None;
self
}
pub fn max_retries(mut self, max: u32) -> Self {
if self.retry_policy.is_none() {
self.retry_policy = Some(LLMRetryPolicy::default());
}
if let Some(ref mut policy) = self.retry_policy {
policy.max_attempts = max;
}
self.retry_enabled = true;
self
}
pub async fn send(self) -> LLMResult<ChatCompletionResponse> {
if self.retry_enabled {
let policy = self.retry_policy.unwrap_or_default();
let executor = crate::llm::retry::RetryExecutor::new(self.provider, policy);
executor.chat(self.request).await
} else {
self.provider.chat(self.request).await
}
}
pub async fn send_stream(mut self) -> LLMResult<super::provider::ChatStream> {
self.request.stream = Some(true);
self.provider.chat_stream(self.request).await
}
pub async fn send_with_tools(mut self) -> LLMResult<ChatCompletionResponse> {
let executor = self
.tool_executor
.take()
.ok_or_else(|| LLMError::ConfigError("Tool executor not set".to_string()))?;
if self
.request
.tools
.as_ref()
.map(|tools| tools.is_empty())
.unwrap_or(true)
{
let tools = executor.available_tools().await?;
if !tools.is_empty() {
self.request.tools = Some(tools);
}
}
let max_rounds = self.max_tool_rounds;
let mut round = 0;
loop {
let response = self.provider.chat(self.request.clone()).await?;
if !response.has_tool_calls() {
return Ok(response);
}
round += 1;
if round >= max_rounds {
return Err(LLMError::Other(format!(
"Max tool rounds ({}) exceeded",
max_rounds
)));
}
if let Some(choice) = response.choices.first() {
self.request.messages.push(choice.message.clone());
}
if let Some(tool_calls) = response.tool_calls() {
for tool_call in tool_calls {
let result = executor
.execute(&tool_call.function.name, &tool_call.function.arguments)
.await;
let result_str = match result {
Ok(r) => r,
Err(e) => format!("Error: {}", e),
};
self.request
.messages
.push(ChatMessage::tool_result(&tool_call.id, result_str));
}
}
}
}
}
pub struct ChatSession {
session_id: uuid::Uuid,
user_id: uuid::Uuid,
agent_id: uuid::Uuid,
tenant_id: uuid::Uuid,
client: LLMClient,
messages: Vec<ChatMessage>,
system_prompt: Option<String>,
tools: Vec<Tool>,
tool_executor: Option<Arc<dyn ToolExecutor>>,
created_at: std::time::Instant,
metadata: std::collections::HashMap<String, String>,
message_store: Arc<dyn crate::persistence::MessageStore>,
session_store: Arc<dyn crate::persistence::SessionStore>,
context_window_size: Option<usize>,
last_response_metadata: Option<super::types::LLMResponseMetadata>,
}
impl ChatSession {
pub fn new(client: LLMClient) -> Self {
let store = Arc::new(crate::persistence::InMemoryStore::new());
Self::with_id_and_stores(
Self::generate_session_id(),
client,
uuid::Uuid::now_v7(),
uuid::Uuid::now_v7(),
uuid::Uuid::now_v7(),
store.clone(),
store.clone(),
None,
)
}
pub fn new_with_stores(
client: LLMClient,
user_id: uuid::Uuid,
tenant_id: uuid::Uuid,
agent_id: uuid::Uuid,
message_store: Arc<dyn crate::persistence::MessageStore>,
session_store: Arc<dyn crate::persistence::SessionStore>,
) -> Self {
Self::with_id_and_stores(
Self::generate_session_id(),
client,
user_id,
tenant_id,
agent_id,
message_store,
session_store,
None,
)
}
pub fn with_id(session_id: uuid::Uuid, client: LLMClient) -> Self {
let store = Arc::new(crate::persistence::InMemoryStore::new());
Self {
session_id,
user_id: uuid::Uuid::now_v7(),
agent_id: uuid::Uuid::now_v7(),
tenant_id: uuid::Uuid::now_v7(),
client,
messages: Vec::new(),
system_prompt: None,
tools: Vec::new(),
tool_executor: None,
created_at: std::time::Instant::now(),
metadata: std::collections::HashMap::new(),
message_store: store.clone(),
session_store: store.clone(),
context_window_size: None,
last_response_metadata: None,
}
}
pub fn with_id_str(session_id: &str, client: LLMClient) -> Self {
let session_id = uuid::Uuid::parse_str(session_id).unwrap_or_else(|_| uuid::Uuid::now_v7());
Self::with_id(session_id, client)
}
pub fn with_id_and_stores(
session_id: uuid::Uuid,
client: LLMClient,
user_id: uuid::Uuid,
tenant_id: uuid::Uuid,
agent_id: uuid::Uuid,
message_store: Arc<dyn crate::persistence::MessageStore>,
session_store: Arc<dyn crate::persistence::SessionStore>,
context_window_size: Option<usize>,
) -> Self {
Self {
session_id,
user_id,
tenant_id,
agent_id,
client,
messages: Vec::new(),
system_prompt: None,
tools: Vec::new(),
tool_executor: None,
created_at: std::time::Instant::now(),
metadata: std::collections::HashMap::new(),
message_store,
session_store,
context_window_size,
last_response_metadata: None,
}
}
pub async fn with_id_and_stores_and_persist(
session_id: uuid::Uuid,
client: LLMClient,
user_id: uuid::Uuid,
tenant_id: uuid::Uuid,
agent_id: uuid::Uuid,
message_store: Arc<dyn crate::persistence::MessageStore>,
session_store: Arc<dyn crate::persistence::SessionStore>,
context_window_size: Option<usize>,
) -> crate::persistence::PersistenceResult<Self> {
let session = Self::with_id_and_stores(
session_id,
client,
user_id,
tenant_id,
agent_id,
message_store,
session_store.clone(),
context_window_size,
);
let db_session =
crate::persistence::ChatSession::new(user_id, agent_id).with_id(session_id);
session_store.create_session(&db_session).await?;
Ok(session)
}
fn generate_session_id() -> uuid::Uuid {
uuid::Uuid::now_v7()
}
pub fn session_id(&self) -> uuid::Uuid {
self.session_id
}
pub fn session_id_str(&self) -> String {
self.session_id.to_string()
}
pub fn created_at(&self) -> std::time::Instant {
self.created_at
}
pub async fn load(
session_id: uuid::Uuid,
client: LLMClient,
user_id: uuid::Uuid,
tenant_id: uuid::Uuid,
agent_id: uuid::Uuid,
message_store: Arc<dyn crate::persistence::MessageStore>,
session_store: Arc<dyn crate::persistence::SessionStore>,
context_window_size: Option<usize>,
) -> crate::persistence::PersistenceResult<Self> {
let _db_session = session_store
.get_session(session_id)
.await?
.ok_or_else(|| {
crate::persistence::PersistenceError::NotFound("Session not found".to_string())
})?;
let db_messages = if context_window_size.is_some() {
let total_count = message_store.count_session_messages(session_id).await?;
let rounds = context_window_size.unwrap_or(0);
let limit = (rounds * 2 + 20) as i64;
let offset = std::cmp::max(0, total_count - limit);
message_store
.get_session_messages_paginated(session_id, offset, limit)
.await?
} else {
message_store.get_session_messages(session_id).await?
};
let mut messages = Vec::new();
for db_msg in db_messages {
let domain_role = match db_msg.role {
crate::persistence::MessageRole::System => crate::llm::types::Role::System,
crate::persistence::MessageRole::User => crate::llm::types::Role::User,
crate::persistence::MessageRole::Assistant => crate::llm::types::Role::Assistant,
crate::persistence::MessageRole::Tool => crate::llm::types::Role::Tool,
};
let domain_content = db_msg
.content
.text
.map(crate::llm::types::MessageContent::Text);
let domain_msg = ChatMessage {
role: domain_role,
content: domain_content,
name: None,
tool_calls: None,
tool_call_id: None,
};
messages.push(domain_msg);
}
let messages = Self::apply_sliding_window_static(&messages, context_window_size);
Ok(Self {
session_id,
user_id,
tenant_id,
agent_id,
client,
messages,
system_prompt: None, tools: Vec::new(), tool_executor: None, created_at: std::time::Instant::now(), metadata: std::collections::HashMap::new(), message_store,
session_store,
context_window_size,
last_response_metadata: None,
})
}
pub fn elapsed(&self) -> std::time::Duration {
self.created_at.elapsed()
}
pub fn set_metadata(&mut self, key: impl Into<String>, value: impl Into<String>) {
self.metadata.insert(key.into(), value.into());
}
pub fn get_metadata(&self, key: &str) -> Option<&String> {
self.metadata.get(key)
}
pub fn metadata(&self) -> &std::collections::HashMap<String, String> {
&self.metadata
}
pub fn with_system(mut self, prompt: impl Into<String>) -> Self {
self.system_prompt = Some(prompt.into());
self
}
pub fn with_context_window_size(mut self, size: Option<usize>) -> Self {
self.context_window_size = size;
self
}
pub fn with_tools(mut self, tools: Vec<Tool>, executor: Arc<dyn ToolExecutor>) -> Self {
self.tools = tools;
self.tool_executor = Some(executor);
self
}
pub fn with_tool_executor(mut self, executor: Arc<dyn ToolExecutor>) -> Self {
self.tool_executor = Some(executor);
self
}
pub async fn send(&mut self, content: impl Into<String>) -> LLMResult<String> {
self.messages.push(ChatMessage::user(content));
let mut builder = self.client.chat();
if let Some(ref system) = self.system_prompt {
builder = builder.system(system.clone());
}
let messages_for_context = self.apply_sliding_window();
builder = builder.messages(messages_for_context);
if let Some(ref executor) = self.tool_executor {
let tools = if self.tools.is_empty() {
executor.available_tools().await?
} else {
self.tools.clone()
};
if !tools.is_empty() {
builder = builder.tools(tools);
}
builder = builder.with_tool_executor(executor.clone());
}
let response = if self.tool_executor.is_some() {
builder.send_with_tools().await?
} else {
builder.send().await?
};
self.last_response_metadata = Some(super::types::LLMResponseMetadata::from(&response));
let content = response
.content()
.ok_or_else(|| LLMError::Other("No content in response".to_string()))?
.to_string();
self.messages.push(ChatMessage::assistant(&content));
if self.context_window_size.is_some() {
self.messages =
Self::apply_sliding_window_static(&self.messages, self.context_window_size);
}
Ok(content)
}
pub async fn send_with_content(&mut self, content: MessageContent) -> LLMResult<String> {
self.messages.push(ChatMessage::user_with_content(content));
let mut builder = self.client.chat();
if let Some(ref system) = self.system_prompt {
builder = builder.system(system.clone());
}
let messages_for_context = self.apply_sliding_window();
builder = builder.messages(messages_for_context);
if let Some(ref executor) = self.tool_executor {
let tools = if self.tools.is_empty() {
executor.available_tools().await?
} else {
self.tools.clone()
};
if !tools.is_empty() {
builder = builder.tools(tools);
}
builder = builder.with_tool_executor(executor.clone());
}
let response = if self.tool_executor.is_some() {
builder.send_with_tools().await?
} else {
builder.send().await?
};
self.last_response_metadata = Some(super::types::LLMResponseMetadata::from(&response));
let content = response
.content()
.ok_or_else(|| LLMError::Other("No content in response".to_string()))?
.to_string();
self.messages.push(ChatMessage::assistant(&content));
if self.context_window_size.is_some() {
self.messages =
Self::apply_sliding_window_static(&self.messages, self.context_window_size);
}
Ok(content)
}
pub fn messages(&self) -> &[ChatMessage] {
&self.messages
}
pub fn messages_mut(&mut self) -> &mut Vec<ChatMessage> {
&mut self.messages
}
pub fn clear(&mut self) {
self.messages.clear();
}
pub fn len(&self) -> usize {
self.messages.len()
}
pub fn is_empty(&self) -> bool {
self.messages.is_empty()
}
pub fn set_context_window_size(&mut self, size: Option<usize>) {
self.context_window_size = size;
}
pub fn context_window_size(&self) -> Option<usize> {
self.context_window_size
}
pub fn last_response_metadata(&self) -> Option<&super::types::LLMResponseMetadata> {
self.last_response_metadata.as_ref()
}
fn apply_sliding_window(&self) -> Vec<ChatMessage> {
Self::apply_sliding_window_static(&self.messages, self.context_window_size)
}
pub fn apply_sliding_window_static(
messages: &[ChatMessage],
window_size: Option<usize>,
) -> Vec<ChatMessage> {
let max_rounds = match window_size {
Some(size) if size > 0 => size,
_ => return messages.to_vec(), };
let mut system_messages = Vec::new();
let mut conversation_messages = Vec::new();
for msg in messages {
if msg.role == Role::System {
system_messages.push(msg.clone());
} else {
conversation_messages.push(msg.clone());
}
}
let max_messages = max_rounds * 2;
if conversation_messages.len() <= max_messages {
return messages.to_vec();
}
let start_index = conversation_messages.len() - max_messages;
let limited_conversation: Vec<ChatMessage> = conversation_messages
.into_iter()
.skip(start_index)
.collect();
let mut result = system_messages;
result.extend(limited_conversation);
result
}
pub async fn save(&self) -> crate::persistence::PersistenceResult<()> {
let db_session = crate::persistence::ChatSession::new(self.user_id, self.agent_id)
.with_id(self.session_id)
.with_metadata("client_version", serde_json::json!("0.1.0"));
self.session_store.create_session(&db_session).await?;
for msg in self.messages.iter() {
let persistence_role = match msg.role {
crate::llm::types::Role::System => crate::persistence::MessageRole::System,
crate::llm::types::Role::User => crate::persistence::MessageRole::User,
crate::llm::types::Role::Assistant => crate::persistence::MessageRole::Assistant,
crate::llm::types::Role::Tool => crate::persistence::MessageRole::Tool,
};
let persistence_content = match &msg.content {
Some(crate::llm::types::MessageContent::Text(text)) => {
crate::persistence::MessageContent::text(text)
}
Some(crate::llm::types::MessageContent::Parts(parts)) => {
let text = parts
.iter()
.filter_map(|part| {
if let crate::llm::types::ContentPart::Text { text } = part {
Some(text.clone())
} else {
None
}
})
.collect::<Vec<_>>()
.join("\n");
crate::persistence::MessageContent::text(text)
}
None => crate::persistence::MessageContent::text(""),
};
let llm_message = crate::persistence::LLMMessage::new(
self.session_id,
self.agent_id,
self.user_id,
self.tenant_id,
persistence_role,
persistence_content,
);
self.message_store.save_message(&llm_message).await?;
}
Ok(())
}
pub async fn delete(&self) -> crate::persistence::PersistenceResult<()> {
self.message_store
.delete_session_messages(self.session_id)
.await?;
self.session_store.delete_session(self.session_id).await?;
Ok(())
}
}
pub fn function_tool(
name: impl Into<String>,
description: impl Into<String>,
parameters: serde_json::Value,
) -> Tool {
Tool::function(name, description, parameters)
}