use futures_util::StreamExt;
use llm_connector::types::{ChatRequest, Message, Role};
use llm_connector::LlmClient;
use tracing::{debug, info};
pub struct LlmConnector {
client: LlmClient,
model: String,
}
#[derive(Debug, Clone)]
pub struct LlmResponse {
pub content: String,
pub prompt_tokens: Option<u64>,
pub completion_tokens: Option<u64>,
pub total_tokens: Option<u64>,
}
pub type StreamCallback = Box<dyn Fn(&str) + Send + Sync>;
impl LlmConnector {
pub fn new(client: LlmClient, model: impl Into<String>) -> Self {
Self {
client,
model: model.into(),
}
}
pub fn openai(api_key: &str, model: &str) -> anyhow::Result<Self> {
let client = LlmClient::openai(api_key)?;
Ok(Self::new(client, model))
}
pub fn anthropic(api_key: &str, model: &str) -> anyhow::Result<Self> {
let client = LlmClient::anthropic(api_key)?;
Ok(Self::new(client, model))
}
pub fn deepseek(api_key: &str, model: &str) -> anyhow::Result<Self> {
let client = LlmClient::deepseek(api_key)?;
Ok(Self::new(client, model))
}
pub fn ollama(model: &str) -> anyhow::Result<Self> {
let client = LlmClient::ollama()?;
Ok(Self::new(client, model))
}
pub fn builder() -> llm_connector::builder::LlmClientBuilder {
LlmClient::builder()
}
pub fn model(&self) -> &str {
&self.model
}
pub async fn chat(&self, messages: &[LlmMessage]) -> anyhow::Result<LlmResponse> {
let request = ChatRequest {
model: self.model.clone(),
messages: messages.iter().map(|m| m.to_llm_message()).collect(),
..Default::default()
};
info!(model = %self.model, msg_count = messages.len(), "calling LLM");
let response = self.client.chat(&request).await?;
Ok(LlmResponse {
content: response.content.clone(),
prompt_tokens: response.usage.as_ref().map(|u| u.prompt_tokens as u64),
completion_tokens: response.usage.as_ref().map(|u| u.completion_tokens as u64),
total_tokens: response.usage.as_ref().map(|u| u.total_tokens as u64),
})
}
pub async fn chat_stream(
&self,
messages: &[LlmMessage],
on_chunk: Option<StreamCallback>,
) -> anyhow::Result<LlmResponse> {
let request = ChatRequest {
model: self.model.clone(),
messages: messages.iter().map(|m| m.to_llm_message()).collect(),
stream: Some(true),
..Default::default()
};
info!(model = %self.model, msg_count = messages.len(), "calling LLM (stream)");
let mut stream = self.client.chat_stream(&request).await?;
let mut full_content = String::new();
while let Some(chunk) = stream.next().await {
match chunk {
Ok(resp) => {
if let Some(content) = resp.get_content() {
full_content.push_str(content);
if let Some(cb) = &on_chunk {
cb(content);
}
debug!(chunk_len = content.len(), "stream chunk received");
}
}
Err(e) => {
return Err(anyhow::anyhow!("stream error: {}", e));
}
}
}
Ok(LlmResponse {
content: full_content,
prompt_tokens: None,
completion_tokens: None,
total_tokens: None,
})
}
}
#[derive(Debug, Clone)]
pub struct LlmMessage {
pub role: LlmRole,
pub content: String,
}
#[derive(Debug, Clone)]
pub enum LlmRole {
System,
User,
Assistant,
Tool,
}
impl LlmMessage {
pub fn system(content: impl Into<String>) -> Self {
Self {
role: LlmRole::System,
content: content.into(),
}
}
pub fn user(content: impl Into<String>) -> Self {
Self {
role: LlmRole::User,
content: content.into(),
}
}
pub fn assistant(content: impl Into<String>) -> Self {
Self {
role: LlmRole::Assistant,
content: content.into(),
}
}
fn to_llm_message(&self) -> Message {
let role = match self.role {
LlmRole::System => Role::System,
LlmRole::User => Role::User,
LlmRole::Assistant => Role::Assistant,
LlmRole::Tool => Role::User,
};
Message::text(role, &self.content)
}
}