use std::collections::HashMap;
use std::pin::Pin;
use std::sync::Arc;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use futures_core::Stream;
use serde::{Deserialize, Serialize};
use rust_decimal::Decimal;
use crate::event::TurnUsage;
use crate::message::Message;
use crate::model::{
AgentLlmConfig, Model, Provider, ProviderPreferences, ReasoningLevel, SamplingParams,
};
use crate::session::SessionId;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolDefinition {
pub name: String,
pub description: String,
pub parameters: serde_json::Value,
}
#[derive(Debug)]
pub struct CompletionRequest {
pub model: Model,
pub messages: Vec<Message>,
pub tools: Vec<ToolDefinition>,
pub max_tokens: Option<u32>,
pub reasoning: ReasoningLevel,
pub sampling: SamplingParams,
pub modalities: Vec<Modality>,
pub audio_config: Option<AudioOutputConfig>,
pub image_config: Option<ImageGenConfig>,
pub user: Option<String>,
pub provider_preferences: Option<ProviderPreferences>,
pub session_id: Option<SessionId>,
}
impl CompletionRequest {
pub fn new(model: Model, messages: Vec<Message>) -> Self {
Self::from_config(&AgentLlmConfig::new(model), messages)
}
pub fn from_config(config: &AgentLlmConfig, messages: Vec<Message>) -> Self {
Self {
model: config.model.clone(),
messages,
tools: Vec::new(),
max_tokens: config.max_tokens,
reasoning: config.reasoning,
sampling: config.sampling.clone(),
modalities: Vec::new(),
audio_config: None,
image_config: None,
user: config.user.clone(),
provider_preferences: config.provider_preferences.clone(),
session_id: config.session_id.clone(),
}
}
pub fn with_session_id(mut self, session_id: impl Into<SessionId>) -> Self {
self.session_id = Some(session_id.into());
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display, strum::EnumString)]
#[strum(serialize_all = "snake_case")]
pub enum FinishReason {
Stop,
ToolCalls,
Length,
ContentFilter,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::Display, strum::EnumString)]
#[strum(serialize_all = "snake_case")]
pub enum ModelCategory {
Chat,
Reasoning,
Vision,
ImageGeneration,
Tts,
Stt,
VideoGeneration,
VideoInput,
}
#[derive(Debug, Clone, Default)]
pub struct ModelPricing {
pub prompt: Option<Decimal>,
pub completion: Option<Decimal>,
pub image: Option<Decimal>,
pub cache_read: Option<Decimal>,
pub cache_write: Option<Decimal>,
}
#[derive(Debug, Clone)]
pub struct ModelInfo {
pub id: String,
pub name: Option<String>,
pub created_at: Option<DateTime<Utc>>,
pub context_length: Option<u32>,
pub max_completion_tokens: Option<u32>,
pub capabilities: ModelCapabilities,
pub categories: Vec<ModelCategory>,
pub pricing: ModelPricing,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct ModelCapabilities {
pub tool_calling: bool,
pub images: bool,
pub documents: bool,
pub video: bool,
pub audio: bool,
pub reasoning: bool,
pub audio_output: bool,
pub image_generation: bool,
pub video_generation: bool,
}
impl ModelCapabilities {
pub fn categories(&self) -> Vec<ModelCategory> {
let mut cats = vec![ModelCategory::Chat];
if self.reasoning {
cats.push(ModelCategory::Reasoning);
}
if self.images {
cats.push(ModelCategory::Vision);
}
if self.image_generation {
cats.push(ModelCategory::ImageGeneration);
}
if self.audio_output {
cats.push(ModelCategory::Tts);
}
if self.audio {
cats.push(ModelCategory::Stt);
}
if self.video_generation {
cats.push(ModelCategory::VideoGeneration);
}
if self.video {
cats.push(ModelCategory::VideoInput);
}
cats
}
pub fn all() -> Self {
Self {
tool_calling: true,
images: true,
documents: true,
video: false,
audio: true,
reasoning: true,
audio_output: false,
image_generation: false,
video_generation: false,
}
}
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
pub struct TokenUsage {
pub prompt_tokens: u32,
pub completion_tokens: u32,
pub total_tokens: u32,
pub cache_read_tokens: u32,
pub cache_creation_tokens: u32,
}
impl TokenUsage {
pub fn cost(&self, pricing: &ModelPricing) -> Option<Decimal> {
let prompt_price = pricing.prompt?;
let completion_price = pricing.completion?;
let cache_read_price = pricing.cache_read.unwrap_or(prompt_price);
let cache_write_price = pricing.cache_write.unwrap_or(prompt_price);
let non_cached = self
.prompt_tokens
.saturating_sub(self.cache_read_tokens + self.cache_creation_tokens);
Some(
Decimal::from(non_cached) * prompt_price
+ Decimal::from(self.completion_tokens) * completion_price
+ Decimal::from(self.cache_read_tokens) * cache_read_price
+ Decimal::from(self.cache_creation_tokens) * cache_write_price,
)
}
}
impl From<TurnUsage> for TokenUsage {
fn from(u: TurnUsage) -> Self {
Self {
prompt_tokens: u.prompt_tokens,
completion_tokens: u.completion_tokens,
total_tokens: u.prompt_tokens + u.completion_tokens,
cache_read_tokens: u.cache_read_tokens,
cache_creation_tokens: u.cache_creation_tokens,
}
}
}
#[derive(Debug, Clone)]
pub struct CompletionResponse {
pub message: Message,
pub usage: TokenUsage,
pub finish_reason: FinishReason,
}
#[derive(Debug, Clone)]
pub enum StreamEvent {
ReasoningDelta(String),
ContentDelta(String),
ToolCallStart {
index: usize,
id: String,
name: String,
},
ToolCallDelta { index: usize, arguments: String },
Usage(TokenUsage),
AudioDelta { data: Vec<u8>, format: String },
FileAttachment {
filename: String,
media_type: String,
data: Vec<u8>,
},
Finished(FinishReason),
}
pub type CompletionStream =
Pin<Box<dyn Stream<Item = anyhow::Result<StreamEvent>> + Send + 'static>>;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Voice {
pub id: String,
pub name: String,
}
pub struct TtsRequest {
pub model: String,
pub input: String,
pub voice: String,
pub response_format: AudioFormat,
}
pub struct SttRequest {
pub model: String,
pub audio: Vec<u8>,
pub media_type: String,
pub language: Option<String>,
}
#[derive(
Debug, Clone, Copy, Default, strum::EnumString, strum::Display, Serialize, Deserialize,
)]
#[strum(serialize_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum AudioFormat {
#[default]
Mp3,
Wav,
Opus,
Aac,
Flac,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, strum::Display)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")]
pub enum Modality {
Text,
Audio,
Image,
}
#[derive(Debug, Clone)]
pub struct AudioOutputConfig {
pub voice: String,
pub format: AudioFormat,
}
#[derive(Debug, Clone, Default)]
pub struct ImageGenConfig {
pub aspect_ratio: Option<String>,
pub size: Option<String>,
}
impl ImageGenConfig {
pub fn data_uri_from_path(path: &std::path::Path) -> std::io::Result<String> {
let bytes = std::fs::read(path)?;
let mime = mime_from_extension(path);
Ok(Self::data_uri_from_bytes(&bytes, mime))
}
pub fn data_uri_from_bytes(bytes: &[u8], mime: &str) -> String {
use base64::Engine;
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
format!("data:{mime};base64,{b64}")
}
}
pub fn mime_from_extension(path: &std::path::Path) -> &'static str {
match path
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.as_deref()
{
Some("png") => "image/png",
Some("jpg" | "jpeg") => "image/jpeg",
Some("gif") => "image/gif",
Some("webp") => "image/webp",
Some("bmp") => "image/bmp",
Some("svg") => "image/svg+xml",
_ => "application/octet-stream",
}
}
#[derive(Debug, Clone)]
pub struct ImageGenRequest {
pub model: String,
pub prompt: String,
pub size: Option<String>,
pub aspect_ratio: Option<String>,
pub quality: Option<String>,
pub style: Option<String>,
pub n: Option<u32>,
}
#[derive(Debug, Clone)]
pub struct VideoGenRequest {
pub model: String,
pub description: String,
pub resolution: Option<String>,
pub aspect_ratio: Option<String>,
pub duration: Option<u32>,
pub generate_audio: Option<bool>,
pub frame_images: Vec<FrameImage>,
pub input_references: Vec<String>,
}
#[derive(Debug, Clone)]
pub struct FrameImage {
pub url: String,
pub frame_type: String,
}
#[async_trait]
pub trait LlmProvider: Send + Sync {
fn name(&self) -> &str;
fn provider(&self) -> Provider;
async fn context_window(&self, _model: &Model) -> Option<u32> {
None
}
async fn capabilities(&self, _model: &Model) -> ModelCapabilities {
ModelCapabilities {
tool_calling: true,
images: true,
documents: true,
video: false,
audio: false,
reasoning: false,
..Default::default()
}
}
async fn list_models(&self) -> Option<Vec<ModelInfo>> {
None
}
fn complete(&self, request: CompletionRequest) -> CompletionStream;
fn text_to_speech(&self, _request: TtsRequest) -> CompletionStream {
let name = self.name().to_string();
Box::pin(futures_util::stream::once(async move {
Err(anyhow::anyhow!(
"Provider '{name}' does not support text-to-speech"
))
}))
}
fn transcribe(&self, _request: SttRequest) -> CompletionStream {
let name = self.name().to_string();
Box::pin(futures_util::stream::once(async move {
Err(anyhow::anyhow!(
"Provider '{name}' does not support speech-to-text"
))
}))
}
async fn list_voices(&self, _model: &str) -> Option<Vec<Voice>> {
None
}
fn generate_image(&self, _request: ImageGenRequest) -> CompletionStream {
Box::pin(futures_util::stream::once(async move {
Err(anyhow::anyhow!(
"Provider does not support image generation"
))
}))
}
fn generate_video(&self, _request: VideoGenRequest) -> CompletionStream {
Box::pin(futures_util::stream::once(async move {
Err(anyhow::anyhow!(
"Provider does not support video generation"
))
}))
}
fn update_routing(&self, _routing: &HashMap<String, String>) {}
fn into_arc(self) -> Arc<dyn LlmProvider>
where
Self: Sized + 'static,
{
Arc::new(self)
}
}
pub type ProviderRegistry = Arc<HashMap<Provider, Arc<dyn LlmProvider>>>;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn requests_keep_the_sessions_config_and_identity() {
use uuid::Uuid;
let uuid = Uuid::new_v4();
let mut config = AgentLlmConfig::new("openrouter:example/model".parse().unwrap())
.with_session_id(uuid)
.with_user("example-user")
.with_reasoning(ReasoningLevel::Low)
.with_provider_preferences(ProviderPreferences {
zdr: Some(true),
..Default::default()
});
config.max_tokens = Some(100);
config.sampling.temperature = Some(Decimal::ONE);
let first = CompletionRequest::from_config(&config, vec![Message::user("hello")]);
let second = CompletionRequest::from_config(&config, vec![Message::user("continue")]);
assert_eq!(first.session_id, second.session_id);
assert_eq!(
first.session_id.as_ref().map(SessionId::as_str),
Some(uuid.to_string().as_str())
);
assert_eq!(first.user, config.user);
assert_eq!(first.reasoning, config.reasoning);
assert_eq!(first.max_tokens, config.max_tokens);
assert_eq!(first.sampling.temperature, config.sampling.temperature);
assert_eq!(first.provider_preferences.unwrap().zdr, Some(true));
let one_off = CompletionRequest::new(config.model.clone(), vec![Message::user("hello")]);
assert!(one_off.session_id.is_none());
let one_off = one_off.with_session_id(uuid);
assert_eq!(one_off.session_id, second.session_id);
}
use crate::event::TurnUsage;
#[test]
fn token_usage_from_turn_usage() {
let turn = TurnUsage {
prompt_tokens: 100,
completion_tokens: 50,
..TurnUsage::default()
};
let token: TokenUsage = turn.into();
assert_eq!(token.prompt_tokens, 100);
assert_eq!(token.completion_tokens, 50);
assert_eq!(token.total_tokens, 150);
}
#[test]
fn token_usage_default_is_zero() {
let usage = TokenUsage::default();
assert_eq!(usage.total_tokens, 0);
}
#[test]
fn model_capabilities_all() {
let caps = ModelCapabilities::all();
assert!(caps.tool_calling);
assert!(caps.images);
assert!(caps.reasoning);
}
#[test]
fn model_capabilities_default_is_conservative() {
let caps = ModelCapabilities::default();
assert!(!caps.tool_calling);
assert!(!caps.reasoning);
}
#[test]
fn finish_reason_display() {
assert_eq!(FinishReason::Stop.to_string(), "stop");
assert_eq!(FinishReason::ToolCalls.to_string(), "tool_calls");
assert_eq!(FinishReason::Length.to_string(), "length");
}
#[test]
fn finish_reason_from_str() {
let stop: FinishReason = "stop".parse().unwrap();
assert_eq!(stop, FinishReason::Stop);
let tc: FinishReason = "tool_calls".parse().unwrap();
assert_eq!(tc, FinishReason::ToolCalls);
}
#[test]
fn tool_definition_serde() {
let def = ToolDefinition {
name: "file_read".into(),
description: "Read a file".into(),
parameters: serde_json::json!({"type": "object", "properties": {}}),
};
let json = serde_json::to_string(&def).unwrap();
let parsed: ToolDefinition = serde_json::from_str(&json).unwrap();
assert_eq!(parsed.name, "file_read");
}
#[test]
fn audio_format_default() {
assert_eq!(AudioFormat::default().to_string(), "mp3");
}
}