coro_core/llm/
client.rs

1//! LLM client trait and response structures
2
3use crate::error::{LlmError, Result};
4use crate::tools::ToolCall;
5use async_trait::async_trait;
6use serde::{Deserialize, Serialize};
7use std::collections::HashMap;
8
9use super::message::LlmMessage;
10
11/// Trait for LLM clients
12#[async_trait]
13pub trait LlmClient: Send + Sync {
14    /// Send a chat completion request
15    async fn chat_completion(
16        &self,
17        messages: Vec<LlmMessage>,
18        tools: Option<Vec<ToolDefinition>>,
19        options: Option<ChatOptions>,
20    ) -> Result<LlmResponse>;
21
22    /// Get the model name
23    fn model_name(&self) -> &str;
24
25    /// Get the provider name
26    fn provider_name(&self) -> &str;
27
28    /// Check if the client supports streaming
29    fn supports_streaming(&self) -> bool {
30        false
31    }
32
33    /// Send a streaming chat completion request
34    async fn chat_completion_stream(
35        &self,
36        _messages: Vec<LlmMessage>,
37        _tools: Option<Vec<ToolDefinition>>,
38        _options: Option<ChatOptions>,
39    ) -> Result<Box<dyn futures::Stream<Item = Result<LlmStreamChunk>> + Send + Unpin + '_>> {
40        Err((LlmError::InvalidRequest {
41            message: "Streaming not supported by this client".to_string(),
42        })
43        .into())
44    }
45}
46
47/// Response from an LLM
48#[derive(Debug, Clone, Serialize, Deserialize)]
49pub struct LlmResponse {
50    /// The generated message
51    pub message: LlmMessage,
52
53    /// Usage statistics
54    pub usage: Option<Usage>,
55
56    /// Model used for generation
57    pub model: String,
58
59    /// Finish reason
60    pub finish_reason: Option<FinishReason>,
61
62    /// Additional metadata
63    pub metadata: Option<HashMap<String, serde_json::Value>>,
64}
65
66/// Streaming chunk from an LLM
67#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct LlmStreamChunk {
69    /// Delta content
70    pub delta: Option<String>,
71
72    /// Tool calls in this chunk
73    pub tool_calls: Option<Vec<ToolCall>>,
74
75    /// Finish reason if this is the last chunk
76    pub finish_reason: Option<FinishReason>,
77
78    /// Usage statistics (usually only in the last chunk)
79    pub usage: Option<Usage>,
80}
81
82/// Usage statistics for a request
83#[derive(Debug, Clone, Serialize, Deserialize)]
84pub struct Usage {
85    /// Number of tokens in the prompt
86    pub prompt_tokens: u32,
87
88    /// Number of tokens in the completion
89    pub completion_tokens: u32,
90
91    /// Total number of tokens
92    pub total_tokens: u32,
93}
94
95/// Reason why generation finished
96#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
97#[serde(rename_all = "snake_case")]
98pub enum FinishReason {
99    /// Generation completed naturally
100    Stop,
101
102    /// Hit the maximum token limit
103    Length,
104
105    /// Model decided to call a tool
106    ToolCalls,
107
108    /// Content was filtered
109    ContentFilter,
110
111    /// Other reason
112    Other(String),
113}
114
115/// Tool definition for function calling
116#[derive(Debug, Clone, Serialize, Deserialize)]
117pub struct ToolDefinition {
118    /// Type of tool (usually "function")
119    #[serde(rename = "type")]
120    pub tool_type: String,
121
122    /// Function definition
123    pub function: FunctionDefinition,
124}
125
126/// Function definition for tool calling
127#[derive(Debug, Clone, Serialize, Deserialize)]
128pub struct FunctionDefinition {
129    /// Name of the function
130    pub name: String,
131
132    /// Description of what the function does
133    pub description: String,
134
135    /// JSON schema for the function parameters
136    pub parameters: serde_json::Value,
137}
138
139/// Options for chat completion
140#[derive(Debug, Clone, Serialize, Deserialize)]
141pub struct ChatOptions {
142    /// Maximum tokens to generate
143    pub max_tokens: Option<u32>,
144
145    /// Temperature for generation
146    pub temperature: Option<f32>,
147
148    /// Top-p sampling parameter
149    pub top_p: Option<f32>,
150
151    /// Top-k sampling parameter
152    pub top_k: Option<u32>,
153
154    /// Stop sequences
155    pub stop: Option<Vec<String>>,
156
157    /// Whether to stream the response
158    pub stream: Option<bool>,
159
160    /// Tool choice strategy
161    pub tool_choice: Option<ToolChoice>,
162}
163
164/// Tool choice strategy
165#[derive(Debug, Clone, Serialize, Deserialize)]
166#[serde(untagged)]
167pub enum ToolChoice {
168    /// Let the model decide
169    Auto,
170
171    /// Never use tools
172    None,
173
174    /// Force use of a specific tool
175    Required { name: String },
176}
177
178impl Default for ChatOptions {
179    fn default() -> Self {
180        Self {
181            max_tokens: Some(8192),
182            temperature: Some(0.7),
183            top_p: Some(1.0),
184            top_k: None,
185            stop: None,
186            stream: Some(false),
187            tool_choice: Some(ToolChoice::Auto),
188        }
189    }
190}
coro_core/llm/client.rs

coro_core/llm/
client.rs
