1use crate::ProviderCapabilities;
13use async_trait::async_trait;
14use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
15use serde::{Deserialize, Serialize};
16use serde_json::Value;
17use std::{fmt, time::Duration};
18
19pub const DEFAULT_MAX_OUTPUT_BYTES: usize = 4 * 1024 * 1024;
21
22#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
23#[serde(rename_all = "snake_case")]
24pub enum ChatRole {
25 System,
26 User,
27 Assistant,
28 Tool,
30}
31
32impl ChatRole {
33 pub fn as_str(self) -> &'static str {
34 match self {
35 Self::System => "system",
36 Self::User => "user",
37 Self::Assistant => "assistant",
38 Self::Tool => "tool",
39 }
40 }
41}
42
43pub const MAX_TOOL_NAME_BYTES: usize = 64;
45const MAX_TOOLS: usize = 128;
46
47pub fn is_valid_tool_name(name: &str) -> bool {
50 !name.is_empty()
51 && name.len() <= MAX_TOOL_NAME_BYTES
52 && name
53 .chars()
54 .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
55}
56
57#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
60pub struct ToolDefinition {
61 pub name: String,
62 #[serde(default, skip_serializing_if = "Option::is_none")]
63 pub description: Option<String>,
64 pub input_schema: Value,
66}
67
68impl ToolDefinition {
69 pub fn new(name: impl Into<String>, input_schema: Value) -> Self {
70 Self {
71 name: name.into(),
72 description: None,
73 input_schema,
74 }
75 }
76
77 pub fn with_description(mut self, description: impl Into<String>) -> Self {
78 self.description = Some(description.into());
79 self
80 }
81}
82
83#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
85#[serde(tag = "type", rename_all = "snake_case")]
86pub enum ToolChoice {
87 Auto,
89 None,
91 Tool { name: String },
93}
94
95#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
99pub struct ToolCall {
100 pub id: String,
102 pub name: String,
103 pub arguments: Value,
105}
106
107impl ToolCall {
108 pub fn new(id: impl Into<String>, name: impl Into<String>, arguments: Value) -> Self {
109 Self {
110 id: id.into(),
111 name: name.into(),
112 arguments,
113 }
114 }
115}
116
117impl fmt::Debug for ToolCall {
119 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
120 f.debug_struct("ToolCall")
121 .field("id", &self.id)
122 .field("name", &self.name)
123 .finish_non_exhaustive()
124 }
125}
126
127#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
128pub struct ChatMessage {
129 pub role: ChatRole,
130 pub content: String,
131 #[serde(default, skip_serializing_if = "Vec::is_empty")]
134 pub tool_calls: Vec<ToolCall>,
135 #[serde(default, skip_serializing_if = "Option::is_none")]
137 pub tool_call_id: Option<String>,
138}
139
140impl ChatMessage {
141 pub fn new(role: ChatRole, content: impl Into<String>) -> Self {
142 Self {
143 role,
144 content: content.into(),
145 tool_calls: Vec::new(),
146 tool_call_id: None,
147 }
148 }
149 pub fn system(content: impl Into<String>) -> Self {
150 Self::new(ChatRole::System, content)
151 }
152 pub fn user(content: impl Into<String>) -> Self {
153 Self::new(ChatRole::User, content)
154 }
155 pub fn assistant(content: impl Into<String>) -> Self {
156 Self::new(ChatRole::Assistant, content)
157 }
158 pub fn assistant_with_tool_calls(content: impl Into<String>, calls: Vec<ToolCall>) -> Self {
160 Self {
161 tool_calls: calls,
162 ..Self::new(ChatRole::Assistant, content)
163 }
164 }
165 pub fn tool_result(call_id: impl Into<String>, content: impl Into<String>) -> Self {
167 Self {
168 tool_call_id: Some(call_id.into()),
169 ..Self::new(ChatRole::Tool, content)
170 }
171 }
172}
173
174impl fmt::Debug for ChatMessage {
176 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
177 f.debug_struct("ChatMessage")
178 .field("role", &self.role)
179 .field("content_bytes", &self.content.len())
180 .field("tool_calls", &self.tool_calls.len())
181 .field("tool_call_id", &self.tool_call_id)
182 .finish()
183 }
184}
185
186#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
189#[serde(tag = "type", rename_all = "snake_case")]
190pub enum ResponseFormat {
191 #[default]
192 Text,
193 JsonObject,
194 JsonSchema {
195 name: String,
196 schema: Value,
197 #[serde(default)]
198 strict: bool,
199 },
200}
201
202#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
203pub struct GenerationOptions {
204 pub temperature: Option<f32>,
205 pub top_p: Option<f32>,
206 pub max_tokens: Option<u32>,
207 #[serde(default)]
208 pub stop: Vec<String>,
209 pub seed: Option<i64>,
210 #[serde(default)]
211 pub response_format: ResponseFormat,
212 #[serde(default)]
214 pub tools: Vec<ToolDefinition>,
215 #[serde(default)]
216 pub tool_choice: Option<ToolChoice>,
217}
218
219#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
220pub struct ChatRequest {
221 pub model: String,
223 pub messages: Vec<ChatMessage>,
224 #[serde(default)]
225 pub options: GenerationOptions,
226}
227
228impl ChatRequest {
229 pub fn new(model: impl Into<String>, messages: Vec<ChatMessage>) -> Self {
230 Self {
231 model: model.into(),
232 messages,
233 options: GenerationOptions::default(),
234 }
235 }
236
237 pub fn with_options(mut self, options: GenerationOptions) -> Self {
238 self.options = options;
239 self
240 }
241
242 pub fn validate(&self) -> RuntimeResult<()> {
245 use ml_runtime_common::RuntimeError::InvalidRequest;
246 let invalid = |reason: &str| {
247 Err(InvalidRequest {
248 reason: reason.to_owned(),
249 })
250 };
251 if self.model.trim().is_empty() {
252 return invalid("chat request requires an explicit model");
253 }
254 if self.messages.is_empty() {
255 return invalid("chat request requires at least one message");
256 }
257 if !self.messages.iter().any(|m| m.role == ChatRole::User) {
258 return invalid("chat request requires at least one user message");
259 }
260 let o = &self.options;
261 if o.temperature.is_some_and(|t| !t.is_finite() || t < 0.0) {
262 return invalid("temperature must be a finite non-negative number");
263 }
264 if o.top_p.is_some_and(|p| !(0.0..=1.0).contains(&p)) {
265 return invalid("top_p must be within 0.0..=1.0");
266 }
267 if o.max_tokens == Some(0) {
268 return invalid("max_tokens must be greater than zero");
269 }
270 if o.stop.iter().any(|s| s.is_empty()) {
271 return invalid("stop sequences must not be empty");
272 }
273 if let ResponseFormat::JsonSchema { name, .. } = &o.response_format {
274 if name.is_empty()
275 || !name
276 .chars()
277 .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
278 {
279 return invalid("json schema name must be non-empty [A-Za-z0-9_-]");
280 }
281 }
282 self.validate_tools()?;
283 self.validate_messages()
284 }
285
286 fn validate_tools(&self) -> RuntimeResult<()> {
287 use ml_runtime_common::RuntimeError::InvalidRequest;
288 let invalid = |reason: String| Err(InvalidRequest { reason });
289 let o = &self.options;
290 if o.tools.len() > MAX_TOOLS {
291 return invalid(format!("at most {MAX_TOOLS} tools may be offered"));
292 }
293 let mut seen = std::collections::HashSet::new();
294 for tool in &o.tools {
295 if !is_valid_tool_name(&tool.name) {
296 return invalid(format!(
297 "tool name must match [A-Za-z0-9_-]{{1,{MAX_TOOL_NAME_BYTES}}}"
298 ));
299 }
300 if !seen.insert(tool.name.as_str()) {
301 return invalid(format!("duplicate tool name {:?}", tool.name));
302 }
303 if !tool.input_schema.is_object() {
304 return invalid(format!(
305 "input schema of tool {:?} must be a JSON object",
306 tool.name
307 ));
308 }
309 }
310 match &o.tool_choice {
311 None => {}
312 Some(_) if o.tools.is_empty() => {
313 return invalid("tool_choice requires at least one tool".to_owned())
314 }
315 Some(ToolChoice::Tool { name }) if !seen.contains(name.as_str()) => {
316 return invalid(format!("tool_choice names undefined tool {name:?}"))
317 }
318 Some(_) => {}
319 }
320 Ok(())
321 }
322
323 fn validate_messages(&self) -> RuntimeResult<()> {
326 use ml_runtime_common::RuntimeError::InvalidRequest;
327 let invalid = |reason: &str| {
328 Err(InvalidRequest {
329 reason: reason.to_owned(),
330 })
331 };
332 let mut open: std::collections::HashSet<&str> = std::collections::HashSet::new();
333 let mut proposed: std::collections::HashSet<&str> = std::collections::HashSet::new();
334 for message in &self.messages {
335 match message.role {
336 ChatRole::Assistant => {
337 if message.tool_call_id.is_some() {
338 return invalid("only tool messages may carry tool_call_id");
339 }
340 for call in &message.tool_calls {
341 if call.id.is_empty() || !is_valid_tool_name(&call.name) {
342 return invalid(
343 "tool call requires a non-empty id and a valid tool name",
344 );
345 }
346 if !call.arguments.is_object() {
347 return invalid("tool call arguments must be a JSON object");
348 }
349 if !proposed.insert(call.id.as_str()) {
350 return invalid("duplicate tool call id");
351 }
352 open.insert(call.id.as_str());
353 }
354 }
355 ChatRole::Tool => {
356 if !message.tool_calls.is_empty() {
357 return invalid("tool messages cannot propose tool calls");
358 }
359 let Some(id) = message.tool_call_id.as_deref().filter(|id| !id.is_empty())
360 else {
361 return invalid("tool message requires tool_call_id");
362 };
363 if !open.remove(id) {
364 return invalid(
365 "tool message does not answer an earlier unanswered tool call",
366 );
367 }
368 }
369 ChatRole::System | ChatRole::User => {
370 if !message.tool_calls.is_empty() || message.tool_call_id.is_some() {
371 return invalid("only assistant and tool messages carry tool fields");
372 }
373 }
374 }
375 }
376 Ok(())
377 }
378}
379
380#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
383#[serde(rename_all = "snake_case")]
384pub enum FinishReason {
385 Stop,
386 Length,
387 ContentFilter,
388 ToolCalls,
390 Other(String),
391}
392
393#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
395pub struct Usage {
396 pub prompt_tokens: Option<u64>,
397 pub completion_tokens: Option<u64>,
398 pub total_tokens: Option<u64>,
399}
400
401#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
402#[serde(rename_all = "snake_case")]
403pub enum ExecutionLocation {
404 Local,
405 Remote,
406}
407
408#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
410pub struct ChatMetadata {
411 pub provider: String,
412 pub transport: String,
413 pub execution: ExecutionLocation,
414 pub execution_inferred: bool,
417 pub endpoint: String,
419 pub requested_model: String,
420 pub reported_model: Option<String>,
421 pub response_id: Option<String>,
422 pub streamed: bool,
423 pub stream_events: Option<u64>,
425 pub time_to_first_token: Option<Duration>,
426 pub duration: Duration,
427 pub runtime_version: String,
428}
429
430#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
432#[serde(tag = "status", rename_all = "snake_case")]
433pub enum StructuredOutput {
434 Validated {
436 value: Value,
437 },
438 InvalidJson {
439 error: String,
440 },
441 SchemaViolation {
442 errors: Vec<String>,
443 },
444}
445
446impl StructuredOutput {
447 pub fn is_validated(&self) -> bool {
448 matches!(self, Self::Validated { .. })
449 }
450}
451
452#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
453pub struct ChatResponse {
454 pub message: ChatMessage,
455 pub finish_reason: Option<FinishReason>,
456 pub usage: Option<Usage>,
457 pub metadata: ChatMetadata,
458 pub structured: Option<StructuredOutput>,
461}
462
463#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
466pub struct ChatStreamSummary {
467 pub finish_reason: Option<FinishReason>,
468 pub usage: Option<Usage>,
469 pub metadata: ChatMetadata,
470 pub structured: Option<StructuredOutput>,
471}
472
473#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
474#[serde(tag = "type", rename_all = "snake_case")]
475pub enum ChatStreamEvent {
476 Started { provider: String, model: String },
478 Delta { content: String },
480 ToolCallDelta {
483 index: u32,
484 id: Option<String>,
485 name: Option<String>,
486 arguments_fragment: Option<String>,
487 },
488 ToolCallCompleted { call: ToolCall },
491 Completed(Box<ChatStreamSummary>),
493}
494
495#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
498pub struct GenerationCapabilities {
499 pub streaming: bool,
500 pub stop_sequences: bool,
501 pub seed: bool,
502 pub json_object: bool,
503 pub json_schema: bool,
504 pub tools: bool,
506 pub tool_choice: bool,
508}
509
510impl GenerationCapabilities {
511 pub fn check(&self, request: &ChatRequest, stream: bool) -> RuntimeResult<()> {
513 use ml_runtime_common::RuntimeError::CapabilityMismatch;
514 let unsupported = |what: &str| {
515 Err(CapabilityMismatch {
516 reason: format!("provider does not support {what}"),
517 })
518 };
519 if stream && !self.streaming {
520 return unsupported("streaming");
521 }
522 let o = &request.options;
523 if !o.stop.is_empty() && !self.stop_sequences {
524 return unsupported("stop sequences");
525 }
526 if o.seed.is_some() && !self.seed {
527 return unsupported("seed");
528 }
529 if (!o.tools.is_empty()
530 || request
531 .messages
532 .iter()
533 .any(|m| m.role == ChatRole::Tool || !m.tool_calls.is_empty()))
534 && !self.tools
535 {
536 return unsupported("tool calling");
537 }
538 if o.tool_choice.is_some() && !self.tool_choice {
539 return unsupported("tool_choice");
540 }
541 match o.response_format {
542 ResponseFormat::JsonObject if !self.json_object => unsupported("JSON object output"),
543 ResponseFormat::JsonSchema { .. } if !self.json_schema => {
544 unsupported("JSON schema output")
545 }
546 _ => Ok(()),
547 }
548 }
549}
550
551#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
552pub struct GenerationModelInfo {
553 pub id: String,
554 pub owned_by: Option<String>,
555}
556
557#[async_trait]
560pub trait GenerationProvider: Send + Sync {
561 fn name(&self) -> &str;
562 fn capabilities(&self) -> ProviderCapabilities;
564 fn generation_capabilities(&self) -> GenerationCapabilities;
565 async fn list_models(
568 &self,
569 cancellation: Option<CancellationToken>,
570 ) -> RuntimeResult<Vec<GenerationModelInfo>>;
571 async fn chat(
572 &self,
573 request: ChatRequest,
574 cancellation: Option<CancellationToken>,
575 ) -> RuntimeResult<ChatResponse>;
576 async fn chat_stream(
578 &self,
579 request: ChatRequest,
580 cancellation: Option<CancellationToken>,
581 ) -> RuntimeResult<BoxStream<RuntimeResult<ChatStreamEvent>>>;
582}