Skip to main content

ml_runtime_provider/
generation.rs

1//! Model-neutral generative inference contract.
2//!
3//! This is a separate contract from typed inference (`InferenceRequest` /
4//! `InferenceResult`). Provider wire formats (for example OpenAI-compatible
5//! JSON) never appear here; adapters translate at their own boundary.
6//!
7//! Metadata is reported honestly: when a provider does not report token
8//! usage, a finish reason, or a model identity, the corresponding field is
9//! `None` rather than an estimate. Generated text is model output and is not
10//! execution evidence; the evidence is [`ChatMetadata`].
11
12use crate::ProviderCapabilities;
13use async_trait::async_trait;
14use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
15use serde::{Deserialize, Serialize};
16use serde_json::Value;
17use std::{fmt, time::Duration};
18
19/// Default upper bound on accumulated generated output (4 MiB).
20pub const DEFAULT_MAX_OUTPUT_BYTES: usize = 4 * 1024 * 1024;
21
22#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
23#[serde(rename_all = "snake_case")]
24pub enum ChatRole {
25    System,
26    User,
27    Assistant,
28    /// A caller-supplied result for an earlier [`ToolCall`] proposal.
29    Tool,
30}
31
32impl ChatRole {
33    pub fn as_str(self) -> &'static str {
34        match self {
35            Self::System => "system",
36            Self::User => "user",
37            Self::Assistant => "assistant",
38            Self::Tool => "tool",
39        }
40    }
41}
42
43/// Longest tool name accepted (matches common provider limits).
44pub const MAX_TOOL_NAME_BYTES: usize = 64;
45const MAX_TOOLS: usize = 128;
46
47/// Tool names are restricted to `[A-Za-z0-9_-]{1,64}` so they are safe on every
48/// provider wire and in logs.
49pub fn is_valid_tool_name(name: &str) -> bool {
50    !name.is_empty()
51        && name.len() <= MAX_TOOL_NAME_BYTES
52        && name
53            .chars()
54            .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
55}
56
57/// A tool the model may *propose* to call. The runtime never executes it: the
58/// schema constrains the proposed arguments and grants no authority.
59#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
60pub struct ToolDefinition {
61    pub name: String,
62    #[serde(default, skip_serializing_if = "Option::is_none")]
63    pub description: Option<String>,
64    /// JSON Schema (an object) for the tool's arguments.
65    pub input_schema: Value,
66}
67
68impl ToolDefinition {
69    pub fn new(name: impl Into<String>, input_schema: Value) -> Self {
70        Self {
71            name: name.into(),
72            description: None,
73            input_schema,
74        }
75    }
76
77    pub fn with_description(mut self, description: impl Into<String>) -> Self {
78        self.description = Some(description.into());
79        self
80    }
81}
82
83/// Provider-neutral tool selection policy. Absent means the provider default.
84#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
85#[serde(tag = "type", rename_all = "snake_case")]
86pub enum ToolChoice {
87    /// The model decides whether to propose tool calls.
88    Auto,
89    /// The model must not propose tool calls.
90    None,
91    /// The model must propose a call to this named tool.
92    Tool { name: String },
93}
94
95/// A tool-call **proposal** produced by a model. It is not evidence that the
96/// operation ran, was authorized, or would succeed; the calling application
97/// decides whether and how to execute it.
98#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
99pub struct ToolCall {
100    /// Provider call ID; a [`ChatRole::Tool`] message echoes it in `tool_call_id`.
101    pub id: String,
102    pub name: String,
103    /// Parsed arguments (always a JSON object when produced by a provider).
104    pub arguments: Value,
105}
106
107impl ToolCall {
108    pub fn new(id: impl Into<String>, name: impl Into<String>, arguments: Value) -> Self {
109        Self {
110            id: id.into(),
111            name: name.into(),
112            arguments,
113        }
114    }
115}
116
117/// Arguments may contain sensitive data, so `Debug` omits them.
118impl fmt::Debug for ToolCall {
119    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
120        f.debug_struct("ToolCall")
121            .field("id", &self.id)
122            .field("name", &self.name)
123            .finish_non_exhaustive()
124    }
125}
126
127#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
128pub struct ChatMessage {
129    pub role: ChatRole,
130    pub content: String,
131    /// Tool-call proposals; only on `Assistant` messages. Text and calls may
132    /// both be present where the provider produced both.
133    #[serde(default, skip_serializing_if = "Vec::is_empty")]
134    pub tool_calls: Vec<ToolCall>,
135    /// Only on `Tool` messages: the [`ToolCall::id`] this content answers.
136    #[serde(default, skip_serializing_if = "Option::is_none")]
137    pub tool_call_id: Option<String>,
138}
139
140impl ChatMessage {
141    pub fn new(role: ChatRole, content: impl Into<String>) -> Self {
142        Self {
143            role,
144            content: content.into(),
145            tool_calls: Vec::new(),
146            tool_call_id: None,
147        }
148    }
149    pub fn system(content: impl Into<String>) -> Self {
150        Self::new(ChatRole::System, content)
151    }
152    pub fn user(content: impl Into<String>) -> Self {
153        Self::new(ChatRole::User, content)
154    }
155    pub fn assistant(content: impl Into<String>) -> Self {
156        Self::new(ChatRole::Assistant, content)
157    }
158    /// An assistant turn that proposed tool calls (replayed in the next request).
159    pub fn assistant_with_tool_calls(content: impl Into<String>, calls: Vec<ToolCall>) -> Self {
160        Self {
161            tool_calls: calls,
162            ..Self::new(ChatRole::Assistant, content)
163        }
164    }
165    /// The caller-supplied result for the call with `call_id`.
166    pub fn tool_result(call_id: impl Into<String>, content: impl Into<String>) -> Self {
167        Self {
168            tool_call_id: Some(call_id.into()),
169            ..Self::new(ChatRole::Tool, content)
170        }
171    }
172}
173
174/// Message content may be sensitive, so `Debug` reports only its length.
175impl fmt::Debug for ChatMessage {
176    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
177        f.debug_struct("ChatMessage")
178            .field("role", &self.role)
179            .field("content_bytes", &self.content.len())
180            .field("tool_calls", &self.tool_calls.len())
181            .field("tool_call_id", &self.tool_call_id)
182            .finish()
183    }
184}
185
186/// Requested output shape. `JsonSchema` is a request to the provider; it is
187/// never a guarantee. The runtime validates the result separately.
188#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
189#[serde(tag = "type", rename_all = "snake_case")]
190pub enum ResponseFormat {
191    #[default]
192    Text,
193    JsonObject,
194    JsonSchema {
195        name: String,
196        schema: Value,
197        #[serde(default)]
198        strict: bool,
199    },
200}
201
202#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
203pub struct GenerationOptions {
204    pub temperature: Option<f32>,
205    pub top_p: Option<f32>,
206    pub max_tokens: Option<u32>,
207    #[serde(default)]
208    pub stop: Vec<String>,
209    pub seed: Option<i64>,
210    #[serde(default)]
211    pub response_format: ResponseFormat,
212    /// Tools the model may propose to call. Empty means none are offered.
213    #[serde(default)]
214    pub tools: Vec<ToolDefinition>,
215    #[serde(default)]
216    pub tool_choice: Option<ToolChoice>,
217}
218
219#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
220pub struct ChatRequest {
221    /// Explicit model identity; the provider never substitutes another model.
222    pub model: String,
223    pub messages: Vec<ChatMessage>,
224    #[serde(default)]
225    pub options: GenerationOptions,
226}
227
228impl ChatRequest {
229    pub fn new(model: impl Into<String>, messages: Vec<ChatMessage>) -> Self {
230        Self {
231            model: model.into(),
232            messages,
233            options: GenerationOptions::default(),
234        }
235    }
236
237    pub fn with_options(mut self, options: GenerationOptions) -> Self {
238        self.options = options;
239        self
240    }
241
242    /// Provider-independent request validation. Providers still reject
243    /// options they do not support.
244    pub fn validate(&self) -> RuntimeResult<()> {
245        use ml_runtime_common::RuntimeError::InvalidRequest;
246        let invalid = |reason: &str| {
247            Err(InvalidRequest {
248                reason: reason.to_owned(),
249            })
250        };
251        if self.model.trim().is_empty() {
252            return invalid("chat request requires an explicit model");
253        }
254        if self.messages.is_empty() {
255            return invalid("chat request requires at least one message");
256        }
257        if !self.messages.iter().any(|m| m.role == ChatRole::User) {
258            return invalid("chat request requires at least one user message");
259        }
260        let o = &self.options;
261        if o.temperature.is_some_and(|t| !t.is_finite() || t < 0.0) {
262            return invalid("temperature must be a finite non-negative number");
263        }
264        if o.top_p.is_some_and(|p| !(0.0..=1.0).contains(&p)) {
265            return invalid("top_p must be within 0.0..=1.0");
266        }
267        if o.max_tokens == Some(0) {
268            return invalid("max_tokens must be greater than zero");
269        }
270        if o.stop.iter().any(|s| s.is_empty()) {
271            return invalid("stop sequences must not be empty");
272        }
273        if let ResponseFormat::JsonSchema { name, .. } = &o.response_format {
274            if name.is_empty()
275                || !name
276                    .chars()
277                    .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
278            {
279                return invalid("json schema name must be non-empty [A-Za-z0-9_-]");
280            }
281        }
282        self.validate_tools()?;
283        self.validate_messages()
284    }
285
286    fn validate_tools(&self) -> RuntimeResult<()> {
287        use ml_runtime_common::RuntimeError::InvalidRequest;
288        let invalid = |reason: String| Err(InvalidRequest { reason });
289        let o = &self.options;
290        if o.tools.len() > MAX_TOOLS {
291            return invalid(format!("at most {MAX_TOOLS} tools may be offered"));
292        }
293        let mut seen = std::collections::HashSet::new();
294        for tool in &o.tools {
295            if !is_valid_tool_name(&tool.name) {
296                return invalid(format!(
297                    "tool name must match [A-Za-z0-9_-]{{1,{MAX_TOOL_NAME_BYTES}}}"
298                ));
299            }
300            if !seen.insert(tool.name.as_str()) {
301                return invalid(format!("duplicate tool name {:?}", tool.name));
302            }
303            if !tool.input_schema.is_object() {
304                return invalid(format!(
305                    "input schema of tool {:?} must be a JSON object",
306                    tool.name
307                ));
308            }
309        }
310        match &o.tool_choice {
311            None => {}
312            Some(_) if o.tools.is_empty() => {
313                return invalid("tool_choice requires at least one tool".to_owned())
314            }
315            Some(ToolChoice::Tool { name }) if !seen.contains(name.as_str()) => {
316                return invalid(format!("tool_choice names undefined tool {name:?}"))
317            }
318            Some(_) => {}
319        }
320        Ok(())
321    }
322
323    /// Checks message shape and that every tool result answers an earlier
324    /// proposal exactly once. Nothing here implies a tool ever ran.
325    fn validate_messages(&self) -> RuntimeResult<()> {
326        use ml_runtime_common::RuntimeError::InvalidRequest;
327        let invalid = |reason: &str| {
328            Err(InvalidRequest {
329                reason: reason.to_owned(),
330            })
331        };
332        let mut open: std::collections::HashSet<&str> = std::collections::HashSet::new();
333        let mut proposed: std::collections::HashSet<&str> = std::collections::HashSet::new();
334        for message in &self.messages {
335            match message.role {
336                ChatRole::Assistant => {
337                    if message.tool_call_id.is_some() {
338                        return invalid("only tool messages may carry tool_call_id");
339                    }
340                    for call in &message.tool_calls {
341                        if call.id.is_empty() || !is_valid_tool_name(&call.name) {
342                            return invalid(
343                                "tool call requires a non-empty id and a valid tool name",
344                            );
345                        }
346                        if !call.arguments.is_object() {
347                            return invalid("tool call arguments must be a JSON object");
348                        }
349                        if !proposed.insert(call.id.as_str()) {
350                            return invalid("duplicate tool call id");
351                        }
352                        open.insert(call.id.as_str());
353                    }
354                }
355                ChatRole::Tool => {
356                    if !message.tool_calls.is_empty() {
357                        return invalid("tool messages cannot propose tool calls");
358                    }
359                    let Some(id) = message.tool_call_id.as_deref().filter(|id| !id.is_empty())
360                    else {
361                        return invalid("tool message requires tool_call_id");
362                    };
363                    if !open.remove(id) {
364                        return invalid(
365                            "tool message does not answer an earlier unanswered tool call",
366                        );
367                    }
368                }
369                ChatRole::System | ChatRole::User => {
370                    if !message.tool_calls.is_empty() || message.tool_call_id.is_some() {
371                        return invalid("only assistant and tool messages carry tool fields");
372                    }
373                }
374            }
375        }
376        Ok(())
377    }
378}
379
380/// Why generation stopped, when the provider says. `Other` preserves an
381/// unrecognized provider value verbatim.
382#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
383#[serde(rename_all = "snake_case")]
384pub enum FinishReason {
385    Stop,
386    Length,
387    ContentFilter,
388    /// The model stopped to propose tool calls. Proposals only: nothing ran.
389    ToolCalls,
390    Other(String),
391}
392
393/// Token accounting as reported by the provider; never estimated.
394#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
395pub struct Usage {
396    pub prompt_tokens: Option<u64>,
397    pub completion_tokens: Option<u64>,
398    pub total_tokens: Option<u64>,
399}
400
401#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
402#[serde(rename_all = "snake_case")]
403pub enum ExecutionLocation {
404    Local,
405    Remote,
406}
407
408/// Execution evidence recorded by the runtime/provider (not by the model).
409#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
410pub struct ChatMetadata {
411    pub provider: String,
412    pub transport: String,
413    pub execution: ExecutionLocation,
414    /// True when `execution` was inferred from the endpoint host rather than
415    /// configured explicitly.
416    pub execution_inferred: bool,
417    /// Endpoint with userinfo, query, and fragment removed.
418    pub endpoint: String,
419    pub requested_model: String,
420    pub reported_model: Option<String>,
421    pub response_id: Option<String>,
422    pub streamed: bool,
423    /// Number of content deltas received when streamed.
424    pub stream_events: Option<u64>,
425    pub time_to_first_token: Option<Duration>,
426    pub duration: Duration,
427    pub runtime_version: String,
428}
429
430/// Result of checking generated text against the requested format.
431#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
432#[serde(tag = "status", rename_all = "snake_case")]
433pub enum StructuredOutput {
434    /// Parsed as JSON and, for `JsonSchema`, validated against the schema.
435    Validated {
436        value: Value,
437    },
438    InvalidJson {
439        error: String,
440    },
441    SchemaViolation {
442        errors: Vec<String>,
443    },
444}
445
446impl StructuredOutput {
447    pub fn is_validated(&self) -> bool {
448        matches!(self, Self::Validated { .. })
449    }
450}
451
452#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
453pub struct ChatResponse {
454    pub message: ChatMessage,
455    pub finish_reason: Option<FinishReason>,
456    pub usage: Option<Usage>,
457    pub metadata: ChatMetadata,
458    /// Present only when a JSON response format was requested. `None` means
459    /// the output was not checked, never that it passed.
460    pub structured: Option<StructuredOutput>,
461}
462
463/// Terminal stream summary. It deliberately omits the full text, so a stream
464/// never forces the runtime to buffer unbounded output.
465#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
466pub struct ChatStreamSummary {
467    pub finish_reason: Option<FinishReason>,
468    pub usage: Option<Usage>,
469    pub metadata: ChatMetadata,
470    pub structured: Option<StructuredOutput>,
471}
472
473#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
474#[serde(tag = "type", rename_all = "snake_case")]
475pub enum ChatStreamEvent {
476    /// Exactly once, first.
477    Started { provider: String, model: String },
478    /// Zero or more incremental content fragments, in order.
479    Delta { content: String },
480    /// Raw, unvalidated fragment of a tool call being assembled. For display
481    /// only: it is never a usable or authorized proposal.
482    ToolCallDelta {
483        index: u32,
484        id: Option<String>,
485        name: Option<String>,
486        arguments_fragment: Option<String>,
487    },
488    /// A fully assembled and validated proposal. Emitted only after the whole
489    /// response was received, immediately before `Completed`, in call order.
490    ToolCallCompleted { call: ToolCall },
491    /// Exactly once on success. Never emitted after an error or cancellation.
492    Completed(Box<ChatStreamSummary>),
493}
494
495/// Optional features of a generation provider. The OpenAI-compatible adapter
496/// cannot discover these, so they are configured and enforced up front.
497#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
498pub struct GenerationCapabilities {
499    pub streaming: bool,
500    pub stop_sequences: bool,
501    pub seed: bool,
502    pub json_object: bool,
503    pub json_schema: bool,
504    /// Tool definitions, assistant tool calls, and tool-result messages.
505    pub tools: bool,
506    /// The `tool_choice` control (`auto` / `none` / named tool).
507    pub tool_choice: bool,
508}
509
510impl GenerationCapabilities {
511    /// Rejects options the provider is not declared to support.
512    pub fn check(&self, request: &ChatRequest, stream: bool) -> RuntimeResult<()> {
513        use ml_runtime_common::RuntimeError::CapabilityMismatch;
514        let unsupported = |what: &str| {
515            Err(CapabilityMismatch {
516                reason: format!("provider does not support {what}"),
517            })
518        };
519        if stream && !self.streaming {
520            return unsupported("streaming");
521        }
522        let o = &request.options;
523        if !o.stop.is_empty() && !self.stop_sequences {
524            return unsupported("stop sequences");
525        }
526        if o.seed.is_some() && !self.seed {
527            return unsupported("seed");
528        }
529        if (!o.tools.is_empty()
530            || request
531                .messages
532                .iter()
533                .any(|m| m.role == ChatRole::Tool || !m.tool_calls.is_empty()))
534            && !self.tools
535        {
536            return unsupported("tool calling");
537        }
538        if o.tool_choice.is_some() && !self.tool_choice {
539            return unsupported("tool_choice");
540        }
541        match o.response_format {
542            ResponseFormat::JsonObject if !self.json_object => unsupported("JSON object output"),
543            ResponseFormat::JsonSchema { .. } if !self.json_schema => {
544                unsupported("JSON schema output")
545            }
546            _ => Ok(()),
547        }
548    }
549}
550
551#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
552pub struct GenerationModelInfo {
553    pub id: String,
554    pub owned_by: Option<String>,
555}
556
557/// A provider of generative inference. Separate from [`crate::Provider`],
558/// which serves typed inference.
559#[async_trait]
560pub trait GenerationProvider: Send + Sync {
561    fn name(&self) -> &str;
562    /// Shared provider descriptor (local/remote, endpoint, streaming, ...).
563    fn capabilities(&self) -> ProviderCapabilities;
564    fn generation_capabilities(&self) -> GenerationCapabilities;
565    /// Lists models when the upstream supports discovery; otherwise returns
566    /// a `CapabilityMismatch` error.
567    async fn list_models(
568        &self,
569        cancellation: Option<CancellationToken>,
570    ) -> RuntimeResult<Vec<GenerationModelInfo>>;
571    async fn chat(
572        &self,
573        request: ChatRequest,
574        cancellation: Option<CancellationToken>,
575    ) -> RuntimeResult<ChatResponse>;
576    /// Dropping the returned stream must release the upstream connection.
577    async fn chat_stream(
578        &self,
579        request: ChatRequest,
580        cancellation: Option<CancellationToken>,
581    ) -> RuntimeResult<BoxStream<RuntimeResult<ChatStreamEvent>>>;
582}