Skip to main content

rust_ml_runtime/
generation.rs

1//! Generative inference entry points on [`Runtime`].
2//!
3//! The caller names the provider explicitly; there is no routing and no
4//! fallback to another provider, model, or execution mode. Providers own
5//! transport; the runtime owns request validation, concurrency admission,
6//! structured-output validation, and the stream termination contract.
7
8use super::Runtime;
9use futures_util::StreamExt;
10use jsonschema::Validator;
11use ml_runtime_common::{BoxStream, CancellationToken, RuntimeError, RuntimeResult};
12use ml_runtime_provider::{
13    ChatRequest, ChatResponse, ChatStreamEvent, GenerationModelInfo, GenerationProvider,
14    ProviderCapability, ResponseFormat, StructuredOutput, ToolCall, ToolChoice, ToolDefinition,
15    DEFAULT_MAX_OUTPUT_BYTES,
16};
17use serde_json::Value;
18use std::{
19    collections::{BTreeMap, VecDeque},
20    sync::Arc,
21};
22use tokio::sync::OwnedSemaphorePermit;
23
24const MAX_REPORTED_VIOLATIONS: usize = 16;
25const MAX_VIOLATION_CHARS: usize = 200;
26
27/// How generated text is checked for a requested response format.
28enum OutputCheck {
29    /// Valid JSON of any shape.
30    Json,
31    Schema(Arc<Validator>),
32}
33
34impl OutputCheck {
35    /// Compiles the schema up front so an unusable schema is an
36    /// `InvalidRequest` rather than a late generation failure.
37    fn for_format(format: &ResponseFormat) -> RuntimeResult<Option<Self>> {
38        match format {
39            ResponseFormat::Text => Ok(None),
40            ResponseFormat::JsonObject => Ok(Some(Self::Json)),
41            ResponseFormat::JsonSchema { schema, .. } => jsonschema::validator_for(schema)
42                .map(|validator| Some(Self::Schema(Arc::new(validator))))
43                .map_err(|error| RuntimeError::InvalidRequest {
44                    reason: format!("unusable JSON schema: {}", clip(&error.to_string())),
45                }),
46        }
47    }
48
49    fn check(&self, text: &str, format: &ResponseFormat) -> StructuredOutput {
50        let value: Value = match serde_json::from_str(text) {
51            Ok(value) => value,
52            Err(error) => {
53                return StructuredOutput::InvalidJson {
54                    error: error.to_string(),
55                }
56            }
57        };
58        // `json_object` promises an object; a bare scalar or array is a
59        // violation of the requested format.
60        if matches!(format, ResponseFormat::JsonObject) && !value.is_object() {
61            return StructuredOutput::SchemaViolation {
62                errors: vec![
63                    "JSON object output was requested but the value is not an object".to_owned(),
64                ],
65            };
66        }
67        match self {
68            Self::Json => StructuredOutput::Validated { value },
69            Self::Schema(validator) => {
70                let errors: Vec<String> = validator
71                    .iter_errors(&value)
72                    .take(MAX_REPORTED_VIOLATIONS)
73                    .map(|error| clip(&error.to_string()))
74                    .collect();
75                if errors.is_empty() {
76                    StructuredOutput::Validated { value }
77                } else {
78                    StructuredOutput::SchemaViolation { errors }
79                }
80            }
81        }
82    }
83}
84
85/// Validates tool-call *proposals* against the definitions the caller offered.
86/// A schema constrains arguments; passing it grants no authority and does not
87/// mean the operation ran.
88struct ToolChecks {
89    validators: BTreeMap<String, Validator>,
90    choice: Option<ToolChoice>,
91}
92
93impl ToolChecks {
94    /// Compiles every tool schema up front so an unusable one is an
95    /// `InvalidRequest` before anything is sent.
96    fn compile(tools: &[ToolDefinition], choice: &Option<ToolChoice>) -> RuntimeResult<Self> {
97        let mut validators = BTreeMap::new();
98        for tool in tools {
99            let validator = jsonschema::validator_for(&tool.input_schema).map_err(|error| {
100                RuntimeError::InvalidRequest {
101                    reason: format!(
102                        "unusable input schema for tool {:?}: {}",
103                        tool.name,
104                        clip(&error.to_string())
105                    ),
106                }
107            })?;
108            validators.insert(tool.name.clone(), validator);
109        }
110        Ok(Self {
111            validators,
112            choice: choice.clone(),
113        })
114    }
115
116    fn check(&self, calls: &[ToolCall]) -> RuntimeResult<()> {
117        let invalid = |reason: String| Err(RuntimeError::InvalidOutput { reason });
118        if calls.is_empty() {
119            return Ok(());
120        }
121        if matches!(self.choice, Some(ToolChoice::None)) {
122            return invalid("model proposed tool calls although tool_choice is none".to_owned());
123        }
124        for call in calls {
125            let Some(validator) = self.validators.get(&call.name) else {
126                return invalid(format!(
127                    "model proposed a call to undefined tool {:?}",
128                    call.name
129                ));
130            };
131            if let Some(ToolChoice::Tool { name }) = &self.choice {
132                if *name != call.name {
133                    return invalid(format!(
134                        "model proposed tool {:?} although {name:?} was required",
135                        call.name
136                    ));
137                }
138            }
139            let paths: Vec<String> = validator
140                .iter_errors(&call.arguments)
141                .take(MAX_REPORTED_VIOLATIONS)
142                .map(|error| error.instance_path().to_string())
143                .collect();
144            if !paths.is_empty() {
145                // Locations only: argument values are not echoed into errors.
146                return invalid(format!(
147                    "arguments proposed for tool {:?} violate its input schema at: {}",
148                    call.name,
149                    paths.join(", ")
150                ));
151            }
152        }
153        Ok(())
154    }
155}
156
157fn clip(text: &str) -> String {
158    text.chars().take(MAX_VIOLATION_CHARS).collect()
159}
160
161impl Runtime {
162    fn generation_provider(&self, name: &str) -> RuntimeResult<&Arc<dyn GenerationProvider>> {
163        self.generation_providers.get(name).ok_or_else(|| {
164            RuntimeError::provider_unavailable(name, ": no such generation provider is registered")
165        })
166    }
167
168    /// Descriptors of registered generation providers. These are separate
169    /// from typed-inference providers and never take part in their routing.
170    pub fn generation_providers(&self) -> Vec<ProviderCapability> {
171        self.generation_providers
172            .iter()
173            .map(|(name, provider)| {
174                ProviderCapability::from_parts(name.clone(), provider.capabilities())
175            })
176            .collect()
177    }
178
179    /// Lists models from the named provider, where the upstream supports it.
180    pub async fn list_generation_models(
181        &self,
182        provider: &str,
183        cancellation: Option<CancellationToken>,
184    ) -> RuntimeResult<Vec<GenerationModelInfo>> {
185        self.generation_provider(provider)?
186            .list_models(cancellation)
187            .await
188    }
189
190    async fn admit(
191        &self,
192        cancellation: &Option<CancellationToken>,
193    ) -> RuntimeResult<OwnedSemaphorePermit> {
194        let acquire = self.inference_semaphore.clone().acquire_owned();
195        let permit = match cancellation {
196            Some(token) => tokio::select! {
197                biased;
198                _ = token.cancelled() => return Err(RuntimeError::Cancelled),
199                permit = acquire => permit,
200            },
201            None => acquire.await,
202        };
203        permit.map_err(|_| RuntimeError::ResourceLimit {
204            reason: "runtime is shut down".to_owned(),
205        })
206    }
207
208    /// Single-response generation through the explicitly named provider.
209    ///
210    /// When a JSON response format is requested the returned
211    /// [`ChatResponse::structured`] states whether the text parsed and
212    /// validated; invalid output is reported, never repaired.
213    pub async fn chat(
214        &self,
215        provider: &str,
216        request: ChatRequest,
217        cancellation: Option<CancellationToken>,
218    ) -> RuntimeResult<ChatResponse> {
219        request.validate()?;
220        let format = request.options.response_format.clone();
221        let check = OutputCheck::for_format(&format)?;
222        let tools = ToolChecks::compile(&request.options.tools, &request.options.tool_choice)?;
223        let provider = self.generation_provider(provider)?;
224        let _permit = self.admit(&cancellation).await?;
225        let mut response = provider.chat(request, cancellation).await?;
226        tools.check(&response.message.tool_calls)?;
227        // A response that proposes tool calls is not an answer in the requested
228        // format, so it is not checked as one (`structured` stays `None`).
229        if let (Some(check), true) = (check, response.message.tool_calls.is_empty()) {
230            response.structured = Some(check.check(&response.message.content, &format));
231        }
232        Ok(response)
233    }
234
235    /// Streaming generation. Yields `Started`, zero or more `Delta`, then
236    /// exactly one `Completed` on success. Any failure or cancellation ends
237    /// the stream with an error and never with `Completed`; a stream that ends
238    /// without either is reported as a protocol error. Dropping the stream
239    /// releases the upstream connection and the concurrency permit.
240    pub async fn chat_stream(
241        &self,
242        provider: &str,
243        request: ChatRequest,
244        cancellation: Option<CancellationToken>,
245    ) -> RuntimeResult<BoxStream<RuntimeResult<ChatStreamEvent>>> {
246        request.validate()?;
247        let format = request.options.response_format.clone();
248        let check = OutputCheck::for_format(&format)?;
249        let tools = ToolChecks::compile(&request.options.tools, &request.options.tool_choice)?;
250        let provider = self.generation_provider(provider)?;
251        let permit = self.admit(&cancellation).await?;
252        let inner = provider.chat_stream(request, cancellation.clone()).await?;
253
254        struct State {
255            inner: BoxStream<RuntimeResult<ChatStreamEvent>>,
256            // Text is retained only when it must be validated, and is bounded.
257            accumulated: Option<String>,
258            check: Option<OutputCheck>,
259            tools: ToolChecks,
260            // Proposals are held until the whole response validates, so a
261            // consumer never sees a partial set.
262            held: Vec<ToolCall>,
263            outbox: VecDeque<RuntimeResult<ChatStreamEvent>>,
264            format: ResponseFormat,
265            cancellation: Option<CancellationToken>,
266            completed: bool,
267            finished: bool,
268            _permit: OwnedSemaphorePermit,
269        }
270        let state = State {
271            inner,
272            accumulated: check.is_some().then(String::new),
273            check,
274            tools,
275            held: Vec::new(),
276            outbox: VecDeque::new(),
277            format,
278            cancellation,
279            completed: false,
280            finished: false,
281            _permit: permit,
282        };
283        Ok(Box::pin(futures_util::stream::unfold(
284            state,
285            |mut s| async move {
286                loop {
287                    if let Some(item) = s.outbox.pop_front() {
288                        return Some((item, s));
289                    }
290                    if s.finished {
291                        return None;
292                    }
293                    let item = match s.inner.next().await {
294                        Some(Ok(ChatStreamEvent::Delta { content })) => {
295                            if let Some(text) = &mut s.accumulated {
296                                if text.len() + content.len() > DEFAULT_MAX_OUTPUT_BYTES {
297                                    s.finished = true;
298                                    return Some((
299                                        Err(RuntimeError::ResourceLimit {
300                                            reason:
301                                                "structured output exceeds the validation buffer"
302                                                    .to_owned(),
303                                        }),
304                                        s,
305                                    ));
306                                }
307                                text.push_str(&content);
308                            }
309                            Ok(ChatStreamEvent::Delta { content })
310                        }
311                        Some(Ok(ChatStreamEvent::ToolCallCompleted { call })) => {
312                            s.held.push(call);
313                            continue;
314                        }
315                        Some(Ok(ChatStreamEvent::Completed(mut summary))) => {
316                            s.completed = true;
317                            s.finished = true;
318                            match s.tools.check(&s.held) {
319                                Err(error) => Err(error),
320                                Ok(()) => {
321                                    if s.held.is_empty() {
322                                        if let (Some(check), Some(text)) =
323                                            (&s.check, &s.accumulated)
324                                        {
325                                            summary.structured = Some(check.check(text, &s.format));
326                                        }
327                                    }
328                                    let held = std::mem::take(&mut s.held);
329                                    s.outbox.extend(held.into_iter().map(|call| {
330                                        Ok(ChatStreamEvent::ToolCallCompleted { call })
331                                    }));
332                                    s.outbox.push_back(Ok(ChatStreamEvent::Completed(summary)));
333                                    continue;
334                                }
335                            }
336                        }
337                        Some(Ok(event)) => Ok(event),
338                        Some(Err(error)) => {
339                            s.finished = true;
340                            Err(error)
341                        }
342                        None if s.completed => return None,
343                        None => {
344                            s.finished = true;
345                            Err(
346                                if s.cancellation.as_ref().is_some_and(|t| t.is_cancelled()) {
347                                    RuntimeError::Cancelled
348                                } else {
349                                    RuntimeError::Protocol {
350                                        operation: "chat stream".to_owned(),
351                                        reason: "stream ended without completion".to_owned(),
352                                    }
353                                },
354                            )
355                        }
356                    };
357                    return Some((item, s));
358                }
359            },
360        )))
361    }
362}