1use super::Runtime;
9use futures_util::StreamExt;
10use jsonschema::Validator;
11use ml_runtime_common::{BoxStream, CancellationToken, RuntimeError, RuntimeResult};
12use ml_runtime_provider::{
13 ChatRequest, ChatResponse, ChatStreamEvent, GenerationModelInfo, GenerationProvider,
14 ProviderCapability, ResponseFormat, StructuredOutput, ToolCall, ToolChoice, ToolDefinition,
15 DEFAULT_MAX_OUTPUT_BYTES,
16};
17use serde_json::Value;
18use std::{
19 collections::{BTreeMap, VecDeque},
20 sync::Arc,
21};
22use tokio::sync::OwnedSemaphorePermit;
23
24const MAX_REPORTED_VIOLATIONS: usize = 16;
25const MAX_VIOLATION_CHARS: usize = 200;
26
27enum OutputCheck {
29 Json,
31 Schema(Arc<Validator>),
32}
33
34impl OutputCheck {
35 fn for_format(format: &ResponseFormat) -> RuntimeResult<Option<Self>> {
38 match format {
39 ResponseFormat::Text => Ok(None),
40 ResponseFormat::JsonObject => Ok(Some(Self::Json)),
41 ResponseFormat::JsonSchema { schema, .. } => jsonschema::validator_for(schema)
42 .map(|validator| Some(Self::Schema(Arc::new(validator))))
43 .map_err(|error| RuntimeError::InvalidRequest {
44 reason: format!("unusable JSON schema: {}", clip(&error.to_string())),
45 }),
46 }
47 }
48
49 fn check(&self, text: &str, format: &ResponseFormat) -> StructuredOutput {
50 let value: Value = match serde_json::from_str(text) {
51 Ok(value) => value,
52 Err(error) => {
53 return StructuredOutput::InvalidJson {
54 error: error.to_string(),
55 }
56 }
57 };
58 if matches!(format, ResponseFormat::JsonObject) && !value.is_object() {
61 return StructuredOutput::SchemaViolation {
62 errors: vec![
63 "JSON object output was requested but the value is not an object".to_owned(),
64 ],
65 };
66 }
67 match self {
68 Self::Json => StructuredOutput::Validated { value },
69 Self::Schema(validator) => {
70 let errors: Vec<String> = validator
71 .iter_errors(&value)
72 .take(MAX_REPORTED_VIOLATIONS)
73 .map(|error| clip(&error.to_string()))
74 .collect();
75 if errors.is_empty() {
76 StructuredOutput::Validated { value }
77 } else {
78 StructuredOutput::SchemaViolation { errors }
79 }
80 }
81 }
82 }
83}
84
85struct ToolChecks {
89 validators: BTreeMap<String, Validator>,
90 choice: Option<ToolChoice>,
91}
92
93impl ToolChecks {
94 fn compile(tools: &[ToolDefinition], choice: &Option<ToolChoice>) -> RuntimeResult<Self> {
97 let mut validators = BTreeMap::new();
98 for tool in tools {
99 let validator = jsonschema::validator_for(&tool.input_schema).map_err(|error| {
100 RuntimeError::InvalidRequest {
101 reason: format!(
102 "unusable input schema for tool {:?}: {}",
103 tool.name,
104 clip(&error.to_string())
105 ),
106 }
107 })?;
108 validators.insert(tool.name.clone(), validator);
109 }
110 Ok(Self {
111 validators,
112 choice: choice.clone(),
113 })
114 }
115
116 fn check(&self, calls: &[ToolCall]) -> RuntimeResult<()> {
117 let invalid = |reason: String| Err(RuntimeError::InvalidOutput { reason });
118 if calls.is_empty() {
119 return Ok(());
120 }
121 if matches!(self.choice, Some(ToolChoice::None)) {
122 return invalid("model proposed tool calls although tool_choice is none".to_owned());
123 }
124 for call in calls {
125 let Some(validator) = self.validators.get(&call.name) else {
126 return invalid(format!(
127 "model proposed a call to undefined tool {:?}",
128 call.name
129 ));
130 };
131 if let Some(ToolChoice::Tool { name }) = &self.choice {
132 if *name != call.name {
133 return invalid(format!(
134 "model proposed tool {:?} although {name:?} was required",
135 call.name
136 ));
137 }
138 }
139 let paths: Vec<String> = validator
140 .iter_errors(&call.arguments)
141 .take(MAX_REPORTED_VIOLATIONS)
142 .map(|error| error.instance_path().to_string())
143 .collect();
144 if !paths.is_empty() {
145 return invalid(format!(
147 "arguments proposed for tool {:?} violate its input schema at: {}",
148 call.name,
149 paths.join(", ")
150 ));
151 }
152 }
153 Ok(())
154 }
155}
156
157fn clip(text: &str) -> String {
158 text.chars().take(MAX_VIOLATION_CHARS).collect()
159}
160
161impl Runtime {
162 fn generation_provider(&self, name: &str) -> RuntimeResult<&Arc<dyn GenerationProvider>> {
163 self.generation_providers.get(name).ok_or_else(|| {
164 RuntimeError::provider_unavailable(name, ": no such generation provider is registered")
165 })
166 }
167
168 pub fn generation_providers(&self) -> Vec<ProviderCapability> {
171 self.generation_providers
172 .iter()
173 .map(|(name, provider)| {
174 ProviderCapability::from_parts(name.clone(), provider.capabilities())
175 })
176 .collect()
177 }
178
179 pub async fn list_generation_models(
181 &self,
182 provider: &str,
183 cancellation: Option<CancellationToken>,
184 ) -> RuntimeResult<Vec<GenerationModelInfo>> {
185 self.generation_provider(provider)?
186 .list_models(cancellation)
187 .await
188 }
189
190 async fn admit(
191 &self,
192 cancellation: &Option<CancellationToken>,
193 ) -> RuntimeResult<OwnedSemaphorePermit> {
194 let acquire = self.inference_semaphore.clone().acquire_owned();
195 let permit = match cancellation {
196 Some(token) => tokio::select! {
197 biased;
198 _ = token.cancelled() => return Err(RuntimeError::Cancelled),
199 permit = acquire => permit,
200 },
201 None => acquire.await,
202 };
203 permit.map_err(|_| RuntimeError::ResourceLimit {
204 reason: "runtime is shut down".to_owned(),
205 })
206 }
207
208 pub async fn chat(
214 &self,
215 provider: &str,
216 request: ChatRequest,
217 cancellation: Option<CancellationToken>,
218 ) -> RuntimeResult<ChatResponse> {
219 request.validate()?;
220 let format = request.options.response_format.clone();
221 let check = OutputCheck::for_format(&format)?;
222 let tools = ToolChecks::compile(&request.options.tools, &request.options.tool_choice)?;
223 let provider = self.generation_provider(provider)?;
224 let _permit = self.admit(&cancellation).await?;
225 let mut response = provider.chat(request, cancellation).await?;
226 tools.check(&response.message.tool_calls)?;
227 if let (Some(check), true) = (check, response.message.tool_calls.is_empty()) {
230 response.structured = Some(check.check(&response.message.content, &format));
231 }
232 Ok(response)
233 }
234
235 pub async fn chat_stream(
241 &self,
242 provider: &str,
243 request: ChatRequest,
244 cancellation: Option<CancellationToken>,
245 ) -> RuntimeResult<BoxStream<RuntimeResult<ChatStreamEvent>>> {
246 request.validate()?;
247 let format = request.options.response_format.clone();
248 let check = OutputCheck::for_format(&format)?;
249 let tools = ToolChecks::compile(&request.options.tools, &request.options.tool_choice)?;
250 let provider = self.generation_provider(provider)?;
251 let permit = self.admit(&cancellation).await?;
252 let inner = provider.chat_stream(request, cancellation.clone()).await?;
253
254 struct State {
255 inner: BoxStream<RuntimeResult<ChatStreamEvent>>,
256 accumulated: Option<String>,
258 check: Option<OutputCheck>,
259 tools: ToolChecks,
260 held: Vec<ToolCall>,
263 outbox: VecDeque<RuntimeResult<ChatStreamEvent>>,
264 format: ResponseFormat,
265 cancellation: Option<CancellationToken>,
266 completed: bool,
267 finished: bool,
268 _permit: OwnedSemaphorePermit,
269 }
270 let state = State {
271 inner,
272 accumulated: check.is_some().then(String::new),
273 check,
274 tools,
275 held: Vec::new(),
276 outbox: VecDeque::new(),
277 format,
278 cancellation,
279 completed: false,
280 finished: false,
281 _permit: permit,
282 };
283 Ok(Box::pin(futures_util::stream::unfold(
284 state,
285 |mut s| async move {
286 loop {
287 if let Some(item) = s.outbox.pop_front() {
288 return Some((item, s));
289 }
290 if s.finished {
291 return None;
292 }
293 let item = match s.inner.next().await {
294 Some(Ok(ChatStreamEvent::Delta { content })) => {
295 if let Some(text) = &mut s.accumulated {
296 if text.len() + content.len() > DEFAULT_MAX_OUTPUT_BYTES {
297 s.finished = true;
298 return Some((
299 Err(RuntimeError::ResourceLimit {
300 reason:
301 "structured output exceeds the validation buffer"
302 .to_owned(),
303 }),
304 s,
305 ));
306 }
307 text.push_str(&content);
308 }
309 Ok(ChatStreamEvent::Delta { content })
310 }
311 Some(Ok(ChatStreamEvent::ToolCallCompleted { call })) => {
312 s.held.push(call);
313 continue;
314 }
315 Some(Ok(ChatStreamEvent::Completed(mut summary))) => {
316 s.completed = true;
317 s.finished = true;
318 match s.tools.check(&s.held) {
319 Err(error) => Err(error),
320 Ok(()) => {
321 if s.held.is_empty() {
322 if let (Some(check), Some(text)) =
323 (&s.check, &s.accumulated)
324 {
325 summary.structured = Some(check.check(text, &s.format));
326 }
327 }
328 let held = std::mem::take(&mut s.held);
329 s.outbox.extend(held.into_iter().map(|call| {
330 Ok(ChatStreamEvent::ToolCallCompleted { call })
331 }));
332 s.outbox.push_back(Ok(ChatStreamEvent::Completed(summary)));
333 continue;
334 }
335 }
336 }
337 Some(Ok(event)) => Ok(event),
338 Some(Err(error)) => {
339 s.finished = true;
340 Err(error)
341 }
342 None if s.completed => return None,
343 None => {
344 s.finished = true;
345 Err(
346 if s.cancellation.as_ref().is_some_and(|t| t.is_cancelled()) {
347 RuntimeError::Cancelled
348 } else {
349 RuntimeError::Protocol {
350 operation: "chat stream".to_owned(),
351 reason: "stream ended without completion".to_owned(),
352 }
353 },
354 )
355 }
356 };
357 return Some((item, s));
358 }
359 },
360 )))
361 }
362}