1use async_trait::async_trait;
9use futures_util::StreamExt;
10use ml_runtime_common::{BoxStream, CancellationToken, RuntimeError, RuntimeResult};
11use ml_runtime_provider::is_valid_tool_name;
12use ml_runtime_provider::{
13 ChatMessage, ChatMetadata, ChatRequest, ChatResponse, ChatStreamEvent, ChatStreamSummary,
14 ExecutionLocation, FinishReason, GenerationCapabilities, GenerationModelInfo,
15 GenerationProvider, ProviderCapabilities, ResponseFormat, ToolCall, ToolChoice, Usage,
16 DEFAULT_MAX_OUTPUT_BYTES,
17};
18use reqwest::{Response, StatusCode, Url};
19use serde::{Deserialize, Serialize};
20use serde_json::Value;
21use std::{
22 fmt,
23 future::Future,
24 time::{Duration, Instant},
25};
26use tokio::sync::mpsc;
27use tokio_stream::wrappers::ReceiverStream;
28
29pub const PROVIDER_NAME: &str = "openai-compatible";
30const TRANSPORT: &str = "http";
31const ERROR_BODY_LIMIT: usize = 64 * 1024;
32const ERROR_MESSAGE_CHARS: usize = 300;
33
34#[derive(Clone)]
36pub struct ApiKey(String);
37
38impl ApiKey {
39 pub fn new(key: impl Into<String>) -> RuntimeResult<Self> {
40 let key = key.into();
41 if key.is_empty() || key.chars().any(|c| c.is_control() || c.is_whitespace()) {
42 return Err(RuntimeError::InvalidRequest {
43 reason: "API key must be non-empty and contain no whitespace".to_owned(),
44 });
45 }
46 Ok(Self(key))
47 }
48}
49
50impl fmt::Debug for ApiKey {
51 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52 f.write_str("ApiKey(<redacted>)")
53 }
54}
55
56#[derive(Clone, Debug)]
57pub struct OpenAiCompatibleConfig {
58 pub base_url: String,
60 pub api_key: Option<ApiKey>,
61 pub location: Option<ExecutionLocation>,
64 pub capabilities: GenerationCapabilities,
65 pub stream_usage: bool,
67 pub connect_timeout: Duration,
68 pub request_timeout: Duration,
71 pub stream_idle_timeout: Duration,
73 pub max_output_bytes: usize,
74 pub max_response_bytes: usize,
75 pub max_event_bytes: usize,
76 pub max_tool_calls: usize,
78 pub stream_buffer: usize,
80}
81
82impl OpenAiCompatibleConfig {
83 pub fn new(base_url: impl Into<String>) -> Self {
84 Self {
85 base_url: base_url.into(),
86 api_key: None,
87 location: None,
88 capabilities: GenerationCapabilities {
89 streaming: true,
90 stop_sequences: true,
91 seed: true,
92 json_object: false,
95 json_schema: false,
96 tools: false,
98 tool_choice: false,
99 },
100 stream_usage: true,
101 connect_timeout: Duration::from_secs(10),
102 request_timeout: Duration::from_secs(120),
103 stream_idle_timeout: Duration::from_secs(60),
104 max_output_bytes: DEFAULT_MAX_OUTPUT_BYTES,
105 max_response_bytes: 16 * 1024 * 1024,
106 max_event_bytes: 1024 * 1024,
107 max_tool_calls: 32,
108 stream_buffer: 32,
109 }
110 }
111
112 pub fn with_api_key(mut self, key: ApiKey) -> Self {
113 self.api_key = Some(key);
114 self
115 }
116}
117
118#[derive(Clone)]
119pub struct OpenAiCompatibleProvider {
120 base: String,
121 config: OpenAiCompatibleConfig,
122 location: ExecutionLocation,
123 location_inferred: bool,
124 client: reqwest::Client,
125}
126
127impl fmt::Debug for OpenAiCompatibleProvider {
128 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
129 f.debug_struct("OpenAiCompatibleProvider")
130 .field("endpoint", &self.base)
131 .field("location", &self.location)
132 .field("authenticated", &self.config.api_key.is_some())
133 .finish()
134 }
135}
136
137fn config_error(reason: impl Into<String>) -> RuntimeError {
138 RuntimeError::InvalidRequest {
139 reason: reason.into(),
140 }
141}
142
143fn is_loopback(url: &Url) -> bool {
144 let Some(host) = url.host_str() else {
145 return false;
146 };
147 let host = host.trim_start_matches('[').trim_end_matches(']');
148 host.eq_ignore_ascii_case("localhost")
149 || host
150 .parse::<std::net::IpAddr>()
151 .is_ok_and(|ip| ip.is_loopback())
152}
153
154impl OpenAiCompatibleProvider {
155 pub fn new(config: OpenAiCompatibleConfig) -> RuntimeResult<Self> {
156 let url = Url::parse(&config.base_url)
157 .map_err(|_| config_error("endpoint is not a valid URL"))?;
158 if !matches!(url.scheme(), "http" | "https") {
159 return Err(config_error("endpoint scheme must be http or https"));
160 }
161 if url.host().is_none() {
162 return Err(config_error("endpoint must include a host"));
163 }
164 if !url.username().is_empty() || url.password().is_some() {
165 return Err(config_error(
166 "endpoint must not embed credentials; supply the API key separately",
167 ));
168 }
169 if url.query().is_some() || url.fragment().is_some() {
170 return Err(config_error(
171 "endpoint must not include a query or fragment",
172 ));
173 }
174 let loopback = is_loopback(&url);
175 if config.api_key.is_some() && url.scheme() == "http" && !loopback {
176 return Err(config_error(
177 "refusing to send an API key over plain http to a non-loopback host",
178 ));
179 }
180 for (name, value) in [
181 ("max_output_bytes", config.max_output_bytes),
182 ("max_response_bytes", config.max_response_bytes),
183 ("max_event_bytes", config.max_event_bytes),
184 ("max_tool_calls", config.max_tool_calls),
185 ("stream_buffer", config.stream_buffer),
186 ] {
187 if value == 0 {
188 return Err(config_error(format!("{name} must be greater than zero")));
189 }
190 }
191 let (location, location_inferred) = match config.location {
192 Some(location) => (location, false),
193 None => (
194 if loopback {
195 ExecutionLocation::Local
196 } else {
197 ExecutionLocation::Remote
198 },
199 true,
200 ),
201 };
202 let client = reqwest::Client::builder()
203 .connect_timeout(config.connect_timeout)
204 .redirect(reqwest::redirect::Policy::none())
206 .build()
207 .map_err(|error| {
208 RuntimeError::transport(PROVIDER_NAME, error.without_url().to_string())
209 })?;
210 Ok(Self {
211 base: url.as_str().trim_end_matches('/').to_owned(),
212 config,
213 location,
214 location_inferred,
215 client,
216 })
217 }
218
219 pub fn endpoint(&self) -> &str {
220 &self.base
221 }
222
223 pub fn location(&self) -> ExecutionLocation {
224 self.location
225 }
226
227 fn request(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder {
228 let builder = self
229 .client
230 .request(method, format!("{}{}", self.base, path));
231 match &self.config.api_key {
232 Some(key) => builder.bearer_auth(&key.0),
233 None => builder,
234 }
235 }
236
237 fn metadata(&self, request_model: &str, tracker: &Tracker, streamed: bool) -> ChatMetadata {
238 ChatMetadata {
239 provider: PROVIDER_NAME.to_owned(),
240 transport: TRANSPORT.to_owned(),
241 execution: self.location,
242 execution_inferred: self.location_inferred,
243 endpoint: self.base.clone(),
244 requested_model: request_model.to_owned(),
245 reported_model: tracker.model.clone(),
246 response_id: tracker.id.clone(),
247 streamed,
248 stream_events: streamed.then_some(tracker.deltas),
249 time_to_first_token: tracker.first_token,
250 duration: tracker.started.elapsed(),
251 runtime_version: env!("CARGO_PKG_VERSION").to_owned(),
252 }
253 }
254
255 fn wire_request<'a>(&self, request: &'a ChatRequest, stream: bool) -> WireRequest<'a> {
256 let o = &request.options;
257 WireRequest {
258 model: &request.model,
259 messages: request
260 .messages
261 .iter()
262 .map(|m| WireMessage {
263 role: m.role.as_str(),
264 content: (m.tool_calls.is_empty() || !m.content.is_empty())
265 .then_some(m.content.as_str()),
266 tool_calls: (!m.tool_calls.is_empty()).then(|| {
267 m.tool_calls
268 .iter()
269 .map(|c| WireToolCallIn {
270 id: &c.id,
271 r#type: "function",
272 function: WireFunctionIn {
273 name: &c.name,
274 arguments: c.arguments.to_string(),
275 },
276 })
277 .collect()
278 }),
279 tool_call_id: m.tool_call_id.as_deref(),
280 })
281 .collect(),
282 stream,
283 stream_options: (stream && self.config.stream_usage).then_some(WireStreamOptions {
284 include_usage: true,
285 }),
286 temperature: o.temperature,
287 top_p: o.top_p,
288 max_tokens: o.max_tokens,
289 stop: (!o.stop.is_empty()).then_some(&o.stop),
290 seed: o.seed,
291 response_format: match &o.response_format {
292 ResponseFormat::Text => None,
293 ResponseFormat::JsonObject => Some(WireResponseFormat::JsonObject),
294 ResponseFormat::JsonSchema {
295 name,
296 schema,
297 strict,
298 } => Some(WireResponseFormat::JsonSchema {
299 json_schema: WireJsonSchema {
300 name,
301 schema,
302 strict: *strict,
303 },
304 }),
305 },
306 tools: (!o.tools.is_empty()).then(|| {
307 o.tools
308 .iter()
309 .map(|t| WireTool {
310 r#type: "function",
311 function: WireToolFunction {
312 name: &t.name,
313 description: t.description.as_deref(),
314 parameters: &t.input_schema,
315 },
316 })
317 .collect()
318 }),
319 tool_choice: o.tool_choice.as_ref().map(|choice| match choice {
320 ToolChoice::Auto => WireToolChoice::Mode("auto"),
321 ToolChoice::None => WireToolChoice::Mode("none"),
322 ToolChoice::Tool { name } => WireToolChoice::Named {
323 r#type: "function",
324 function: WireFunctionName { name },
325 },
326 }),
327 }
328 }
329
330 async fn error_for_status(&self, response: Response) -> RuntimeError {
331 let status = response.status();
332 let body = read_limited(response, ERROR_BODY_LIMIT)
333 .await
334 .unwrap_or_default();
335 map_http_error(status, &body)
336 }
337}
338
339#[derive(Serialize)]
342struct WireMessage<'a> {
343 role: &'static str,
344 content: Option<&'a str>,
346 #[serde(skip_serializing_if = "Option::is_none")]
347 tool_calls: Option<Vec<WireToolCallIn<'a>>>,
348 #[serde(skip_serializing_if = "Option::is_none")]
349 tool_call_id: Option<&'a str>,
350}
351
352#[derive(Serialize)]
353struct WireToolCallIn<'a> {
354 id: &'a str,
355 r#type: &'static str,
356 function: WireFunctionIn<'a>,
357}
358
359#[derive(Serialize)]
360struct WireFunctionIn<'a> {
361 name: &'a str,
362 arguments: String,
364}
365
366#[derive(Serialize)]
367struct WireTool<'a> {
368 r#type: &'static str,
369 function: WireToolFunction<'a>,
370}
371
372#[derive(Serialize)]
373struct WireToolFunction<'a> {
374 name: &'a str,
375 #[serde(skip_serializing_if = "Option::is_none")]
376 description: Option<&'a str>,
377 parameters: &'a Value,
378}
379
380#[derive(Serialize)]
381#[serde(untagged)]
382enum WireToolChoice<'a> {
383 Mode(&'static str),
384 Named {
385 r#type: &'static str,
386 function: WireFunctionName<'a>,
387 },
388}
389
390#[derive(Serialize)]
391struct WireFunctionName<'a> {
392 name: &'a str,
393}
394
395#[derive(Serialize)]
396struct WireStreamOptions {
397 include_usage: bool,
398}
399
400#[derive(Serialize)]
401#[serde(tag = "type", rename_all = "snake_case")]
402enum WireResponseFormat<'a> {
403 JsonObject,
404 JsonSchema { json_schema: WireJsonSchema<'a> },
405}
406
407#[derive(Serialize)]
408struct WireJsonSchema<'a> {
409 name: &'a str,
410 schema: &'a Value,
411 strict: bool,
412}
413
414#[derive(Serialize)]
415struct WireRequest<'a> {
416 model: &'a str,
417 messages: Vec<WireMessage<'a>>,
418 stream: bool,
419 #[serde(skip_serializing_if = "Option::is_none")]
420 stream_options: Option<WireStreamOptions>,
421 #[serde(skip_serializing_if = "Option::is_none")]
422 temperature: Option<f32>,
423 #[serde(skip_serializing_if = "Option::is_none")]
424 top_p: Option<f32>,
425 #[serde(skip_serializing_if = "Option::is_none")]
426 max_tokens: Option<u32>,
427 #[serde(skip_serializing_if = "Option::is_none")]
428 stop: Option<&'a Vec<String>>,
429 #[serde(skip_serializing_if = "Option::is_none")]
430 seed: Option<i64>,
431 #[serde(skip_serializing_if = "Option::is_none")]
432 response_format: Option<WireResponseFormat<'a>>,
433 #[serde(skip_serializing_if = "Option::is_none")]
434 tools: Option<Vec<WireTool<'a>>>,
435 #[serde(skip_serializing_if = "Option::is_none")]
436 tool_choice: Option<WireToolChoice<'a>>,
437}
438
439#[derive(Deserialize)]
440struct WireUsage {
441 prompt_tokens: Option<u64>,
442 completion_tokens: Option<u64>,
443 total_tokens: Option<u64>,
444}
445
446impl From<WireUsage> for Usage {
447 fn from(w: WireUsage) -> Self {
448 Usage {
449 prompt_tokens: w.prompt_tokens,
450 completion_tokens: w.completion_tokens,
451 total_tokens: w.total_tokens,
452 }
453 }
454}
455
456#[derive(Deserialize)]
457struct WireCompletion {
458 id: Option<String>,
459 model: Option<String>,
460 #[serde(default)]
461 choices: Vec<WireChoice>,
462 usage: Option<WireUsage>,
463}
464
465#[derive(Deserialize)]
466struct WireChoice {
467 message: Option<WireResponseMessage>,
468 delta: Option<WireResponseMessage>,
469 finish_reason: Option<String>,
470}
471
472#[derive(Deserialize)]
473struct WireResponseMessage {
474 role: Option<String>,
475 content: Option<String>,
476 tool_calls: Option<Vec<WireToolCallOut>>,
477}
478
479#[derive(Deserialize)]
480struct WireToolCallOut {
481 index: Option<u32>,
482 id: Option<String>,
483 function: Option<WireFunctionOut>,
484}
485
486#[derive(Deserialize)]
487struct WireFunctionOut {
488 name: Option<String>,
489 arguments: Option<Value>,
490}
491
492#[derive(Deserialize)]
493struct WireModels {
494 data: Vec<WireModel>,
495}
496
497#[derive(Deserialize)]
498struct WireModel {
499 id: String,
500 owned_by: Option<String>,
501}
502
503fn finish_reason(raw: String) -> FinishReason {
504 match raw.as_str() {
505 "stop" => FinishReason::Stop,
506 "length" => FinishReason::Length,
507 "content_filter" => FinishReason::ContentFilter,
508 "tool_calls" => FinishReason::ToolCalls,
509 _ => FinishReason::Other(raw),
510 }
511}
512
513fn protocol(operation: &str, reason: impl Into<String>) -> RuntimeError {
514 RuntimeError::Protocol {
515 operation: operation.to_owned(),
516 reason: reason.into(),
517 }
518}
519
520struct PendingCall {
523 index: u32,
524 id: String,
525 name: String,
526 arguments: String,
527}
528
529struct CallFragment {
531 index: u32,
532 id: Option<String>,
533 name: Option<String>,
534 arguments: Option<String>,
535}
536
537struct ToolAssembler {
542 calls: Vec<PendingCall>,
543 max_calls: usize,
544 bytes: usize,
545}
546
547impl ToolAssembler {
548 fn new(max_calls: usize) -> Self {
549 Self {
550 calls: Vec::new(),
551 max_calls,
552 bytes: 0,
553 }
554 }
555
556 fn feed(
557 &mut self,
558 wire: Vec<WireToolCallOut>,
559 operation: &str,
560 ) -> RuntimeResult<Vec<CallFragment>> {
561 let mut fragments = Vec::new();
562 for call in wire {
563 let function = call.function;
564 let name = function.as_ref().and_then(|f| f.name.clone());
565 let arguments = match function.and_then(|f| f.arguments) {
566 None | Some(Value::Null) => None,
567 Some(Value::String(text)) => Some(text),
568 Some(_) => {
569 return Err(protocol(
570 operation,
571 "tool call arguments must be a JSON-encoded string",
572 ))
573 }
574 };
575 let slot = match (
576 call.index,
577 self.calls.iter().position(|c| Some(c.index) == call.index),
578 ) {
579 (_, Some(slot)) => slot,
580 (None, None) if call.id.is_none() && !self.calls.is_empty() => self.calls.len() - 1,
583 (index, None) => {
584 if self.calls.len() >= self.max_calls {
585 return Err(RuntimeError::ResourceLimit {
586 reason: format!(
587 "response proposes more than {} tool calls",
588 self.max_calls
589 ),
590 });
591 }
592 let index = index.unwrap_or(self.calls.len() as u32);
593 self.calls.push(PendingCall {
594 index,
595 id: String::new(),
596 name: String::new(),
597 arguments: String::new(),
598 });
599 self.calls.len() - 1
600 }
601 };
602 let pending = &mut self.calls[slot];
603 let id = call.id.filter(|f| !f.is_empty());
607 let name = name.filter(|f| !f.is_empty());
608 let arguments = arguments.filter(|f| !f.is_empty());
609 self.bytes += id.as_deref().map_or(0, str::len)
610 + name.as_deref().map_or(0, str::len)
611 + arguments.as_deref().map_or(0, str::len);
612 if let Some(id) = &id {
613 pending.id.push_str(id);
614 }
615 if let Some(name) = &name {
616 pending.name.push_str(name);
617 }
618 if let Some(arguments) = &arguments {
619 pending.arguments.push_str(arguments);
620 }
621 if id.is_some() || name.is_some() || arguments.is_some() {
622 fragments.push(CallFragment {
623 index: pending.index,
624 id,
625 name,
626 arguments,
627 });
628 }
629 }
630 Ok(fragments)
631 }
632
633 fn finish(self, operation: &str) -> RuntimeResult<Vec<ToolCall>> {
634 let mut seen = std::collections::HashSet::new();
635 let mut out = Vec::with_capacity(self.calls.len());
636 for call in self.calls {
637 if call.id.is_empty() || call.name.is_empty() {
638 return Err(protocol(operation, "tool call is missing its id or name"));
639 }
640 if !seen.insert(call.id.clone()) {
641 return Err(protocol(operation, "duplicate tool call id"));
642 }
643 if !is_valid_tool_name(&call.name) {
644 return Err(RuntimeError::InvalidOutput {
645 reason: "model proposed a tool call with a malformed tool name".to_owned(),
646 });
647 }
648 let invalid = |why: &str| RuntimeError::InvalidOutput {
649 reason: format!("arguments for tool call to {:?} {why}", call.name),
650 };
651 let arguments: Value = serde_json::from_str(&call.arguments)
652 .map_err(|_| invalid("are not complete, valid JSON"))?;
653 if !arguments.is_object() {
654 return Err(invalid("are not a JSON object"));
655 }
656 out.push(ToolCall::new(call.id, call.name, arguments));
657 }
658 Ok(out)
659 }
660}
661
662fn truncate(text: &str) -> String {
665 let mut out: String = text.chars().take(ERROR_MESSAGE_CHARS).collect();
666 if text.chars().count() > ERROR_MESSAGE_CHARS {
667 out.push('…');
668 }
669 out
670}
671
672fn provider_error_text(body: &[u8]) -> (Option<String>, String) {
674 let Ok(value) = serde_json::from_slice::<Value>(body) else {
675 return (None, truncate(String::from_utf8_lossy(body).trim()));
676 };
677 let error = value.get("error").unwrap_or(&value);
678 if let Some(text) = error.as_str() {
679 return (None, truncate(text));
680 }
681 let message = error
682 .get("message")
683 .and_then(Value::as_str)
684 .unwrap_or("no error message");
685 let code = error
686 .get("code")
687 .and_then(|c| c.as_str().map(str::to_owned))
688 .or_else(|| error.get("type").and_then(Value::as_str).map(str::to_owned));
689 (code, truncate(message))
690}
691
692fn map_http_error(status: StatusCode, body: &[u8]) -> RuntimeError {
695 let (code, message) = provider_error_text(body);
696 let detail = format!("HTTP {}: {message}", status.as_u16());
697 match status.as_u16() {
698 401 | 403 => RuntimeError::provider_unavailable(
699 PROVIDER_NAME,
700 format!("authentication or authorization failed ({detail})"),
701 ),
702 404 if code.as_deref() == Some("model_not_found")
703 || message.to_ascii_lowercase().contains("model") =>
704 {
705 RuntimeError::ModelNotFound { model: message }
706 }
707 408 | 504 => RuntimeError::Timeout {
708 operation: format!("provider response ({detail})"),
709 },
710 400 | 404 | 405 | 409 | 413 | 415 | 422 => RuntimeError::InvalidRequest {
711 reason: format!("provider rejected request ({detail})"),
712 },
713 429 | 500..=599 => RuntimeError::provider_unavailable(PROVIDER_NAME, detail),
714 _ => protocol("provider request", detail),
715 }
716}
717
718fn map_send_error(error: reqwest::Error) -> RuntimeError {
719 let error = error.without_url();
720 if error.is_timeout() {
721 RuntimeError::Timeout {
722 operation: "connecting to provider".to_owned(),
723 }
724 } else {
725 RuntimeError::transport(PROVIDER_NAME, error.to_string())
726 }
727}
728
729async fn read_limited(response: Response, limit: usize) -> RuntimeResult<Vec<u8>> {
731 if response
732 .content_length()
733 .is_some_and(|n| n as usize > limit)
734 {
735 return Err(RuntimeError::ResourceLimit {
736 reason: format!("provider response exceeds {limit} bytes"),
737 });
738 }
739 let mut body = Vec::new();
740 let mut stream = response.bytes_stream();
741 while let Some(chunk) = stream.next().await {
742 let chunk = chunk.map_err(map_send_error)?;
743 if body.len() + chunk.len() > limit {
744 return Err(RuntimeError::ResourceLimit {
745 reason: format!("provider response exceeds {limit} bytes"),
746 });
747 }
748 body.extend_from_slice(&chunk);
749 }
750 Ok(body)
751}
752
753async fn cancelled(token: &Option<CancellationToken>) {
754 match token {
755 Some(token) => token.cancelled().await,
756 None => std::future::pending().await,
757 }
758}
759
760async fn bounded<T>(
762 token: &Option<CancellationToken>,
763 deadline: Duration,
764 operation: &str,
765 future: impl Future<Output = RuntimeResult<T>>,
766) -> RuntimeResult<T> {
767 tokio::select! {
768 biased;
769 _ = cancelled(token) => Err(RuntimeError::Cancelled),
770 result = tokio::time::timeout(deadline, future) => match result {
771 Ok(result) => result,
772 Err(_) => Err(RuntimeError::Timeout { operation: operation.to_owned() }),
773 },
774 }
775}
776
777struct Tracker {
778 started: Instant,
779 id: Option<String>,
780 model: Option<String>,
781 first_token: Option<Duration>,
782 deltas: u64,
783}
784
785impl Tracker {
786 fn new() -> Self {
787 Self {
788 started: Instant::now(),
789 id: None,
790 model: None,
791 first_token: None,
792 deltas: 0,
793 }
794 }
795
796 fn observe(&mut self, id: &Option<String>, model: &Option<String>) {
797 if self.id.is_none() {
798 self.id.clone_from(id);
799 }
800 if self.model.is_none() {
801 self.model.clone_from(model);
802 }
803 }
804}
805
806#[async_trait]
809impl GenerationProvider for OpenAiCompatibleProvider {
810 fn name(&self) -> &str {
811 PROVIDER_NAME
812 }
813
814 fn capabilities(&self) -> ProviderCapabilities {
815 let g = &self.config.capabilities;
816 ProviderCapabilities {
817 available: true,
818 local: self.location == ExecutionLocation::Local,
819 remote: self.location == ExecutionLocation::Remote,
820 streaming: g.streaming,
821 cancellation: true,
822 structured_output: g.json_object || g.json_schema,
823 batching: false,
824 endpoint: Some(self.base.clone()),
825 notes: vec!["OpenAI-compatible chat completions over HTTP".to_owned()],
826 }
827 }
828
829 fn generation_capabilities(&self) -> GenerationCapabilities {
830 self.config.capabilities.clone()
831 }
832
833 async fn list_models(
834 &self,
835 cancellation: Option<CancellationToken>,
836 ) -> RuntimeResult<Vec<GenerationModelInfo>> {
837 let operation = "listing models";
838 bounded(
839 &cancellation,
840 self.config.request_timeout,
841 operation,
842 async {
843 let response = self
844 .request(reqwest::Method::GET, "/models")
845 .send()
846 .await
847 .map_err(map_send_error)?;
848 let status = response.status();
849 if matches!(status.as_u16(), 404 | 405 | 501) {
850 return Err(RuntimeError::CapabilityMismatch {
851 reason: "upstream does not implement model listing".to_owned(),
852 });
853 }
854 if !status.is_success() {
855 return Err(self.error_for_status(response).await);
856 }
857 let body = read_limited(response, self.config.max_response_bytes).await?;
858 let models: WireModels = serde_json::from_slice(&body)
859 .map_err(|_| protocol(operation, "malformed model list"))?;
860 Ok(models
861 .data
862 .into_iter()
863 .map(|m| GenerationModelInfo {
864 id: m.id,
865 owned_by: m.owned_by,
866 })
867 .collect())
868 },
869 )
870 .await
871 }
872
873 async fn chat(
874 &self,
875 request: ChatRequest,
876 cancellation: Option<CancellationToken>,
877 ) -> RuntimeResult<ChatResponse> {
878 request.validate()?;
879 self.config.capabilities.check(&request, false)?;
880 let operation = "chat completion";
881 let mut tracker = Tracker::new();
882 let completion = bounded(
883 &cancellation,
884 self.config.request_timeout,
885 operation,
886 async {
887 let response = self
888 .request(reqwest::Method::POST, "/chat/completions")
889 .json(&self.wire_request(&request, false))
890 .send()
891 .await
892 .map_err(map_send_error)?;
893 if !response.status().is_success() {
894 return Err(self.error_for_status(response).await);
895 }
896 let body = read_limited(response, self.config.max_response_bytes).await?;
897 serde_json::from_slice::<WireCompletion>(&body)
898 .map_err(|_| protocol(operation, "malformed completion payload"))
899 },
900 )
901 .await?;
902
903 tracker.observe(&completion.id, &completion.model);
904 let choice = completion
905 .choices
906 .into_iter()
907 .next()
908 .ok_or_else(|| protocol(operation, "completion contained no choices"))?;
909 let message = choice
910 .message
911 .ok_or_else(|| protocol(operation, "choice contained no message"))?;
912 if message.role.as_deref().is_some_and(|r| r != "assistant") {
913 return Err(protocol(
914 operation,
915 "response message role is not assistant",
916 ));
917 }
918 let content = message.content.unwrap_or_default();
919 let mut assembler = ToolAssembler::new(self.config.max_tool_calls);
920 assembler.feed(message.tool_calls.unwrap_or_default(), operation)?;
921 if content.len() + assembler.bytes > self.config.max_output_bytes {
922 return Err(RuntimeError::ResourceLimit {
923 reason: format!(
924 "generated output exceeds {} bytes",
925 self.config.max_output_bytes
926 ),
927 });
928 }
929 let tool_calls = assembler.finish(operation)?;
930 Ok(ChatResponse {
931 message: ChatMessage::assistant_with_tool_calls(content, tool_calls),
932 finish_reason: choice.finish_reason.map(finish_reason),
933 usage: completion.usage.map(Usage::from),
934 metadata: self.metadata(&request.model, &tracker, false),
935 structured: None,
936 })
937 }
938
939 async fn chat_stream(
940 &self,
941 request: ChatRequest,
942 cancellation: Option<CancellationToken>,
943 ) -> RuntimeResult<BoxStream<RuntimeResult<ChatStreamEvent>>> {
944 request.validate()?;
945 self.config.capabilities.check(&request, true)?;
946 let tracker = Tracker::new();
947 let response = bounded(
948 &cancellation,
949 self.config.request_timeout,
950 "waiting for stream response",
951 async {
952 let response = self
953 .request(reqwest::Method::POST, "/chat/completions")
954 .json(&self.wire_request(&request, true))
955 .send()
956 .await
957 .map_err(map_send_error)?;
958 if !response.status().is_success() {
959 return Err(self.error_for_status(response).await);
960 }
961 Ok(response)
962 },
963 )
964 .await?;
965 if response
966 .headers()
967 .get(reqwest::header::CONTENT_TYPE)
968 .and_then(|v| v.to_str().ok())
969 .is_some_and(|v| v.trim_start().starts_with("application/json"))
970 {
971 return Err(protocol(
972 "chat stream",
973 "provider returned a non-streaming response to a streaming request",
974 ));
975 }
976
977 let (sender, receiver) = mpsc::channel(self.config.stream_buffer);
978 let provider = self.clone();
979 tokio::spawn(async move {
980 provider
981 .pump_stream(request, response, tracker, cancellation, sender)
982 .await
983 });
984 Ok(Box::pin(ReceiverStream::new(receiver)))
985 }
986}
987
988type EventSender = mpsc::Sender<RuntimeResult<ChatStreamEvent>>;
989
990impl OpenAiCompatibleProvider {
991 async fn pump_stream(
995 &self,
996 request: ChatRequest,
997 response: Response,
998 mut tracker: Tracker,
999 cancellation: Option<CancellationToken>,
1000 sender: EventSender,
1001 ) {
1002 let outcome = self
1003 .read_stream(&request, response, &mut tracker, &cancellation, &sender)
1004 .await;
1005 if let Err(error) = outcome {
1006 let _ = sender.try_send(Err(error.clone()));
1008 if sender.capacity() == 0 && !sender.is_closed() {
1009 tokio::select! {
1011 _ = cancelled(&cancellation) => {}
1012 _ = sender.send(Err(error)) => {}
1013 }
1014 }
1015 }
1016 }
1017
1018 async fn emit(
1019 sender: &EventSender,
1020 cancellation: &Option<CancellationToken>,
1021 event: ChatStreamEvent,
1022 ) -> RuntimeResult<()> {
1023 tokio::select! {
1024 biased;
1025 _ = cancelled(cancellation) => Err(RuntimeError::Cancelled),
1026 sent = sender.send(Ok(event)) => sent.map_err(|_| RuntimeError::Cancelled),
1027 }
1028 }
1029
1030 async fn read_stream(
1031 &self,
1032 request: &ChatRequest,
1033 response: Response,
1034 tracker: &mut Tracker,
1035 cancellation: &Option<CancellationToken>,
1036 sender: &EventSender,
1037 ) -> RuntimeResult<()> {
1038 let operation = "chat stream";
1039 Self::emit(
1040 sender,
1041 cancellation,
1042 ChatStreamEvent::Started {
1043 provider: PROVIDER_NAME.to_owned(),
1044 model: request.model.clone(),
1045 },
1046 )
1047 .await?;
1048
1049 let mut parser = SseParser::new(self.config.max_event_bytes);
1050 let mut bytes = response.bytes_stream();
1051 let mut finish = None;
1052 let mut usage = None;
1053 let mut output_bytes = 0usize;
1054 let mut done = false;
1055 let mut assembler = ToolAssembler::new(self.config.max_tool_calls);
1056
1057 'read: loop {
1058 let chunk = tokio::select! {
1059 biased;
1060 _ = sender.closed() => return Ok(()),
1061 _ = cancelled(cancellation) => return Err(RuntimeError::Cancelled),
1062 chunk = tokio::time::timeout(self.config.stream_idle_timeout, bytes.next()) => chunk,
1063 };
1064 let events = match chunk {
1065 Err(_) if finish.is_some() => break 'read, Err(_) => {
1067 return Err(RuntimeError::Timeout {
1068 operation: "waiting for stream data".to_owned(),
1069 })
1070 }
1071 Ok(None) => parser.finish(),
1072 Ok(Some(Err(error))) => return Err(map_send_error(error)),
1073 Ok(Some(Ok(chunk))) => parser.push(&chunk),
1074 }
1075 .map_err(|e| protocol(operation, e))?;
1076 let at_eof = events.eof;
1077
1078 for event in events.events {
1079 if event.data.trim() == "[DONE]" {
1080 done = true;
1081 break 'read;
1082 }
1083 let value: Value = serde_json::from_str(&event.data)
1084 .map_err(|_| protocol(operation, "malformed stream event"))?;
1085 if let Some(error) = value.get("error") {
1086 let body = serde_json::to_vec(&serde_json::json!({ "error": error }))
1087 .unwrap_or_default();
1088 return Err(map_stream_error(&body));
1089 }
1090 let chunk: WireCompletion = serde_json::from_value(value)
1091 .map_err(|_| protocol(operation, "malformed stream chunk"))?;
1092 tracker.observe(&chunk.id, &chunk.model);
1093 if let Some(u) = chunk.usage {
1094 usage = Some(Usage::from(u));
1095 }
1096 if let Some(choice) = chunk.choices.into_iter().next() {
1097 if let Some(reason) = choice.finish_reason {
1098 finish = Some(finish_reason(reason));
1099 }
1100 let mut delta = choice.delta;
1101 if let Some(wire) = delta.as_mut().and_then(|d| d.tool_calls.take()) {
1102 let fragments = assembler.feed(wire, operation)?;
1103 if output_bytes + assembler.bytes > self.config.max_output_bytes {
1104 return Err(RuntimeError::ResourceLimit {
1105 reason: format!(
1106 "generated output exceeds {} bytes",
1107 self.config.max_output_bytes
1108 ),
1109 });
1110 }
1111 for fragment in fragments {
1112 Self::emit(
1113 sender,
1114 cancellation,
1115 ChatStreamEvent::ToolCallDelta {
1116 index: fragment.index,
1117 id: fragment.id,
1118 name: fragment.name,
1119 arguments_fragment: fragment.arguments,
1120 },
1121 )
1122 .await?;
1123 }
1124 }
1125 if let Some(content) = delta.and_then(|d| d.content) {
1126 if !content.is_empty() {
1127 output_bytes += content.len();
1128 if output_bytes + assembler.bytes > self.config.max_output_bytes {
1129 return Err(RuntimeError::ResourceLimit {
1130 reason: format!(
1131 "generated output exceeds {} bytes",
1132 self.config.max_output_bytes
1133 ),
1134 });
1135 }
1136 tracker.deltas += 1;
1137 tracker
1138 .first_token
1139 .get_or_insert_with(|| tracker.started.elapsed());
1140 Self::emit(sender, cancellation, ChatStreamEvent::Delta { content })
1141 .await?;
1142 }
1143 }
1144 }
1145 }
1146 if at_eof {
1147 break;
1148 }
1149 }
1150
1151 if !done && finish.is_none() {
1152 return Err(protocol(
1153 operation,
1154 "stream ended before the provider signalled completion",
1155 ));
1156 }
1157 for call in assembler.finish(operation)? {
1159 Self::emit(
1160 sender,
1161 cancellation,
1162 ChatStreamEvent::ToolCallCompleted { call },
1163 )
1164 .await?;
1165 }
1166 Self::emit(
1167 sender,
1168 cancellation,
1169 ChatStreamEvent::Completed(Box::new(ChatStreamSummary {
1170 finish_reason: finish,
1171 usage,
1172 metadata: self.metadata(&request.model, tracker, true),
1173 structured: None,
1174 })),
1175 )
1176 .await
1177 }
1178}
1179
1180fn map_stream_error(body: &[u8]) -> RuntimeError {
1181 let (_, message) = provider_error_text(body);
1183 RuntimeError::provider_unavailable(PROVIDER_NAME, format!("stream error: {message}"))
1184}
1185
1186#[derive(Debug, PartialEq, Eq)]
1189pub(crate) struct SseEvent {
1190 #[allow(dead_code)]
1191 pub event: Option<String>,
1192 pub data: String,
1193}
1194
1195#[derive(Debug, Default)]
1196pub(crate) struct SseBatch {
1197 pub events: Vec<SseEvent>,
1198 pub eof: bool,
1199}
1200
1201pub(crate) struct SseParser {
1205 line: Vec<u8>,
1206 data: String,
1207 has_data: bool,
1208 event: Option<String>,
1209 pending_cr: bool,
1210 first_line: bool,
1211 max: usize,
1212}
1213
1214impl SseParser {
1215 pub fn new(max: usize) -> Self {
1216 Self {
1217 line: Vec::new(),
1218 data: String::new(),
1219 has_data: false,
1220 event: None,
1221 pending_cr: false,
1222 first_line: true,
1223 max,
1224 }
1225 }
1226
1227 pub fn push(&mut self, bytes: &[u8]) -> Result<SseBatch, String> {
1228 let mut batch = SseBatch::default();
1229 for &byte in bytes {
1230 if std::mem::take(&mut self.pending_cr) && byte == b'\n' {
1231 continue;
1232 }
1233 match byte {
1234 b'\r' => {
1235 self.pending_cr = true;
1236 self.end_line(&mut batch)?;
1237 }
1238 b'\n' => self.end_line(&mut batch)?,
1239 _ => {
1240 self.line.push(byte);
1241 if self.line.len() > self.max {
1242 return Err("stream line exceeds the configured size limit".to_owned());
1243 }
1244 }
1245 }
1246 }
1247 Ok(batch)
1248 }
1249
1250 pub fn finish(&mut self) -> Result<SseBatch, String> {
1254 let mut batch = SseBatch::default();
1255 if !self.line.is_empty() {
1256 self.end_line(&mut batch)?;
1257 }
1258 self.dispatch(&mut batch);
1259 batch.eof = true;
1260 Ok(batch)
1261 }
1262
1263 fn end_line(&mut self, batch: &mut SseBatch) -> Result<(), String> {
1264 let bytes = std::mem::take(&mut self.line);
1265 let mut line = String::from_utf8(bytes).map_err(|_| "stream is not valid UTF-8")?;
1266 if std::mem::take(&mut self.first_line) {
1267 if let Some(stripped) = line.strip_prefix('\u{feff}') {
1268 line = stripped.to_owned();
1269 }
1270 }
1271 if line.is_empty() {
1272 self.dispatch(batch);
1273 return Ok(());
1274 }
1275 if line.starts_with(':') {
1276 return Ok(());
1277 }
1278 let (field, value) = match line.split_once(':') {
1279 Some((field, value)) => (field, value.strip_prefix(' ').unwrap_or(value)),
1280 None => (line.as_str(), ""),
1281 };
1282 match field {
1283 "data" => {
1284 if self.data.len() + value.len() + 1 > self.max {
1285 return Err("stream event exceeds the configured size limit".to_owned());
1286 }
1287 if self.has_data {
1288 self.data.push('\n');
1289 }
1290 self.data.push_str(value);
1291 self.has_data = true;
1292 }
1293 "event" => self.event = Some(value.to_owned()),
1294 _ => {} }
1296 Ok(())
1297 }
1298
1299 fn dispatch(&mut self, batch: &mut SseBatch) {
1300 if self.has_data {
1301 batch.events.push(SseEvent {
1302 event: self.event.take(),
1303 data: std::mem::take(&mut self.data),
1304 });
1305 }
1306 self.has_data = false;
1307 self.event = None;
1308 }
1309}
1310
1311#[cfg(test)]
1312mod tests {
1313 use super::*;
1314
1315 fn parse_split(input: &[u8], step: usize) -> Vec<String> {
1316 let mut parser = SseParser::new(1024);
1317 let mut out = Vec::new();
1318 for piece in input.chunks(step) {
1319 out.extend(
1320 parser
1321 .push(piece)
1322 .unwrap()
1323 .events
1324 .into_iter()
1325 .map(|e| e.data),
1326 );
1327 }
1328 out.extend(parser.finish().unwrap().events.into_iter().map(|e| e.data));
1329 out
1330 }
1331
1332 #[test]
1333 fn events_are_identical_at_every_chunk_size() {
1334 let input = "data: {\"a\":\"é😀\"}\n\n: comment\r\ndata: two\r\ndata: lines\r\n\r\ndata: [DONE]\n\n";
1335 let expected = vec!["{\"a\":\"é😀\"}", "two\nlines", "[DONE]"];
1336 for step in 1..=input.len() {
1337 assert_eq!(parse_split(input.as_bytes(), step), expected, "step {step}");
1338 }
1339 }
1340
1341 #[test]
1342 fn lone_cr_terminators_and_bom() {
1343 assert_eq!(
1344 parse_split("\u{feff}data: x\r\rdata: y\r\r".as_bytes(), 2),
1345 vec!["x", "y"]
1346 );
1347 }
1348
1349 #[test]
1350 fn unterminated_final_event_is_dispatched_at_eof() {
1351 assert_eq!(parse_split(b"data: [DONE]", 3), vec!["[DONE]"]);
1352 }
1353
1354 #[test]
1355 fn oversized_lines_and_events_are_rejected() {
1356 let mut parser = SseParser::new(8);
1357 assert!(parser.push(b"data: 0123456789").is_err());
1358 let mut parser = SseParser::new(8);
1359 assert!(parser.push(b"data: 1234\ndata: 5678\n").is_err());
1360 }
1361
1362 #[test]
1363 fn invalid_utf8_is_rejected() {
1364 let mut parser = SseParser::new(64);
1365 assert!(parser.push(b"data: \xff\xfe\n").is_err());
1366 }
1367}