Skip to main content

ml_runtime_http_provider/
openai.rs

1//! OpenAI-compatible chat-completions adapter.
2//!
3//! All OpenAI wire types live in this module and never escape it; callers see
4//! only the model-neutral contract from `ml-runtime-provider`. The adapter
5//! never logs, and its `Debug` output and errors never contain the API key or
6//! message contents.
7
8use async_trait::async_trait;
9use futures_util::StreamExt;
10use ml_runtime_common::{BoxStream, CancellationToken, RuntimeError, RuntimeResult};
11use ml_runtime_provider::is_valid_tool_name;
12use ml_runtime_provider::{
13    ChatMessage, ChatMetadata, ChatRequest, ChatResponse, ChatStreamEvent, ChatStreamSummary,
14    ExecutionLocation, FinishReason, GenerationCapabilities, GenerationModelInfo,
15    GenerationProvider, ProviderCapabilities, ResponseFormat, ToolCall, ToolChoice, Usage,
16    DEFAULT_MAX_OUTPUT_BYTES,
17};
18use reqwest::{Response, StatusCode, Url};
19use serde::{Deserialize, Serialize};
20use serde_json::Value;
21use std::{
22    fmt,
23    future::Future,
24    time::{Duration, Instant},
25};
26use tokio::sync::mpsc;
27use tokio_stream::wrappers::ReceiverStream;
28
29pub const PROVIDER_NAME: &str = "openai-compatible";
30const TRANSPORT: &str = "http";
31const ERROR_BODY_LIMIT: usize = 64 * 1024;
32const ERROR_MESSAGE_CHARS: usize = 300;
33
34/// A bearer credential. `Debug` is redacted and the type is not serializable.
35#[derive(Clone)]
36pub struct ApiKey(String);
37
38impl ApiKey {
39    pub fn new(key: impl Into<String>) -> RuntimeResult<Self> {
40        let key = key.into();
41        if key.is_empty() || key.chars().any(|c| c.is_control() || c.is_whitespace()) {
42            return Err(RuntimeError::InvalidRequest {
43                reason: "API key must be non-empty and contain no whitespace".to_owned(),
44            });
45        }
46        Ok(Self(key))
47    }
48}
49
50impl fmt::Debug for ApiKey {
51    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52        f.write_str("ApiKey(<redacted>)")
53    }
54}
55
56#[derive(Clone, Debug)]
57pub struct OpenAiCompatibleConfig {
58    /// Base URL including the API prefix, e.g. `http://127.0.0.1:11434/v1`.
59    pub base_url: String,
60    pub api_key: Option<ApiKey>,
61    /// Explicit local/remote classification. When `None`, loopback hosts are
62    /// reported as local and everything else as remote (flagged as inferred).
63    pub location: Option<ExecutionLocation>,
64    pub capabilities: GenerationCapabilities,
65    /// Ask streaming servers to include token usage in the final chunk.
66    pub stream_usage: bool,
67    pub connect_timeout: Duration,
68    /// Bound on time-to-response-headers for streams and on the whole
69    /// operation for non-streaming requests and model listing.
70    pub request_timeout: Duration,
71    /// Maximum silence between stream chunks.
72    pub stream_idle_timeout: Duration,
73    pub max_output_bytes: usize,
74    pub max_response_bytes: usize,
75    pub max_event_bytes: usize,
76    /// Maximum tool calls accepted in one response.
77    pub max_tool_calls: usize,
78    /// Bounded channel depth between the network task and the consumer.
79    pub stream_buffer: usize,
80}
81
82impl OpenAiCompatibleConfig {
83    pub fn new(base_url: impl Into<String>) -> Self {
84        Self {
85            base_url: base_url.into(),
86            api_key: None,
87            location: None,
88            capabilities: GenerationCapabilities {
89                streaming: true,
90                stop_sequences: true,
91                seed: true,
92                // Structured output support varies by server and cannot be
93                // discovered; it must be declared.
94                json_object: false,
95                json_schema: false,
96                // Tool support is likewise per-server and must be declared.
97                tools: false,
98                tool_choice: false,
99            },
100            stream_usage: true,
101            connect_timeout: Duration::from_secs(10),
102            request_timeout: Duration::from_secs(120),
103            stream_idle_timeout: Duration::from_secs(60),
104            max_output_bytes: DEFAULT_MAX_OUTPUT_BYTES,
105            max_response_bytes: 16 * 1024 * 1024,
106            max_event_bytes: 1024 * 1024,
107            max_tool_calls: 32,
108            stream_buffer: 32,
109        }
110    }
111
112    pub fn with_api_key(mut self, key: ApiKey) -> Self {
113        self.api_key = Some(key);
114        self
115    }
116}
117
118#[derive(Clone)]
119pub struct OpenAiCompatibleProvider {
120    base: String,
121    config: OpenAiCompatibleConfig,
122    location: ExecutionLocation,
123    location_inferred: bool,
124    client: reqwest::Client,
125}
126
127impl fmt::Debug for OpenAiCompatibleProvider {
128    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
129        f.debug_struct("OpenAiCompatibleProvider")
130            .field("endpoint", &self.base)
131            .field("location", &self.location)
132            .field("authenticated", &self.config.api_key.is_some())
133            .finish()
134    }
135}
136
137fn config_error(reason: impl Into<String>) -> RuntimeError {
138    RuntimeError::InvalidRequest {
139        reason: reason.into(),
140    }
141}
142
143fn is_loopback(url: &Url) -> bool {
144    let Some(host) = url.host_str() else {
145        return false;
146    };
147    let host = host.trim_start_matches('[').trim_end_matches(']');
148    host.eq_ignore_ascii_case("localhost")
149        || host
150            .parse::<std::net::IpAddr>()
151            .is_ok_and(|ip| ip.is_loopback())
152}
153
154impl OpenAiCompatibleProvider {
155    pub fn new(config: OpenAiCompatibleConfig) -> RuntimeResult<Self> {
156        let url = Url::parse(&config.base_url)
157            .map_err(|_| config_error("endpoint is not a valid URL"))?;
158        if !matches!(url.scheme(), "http" | "https") {
159            return Err(config_error("endpoint scheme must be http or https"));
160        }
161        if url.host().is_none() {
162            return Err(config_error("endpoint must include a host"));
163        }
164        if !url.username().is_empty() || url.password().is_some() {
165            return Err(config_error(
166                "endpoint must not embed credentials; supply the API key separately",
167            ));
168        }
169        if url.query().is_some() || url.fragment().is_some() {
170            return Err(config_error(
171                "endpoint must not include a query or fragment",
172            ));
173        }
174        let loopback = is_loopback(&url);
175        if config.api_key.is_some() && url.scheme() == "http" && !loopback {
176            return Err(config_error(
177                "refusing to send an API key over plain http to a non-loopback host",
178            ));
179        }
180        for (name, value) in [
181            ("max_output_bytes", config.max_output_bytes),
182            ("max_response_bytes", config.max_response_bytes),
183            ("max_event_bytes", config.max_event_bytes),
184            ("max_tool_calls", config.max_tool_calls),
185            ("stream_buffer", config.stream_buffer),
186        ] {
187            if value == 0 {
188                return Err(config_error(format!("{name} must be greater than zero")));
189            }
190        }
191        let (location, location_inferred) = match config.location {
192            Some(location) => (location, false),
193            None => (
194                if loopback {
195                    ExecutionLocation::Local
196                } else {
197                    ExecutionLocation::Remote
198                },
199                true,
200            ),
201        };
202        let client = reqwest::Client::builder()
203            .connect_timeout(config.connect_timeout)
204            // A POST carrying a bearer token must not be silently re-sent.
205            .redirect(reqwest::redirect::Policy::none())
206            .build()
207            .map_err(|error| {
208                RuntimeError::transport(PROVIDER_NAME, error.without_url().to_string())
209            })?;
210        Ok(Self {
211            base: url.as_str().trim_end_matches('/').to_owned(),
212            config,
213            location,
214            location_inferred,
215            client,
216        })
217    }
218
219    pub fn endpoint(&self) -> &str {
220        &self.base
221    }
222
223    pub fn location(&self) -> ExecutionLocation {
224        self.location
225    }
226
227    fn request(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder {
228        let builder = self
229            .client
230            .request(method, format!("{}{}", self.base, path));
231        match &self.config.api_key {
232            Some(key) => builder.bearer_auth(&key.0),
233            None => builder,
234        }
235    }
236
237    fn metadata(&self, request_model: &str, tracker: &Tracker, streamed: bool) -> ChatMetadata {
238        ChatMetadata {
239            provider: PROVIDER_NAME.to_owned(),
240            transport: TRANSPORT.to_owned(),
241            execution: self.location,
242            execution_inferred: self.location_inferred,
243            endpoint: self.base.clone(),
244            requested_model: request_model.to_owned(),
245            reported_model: tracker.model.clone(),
246            response_id: tracker.id.clone(),
247            streamed,
248            stream_events: streamed.then_some(tracker.deltas),
249            time_to_first_token: tracker.first_token,
250            duration: tracker.started.elapsed(),
251            runtime_version: env!("CARGO_PKG_VERSION").to_owned(),
252        }
253    }
254
255    fn wire_request<'a>(&self, request: &'a ChatRequest, stream: bool) -> WireRequest<'a> {
256        let o = &request.options;
257        WireRequest {
258            model: &request.model,
259            messages: request
260                .messages
261                .iter()
262                .map(|m| WireMessage {
263                    role: m.role.as_str(),
264                    content: (m.tool_calls.is_empty() || !m.content.is_empty())
265                        .then_some(m.content.as_str()),
266                    tool_calls: (!m.tool_calls.is_empty()).then(|| {
267                        m.tool_calls
268                            .iter()
269                            .map(|c| WireToolCallIn {
270                                id: &c.id,
271                                r#type: "function",
272                                function: WireFunctionIn {
273                                    name: &c.name,
274                                    arguments: c.arguments.to_string(),
275                                },
276                            })
277                            .collect()
278                    }),
279                    tool_call_id: m.tool_call_id.as_deref(),
280                })
281                .collect(),
282            stream,
283            stream_options: (stream && self.config.stream_usage).then_some(WireStreamOptions {
284                include_usage: true,
285            }),
286            temperature: o.temperature,
287            top_p: o.top_p,
288            max_tokens: o.max_tokens,
289            stop: (!o.stop.is_empty()).then_some(&o.stop),
290            seed: o.seed,
291            response_format: match &o.response_format {
292                ResponseFormat::Text => None,
293                ResponseFormat::JsonObject => Some(WireResponseFormat::JsonObject),
294                ResponseFormat::JsonSchema {
295                    name,
296                    schema,
297                    strict,
298                } => Some(WireResponseFormat::JsonSchema {
299                    json_schema: WireJsonSchema {
300                        name,
301                        schema,
302                        strict: *strict,
303                    },
304                }),
305            },
306            tools: (!o.tools.is_empty()).then(|| {
307                o.tools
308                    .iter()
309                    .map(|t| WireTool {
310                        r#type: "function",
311                        function: WireToolFunction {
312                            name: &t.name,
313                            description: t.description.as_deref(),
314                            parameters: &t.input_schema,
315                        },
316                    })
317                    .collect()
318            }),
319            tool_choice: o.tool_choice.as_ref().map(|choice| match choice {
320                ToolChoice::Auto => WireToolChoice::Mode("auto"),
321                ToolChoice::None => WireToolChoice::Mode("none"),
322                ToolChoice::Tool { name } => WireToolChoice::Named {
323                    r#type: "function",
324                    function: WireFunctionName { name },
325                },
326            }),
327        }
328    }
329
330    async fn error_for_status(&self, response: Response) -> RuntimeError {
331        let status = response.status();
332        let body = read_limited(response, ERROR_BODY_LIMIT)
333            .await
334            .unwrap_or_default();
335        map_http_error(status, &body)
336    }
337}
338
339// ---- wire types (private to this adapter) --------------------------------
340
341#[derive(Serialize)]
342struct WireMessage<'a> {
343    role: &'static str,
344    /// `null` for an assistant turn that only proposed tool calls.
345    content: Option<&'a str>,
346    #[serde(skip_serializing_if = "Option::is_none")]
347    tool_calls: Option<Vec<WireToolCallIn<'a>>>,
348    #[serde(skip_serializing_if = "Option::is_none")]
349    tool_call_id: Option<&'a str>,
350}
351
352#[derive(Serialize)]
353struct WireToolCallIn<'a> {
354    id: &'a str,
355    r#type: &'static str,
356    function: WireFunctionIn<'a>,
357}
358
359#[derive(Serialize)]
360struct WireFunctionIn<'a> {
361    name: &'a str,
362    /// The wire carries arguments as a JSON-encoded string.
363    arguments: String,
364}
365
366#[derive(Serialize)]
367struct WireTool<'a> {
368    r#type: &'static str,
369    function: WireToolFunction<'a>,
370}
371
372#[derive(Serialize)]
373struct WireToolFunction<'a> {
374    name: &'a str,
375    #[serde(skip_serializing_if = "Option::is_none")]
376    description: Option<&'a str>,
377    parameters: &'a Value,
378}
379
380#[derive(Serialize)]
381#[serde(untagged)]
382enum WireToolChoice<'a> {
383    Mode(&'static str),
384    Named {
385        r#type: &'static str,
386        function: WireFunctionName<'a>,
387    },
388}
389
390#[derive(Serialize)]
391struct WireFunctionName<'a> {
392    name: &'a str,
393}
394
395#[derive(Serialize)]
396struct WireStreamOptions {
397    include_usage: bool,
398}
399
400#[derive(Serialize)]
401#[serde(tag = "type", rename_all = "snake_case")]
402enum WireResponseFormat<'a> {
403    JsonObject,
404    JsonSchema { json_schema: WireJsonSchema<'a> },
405}
406
407#[derive(Serialize)]
408struct WireJsonSchema<'a> {
409    name: &'a str,
410    schema: &'a Value,
411    strict: bool,
412}
413
414#[derive(Serialize)]
415struct WireRequest<'a> {
416    model: &'a str,
417    messages: Vec<WireMessage<'a>>,
418    stream: bool,
419    #[serde(skip_serializing_if = "Option::is_none")]
420    stream_options: Option<WireStreamOptions>,
421    #[serde(skip_serializing_if = "Option::is_none")]
422    temperature: Option<f32>,
423    #[serde(skip_serializing_if = "Option::is_none")]
424    top_p: Option<f32>,
425    #[serde(skip_serializing_if = "Option::is_none")]
426    max_tokens: Option<u32>,
427    #[serde(skip_serializing_if = "Option::is_none")]
428    stop: Option<&'a Vec<String>>,
429    #[serde(skip_serializing_if = "Option::is_none")]
430    seed: Option<i64>,
431    #[serde(skip_serializing_if = "Option::is_none")]
432    response_format: Option<WireResponseFormat<'a>>,
433    #[serde(skip_serializing_if = "Option::is_none")]
434    tools: Option<Vec<WireTool<'a>>>,
435    #[serde(skip_serializing_if = "Option::is_none")]
436    tool_choice: Option<WireToolChoice<'a>>,
437}
438
439#[derive(Deserialize)]
440struct WireUsage {
441    prompt_tokens: Option<u64>,
442    completion_tokens: Option<u64>,
443    total_tokens: Option<u64>,
444}
445
446impl From<WireUsage> for Usage {
447    fn from(w: WireUsage) -> Self {
448        Usage {
449            prompt_tokens: w.prompt_tokens,
450            completion_tokens: w.completion_tokens,
451            total_tokens: w.total_tokens,
452        }
453    }
454}
455
456#[derive(Deserialize)]
457struct WireCompletion {
458    id: Option<String>,
459    model: Option<String>,
460    #[serde(default)]
461    choices: Vec<WireChoice>,
462    usage: Option<WireUsage>,
463}
464
465#[derive(Deserialize)]
466struct WireChoice {
467    message: Option<WireResponseMessage>,
468    delta: Option<WireResponseMessage>,
469    finish_reason: Option<String>,
470}
471
472#[derive(Deserialize)]
473struct WireResponseMessage {
474    role: Option<String>,
475    content: Option<String>,
476    tool_calls: Option<Vec<WireToolCallOut>>,
477}
478
479#[derive(Deserialize)]
480struct WireToolCallOut {
481    index: Option<u32>,
482    id: Option<String>,
483    function: Option<WireFunctionOut>,
484}
485
486#[derive(Deserialize)]
487struct WireFunctionOut {
488    name: Option<String>,
489    arguments: Option<Value>,
490}
491
492#[derive(Deserialize)]
493struct WireModels {
494    data: Vec<WireModel>,
495}
496
497#[derive(Deserialize)]
498struct WireModel {
499    id: String,
500    owned_by: Option<String>,
501}
502
503fn finish_reason(raw: String) -> FinishReason {
504    match raw.as_str() {
505        "stop" => FinishReason::Stop,
506        "length" => FinishReason::Length,
507        "content_filter" => FinishReason::ContentFilter,
508        "tool_calls" => FinishReason::ToolCalls,
509        _ => FinishReason::Other(raw),
510    }
511}
512
513fn protocol(operation: &str, reason: impl Into<String>) -> RuntimeError {
514    RuntimeError::Protocol {
515        operation: operation.to_owned(),
516        reason: reason.into(),
517    }
518}
519
520// ---- tool-call assembly ----------------------------------------------------------
521
522struct PendingCall {
523    index: u32,
524    id: String,
525    name: String,
526    arguments: String,
527}
528
529/// Raw fragment, reported to the consumer for display only.
530struct CallFragment {
531    index: u32,
532    id: Option<String>,
533    name: Option<String>,
534    arguments: Option<String>,
535}
536
537/// Accumulates tool-call fragments (streamed or whole) and turns them into
538/// validated proposals only in [`ToolAssembler::finish`]. Nothing is invented:
539/// a call without an id or name, with non-JSON or non-object arguments, or a
540/// duplicate id fails the whole response.
541struct ToolAssembler {
542    calls: Vec<PendingCall>,
543    max_calls: usize,
544    bytes: usize,
545}
546
547impl ToolAssembler {
548    fn new(max_calls: usize) -> Self {
549        Self {
550            calls: Vec::new(),
551            max_calls,
552            bytes: 0,
553        }
554    }
555
556    fn feed(
557        &mut self,
558        wire: Vec<WireToolCallOut>,
559        operation: &str,
560    ) -> RuntimeResult<Vec<CallFragment>> {
561        let mut fragments = Vec::new();
562        for call in wire {
563            let function = call.function;
564            let name = function.as_ref().and_then(|f| f.name.clone());
565            let arguments = match function.and_then(|f| f.arguments) {
566                None | Some(Value::Null) => None,
567                Some(Value::String(text)) => Some(text),
568                Some(_) => {
569                    return Err(protocol(
570                        operation,
571                        "tool call arguments must be a JSON-encoded string",
572                    ))
573                }
574            };
575            let slot = match (
576                call.index,
577                self.calls.iter().position(|c| Some(c.index) == call.index),
578            ) {
579                (_, Some(slot)) => slot,
580                // Servers that omit `index`: an id starts a new call, anything
581                // else continues the latest one.
582                (None, None) if call.id.is_none() && !self.calls.is_empty() => self.calls.len() - 1,
583                (index, None) => {
584                    if self.calls.len() >= self.max_calls {
585                        return Err(RuntimeError::ResourceLimit {
586                            reason: format!(
587                                "response proposes more than {} tool calls",
588                                self.max_calls
589                            ),
590                        });
591                    }
592                    let index = index.unwrap_or(self.calls.len() as u32);
593                    self.calls.push(PendingCall {
594                        index,
595                        id: String::new(),
596                        name: String::new(),
597                        arguments: String::new(),
598                    });
599                    self.calls.len() - 1
600                }
601            };
602            let pending = &mut self.calls[slot];
603            // Fragments are concatenated as the wire specifies. A server that
604            // repeats a whole name/id per chunk corrupts the value, which is
605            // then rejected (unknown tool / failed correlation), not guessed.
606            let id = call.id.filter(|f| !f.is_empty());
607            let name = name.filter(|f| !f.is_empty());
608            let arguments = arguments.filter(|f| !f.is_empty());
609            self.bytes += id.as_deref().map_or(0, str::len)
610                + name.as_deref().map_or(0, str::len)
611                + arguments.as_deref().map_or(0, str::len);
612            if let Some(id) = &id {
613                pending.id.push_str(id);
614            }
615            if let Some(name) = &name {
616                pending.name.push_str(name);
617            }
618            if let Some(arguments) = &arguments {
619                pending.arguments.push_str(arguments);
620            }
621            if id.is_some() || name.is_some() || arguments.is_some() {
622                fragments.push(CallFragment {
623                    index: pending.index,
624                    id,
625                    name,
626                    arguments,
627                });
628            }
629        }
630        Ok(fragments)
631    }
632
633    fn finish(self, operation: &str) -> RuntimeResult<Vec<ToolCall>> {
634        let mut seen = std::collections::HashSet::new();
635        let mut out = Vec::with_capacity(self.calls.len());
636        for call in self.calls {
637            if call.id.is_empty() || call.name.is_empty() {
638                return Err(protocol(operation, "tool call is missing its id or name"));
639            }
640            if !seen.insert(call.id.clone()) {
641                return Err(protocol(operation, "duplicate tool call id"));
642            }
643            if !is_valid_tool_name(&call.name) {
644                return Err(RuntimeError::InvalidOutput {
645                    reason: "model proposed a tool call with a malformed tool name".to_owned(),
646                });
647            }
648            let invalid = |why: &str| RuntimeError::InvalidOutput {
649                reason: format!("arguments for tool call to {:?} {why}", call.name),
650            };
651            let arguments: Value = serde_json::from_str(&call.arguments)
652                .map_err(|_| invalid("are not complete, valid JSON"))?;
653            if !arguments.is_object() {
654                return Err(invalid("are not a JSON object"));
655            }
656            out.push(ToolCall::new(call.id, call.name, arguments));
657        }
658        Ok(out)
659    }
660}
661
662// ---- errors ----------------------------------------------------------------
663
664fn truncate(text: &str) -> String {
665    let mut out: String = text.chars().take(ERROR_MESSAGE_CHARS).collect();
666    if text.chars().count() > ERROR_MESSAGE_CHARS {
667        out.push('…');
668    }
669    out
670}
671
672/// Extracts a provider's error text from the common body shapes.
673fn provider_error_text(body: &[u8]) -> (Option<String>, String) {
674    let Ok(value) = serde_json::from_slice::<Value>(body) else {
675        return (None, truncate(String::from_utf8_lossy(body).trim()));
676    };
677    let error = value.get("error").unwrap_or(&value);
678    if let Some(text) = error.as_str() {
679        return (None, truncate(text));
680    }
681    let message = error
682        .get("message")
683        .and_then(Value::as_str)
684        .unwrap_or("no error message");
685    let code = error
686        .get("code")
687        .and_then(|c| c.as_str().map(str::to_owned))
688        .or_else(|| error.get("type").and_then(Value::as_str).map(str::to_owned));
689    (code, truncate(message))
690}
691
692/// Maps an upstream HTTP error to a structured runtime error. The upstream
693/// message is untrusted text and is truncated.
694fn map_http_error(status: StatusCode, body: &[u8]) -> RuntimeError {
695    let (code, message) = provider_error_text(body);
696    let detail = format!("HTTP {}: {message}", status.as_u16());
697    match status.as_u16() {
698        401 | 403 => RuntimeError::provider_unavailable(
699            PROVIDER_NAME,
700            format!("authentication or authorization failed ({detail})"),
701        ),
702        404 if code.as_deref() == Some("model_not_found")
703            || message.to_ascii_lowercase().contains("model") =>
704        {
705            RuntimeError::ModelNotFound { model: message }
706        }
707        408 | 504 => RuntimeError::Timeout {
708            operation: format!("provider response ({detail})"),
709        },
710        400 | 404 | 405 | 409 | 413 | 415 | 422 => RuntimeError::InvalidRequest {
711            reason: format!("provider rejected request ({detail})"),
712        },
713        429 | 500..=599 => RuntimeError::provider_unavailable(PROVIDER_NAME, detail),
714        _ => protocol("provider request", detail),
715    }
716}
717
718fn map_send_error(error: reqwest::Error) -> RuntimeError {
719    let error = error.without_url();
720    if error.is_timeout() {
721        RuntimeError::Timeout {
722            operation: "connecting to provider".to_owned(),
723        }
724    } else {
725        RuntimeError::transport(PROVIDER_NAME, error.to_string())
726    }
727}
728
729/// Reads at most `limit` bytes of a response body.
730async fn read_limited(response: Response, limit: usize) -> RuntimeResult<Vec<u8>> {
731    if response
732        .content_length()
733        .is_some_and(|n| n as usize > limit)
734    {
735        return Err(RuntimeError::ResourceLimit {
736            reason: format!("provider response exceeds {limit} bytes"),
737        });
738    }
739    let mut body = Vec::new();
740    let mut stream = response.bytes_stream();
741    while let Some(chunk) = stream.next().await {
742        let chunk = chunk.map_err(map_send_error)?;
743        if body.len() + chunk.len() > limit {
744            return Err(RuntimeError::ResourceLimit {
745                reason: format!("provider response exceeds {limit} bytes"),
746            });
747        }
748        body.extend_from_slice(&chunk);
749    }
750    Ok(body)
751}
752
753async fn cancelled(token: &Option<CancellationToken>) {
754    match token {
755        Some(token) => token.cancelled().await,
756        None => std::future::pending().await,
757    }
758}
759
760/// Runs `future` under a deadline and cooperative cancellation.
761async fn bounded<T>(
762    token: &Option<CancellationToken>,
763    deadline: Duration,
764    operation: &str,
765    future: impl Future<Output = RuntimeResult<T>>,
766) -> RuntimeResult<T> {
767    tokio::select! {
768        biased;
769        _ = cancelled(token) => Err(RuntimeError::Cancelled),
770        result = tokio::time::timeout(deadline, future) => match result {
771            Ok(result) => result,
772            Err(_) => Err(RuntimeError::Timeout { operation: operation.to_owned() }),
773        },
774    }
775}
776
777struct Tracker {
778    started: Instant,
779    id: Option<String>,
780    model: Option<String>,
781    first_token: Option<Duration>,
782    deltas: u64,
783}
784
785impl Tracker {
786    fn new() -> Self {
787        Self {
788            started: Instant::now(),
789            id: None,
790            model: None,
791            first_token: None,
792            deltas: 0,
793        }
794    }
795
796    fn observe(&mut self, id: &Option<String>, model: &Option<String>) {
797        if self.id.is_none() {
798            self.id.clone_from(id);
799        }
800        if self.model.is_none() {
801            self.model.clone_from(model);
802        }
803    }
804}
805
806// ---- provider ----------------------------------------------------------------
807
808#[async_trait]
809impl GenerationProvider for OpenAiCompatibleProvider {
810    fn name(&self) -> &str {
811        PROVIDER_NAME
812    }
813
814    fn capabilities(&self) -> ProviderCapabilities {
815        let g = &self.config.capabilities;
816        ProviderCapabilities {
817            available: true,
818            local: self.location == ExecutionLocation::Local,
819            remote: self.location == ExecutionLocation::Remote,
820            streaming: g.streaming,
821            cancellation: true,
822            structured_output: g.json_object || g.json_schema,
823            batching: false,
824            endpoint: Some(self.base.clone()),
825            notes: vec!["OpenAI-compatible chat completions over HTTP".to_owned()],
826        }
827    }
828
829    fn generation_capabilities(&self) -> GenerationCapabilities {
830        self.config.capabilities.clone()
831    }
832
833    async fn list_models(
834        &self,
835        cancellation: Option<CancellationToken>,
836    ) -> RuntimeResult<Vec<GenerationModelInfo>> {
837        let operation = "listing models";
838        bounded(
839            &cancellation,
840            self.config.request_timeout,
841            operation,
842            async {
843                let response = self
844                    .request(reqwest::Method::GET, "/models")
845                    .send()
846                    .await
847                    .map_err(map_send_error)?;
848                let status = response.status();
849                if matches!(status.as_u16(), 404 | 405 | 501) {
850                    return Err(RuntimeError::CapabilityMismatch {
851                        reason: "upstream does not implement model listing".to_owned(),
852                    });
853                }
854                if !status.is_success() {
855                    return Err(self.error_for_status(response).await);
856                }
857                let body = read_limited(response, self.config.max_response_bytes).await?;
858                let models: WireModels = serde_json::from_slice(&body)
859                    .map_err(|_| protocol(operation, "malformed model list"))?;
860                Ok(models
861                    .data
862                    .into_iter()
863                    .map(|m| GenerationModelInfo {
864                        id: m.id,
865                        owned_by: m.owned_by,
866                    })
867                    .collect())
868            },
869        )
870        .await
871    }
872
873    async fn chat(
874        &self,
875        request: ChatRequest,
876        cancellation: Option<CancellationToken>,
877    ) -> RuntimeResult<ChatResponse> {
878        request.validate()?;
879        self.config.capabilities.check(&request, false)?;
880        let operation = "chat completion";
881        let mut tracker = Tracker::new();
882        let completion = bounded(
883            &cancellation,
884            self.config.request_timeout,
885            operation,
886            async {
887                let response = self
888                    .request(reqwest::Method::POST, "/chat/completions")
889                    .json(&self.wire_request(&request, false))
890                    .send()
891                    .await
892                    .map_err(map_send_error)?;
893                if !response.status().is_success() {
894                    return Err(self.error_for_status(response).await);
895                }
896                let body = read_limited(response, self.config.max_response_bytes).await?;
897                serde_json::from_slice::<WireCompletion>(&body)
898                    .map_err(|_| protocol(operation, "malformed completion payload"))
899            },
900        )
901        .await?;
902
903        tracker.observe(&completion.id, &completion.model);
904        let choice = completion
905            .choices
906            .into_iter()
907            .next()
908            .ok_or_else(|| protocol(operation, "completion contained no choices"))?;
909        let message = choice
910            .message
911            .ok_or_else(|| protocol(operation, "choice contained no message"))?;
912        if message.role.as_deref().is_some_and(|r| r != "assistant") {
913            return Err(protocol(
914                operation,
915                "response message role is not assistant",
916            ));
917        }
918        let content = message.content.unwrap_or_default();
919        let mut assembler = ToolAssembler::new(self.config.max_tool_calls);
920        assembler.feed(message.tool_calls.unwrap_or_default(), operation)?;
921        if content.len() + assembler.bytes > self.config.max_output_bytes {
922            return Err(RuntimeError::ResourceLimit {
923                reason: format!(
924                    "generated output exceeds {} bytes",
925                    self.config.max_output_bytes
926                ),
927            });
928        }
929        let tool_calls = assembler.finish(operation)?;
930        Ok(ChatResponse {
931            message: ChatMessage::assistant_with_tool_calls(content, tool_calls),
932            finish_reason: choice.finish_reason.map(finish_reason),
933            usage: completion.usage.map(Usage::from),
934            metadata: self.metadata(&request.model, &tracker, false),
935            structured: None,
936        })
937    }
938
939    async fn chat_stream(
940        &self,
941        request: ChatRequest,
942        cancellation: Option<CancellationToken>,
943    ) -> RuntimeResult<BoxStream<RuntimeResult<ChatStreamEvent>>> {
944        request.validate()?;
945        self.config.capabilities.check(&request, true)?;
946        let tracker = Tracker::new();
947        let response = bounded(
948            &cancellation,
949            self.config.request_timeout,
950            "waiting for stream response",
951            async {
952                let response = self
953                    .request(reqwest::Method::POST, "/chat/completions")
954                    .json(&self.wire_request(&request, true))
955                    .send()
956                    .await
957                    .map_err(map_send_error)?;
958                if !response.status().is_success() {
959                    return Err(self.error_for_status(response).await);
960                }
961                Ok(response)
962            },
963        )
964        .await?;
965        if response
966            .headers()
967            .get(reqwest::header::CONTENT_TYPE)
968            .and_then(|v| v.to_str().ok())
969            .is_some_and(|v| v.trim_start().starts_with("application/json"))
970        {
971            return Err(protocol(
972                "chat stream",
973                "provider returned a non-streaming response to a streaming request",
974            ));
975        }
976
977        let (sender, receiver) = mpsc::channel(self.config.stream_buffer);
978        let provider = self.clone();
979        tokio::spawn(async move {
980            provider
981                .pump_stream(request, response, tracker, cancellation, sender)
982                .await
983        });
984        Ok(Box::pin(ReceiverStream::new(receiver)))
985    }
986}
987
988type EventSender = mpsc::Sender<RuntimeResult<ChatStreamEvent>>;
989
990impl OpenAiCompatibleProvider {
991    /// Owns the connection. Returns (dropping the response and so closing the
992    /// connection) when the consumer drops the stream, on cancellation, on
993    /// error, or on completion.
994    async fn pump_stream(
995        &self,
996        request: ChatRequest,
997        response: Response,
998        mut tracker: Tracker,
999        cancellation: Option<CancellationToken>,
1000        sender: EventSender,
1001    ) {
1002        let outcome = self
1003            .read_stream(&request, response, &mut tracker, &cancellation, &sender)
1004            .await;
1005        if let Err(error) = outcome {
1006            // The consumer may already be gone; that is not an error here.
1007            let _ = sender.try_send(Err(error.clone()));
1008            if sender.capacity() == 0 && !sender.is_closed() {
1009                // Buffer full: wait for room, but never past cancellation.
1010                tokio::select! {
1011                    _ = cancelled(&cancellation) => {}
1012                    _ = sender.send(Err(error)) => {}
1013                }
1014            }
1015        }
1016    }
1017
1018    async fn emit(
1019        sender: &EventSender,
1020        cancellation: &Option<CancellationToken>,
1021        event: ChatStreamEvent,
1022    ) -> RuntimeResult<()> {
1023        tokio::select! {
1024            biased;
1025            _ = cancelled(cancellation) => Err(RuntimeError::Cancelled),
1026            sent = sender.send(Ok(event)) => sent.map_err(|_| RuntimeError::Cancelled),
1027        }
1028    }
1029
1030    async fn read_stream(
1031        &self,
1032        request: &ChatRequest,
1033        response: Response,
1034        tracker: &mut Tracker,
1035        cancellation: &Option<CancellationToken>,
1036        sender: &EventSender,
1037    ) -> RuntimeResult<()> {
1038        let operation = "chat stream";
1039        Self::emit(
1040            sender,
1041            cancellation,
1042            ChatStreamEvent::Started {
1043                provider: PROVIDER_NAME.to_owned(),
1044                model: request.model.clone(),
1045            },
1046        )
1047        .await?;
1048
1049        let mut parser = SseParser::new(self.config.max_event_bytes);
1050        let mut bytes = response.bytes_stream();
1051        let mut finish = None;
1052        let mut usage = None;
1053        let mut output_bytes = 0usize;
1054        let mut done = false;
1055        let mut assembler = ToolAssembler::new(self.config.max_tool_calls);
1056
1057        'read: loop {
1058            let chunk = tokio::select! {
1059                biased;
1060                _ = sender.closed() => return Ok(()),
1061                _ = cancelled(cancellation) => return Err(RuntimeError::Cancelled),
1062                chunk = tokio::time::timeout(self.config.stream_idle_timeout, bytes.next()) => chunk,
1063            };
1064            let events = match chunk {
1065                Err(_) if finish.is_some() => break 'read, // finished, server just kept the socket open
1066                Err(_) => {
1067                    return Err(RuntimeError::Timeout {
1068                        operation: "waiting for stream data".to_owned(),
1069                    })
1070                }
1071                Ok(None) => parser.finish(),
1072                Ok(Some(Err(error))) => return Err(map_send_error(error)),
1073                Ok(Some(Ok(chunk))) => parser.push(&chunk),
1074            }
1075            .map_err(|e| protocol(operation, e))?;
1076            let at_eof = events.eof;
1077
1078            for event in events.events {
1079                if event.data.trim() == "[DONE]" {
1080                    done = true;
1081                    break 'read;
1082                }
1083                let value: Value = serde_json::from_str(&event.data)
1084                    .map_err(|_| protocol(operation, "malformed stream event"))?;
1085                if let Some(error) = value.get("error") {
1086                    let body = serde_json::to_vec(&serde_json::json!({ "error": error }))
1087                        .unwrap_or_default();
1088                    return Err(map_stream_error(&body));
1089                }
1090                let chunk: WireCompletion = serde_json::from_value(value)
1091                    .map_err(|_| protocol(operation, "malformed stream chunk"))?;
1092                tracker.observe(&chunk.id, &chunk.model);
1093                if let Some(u) = chunk.usage {
1094                    usage = Some(Usage::from(u));
1095                }
1096                if let Some(choice) = chunk.choices.into_iter().next() {
1097                    if let Some(reason) = choice.finish_reason {
1098                        finish = Some(finish_reason(reason));
1099                    }
1100                    let mut delta = choice.delta;
1101                    if let Some(wire) = delta.as_mut().and_then(|d| d.tool_calls.take()) {
1102                        let fragments = assembler.feed(wire, operation)?;
1103                        if output_bytes + assembler.bytes > self.config.max_output_bytes {
1104                            return Err(RuntimeError::ResourceLimit {
1105                                reason: format!(
1106                                    "generated output exceeds {} bytes",
1107                                    self.config.max_output_bytes
1108                                ),
1109                            });
1110                        }
1111                        for fragment in fragments {
1112                            Self::emit(
1113                                sender,
1114                                cancellation,
1115                                ChatStreamEvent::ToolCallDelta {
1116                                    index: fragment.index,
1117                                    id: fragment.id,
1118                                    name: fragment.name,
1119                                    arguments_fragment: fragment.arguments,
1120                                },
1121                            )
1122                            .await?;
1123                        }
1124                    }
1125                    if let Some(content) = delta.and_then(|d| d.content) {
1126                        if !content.is_empty() {
1127                            output_bytes += content.len();
1128                            if output_bytes + assembler.bytes > self.config.max_output_bytes {
1129                                return Err(RuntimeError::ResourceLimit {
1130                                    reason: format!(
1131                                        "generated output exceeds {} bytes",
1132                                        self.config.max_output_bytes
1133                                    ),
1134                                });
1135                            }
1136                            tracker.deltas += 1;
1137                            tracker
1138                                .first_token
1139                                .get_or_insert_with(|| tracker.started.elapsed());
1140                            Self::emit(sender, cancellation, ChatStreamEvent::Delta { content })
1141                                .await?;
1142                        }
1143                    }
1144                }
1145            }
1146            if at_eof {
1147                break;
1148            }
1149        }
1150
1151        if !done && finish.is_none() {
1152            return Err(protocol(
1153                operation,
1154                "stream ended before the provider signalled completion",
1155            ));
1156        }
1157        // Assemble and validate every call before any proposal is exposed.
1158        for call in assembler.finish(operation)? {
1159            Self::emit(
1160                sender,
1161                cancellation,
1162                ChatStreamEvent::ToolCallCompleted { call },
1163            )
1164            .await?;
1165        }
1166        Self::emit(
1167            sender,
1168            cancellation,
1169            ChatStreamEvent::Completed(Box::new(ChatStreamSummary {
1170                finish_reason: finish,
1171                usage,
1172                metadata: self.metadata(&request.model, tracker, true),
1173                structured: None,
1174            })),
1175        )
1176        .await
1177    }
1178}
1179
1180fn map_stream_error(body: &[u8]) -> RuntimeError {
1181    // Mid-stream errors carry no HTTP status; they are provider failures.
1182    let (_, message) = provider_error_text(body);
1183    RuntimeError::provider_unavailable(PROVIDER_NAME, format!("stream error: {message}"))
1184}
1185
1186// ---- incremental SSE parsing ---------------------------------------------------
1187
1188#[derive(Debug, PartialEq, Eq)]
1189pub(crate) struct SseEvent {
1190    #[allow(dead_code)]
1191    pub event: Option<String>,
1192    pub data: String,
1193}
1194
1195#[derive(Debug, Default)]
1196pub(crate) struct SseBatch {
1197    pub events: Vec<SseEvent>,
1198    pub eof: bool,
1199}
1200
1201/// Incremental Server-Sent Events parser. Input may be split at arbitrary
1202/// byte boundaries (including inside a UTF-8 sequence or between `\r` and
1203/// `\n`); only complete lines are decoded. Line and event size are bounded.
1204pub(crate) struct SseParser {
1205    line: Vec<u8>,
1206    data: String,
1207    has_data: bool,
1208    event: Option<String>,
1209    pending_cr: bool,
1210    first_line: bool,
1211    max: usize,
1212}
1213
1214impl SseParser {
1215    pub fn new(max: usize) -> Self {
1216        Self {
1217            line: Vec::new(),
1218            data: String::new(),
1219            has_data: false,
1220            event: None,
1221            pending_cr: false,
1222            first_line: true,
1223            max,
1224        }
1225    }
1226
1227    pub fn push(&mut self, bytes: &[u8]) -> Result<SseBatch, String> {
1228        let mut batch = SseBatch::default();
1229        for &byte in bytes {
1230            if std::mem::take(&mut self.pending_cr) && byte == b'\n' {
1231                continue;
1232            }
1233            match byte {
1234                b'\r' => {
1235                    self.pending_cr = true;
1236                    self.end_line(&mut batch)?;
1237                }
1238                b'\n' => self.end_line(&mut batch)?,
1239                _ => {
1240                    self.line.push(byte);
1241                    if self.line.len() > self.max {
1242                        return Err("stream line exceeds the configured size limit".to_owned());
1243                    }
1244                }
1245            }
1246        }
1247        Ok(batch)
1248    }
1249
1250    /// Called at end of input. Unlike the strict SSE rules, a final
1251    /// unterminated line/event is still dispatched: truncated JSON then fails
1252    /// to parse and is reported, rather than a `[DONE]` being lost.
1253    pub fn finish(&mut self) -> Result<SseBatch, String> {
1254        let mut batch = SseBatch::default();
1255        if !self.line.is_empty() {
1256            self.end_line(&mut batch)?;
1257        }
1258        self.dispatch(&mut batch);
1259        batch.eof = true;
1260        Ok(batch)
1261    }
1262
1263    fn end_line(&mut self, batch: &mut SseBatch) -> Result<(), String> {
1264        let bytes = std::mem::take(&mut self.line);
1265        let mut line = String::from_utf8(bytes).map_err(|_| "stream is not valid UTF-8")?;
1266        if std::mem::take(&mut self.first_line) {
1267            if let Some(stripped) = line.strip_prefix('\u{feff}') {
1268                line = stripped.to_owned();
1269            }
1270        }
1271        if line.is_empty() {
1272            self.dispatch(batch);
1273            return Ok(());
1274        }
1275        if line.starts_with(':') {
1276            return Ok(());
1277        }
1278        let (field, value) = match line.split_once(':') {
1279            Some((field, value)) => (field, value.strip_prefix(' ').unwrap_or(value)),
1280            None => (line.as_str(), ""),
1281        };
1282        match field {
1283            "data" => {
1284                if self.data.len() + value.len() + 1 > self.max {
1285                    return Err("stream event exceeds the configured size limit".to_owned());
1286                }
1287                if self.has_data {
1288                    self.data.push('\n');
1289                }
1290                self.data.push_str(value);
1291                self.has_data = true;
1292            }
1293            "event" => self.event = Some(value.to_owned()),
1294            _ => {} // id, retry, and unknown fields are irrelevant here
1295        }
1296        Ok(())
1297    }
1298
1299    fn dispatch(&mut self, batch: &mut SseBatch) {
1300        if self.has_data {
1301            batch.events.push(SseEvent {
1302                event: self.event.take(),
1303                data: std::mem::take(&mut self.data),
1304            });
1305        }
1306        self.has_data = false;
1307        self.event = None;
1308    }
1309}
1310
1311#[cfg(test)]
1312mod tests {
1313    use super::*;
1314
1315    fn parse_split(input: &[u8], step: usize) -> Vec<String> {
1316        let mut parser = SseParser::new(1024);
1317        let mut out = Vec::new();
1318        for piece in input.chunks(step) {
1319            out.extend(
1320                parser
1321                    .push(piece)
1322                    .unwrap()
1323                    .events
1324                    .into_iter()
1325                    .map(|e| e.data),
1326            );
1327        }
1328        out.extend(parser.finish().unwrap().events.into_iter().map(|e| e.data));
1329        out
1330    }
1331
1332    #[test]
1333    fn events_are_identical_at_every_chunk_size() {
1334        let input = "data: {\"a\":\"é😀\"}\n\n: comment\r\ndata: two\r\ndata: lines\r\n\r\ndata: [DONE]\n\n";
1335        let expected = vec!["{\"a\":\"é😀\"}", "two\nlines", "[DONE]"];
1336        for step in 1..=input.len() {
1337            assert_eq!(parse_split(input.as_bytes(), step), expected, "step {step}");
1338        }
1339    }
1340
1341    #[test]
1342    fn lone_cr_terminators_and_bom() {
1343        assert_eq!(
1344            parse_split("\u{feff}data: x\r\rdata: y\r\r".as_bytes(), 2),
1345            vec!["x", "y"]
1346        );
1347    }
1348
1349    #[test]
1350    fn unterminated_final_event_is_dispatched_at_eof() {
1351        assert_eq!(parse_split(b"data: [DONE]", 3), vec!["[DONE]"]);
1352    }
1353
1354    #[test]
1355    fn oversized_lines_and_events_are_rejected() {
1356        let mut parser = SseParser::new(8);
1357        assert!(parser.push(b"data: 0123456789").is_err());
1358        let mut parser = SseParser::new(8);
1359        assert!(parser.push(b"data: 1234\ndata: 5678\n").is_err());
1360    }
1361
1362    #[test]
1363    fn invalid_utf8_is_rejected() {
1364        let mut parser = SseParser::new(64);
1365        assert!(parser.push(b"data: \xff\xfe\n").is_err());
1366    }
1367}