Skip to main content

tinyharness_lib/provider/
openai_compat.rs

1use std::collections::HashMap;
2use std::future::Future;
3use std::pin::Pin;
4use std::time::Duration;
5
6use reqwest::Client;
7use serde::{Deserialize, Serialize};
8use tokio_stream::StreamExt;
9
10use crate::{
11    SecretString,
12    provider::{
13        ChatMessage, ChatMessageResponse, Message, Role, ToolCall, ToolCallFunction, ToolDefinition,
14    },
15};
16
17/// Shared inner state for OpenAI-compatible providers (llama.cpp, vLLM, etc.).
18///
19/// Encapsulates the common `{client, base_url, model, api_key}` fields and all
20/// shared logic so that provider implementations only need to differ in
21/// `list_models()`.
22pub struct OpenAiCompatInner {
23    client: Client,
24    base_url: String,
25    model: Option<String>,
26    /// Optional bearer token sent as `Authorization: Bearer <key>` on every
27    /// request. Used by hosted OpenAI-compatible APIs (e.g. OpenRouter,
28    /// Together, self-hosted gateways) that require authentication.
29    api_key: Option<SecretString>,
30}
31
32impl OpenAiCompatInner {
33    pub fn new(base_url: String) -> Self {
34        Self::with_api_key(base_url, None)
35    }
36
37    /// Create a new inner state with an optional bearer token.
38    pub fn with_api_key(base_url: String, api_key: Option<SecretString>) -> Self {
39        let client = Client::builder()
40            .connect_timeout(Duration::from_secs(15))
41            .read_timeout(Duration::from_secs(300))
42            .build()
43            .unwrap_or_else(|_| Client::new());
44        OpenAiCompatInner {
45            client,
46            base_url,
47            model: None,
48            api_key,
49        }
50    }
51
52    /// Perform a health check against the server's `/health` endpoint.
53    pub fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
54        let url = format!("{}/health", self.base_url.trim_end_matches('/'));
55        let client = self.client.clone();
56        let api_key = self.api_key.clone();
57        Box::pin(async move {
58            let mut req = client.get(&url);
59            if let Some(key) = &api_key {
60                req = req.bearer_auth(key.expose_secret());
61            }
62            match req.send().await {
63                Ok(resp) if resp.status().is_success() => Ok(()),
64                Ok(resp) => Err(format!(
65                    "Server returned {}: {}",
66                    resp.status().as_u16(),
67                    resp.text().await.unwrap_or_default()
68                )),
69                Err(e) => Err(format!("Cannot reach {}: {}", url, e)),
70            }
71        })
72    }
73
74    pub fn select_model(&mut self, name: String) {
75        self.model = Some(name);
76    }
77
78    pub fn current_model(&self) -> Option<String> {
79        self.model.clone()
80    }
81
82    /// Return the `/v1/chat/completions` URL for this server.
83    pub fn chat_url(&self) -> String {
84        format!(
85            "{}/v1/chat/completions",
86            self.base_url.trim_end_matches('/').trim_end_matches("/v1")
87        )
88    }
89
90    /// Fetch the model list from the server's `/v1/models` endpoint.
91    /// Returns the list of model IDs, or an empty vec on failure.
92    pub fn fetch_model_list(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
93        let url = format!(
94            "{}/v1/models",
95            self.base_url.trim_end_matches('/').trim_end_matches("/v1")
96        );
97        let client = self.client.clone();
98        let current_model = self.model.clone();
99        let api_key = self.api_key.clone();
100        Box::pin(async move {
101            let mut req = client.get(&url);
102            if let Some(key) = &api_key {
103                req = req.bearer_auth(key.expose_secret());
104            }
105            match req.send().await {
106                Ok(resp) if resp.status().is_success() => {
107                    match resp.json::<ModelListResponse>().await {
108                        Ok(list) => list.data.into_iter().map(|m| m.id).collect(),
109                        Err(_) => current_model.into_iter().collect(),
110                    }
111                }
112                _ => current_model.into_iter().collect(),
113            }
114        })
115    }
116
117    /// Stream chat completions using the OpenAI-compatible API.
118    /// Returns a receiver for streaming response chunks, or an error string
119    /// if the request cannot be started.
120    pub fn chat(
121        &self,
122        messages: Vec<Message>,
123        tools: Vec<ToolDefinition>,
124    ) -> Pin<
125        Box<
126            dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
127                + Send,
128        >,
129    > {
130        let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
131
132        let model = self.model.clone().unwrap_or_default();
133        let openai_messages = messages.into_iter().map(to_openai_message).collect();
134        let openai_tools = tools.into_iter().map(to_openai_tool).collect();
135        let client = self.client.clone();
136        let chat_url = self.chat_url();
137        let api_key = self.api_key.clone();
138
139        let body = ChatRequest {
140            model,
141            messages: openai_messages,
142            stream: true,
143            stream_options: Some(StreamOptions {
144                include_usage: true,
145            }),
146            tools: openai_tools,
147        };
148
149        // Spawn the streaming work on a background task
150        tokio::spawn(async move {
151            let _usage =
152                stream_chat_completions(&client, &chat_url, &body, api_key.as_ref(), &send).await;
153        });
154
155        Box::pin(async move { Ok(recv) })
156    }
157}
158
159// ── OpenAI-compatible request/response types ──
160
161#[derive(Serialize)]
162pub struct ChatRequest {
163    pub model: String,
164    pub messages: Vec<OpenAIMessage>,
165    pub stream: bool,
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub stream_options: Option<StreamOptions>,
168    #[serde(skip_serializing_if = "Vec::is_empty")]
169    pub tools: Vec<OpenAITool>,
170}
171
172#[derive(Serialize)]
173pub struct StreamOptions {
174    pub include_usage: bool,
175}
176
177#[derive(Serialize)]
178pub struct OpenAIMessage {
179    pub role: String,
180    pub content: serde_json::Value,
181    #[serde(skip_serializing_if = "Option::is_none")]
182    pub tool_calls: Option<Vec<OpenAIToolCall>>,
183    #[serde(skip_serializing_if = "Option::is_none")]
184    pub tool_call_id: Option<String>,
185}
186
187#[derive(Serialize)]
188pub struct OpenAITool {
189    #[serde(rename = "type")]
190    pub tool_type: String,
191    pub function: OpenAIToolFunction,
192}
193
194#[derive(Serialize)]
195pub struct OpenAIToolFunction {
196    pub name: String,
197    pub description: String,
198    pub parameters: serde_json::Value,
199}
200
201#[derive(Serialize, Deserialize, Clone)]
202pub struct OpenAIToolCall {
203    #[serde(default)]
204    pub index: usize,
205    #[serde(default)]
206    pub id: String,
207    #[serde(rename = "type", default)]
208    pub call_type: String,
209    #[serde(default)]
210    pub function: OpenAIToolCallFunction,
211}
212
213#[derive(Serialize, Deserialize, Clone, Default)]
214pub struct OpenAIToolCallFunction {
215    #[serde(default)]
216    pub name: String,
217    #[serde(default)]
218    pub arguments: String,
219}
220
221#[derive(Deserialize)]
222pub struct ChunkChoice {
223    pub delta: Delta,
224    #[serde(default, rename = "finish_reason")]
225    pub _finish_reason: Option<String>,
226}
227
228#[derive(Deserialize, Default)]
229pub struct Delta {
230    #[serde(default)]
231    pub content: Option<String>,
232    #[serde(default)]
233    pub tool_calls: Option<Vec<OpenAIToolCall>>,
234}
235
236#[derive(Deserialize)]
237pub struct StreamChunk {
238    #[serde(default)]
239    pub choices: Vec<ChunkChoice>,
240    #[serde(default)]
241    pub usage: Option<OpenAIUsage>,
242}
243
244#[derive(Deserialize, Clone)]
245pub struct OpenAIUsage {
246    #[serde(default)]
247    pub prompt_tokens: u32,
248    #[serde(default)]
249    pub completion_tokens: u32,
250    #[serde(default)]
251    pub total_tokens: u32,
252}
253
254// ── Model list response types ──
255
256#[derive(Deserialize)]
257pub struct ModelListResponse {
258    pub data: Vec<ModelEntry>,
259}
260
261#[derive(Deserialize)]
262pub struct ModelEntry {
263    pub id: String,
264}
265
266// ── Conversion helpers ──
267
268pub fn to_openai_message(msg: Message) -> OpenAIMessage {
269    /// Build the content value: if images are present, use multipart array format;
270    /// otherwise use plain string.
271    fn build_content(msg: &Message) -> serde_json::Value {
272        if msg.images.is_empty() {
273            serde_json::Value::String(msg.content.clone())
274        } else {
275            let mut parts: Vec<serde_json::Value> = Vec::new();
276            // Add text part
277            if !msg.content.is_empty() {
278                parts.push(serde_json::json!({
279                    "type": "text",
280                    "text": msg.content
281                }));
282            }
283            // Add image parts
284            for img in &msg.images {
285                parts.push(serde_json::json!({
286                    "type": "image_url",
287                    "image_url": {
288                        "url": img.data_uri()
289                    }
290                }));
291            }
292            serde_json::Value::Array(parts)
293        }
294    }
295
296    match msg.role {
297        Role::System => OpenAIMessage {
298            role: "system".to_string(),
299            content: serde_json::Value::String(msg.content),
300            tool_calls: None,
301            tool_call_id: None,
302        },
303        Role::User => OpenAIMessage {
304            role: "user".to_string(),
305            content: build_content(&msg),
306            tool_calls: None,
307            tool_call_id: None,
308        },
309        Role::Assistant => {
310            if msg.tool_calls.is_empty() {
311                OpenAIMessage {
312                    role: "assistant".to_string(),
313                    content: serde_json::Value::String(msg.content),
314                    tool_calls: None,
315                    tool_call_id: None,
316                }
317            } else {
318                let tool_calls: Vec<OpenAIToolCall> = msg
319                    .tool_calls
320                    .into_iter()
321                    .enumerate()
322                    .map(|(i, tc)| {
323                        // Synthesize a stable id when the upstream provider
324                        // didn't return one (some local / non-OpenAI servers
325                        // omit it). OpenAI requires a non-empty id.
326                        let id = tc
327                            .id
328                            .filter(|s| !s.is_empty())
329                            .unwrap_or_else(|| format!("call_{}", i));
330                        let args_str = if tc.function.arguments.is_null() {
331                            "{}".to_string()
332                        } else {
333                            tc.function.arguments.to_string()
334                        };
335                        OpenAIToolCall {
336                            index: i,
337                            id,
338                            call_type: "function".to_string(),
339                            function: OpenAIToolCallFunction {
340                                name: tc.function.name,
341                                arguments: args_str,
342                            },
343                        }
344                    })
345                    .collect();
346                OpenAIMessage {
347                    role: "assistant".to_string(),
348                    // MiniMax and some other strict providers reject empty
349                    // strings in assistant messages that contain only tool
350                    // calls.  Send `null` when there is no text content.
351                    content: if msg.content.is_empty() {
352                        serde_json::Value::Null
353                    } else {
354                        serde_json::Value::String(msg.content)
355                    },
356                    tool_calls: Some(tool_calls),
357                    tool_call_id: None,
358                }
359            }
360        }
361        Role::Tool => {
362            // tool_call_id should have been set by the agent loop. If somehow
363            // missing, skip the field rather than emitting a bogus id that the
364            // server will reject.
365            let tool_call_id = msg.tool_call_id.filter(|s| !s.is_empty());
366            OpenAIMessage {
367                role: "tool".to_string(),
368                content: serde_json::Value::String(msg.content),
369                tool_calls: None,
370                tool_call_id,
371            }
372        }
373    }
374}
375
376pub fn to_openai_tool(ti: ToolDefinition) -> OpenAITool {
377    OpenAITool {
378        tool_type: "function".to_string(),
379        function: OpenAIToolFunction {
380            name: ti.name,
381            description: ti.description,
382            parameters: serde_json::to_value(ti.parameters).unwrap_or_default(),
383        },
384    }
385}
386
387/// Stream chat completions from an OpenAI-compatible endpoint.
388/// Returns accumulated tool calls and final content via the sender.
389/// Also returns the token usage if available.
390///
391/// If `api_key` is `Some`, the request includes an
392/// `Authorization: Bearer <key>` header.
393pub async fn stream_chat_completions(
394    client: &reqwest::Client,
395    url: &str,
396    body: &ChatRequest,
397    api_key: Option<&SecretString>,
398    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
399) -> Option<crate::provider::TokenUsage> {
400    let mut request = client.post(url).json(body);
401    if let Some(key) = api_key {
402        request = request.bearer_auth(key.expose_secret());
403    }
404    let response = match request.send().await {
405        Ok(r) => r,
406        Err(e) => {
407            let _ = send
408                .send(ChatMessageResponse {
409                    message: ChatMessage {
410                        content: format!("Error: {}", e),
411                        tool_calls: vec![],
412                        thinking: None,
413                    },
414                    done: true,
415                    is_error: true,
416                    usage: None,
417                })
418                .await;
419            return None;
420        }
421    };
422
423    // If the server returned a non-success status, surface the body as an
424    // error instead of feeding it into the SSE parser (which would silently
425    // drop it).
426    let status = response.status();
427    if !status.is_success() {
428        let body = response.text().await.unwrap_or_default();
429        let _ = send
430            .send(ChatMessageResponse {
431                message: ChatMessage {
432                    content: format!("Error: HTTP {} — {}", status.as_u16(), body),
433                    tool_calls: vec![],
434                    thinking: None,
435                },
436                done: true,
437                is_error: true,
438                usage: None,
439            })
440            .await;
441        return None;
442    }
443
444    let mut stream = response.bytes_stream();
445    let mut buf = String::new();
446
447    let mut acc_tool_calls: HashMap<usize, OpenAIToolCall> = HashMap::new();
448    let mut response_content = String::new();
449    let mut token_usage: Option<crate::provider::TokenUsage> = None;
450
451    while let Some(chunk_result) = stream.next().await {
452        let chunk = match chunk_result {
453            Ok(c) => c,
454            Err(e) => {
455                // Stream read error — surface it rather than silently breaking
456                let _ = send
457                    .send(ChatMessageResponse {
458                        message: ChatMessage {
459                            content: format!("\n\nStream error: {}", e),
460                            tool_calls: vec![],
461                            thinking: None,
462                        },
463                        done: true,
464                        is_error: true,
465                        usage: None,
466                    })
467                    .await;
468                break;
469            }
470        };
471
472        buf.push_str(&String::from_utf8_lossy(&chunk));
473
474        loop {
475            match buf.find('\n') {
476                None => break,
477                Some(pos) => {
478                    let line = buf[..pos].trim().to_string();
479                    buf = buf[pos + 1..].to_string();
480
481                    if line.is_empty() || line == "data: [DONE]" {
482                        continue;
483                    }
484
485                    if let Some(data) = line.strip_prefix("data: ")
486                        && let Ok(chunk) = serde_json::from_str::<StreamChunk>(data)
487                    {
488                        // Capture token usage if present (usually in the final chunk)
489                        if let Some(usage) = &chunk.usage {
490                            token_usage = Some(crate::provider::TokenUsage {
491                                prompt_tokens: usage.prompt_tokens,
492                                completion_tokens: usage.completion_tokens,
493                                total_tokens: usage.total_tokens,
494                            });
495                        }
496
497                        for choice in chunk.choices {
498                            if let Some(content) = &choice.delta.content {
499                                response_content.push_str(content);
500                            }
501
502                            if let Some(tool_calls) = &choice.delta.tool_calls {
503                                for tc in tool_calls {
504                                    let entry =
505                                        acc_tool_calls.entry(tc.index).or_insert(OpenAIToolCall {
506                                            index: tc.index,
507                                            id: String::new(),
508                                            call_type: "function".to_string(),
509                                            function: OpenAIToolCallFunction::default(),
510                                        });
511
512                                    if !tc.id.is_empty() {
513                                        entry.id = tc.id.clone();
514                                    }
515                                    if !tc.function.name.is_empty() {
516                                        entry.function.name = tc.function.name.clone();
517                                    }
518                                    entry.function.arguments.push_str(&tc.function.arguments);
519                                }
520                            }
521                        }
522                    }
523                }
524            }
525        }
526
527        if !response_content.is_empty() {
528            let _ = send
529                .send(ChatMessageResponse {
530                    message: ChatMessage {
531                        content: response_content.clone(),
532                        tool_calls: vec![],
533                        thinking: None,
534                    },
535                    done: false,
536                    is_error: false,
537                    usage: None,
538                })
539                .await;
540            response_content.clear();
541        }
542    }
543
544    let tool_calls: Vec<ToolCall> = if !acc_tool_calls.is_empty() {
545        acc_tool_calls
546            .into_values()
547            .map(|tc| {
548                let args: serde_json::Value =
549                    serde_json::from_str(&tc.function.arguments).unwrap_or(serde_json::Value::Null);
550                ToolCall {
551                    id: if tc.id.is_empty() { None } else { Some(tc.id) },
552                    function: ToolCallFunction {
553                        name: tc.function.name,
554                        arguments: args,
555                        thought_signature: None,
556                    },
557                }
558            })
559            .collect()
560    } else {
561        vec![]
562    };
563
564    // Send the final response with tool calls and token usage
565    let _ = send
566        .send(ChatMessageResponse {
567            message: ChatMessage {
568                content: String::new(),
569                tool_calls,
570                thinking: None,
571            },
572            done: true,
573            is_error: false,
574            usage: token_usage.clone(),
575        })
576        .await;
577
578    token_usage
579}