Skip to main content

tinyharness_lib/provider/
openai_compat.rs

1use std::collections::HashMap;
2use std::future::Future;
3use std::pin::Pin;
4use std::time::Duration;
5
6use reqwest::Client;
7use serde::{Deserialize, Serialize};
8use tokio_stream::StreamExt;
9
10use crate::provider::{
11    ChatMessage, ChatMessageResponse, Message, Role, ToolCall, ToolCallFunction, ToolDefinition,
12};
13
14/// Shared inner state for OpenAI-compatible providers (llama.cpp, vLLM, etc.).
15///
16/// Encapsulates the common `{client, base_url, model, api_key}` fields and all
17/// shared logic so that provider implementations only need to differ in
18/// `list_models()`.
19pub struct OpenAiCompatInner {
20    client: Client,
21    base_url: String,
22    model: Option<String>,
23    /// Optional bearer token sent as `Authorization: Bearer <key>` on every
24    /// request. Used by hosted OpenAI-compatible APIs (e.g. OpenRouter,
25    /// Together, self-hosted gateways) that require authentication.
26    api_key: Option<String>,
27}
28
29impl OpenAiCompatInner {
30    pub fn new(base_url: String) -> Self {
31        Self::with_api_key(base_url, None)
32    }
33
34    /// Create a new inner state with an optional bearer token.
35    pub fn with_api_key(base_url: String, api_key: Option<String>) -> Self {
36        let client = Client::builder()
37            .connect_timeout(Duration::from_secs(15))
38            .read_timeout(Duration::from_secs(300))
39            .build()
40            .unwrap_or_else(|_| Client::new());
41        OpenAiCompatInner {
42            client,
43            base_url,
44            model: None,
45            api_key,
46        }
47    }
48
49    /// Perform a health check against the server's `/health` endpoint.
50    pub fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
51        let url = format!("{}/health", self.base_url.trim_end_matches('/'));
52        let client = self.client.clone();
53        let api_key = self.api_key.clone();
54        Box::pin(async move {
55            let mut req = client.get(&url);
56            if let Some(key) = &api_key {
57                req = req.bearer_auth(key);
58            }
59            match req.send().await {
60                Ok(resp) if resp.status().is_success() => Ok(()),
61                Ok(resp) => Err(format!(
62                    "Server returned {}: {}",
63                    resp.status().as_u16(),
64                    resp.text().await.unwrap_or_default()
65                )),
66                Err(e) => Err(format!("Cannot reach {}: {}", url, e)),
67            }
68        })
69    }
70
71    pub fn select_model(&mut self, name: String) {
72        self.model = Some(name);
73    }
74
75    pub fn current_model(&self) -> Option<String> {
76        self.model.clone()
77    }
78
79    /// Return the `/v1/chat/completions` URL for this server.
80    pub fn chat_url(&self) -> String {
81        format!(
82            "{}/v1/chat/completions",
83            self.base_url.trim_end_matches('/')
84        )
85    }
86
87    /// Fetch the model list from the server's `/v1/models` endpoint.
88    /// Returns the list of model IDs, or an empty vec on failure.
89    pub fn fetch_model_list(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
90        let url = format!("{}/v1/models", self.base_url.trim_end_matches('/'));
91        let client = self.client.clone();
92        let current_model = self.model.clone();
93        let api_key = self.api_key.clone();
94        Box::pin(async move {
95            let mut req = client.get(&url);
96            if let Some(key) = &api_key {
97                req = req.bearer_auth(key);
98            }
99            match req.send().await {
100                Ok(resp) if resp.status().is_success() => {
101                    match resp.json::<ModelListResponse>().await {
102                        Ok(list) => list.data.into_iter().map(|m| m.id).collect(),
103                        Err(_) => current_model.into_iter().collect(),
104                    }
105                }
106                _ => current_model.into_iter().collect(),
107            }
108        })
109    }
110
111    /// Stream chat completions using the OpenAI-compatible API.
112    /// Returns a receiver for streaming response chunks, or an error string
113    /// if the request cannot be started.
114    pub fn chat(
115        &self,
116        messages: Vec<Message>,
117        tools: Vec<ToolDefinition>,
118    ) -> Pin<
119        Box<
120            dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
121                + Send,
122        >,
123    > {
124        let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
125
126        let model = self.model.clone().unwrap_or_default();
127        let openai_messages = messages.into_iter().map(to_openai_message).collect();
128        let openai_tools = tools.into_iter().map(to_openai_tool).collect();
129        let client = self.client.clone();
130        let chat_url = self.chat_url();
131        let api_key = self.api_key.clone();
132
133        let body = ChatRequest {
134            model,
135            messages: openai_messages,
136            stream: true,
137            stream_options: Some(StreamOptions {
138                include_usage: true,
139            }),
140            tools: openai_tools,
141        };
142
143        // Spawn the streaming work on a background task
144        tokio::spawn(async move {
145            let _usage =
146                stream_chat_completions(&client, &chat_url, &body, api_key.as_deref(), &send).await;
147        });
148
149        Box::pin(async move { Ok(recv) })
150    }
151}
152
153// ── OpenAI-compatible request/response types ──
154
155#[derive(Serialize)]
156pub struct ChatRequest {
157    pub model: String,
158    pub messages: Vec<OpenAIMessage>,
159    pub stream: bool,
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub stream_options: Option<StreamOptions>,
162    #[serde(skip_serializing_if = "Vec::is_empty")]
163    pub tools: Vec<OpenAITool>,
164}
165
166#[derive(Serialize)]
167pub struct StreamOptions {
168    pub include_usage: bool,
169}
170
171#[derive(Serialize)]
172pub struct OpenAIMessage {
173    pub role: String,
174    pub content: serde_json::Value,
175    #[serde(skip_serializing_if = "Option::is_none")]
176    pub tool_calls: Option<Vec<OpenAIToolCall>>,
177    #[serde(skip_serializing_if = "Option::is_none")]
178    pub tool_call_id: Option<String>,
179}
180
181#[derive(Serialize)]
182pub struct OpenAITool {
183    #[serde(rename = "type")]
184    pub tool_type: String,
185    pub function: OpenAIToolFunction,
186}
187
188#[derive(Serialize)]
189pub struct OpenAIToolFunction {
190    pub name: String,
191    pub description: String,
192    pub parameters: serde_json::Value,
193}
194
195#[derive(Serialize, Deserialize, Clone)]
196pub struct OpenAIToolCall {
197    #[serde(default)]
198    pub index: usize,
199    #[serde(default)]
200    pub id: String,
201    #[serde(rename = "type", default)]
202    pub call_type: String,
203    #[serde(default)]
204    pub function: OpenAIToolCallFunction,
205}
206
207#[derive(Serialize, Deserialize, Clone, Default)]
208pub struct OpenAIToolCallFunction {
209    #[serde(default)]
210    pub name: String,
211    #[serde(default)]
212    pub arguments: String,
213}
214
215#[derive(Deserialize)]
216pub struct ChunkChoice {
217    pub delta: Delta,
218    #[serde(default, rename = "finish_reason")]
219    pub _finish_reason: Option<String>,
220}
221
222#[derive(Deserialize, Default)]
223pub struct Delta {
224    #[serde(default)]
225    pub content: Option<String>,
226    #[serde(default)]
227    pub tool_calls: Option<Vec<OpenAIToolCall>>,
228}
229
230#[derive(Deserialize)]
231pub struct StreamChunk {
232    #[serde(default)]
233    pub choices: Vec<ChunkChoice>,
234    #[serde(default)]
235    pub usage: Option<OpenAIUsage>,
236}
237
238#[derive(Deserialize, Clone)]
239pub struct OpenAIUsage {
240    #[serde(default)]
241    pub prompt_tokens: u32,
242    #[serde(default)]
243    pub completion_tokens: u32,
244    #[serde(default)]
245    pub total_tokens: u32,
246}
247
248// ── Model list response types ──
249
250#[derive(Deserialize)]
251pub struct ModelListResponse {
252    pub data: Vec<ModelEntry>,
253}
254
255#[derive(Deserialize)]
256pub struct ModelEntry {
257    pub id: String,
258}
259
260// ── Conversion helpers ──
261
262pub fn to_openai_message(msg: Message) -> OpenAIMessage {
263    /// Build the content value: if images are present, use multipart array format;
264    /// otherwise use plain string.
265    fn build_content(msg: &Message) -> serde_json::Value {
266        if msg.images.is_empty() {
267            serde_json::Value::String(msg.content.clone())
268        } else {
269            let mut parts: Vec<serde_json::Value> = Vec::new();
270            // Add text part
271            if !msg.content.is_empty() {
272                parts.push(serde_json::json!({
273                    "type": "text",
274                    "text": msg.content
275                }));
276            }
277            // Add image parts
278            for img in &msg.images {
279                parts.push(serde_json::json!({
280                    "type": "image_url",
281                    "image_url": {
282                        "url": img.data_uri()
283                    }
284                }));
285            }
286            serde_json::Value::Array(parts)
287        }
288    }
289
290    match msg.role {
291        Role::System => OpenAIMessage {
292            role: "system".to_string(),
293            content: serde_json::Value::String(msg.content),
294            tool_calls: None,
295            tool_call_id: None,
296        },
297        Role::User => OpenAIMessage {
298            role: "user".to_string(),
299            content: build_content(&msg),
300            tool_calls: None,
301            tool_call_id: None,
302        },
303        Role::Assistant => {
304            if msg.tool_calls.is_empty() {
305                OpenAIMessage {
306                    role: "assistant".to_string(),
307                    content: serde_json::Value::String(msg.content),
308                    tool_calls: None,
309                    tool_call_id: None,
310                }
311            } else {
312                let tool_calls: Vec<OpenAIToolCall> = msg
313                    .tool_calls
314                    .into_iter()
315                    .enumerate()
316                    .map(|(i, tc)| {
317                        // Synthesize a stable id when the upstream provider
318                        // didn't return one (some local / non-OpenAI servers
319                        // omit it). OpenAI requires a non-empty id.
320                        let id = tc
321                            .id
322                            .filter(|s| !s.is_empty())
323                            .unwrap_or_else(|| format!("call_{}", i));
324                        let args_str = if tc.function.arguments.is_null() {
325                            "{}".to_string()
326                        } else {
327                            tc.function.arguments.to_string()
328                        };
329                        OpenAIToolCall {
330                            index: i,
331                            id,
332                            call_type: "function".to_string(),
333                            function: OpenAIToolCallFunction {
334                                name: tc.function.name,
335                                arguments: args_str,
336                            },
337                        }
338                    })
339                    .collect();
340                OpenAIMessage {
341                    role: "assistant".to_string(),
342                    content: serde_json::Value::String(msg.content),
343                    tool_calls: Some(tool_calls),
344                    tool_call_id: None,
345                }
346            }
347        }
348        Role::Tool => {
349            // tool_call_id should have been set by the agent loop. If somehow
350            // missing, skip the field rather than emitting a bogus id that the
351            // server will reject.
352            let tool_call_id = msg.tool_call_id.filter(|s| !s.is_empty());
353            OpenAIMessage {
354                role: "tool".to_string(),
355                content: serde_json::Value::String(msg.content),
356                tool_calls: None,
357                tool_call_id,
358            }
359        }
360    }
361}
362
363pub fn to_openai_tool(ti: ToolDefinition) -> OpenAITool {
364    OpenAITool {
365        tool_type: "function".to_string(),
366        function: OpenAIToolFunction {
367            name: ti.name,
368            description: ti.description,
369            parameters: serde_json::to_value(ti.parameters).unwrap_or_default(),
370        },
371    }
372}
373
374/// Stream chat completions from an OpenAI-compatible endpoint.
375/// Returns accumulated tool calls and final content via the sender.
376/// Also returns the token usage if available.
377///
378/// If `api_key` is `Some`, the request includes an
379/// `Authorization: Bearer <key>` header.
380pub async fn stream_chat_completions(
381    client: &reqwest::Client,
382    url: &str,
383    body: &ChatRequest,
384    api_key: Option<&str>,
385    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
386) -> Option<crate::provider::TokenUsage> {
387    let mut request = client.post(url).json(body);
388    if let Some(key) = api_key {
389        request = request.bearer_auth(key);
390    }
391    let response = match request.send().await {
392        Ok(r) => r,
393        Err(e) => {
394            let _ = send
395                .send(ChatMessageResponse {
396                    message: ChatMessage {
397                        content: format!("Error: {}", e),
398                        tool_calls: vec![],
399                        thinking: None,
400                    },
401                    done: true,
402                    is_error: true,
403                    usage: None,
404                })
405                .await;
406            return None;
407        }
408    };
409
410    // If the server returned a non-success status, surface the body as an
411    // error instead of feeding it into the SSE parser (which would silently
412    // drop it).
413    let status = response.status();
414    if !status.is_success() {
415        let body = response.text().await.unwrap_or_default();
416        let _ = send
417            .send(ChatMessageResponse {
418                message: ChatMessage {
419                    content: format!("Error: HTTP {} — {}", status.as_u16(), body),
420                    tool_calls: vec![],
421                    thinking: None,
422                },
423                done: true,
424                is_error: true,
425                usage: None,
426            })
427            .await;
428        return None;
429    }
430
431    let mut stream = response.bytes_stream();
432    let mut buf = String::new();
433
434    let mut acc_tool_calls: HashMap<usize, OpenAIToolCall> = HashMap::new();
435    let mut response_content = String::new();
436    let mut token_usage: Option<crate::provider::TokenUsage> = None;
437
438    while let Some(chunk_result) = stream.next().await {
439        let chunk = match chunk_result {
440            Ok(c) => c,
441            Err(e) => {
442                // Stream read error — surface it rather than silently breaking
443                let _ = send
444                    .send(ChatMessageResponse {
445                        message: ChatMessage {
446                            content: format!("\n\nStream error: {}", e),
447                            tool_calls: vec![],
448                            thinking: None,
449                        },
450                        done: true,
451                        is_error: true,
452                        usage: None,
453                    })
454                    .await;
455                break;
456            }
457        };
458
459        buf.push_str(&String::from_utf8_lossy(&chunk));
460
461        loop {
462            match buf.find('\n') {
463                None => break,
464                Some(pos) => {
465                    let line = buf[..pos].trim().to_string();
466                    buf = buf[pos + 1..].to_string();
467
468                    if line.is_empty() || line == "data: [DONE]" {
469                        continue;
470                    }
471
472                    if let Some(data) = line.strip_prefix("data: ")
473                        && let Ok(chunk) = serde_json::from_str::<StreamChunk>(data)
474                    {
475                        // Capture token usage if present (usually in the final chunk)
476                        if let Some(usage) = &chunk.usage {
477                            token_usage = Some(crate::provider::TokenUsage {
478                                prompt_tokens: usage.prompt_tokens,
479                                completion_tokens: usage.completion_tokens,
480                                total_tokens: usage.total_tokens,
481                            });
482                        }
483
484                        for choice in chunk.choices {
485                            if let Some(content) = &choice.delta.content {
486                                response_content.push_str(content);
487                            }
488
489                            if let Some(tool_calls) = &choice.delta.tool_calls {
490                                for tc in tool_calls {
491                                    let entry =
492                                        acc_tool_calls.entry(tc.index).or_insert(OpenAIToolCall {
493                                            index: tc.index,
494                                            id: String::new(),
495                                            call_type: "function".to_string(),
496                                            function: OpenAIToolCallFunction::default(),
497                                        });
498
499                                    if !tc.id.is_empty() {
500                                        entry.id = tc.id.clone();
501                                    }
502                                    if !tc.function.name.is_empty() {
503                                        entry.function.name = tc.function.name.clone();
504                                    }
505                                    entry.function.arguments.push_str(&tc.function.arguments);
506                                }
507                            }
508                        }
509                    }
510                }
511            }
512        }
513
514        if !response_content.is_empty() {
515            let _ = send
516                .send(ChatMessageResponse {
517                    message: ChatMessage {
518                        content: response_content.clone(),
519                        tool_calls: vec![],
520                        thinking: None,
521                    },
522                    done: false,
523                    is_error: false,
524                    usage: None,
525                })
526                .await;
527            response_content.clear();
528        }
529    }
530
531    let tool_calls: Vec<ToolCall> = if !acc_tool_calls.is_empty() {
532        acc_tool_calls
533            .into_values()
534            .map(|tc| {
535                let args: serde_json::Value =
536                    serde_json::from_str(&tc.function.arguments).unwrap_or(serde_json::Value::Null);
537                ToolCall {
538                    id: if tc.id.is_empty() { None } else { Some(tc.id) },
539                    function: ToolCallFunction {
540                        name: tc.function.name,
541                        arguments: args,
542                        thought_signature: None,
543                    },
544                }
545            })
546            .collect()
547    } else {
548        vec![]
549    };
550
551    // Send the final response with tool calls and token usage
552    let _ = send
553        .send(ChatMessageResponse {
554            message: ChatMessage {
555                content: String::new(),
556                tool_calls,
557                thinking: None,
558            },
559            done: true,
560            is_error: false,
561            usage: token_usage.clone(),
562        })
563        .await;
564
565    token_usage
566}