Skip to main content

tinyharness_lib/provider/
openai_compat.rs

1use std::collections::HashMap;
2use std::future::Future;
3use std::pin::Pin;
4use std::time::Duration;
5
6use reqwest::Client;
7use serde::{Deserialize, Serialize};
8use tokio_stream::StreamExt;
9
10use crate::provider::{
11    ChatMessage, ChatMessageResponse, Message, Role, ToolCall, ToolCallFunction, ToolDefinition,
12};
13
14/// Shared inner state for OpenAI-compatible providers (llama.cpp, vLLM, etc.).
15///
16/// Encapsulates the common `{client, base_url, model}` fields and all shared
17/// logic so that provider implementations only need to differ in `list_models()`.
18pub struct OpenAiCompatInner {
19    client: Client,
20    base_url: String,
21    model: Option<String>,
22}
23
24impl OpenAiCompatInner {
25    pub fn new(base_url: String) -> Self {
26        let client = Client::builder()
27            .connect_timeout(Duration::from_secs(15))
28            .read_timeout(Duration::from_secs(300))
29            .build()
30            .unwrap_or_else(|_| Client::new());
31        OpenAiCompatInner {
32            client,
33            base_url,
34            model: None,
35        }
36    }
37
38    /// Perform a health check against the server's `/health` endpoint.
39    pub fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
40        let url = format!("{}/health", self.base_url.trim_end_matches('/'));
41        let client = self.client.clone();
42        Box::pin(async move {
43            match client.get(&url).send().await {
44                Ok(resp) if resp.status().is_success() => Ok(()),
45                Ok(resp) => Err(format!(
46                    "Server returned {}: {}",
47                    resp.status().as_u16(),
48                    resp.text().await.unwrap_or_default()
49                )),
50                Err(e) => Err(format!("Cannot reach {}: {}", url, e)),
51            }
52        })
53    }
54
55    pub fn select_model(&mut self, name: String) {
56        self.model = Some(name);
57    }
58
59    pub fn current_model(&self) -> Option<String> {
60        self.model.clone()
61    }
62
63    /// Return the `/v1/chat/completions` URL for this server.
64    pub fn chat_url(&self) -> String {
65        format!(
66            "{}/v1/chat/completions",
67            self.base_url.trim_end_matches('/')
68        )
69    }
70
71    /// Fetch the model list from the server's `/v1/models` endpoint.
72    /// Returns the list of model IDs, or an empty vec on failure.
73    pub fn fetch_model_list(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
74        let url = format!("{}/v1/models", self.base_url.trim_end_matches('/'));
75        let client = self.client.clone();
76        let current_model = self.model.clone();
77        Box::pin(async move {
78            match client.get(&url).send().await {
79                Ok(resp) if resp.status().is_success() => {
80                    match resp.json::<ModelListResponse>().await {
81                        Ok(list) => list.data.into_iter().map(|m| m.id).collect(),
82                        Err(_) => current_model.into_iter().collect(),
83                    }
84                }
85                _ => current_model.into_iter().collect(),
86            }
87        })
88    }
89
90    /// Stream chat completions using the OpenAI-compatible API.
91    /// Returns a receiver for streaming response chunks, or an error string
92    /// if the request cannot be started.
93    pub fn chat(
94        &self,
95        messages: Vec<Message>,
96        tools: Vec<ToolDefinition>,
97    ) -> Pin<
98        Box<
99            dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
100                + Send,
101        >,
102    > {
103        let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
104
105        let model = self.model.clone().unwrap_or_default();
106        let openai_messages = messages.into_iter().map(to_openai_message).collect();
107        let openai_tools = tools.into_iter().map(to_openai_tool).collect();
108        let client = self.client.clone();
109        let chat_url = self.chat_url();
110
111        let body = ChatRequest {
112            model,
113            messages: openai_messages,
114            stream: true,
115            stream_options: Some(StreamOptions {
116                include_usage: true,
117            }),
118            tools: openai_tools,
119        };
120
121        // Spawn the streaming work on a background task
122        tokio::spawn(async move {
123            let _usage = stream_chat_completions(&client, &chat_url, &body, &send).await;
124        });
125
126        Box::pin(async move { Ok(recv) })
127    }
128}
129
130// ── OpenAI-compatible request/response types ──
131
132#[derive(Serialize)]
133pub struct ChatRequest {
134    pub model: String,
135    pub messages: Vec<OpenAIMessage>,
136    pub stream: bool,
137    #[serde(skip_serializing_if = "Option::is_none")]
138    pub stream_options: Option<StreamOptions>,
139    #[serde(skip_serializing_if = "Vec::is_empty")]
140    pub tools: Vec<OpenAITool>,
141}
142
143#[derive(Serialize)]
144pub struct StreamOptions {
145    pub include_usage: bool,
146}
147
148#[derive(Serialize)]
149pub struct OpenAIMessage {
150    pub role: String,
151    pub content: serde_json::Value,
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub tool_calls: Option<Vec<OpenAIToolCall>>,
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub tool_call_id: Option<String>,
156}
157
158#[derive(Serialize)]
159pub struct OpenAITool {
160    #[serde(rename = "type")]
161    pub tool_type: String,
162    pub function: OpenAIToolFunction,
163}
164
165#[derive(Serialize)]
166pub struct OpenAIToolFunction {
167    pub name: String,
168    pub description: String,
169    pub parameters: serde_json::Value,
170}
171
172#[derive(Serialize, Deserialize, Clone)]
173pub struct OpenAIToolCall {
174    #[serde(default)]
175    pub index: usize,
176    #[serde(default)]
177    pub id: String,
178    #[serde(rename = "type", default)]
179    pub call_type: String,
180    #[serde(default)]
181    pub function: OpenAIToolCallFunction,
182}
183
184#[derive(Serialize, Deserialize, Clone, Default)]
185pub struct OpenAIToolCallFunction {
186    #[serde(default)]
187    pub name: String,
188    #[serde(default)]
189    pub arguments: String,
190}
191
192#[derive(Deserialize)]
193pub struct ChunkChoice {
194    pub delta: Delta,
195    #[serde(default, rename = "finish_reason")]
196    pub _finish_reason: Option<String>,
197}
198
199#[derive(Deserialize, Default)]
200pub struct Delta {
201    #[serde(default)]
202    pub content: Option<String>,
203    #[serde(default)]
204    pub tool_calls: Option<Vec<OpenAIToolCall>>,
205}
206
207#[derive(Deserialize)]
208pub struct StreamChunk {
209    #[serde(default)]
210    pub choices: Vec<ChunkChoice>,
211    #[serde(default)]
212    pub usage: Option<OpenAIUsage>,
213}
214
215#[derive(Deserialize, Clone)]
216pub struct OpenAIUsage {
217    #[serde(default)]
218    pub prompt_tokens: u32,
219    #[serde(default)]
220    pub completion_tokens: u32,
221    #[serde(default)]
222    pub total_tokens: u32,
223}
224
225// ── Model list response types ──
226
227#[derive(Deserialize)]
228pub struct ModelListResponse {
229    pub data: Vec<ModelEntry>,
230}
231
232#[derive(Deserialize)]
233pub struct ModelEntry {
234    pub id: String,
235}
236
237// ── Conversion helpers ──
238
239pub fn to_openai_message(msg: Message) -> OpenAIMessage {
240    /// Build the content value: if images are present, use multipart array format;
241    /// otherwise use plain string.
242    fn build_content(msg: &Message) -> serde_json::Value {
243        if msg.images.is_empty() {
244            serde_json::Value::String(msg.content.clone())
245        } else {
246            let mut parts: Vec<serde_json::Value> = Vec::new();
247            // Add text part
248            if !msg.content.is_empty() {
249                parts.push(serde_json::json!({
250                    "type": "text",
251                    "text": msg.content
252                }));
253            }
254            // Add image parts
255            for img in &msg.images {
256                parts.push(serde_json::json!({
257                    "type": "image_url",
258                    "image_url": {
259                        "url": img.data_uri()
260                    }
261                }));
262            }
263            serde_json::Value::Array(parts)
264        }
265    }
266
267    match msg.role {
268        Role::System => OpenAIMessage {
269            role: "system".to_string(),
270            content: serde_json::Value::String(msg.content),
271            tool_calls: None,
272            tool_call_id: None,
273        },
274        Role::User => OpenAIMessage {
275            role: "user".to_string(),
276            content: build_content(&msg),
277            tool_calls: None,
278            tool_call_id: None,
279        },
280        Role::Assistant => {
281            if msg.tool_calls.is_empty() {
282                OpenAIMessage {
283                    role: "assistant".to_string(),
284                    content: serde_json::Value::String(msg.content),
285                    tool_calls: None,
286                    tool_call_id: None,
287                }
288            } else {
289                let tool_calls: Vec<OpenAIToolCall> = msg
290                    .tool_calls
291                    .into_iter()
292                    .enumerate()
293                    .map(|(i, tc)| OpenAIToolCall {
294                        index: i,
295                        id: String::new(),
296                        call_type: "function".to_string(),
297                        function: OpenAIToolCallFunction {
298                            name: tc.function.name,
299                            arguments: tc.function.arguments.to_string(),
300                        },
301                    })
302                    .collect();
303                OpenAIMessage {
304                    role: "assistant".to_string(),
305                    content: serde_json::Value::String(msg.content),
306                    tool_calls: Some(tool_calls),
307                    tool_call_id: None,
308                }
309            }
310        }
311        Role::Tool => OpenAIMessage {
312            role: "tool".to_string(),
313            content: serde_json::Value::String(msg.content),
314            tool_calls: None,
315            tool_call_id: None,
316        },
317    }
318}
319
320pub fn to_openai_tool(ti: ToolDefinition) -> OpenAITool {
321    OpenAITool {
322        tool_type: "function".to_string(),
323        function: OpenAIToolFunction {
324            name: ti.name,
325            description: ti.description,
326            parameters: serde_json::to_value(ti.parameters).unwrap_or_default(),
327        },
328    }
329}
330
331/// Stream chat completions from an OpenAI-compatible endpoint.
332/// Returns accumulated tool calls and final content via the sender.
333/// Also returns the token usage if available.
334pub async fn stream_chat_completions(
335    client: &reqwest::Client,
336    url: &str,
337    body: &ChatRequest,
338    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
339) -> Option<crate::provider::TokenUsage> {
340    let response = match client.post(url).json(body).send().await {
341        Ok(r) => r,
342        Err(e) => {
343            let _ = send
344                .send(ChatMessageResponse {
345                    message: ChatMessage {
346                        content: format!("Error: {}", e),
347                        tool_calls: vec![],
348                        thinking: None,
349                    },
350                    done: true,
351                    is_error: true,
352                    usage: None,
353                })
354                .await;
355            return None;
356        }
357    };
358
359    let mut stream = response.bytes_stream();
360    let mut buf = String::new();
361
362    let mut acc_tool_calls: HashMap<usize, OpenAIToolCall> = HashMap::new();
363    let mut response_content = String::new();
364    let mut token_usage: Option<crate::provider::TokenUsage> = None;
365
366    while let Some(chunk_result) = stream.next().await {
367        let chunk = match chunk_result {
368            Ok(c) => c,
369            Err(e) => {
370                // Stream read error — surface it rather than silently breaking
371                let _ = send
372                    .send(ChatMessageResponse {
373                        message: ChatMessage {
374                            content: format!("\n\nStream error: {}", e),
375                            tool_calls: vec![],
376                            thinking: None,
377                        },
378                        done: true,
379                        is_error: true,
380                        usage: None,
381                    })
382                    .await;
383                break;
384            }
385        };
386
387        buf.push_str(&String::from_utf8_lossy(&chunk));
388
389        loop {
390            match buf.find('\n') {
391                None => break,
392                Some(pos) => {
393                    let line = buf[..pos].trim().to_string();
394                    buf = buf[pos + 1..].to_string();
395
396                    if line.is_empty() || line == "data: [DONE]" {
397                        continue;
398                    }
399
400                    if let Some(data) = line.strip_prefix("data: ")
401                        && let Ok(chunk) = serde_json::from_str::<StreamChunk>(data)
402                    {
403                        // Capture token usage if present (usually in the final chunk)
404                        if let Some(usage) = &chunk.usage {
405                            token_usage = Some(crate::provider::TokenUsage {
406                                prompt_tokens: usage.prompt_tokens,
407                                completion_tokens: usage.completion_tokens,
408                                total_tokens: usage.total_tokens,
409                            });
410                        }
411
412                        for choice in chunk.choices {
413                            if let Some(content) = &choice.delta.content {
414                                response_content.push_str(content);
415                            }
416
417                            if let Some(tool_calls) = &choice.delta.tool_calls {
418                                for tc in tool_calls {
419                                    let entry =
420                                        acc_tool_calls.entry(tc.index).or_insert(OpenAIToolCall {
421                                            index: tc.index,
422                                            id: String::new(),
423                                            call_type: "function".to_string(),
424                                            function: OpenAIToolCallFunction::default(),
425                                        });
426
427                                    if !tc.id.is_empty() {
428                                        entry.id = tc.id.clone();
429                                    }
430                                    if !tc.function.name.is_empty() {
431                                        entry.function.name = tc.function.name.clone();
432                                    }
433                                    entry.function.arguments.push_str(&tc.function.arguments);
434                                }
435                            }
436                        }
437                    }
438                }
439            }
440        }
441
442        if !response_content.is_empty() {
443            let _ = send
444                .send(ChatMessageResponse {
445                    message: ChatMessage {
446                        content: response_content.clone(),
447                        tool_calls: vec![],
448                        thinking: None,
449                    },
450                    done: false,
451                    is_error: false,
452                    usage: None,
453                })
454                .await;
455            response_content.clear();
456        }
457    }
458
459    let tool_calls: Vec<ToolCall> = if !acc_tool_calls.is_empty() {
460        acc_tool_calls
461            .into_values()
462            .map(|tc| {
463                let args: serde_json::Value =
464                    serde_json::from_str(&tc.function.arguments).unwrap_or(serde_json::Value::Null);
465                ToolCall {
466                    function: ToolCallFunction {
467                        name: tc.function.name,
468                        arguments: args,
469                        thought_signature: None,
470                    },
471                }
472            })
473            .collect()
474    } else {
475        vec![]
476    };
477
478    // Send the final response with tool calls and token usage
479    let _ = send
480        .send(ChatMessageResponse {
481            message: ChatMessage {
482                content: String::new(),
483                tool_calls,
484                thinking: None,
485            },
486            done: true,
487            is_error: false,
488            usage: token_usage.clone(),
489        })
490        .await;
491
492    token_usage
493}