Skip to main content

tinyharness_lib/provider/
ollama.rs

1use std::future::Future;
2use std::pin::Pin;
3use std::time::Duration;
4
5use ollama_rs::{
6    IntoUrlSealed, Ollama,
7    generation::{
8        chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
9        parameters::ThinkType,
10    },
11};
12use serde::Deserialize;
13use tokio_stream::StreamExt;
14
15use crate::config::OllamaThinkType;
16use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
17
18use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
19
20impl From<Message> for OllamaChatMessage {
21    fn from(msg: Message) -> Self {
22        match msg.role {
23            Role::System => OllamaChatMessage::system(msg.content),
24            Role::User => {
25                let mut m = OllamaChatMessage::user(msg.content);
26                if !msg.images.is_empty() {
27                    let images: Vec<ollama_rs::generation::images::Image> = msg
28                        .images
29                        .iter()
30                        .map(|img| {
31                            ollama_rs::generation::images::Image::from_base64(
32                                img.base64_data.clone(),
33                            )
34                        })
35                        .collect();
36                    m.images = Some(images);
37                }
38                m
39            }
40            Role::Assistant => {
41                let mut m = OllamaChatMessage::assistant(msg.content);
42                if !msg.tool_calls.is_empty() {
43                    m.tool_calls = msg
44                        .tool_calls
45                        .into_iter()
46                        .map(|tc| ollama_rs::generation::tools::ToolCall {
47                            function: ollama_rs::generation::tools::ToolCallFunction {
48                                name: tc.function.name,
49                                arguments: tc.function.arguments,
50                            },
51                        })
52                        .collect();
53                }
54                m
55            }
56            Role::Tool => OllamaChatMessage::tool(msg.content),
57        }
58    }
59}
60
61fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
62    ollama_rs::generation::tools::ToolInfo {
63        tool_type: ollama_rs::generation::tools::ToolType::Function,
64        function: ollama_rs::generation::tools::ToolFunctionInfo {
65            name: ti.name,
66            description: ti.description,
67            parameters: ti.parameters,
68        },
69    }
70}
71
72fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
73    match tt {
74        OllamaThinkType::Off => ThinkType::False,
75        OllamaThinkType::Low => ThinkType::Low,
76        OllamaThinkType::Medium => ThinkType::Medium,
77        OllamaThinkType::High => ThinkType::High,
78    }
79}
80
81pub struct OllamaProvider {
82    client: Ollama,
83    http_client: reqwest::Client,
84    base_url: String,
85    model: Option<String>,
86    timeout_secs: u64,
87    max_retries: u32,
88    think_type: OllamaThinkType,
89}
90
91impl OllamaProvider {
92    pub fn new(
93        base: String,
94        timeout_secs: u64,
95        max_retries: u32,
96        think_type: OllamaThinkType,
97    ) -> Self {
98        // Normalize URL: ensure it ends with '/'
99        let base_url = if base.ends_with('/') {
100            base.clone()
101        } else {
102            format!("{base}/")
103        };
104        let client = Ollama::from_url(base.into_url().unwrap());
105        let http_client = reqwest::Client::builder()
106            .connect_timeout(Duration::from_secs(15))
107            .read_timeout(Duration::from_secs(timeout_secs + 60))
108            .build()
109            .unwrap_or_else(|_| reqwest::Client::new());
110        OllamaProvider {
111            client,
112            http_client,
113            base_url,
114            model: None,
115            timeout_secs,
116            max_retries,
117            think_type,
118        }
119    }
120}
121
122impl Provider for OllamaProvider {
123    fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
124        let client = self.client.clone();
125        Box::pin(async move {
126            match client.list_local_models().await {
127                Ok(_) => Ok(()),
128                Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
129            }
130        })
131    }
132
133    fn list_models(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
134        let client = self.client.clone();
135        Box::pin(async move {
136            match client.list_local_models().await {
137                Ok(models) => models.into_iter().map(|m| m.name).collect(),
138                Err(_) => vec![],
139            }
140        })
141    }
142
143    fn select_model(&mut self, name: String) {
144        self.model = Some(name);
145    }
146
147    fn current_model(&self) -> Option<String> {
148        self.model.clone()
149    }
150
151    fn set_timeout(&mut self, timeout_secs: u64) {
152        self.timeout_secs = timeout_secs;
153    }
154
155    fn set_retries(&mut self, max_retries: u32) {
156        self.max_retries = max_retries;
157    }
158
159    fn set_think_type(&mut self, think_type: OllamaThinkType) {
160        self.think_type = think_type;
161    }
162
163    fn chat(
164        &mut self,
165        messages: Vec<Message>,
166        tools: Vec<ToolDefinition>,
167    ) -> Pin<
168        Box<
169            dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
170                + Send,
171        >,
172    > {
173        let model = match self.model.clone() {
174            Some(m) => m,
175            None => {
176                return Box::pin(async move {
177                    Err("No model selected. Use /model <name> to select one.".to_string())
178                });
179            }
180        };
181        let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
182        let timeout_secs = self.timeout_secs;
183        let max_retries = self.max_retries;
184        let http_client = self.http_client.clone();
185        let base_url = self.base_url.clone();
186        let request = {
187            let chat_messages: Vec<OllamaChatMessage> =
188                messages.iter().map(|m| m.clone().into()).collect();
189
190            // Collect Gemini thought_signatures from incoming messages before
191            // ollama-rs conversion drops them. These must be re-injected into
192            // the serialized request so Gemini accepts multi-turn tool calling.
193            let thought_signatures: Vec<Vec<Option<String>>> = messages
194                .iter()
195                .map(|m| {
196                    m.tool_calls
197                        .iter()
198                        .map(|tc| tc.function.thought_signature.clone())
199                        .collect()
200                })
201                .collect();
202
203            let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
204                tools.into_iter().map(to_ollama_tool_info).collect();
205
206            let mut req = ChatMessageRequest::new(model, chat_messages)
207                .think(to_ollama_think_type(self.think_type));
208
209            if !ollama_tools.is_empty() {
210                req = req.tools(ollama_tools);
211                req.think = None;
212            }
213            (req, thought_signatures)
214        };
215
216        // Spawn the streaming work on a background task.
217        //
218        // We use our own raw SSE parser instead of ollama-rs's streaming to handle
219        // both Ollama-native usage format (flat prompt_eval_count/eval_count) and
220        // cloud proxy / OpenAI-compatible format (nested usage object). The ollama-rs
221        // library only recognises the flat format, so cloud proxies that return
222        // {"usage": {"prompt_tokens": ..., "completion_tokens": ..., "total_tokens": ...}}
223        // would lose their usage data.
224        tokio::spawn(async move {
225            let (request, thought_signatures) = request;
226            // Retry loop with exponential backoff
227            let max_attempts = max_retries.max(1);
228
229            let send_err = |msg: String| {
230                let _ = send.try_send(ChatMessageResponse {
231                    message: ChatMessage {
232                        content: msg,
233                        tool_calls: vec![],
234                        thinking: None,
235                    },
236                    done: true,
237                    is_error: true,
238                    usage: None,
239                });
240            };
241
242            for attempt in 1..=max_attempts {
243                let result = tokio::time::timeout(
244                    Duration::from_secs(timeout_secs),
245                    stream_ollama_chat(
246                        &http_client,
247                        &base_url,
248                        &request,
249                        &send,
250                        &thought_signatures,
251                    ),
252                )
253                .await;
254
255                match result {
256                    Ok(Ok(())) => return, // success
257                    Ok(Err(e)) => {
258                        if attempt >= max_attempts {
259                            send_err(format!("Error after {attempt} retries: {e}"));
260                            return;
261                        }
262                    }
263                    Err(_) => {
264                        if attempt >= max_attempts {
265                            send_err(format!(
266                                "Error: Request timed out after {timeout_secs}s ({attempt} retries)"
267                            ));
268                            return;
269                        }
270                    }
271                }
272
273                // Exponential backoff: 1s, 2s, 4s, ...
274                let backoff = Duration::from_secs(1 << (attempt - 1));
275                tokio::time::sleep(backoff).await;
276            }
277        });
278
279        Box::pin(async move { Ok(recv) })
280    }
281}
282
283// ── Raw Ollama SSE streaming ────────────────────────────────────────────────
284
285/// A raw Ollama SSE chunk that accepts **both** native Ollama and
286/// OpenAI-compatible usage formats.
287///
288/// Ollama-native format has flat fields:
289/// ```json
290/// { "prompt_eval_count": 123, "eval_count": 45, ... }
291/// ```
292///
293/// Some cloud proxies return an OpenAI-compatible nested `usage` object:
294/// ```json
295/// { "usage": { "prompt_tokens": 123, "completion_tokens": 45, "total_tokens": 168 } }
296/// ```
297///
298/// We try both sources and prefer the nested `usage` object when present.
299#[derive(Debug, Clone, Deserialize)]
300struct OllamaChunk {
301    #[serde(default)]
302    message: OllamaChunkMessage,
303    #[serde(default)]
304    done: bool,
305    // Native Ollama flat fields
306    #[serde(default)]
307    prompt_eval_count: Option<u64>,
308    #[serde(default)]
309    eval_count: Option<u64>,
310    // OpenAI-compatible nested usage (cloud proxies)
311    #[serde(default)]
312    usage: Option<OllamaUsage>,
313    /// Raw JSON for the tool_calls array from the message, used to extract
314    /// Gemini-specific fields like `thought_signature` that ollama-rs drops.
315    #[serde(skip, default)]
316    raw_tool_calls_json: Option<serde_json::Value>,
317}
318
319#[derive(Debug, Clone, Deserialize)]
320struct OllamaUsage {
321    #[serde(default)]
322    prompt_tokens: u32,
323    #[serde(default)]
324    completion_tokens: u32,
325    #[serde(default)]
326    total_tokens: u32,
327}
328
329#[derive(Debug, Clone, Default, Deserialize)]
330struct OllamaChunkMessage {
331    #[serde(default)]
332    content: String,
333    #[serde(default)]
334    thinking: Option<String>,
335    #[serde(default)]
336    tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
337}
338
339impl OllamaChunk {
340    /// Set raw tool calls JSON from deserialized chunk line.
341    fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
342        if let Some(msg) = raw_line.get("message")
343            && let Some(tcs) = msg.get("tool_calls")
344        {
345            self.raw_tool_calls_json = Some(tcs.clone());
346        }
347    }
348
349    fn to_chat_message_response(&self) -> ChatMessageResponse {
350        let usage = if let Some(u) = &self.usage {
351            Some(TokenUsage {
352                prompt_tokens: u.prompt_tokens,
353                completion_tokens: u.completion_tokens,
354                total_tokens: u.total_tokens,
355            })
356        } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
357            let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
358            let completion = self.eval_count.unwrap_or(0) as u32;
359            Some(TokenUsage {
360                prompt_tokens: prompt,
361                completion_tokens: completion,
362                total_tokens: prompt.saturating_add(completion),
363            })
364        } else {
365            None
366        };
367
368        // Build tool_calls list. Use raw JSON when available to extract
369        // Gemini `thought_signature` fields that ollama-rs drops.
370        let mut raw_tc_iter = self
371            .raw_tool_calls_json
372            .as_ref()
373            .and_then(|v| v.as_array())
374            .map(|a| a.iter())
375            .into_iter()
376            .flatten();
377        let tool_calls: Vec<ToolCall> = self
378            .message
379            .tool_calls
380            .clone()
381            .into_iter()
382            .map(|tc| {
383                let thought_signature = raw_tc_iter
384                    .next()
385                    .and_then(|raw| {
386                        raw.get("function")
387                            .and_then(|f| f.get("thought_signature"))
388                            .and_then(|ts| ts.as_str())
389                    })
390                    .map(|s| s.to_string());
391                ToolCall {
392                    id: None,
393                    function: ToolCallFunction {
394                        name: tc.function.name,
395                        arguments: tc.function.arguments,
396                        thought_signature,
397                    },
398                }
399            })
400            .collect();
401
402        ChatMessageResponse {
403            message: ChatMessage {
404                content: self.message.content.clone(),
405                thinking: self.message.thinking.clone(),
406                tool_calls,
407            },
408            done: self.done,
409            is_error: false,
410            usage,
411        }
412    }
413}
414
415/// Stream chat completions from the Ollama `/api/chat` endpoint using raw SSE parsing.
416///
417/// This bypasses ollama-rs's `send_chat_messages_stream` so we can recognise both
418/// native Ollama and OpenAI-compatible usage formats in the response chunks.
419async fn stream_ollama_chat(
420    client: &reqwest::Client,
421    base_url: &str,
422    request: &ChatMessageRequest,
423    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
424    thought_signatures: &[Vec<Option<String>>],
425) -> Result<(), String> {
426    let url = format!("{base_url}api/chat");
427    let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
428
429    // Fix ollama-rs 0.3.4 serialization quirks for Ollama Cloud / Gemini compatibility:
430    //
431    // 1. ToolType::Function serializes as "Function" (uppercase F), but the
432    //    API spec requires lowercase "function". Ollama Cloud / Gemini rejects
433    //    the uppercase variant with "Invalid tool type". Fix by lowercasing.
434    // Fix 2: "role": "tool" messages need a "name" field (tool_name) so Gemini
435    //    can match results to calls. ollama-rs doesn't add this, so we inject
436    //    it from the preceding assistant message's tool_calls.
437    // 3. Re-inject Gemini `thought_signature` fields captured from tool calls.
438    fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
439        match value {
440            serde_json::Value::Object(map) => {
441                // Fix 1: Lowercase "Function" → "function" in tool definitions
442                if let Some(t) = map.get_mut("type")
443                    && t.as_str() == Some("Function")
444                {
445                    *t = serde_json::Value::String("function".to_string());
446                }
447                for v in map.values_mut() {
448                    fix_request_json(v, thought_signatures);
449                }
450            }
451            serde_json::Value::Array(arr) => {
452                // Fix 2: Add "name" field to tool result messages by tracking
453                // tool_calls from the most recent assistant message.
454                // Also inject "tool_call_id" for OpenAI-compatible backends.
455                // Fix 3: Re-inject thought_signatures into assistant tool_calls.
456                let mut prev_tool_names: Vec<String> = Vec::new();
457                let mut prev_tool_ids: Vec<String> = Vec::new();
458                let mut ts_idx: usize = 0;
459                for msg in arr.iter_mut() {
460                    if let serde_json::Value::Object(msg_map) = msg {
461                        let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
462                        match role {
463                            "assistant" => {
464                                prev_tool_names.clear();
465                                prev_tool_ids.clear();
466                                if let Some(tcs) = msg_map.get_mut("tool_calls")
467                                    && let Some(tc_arr) = tcs.as_array_mut()
468                                {
469                                    // Re-inject thought_signatures
470                                    if let Some(sigs) = thought_signatures.get(ts_idx) {
471                                        for (i, tc) in tc_arr.iter_mut().enumerate() {
472                                            if let Some(sig) = sigs.get(i).and_then(|s| s.clone())
473                                                && let Some(func) = tc.get_mut("function")
474                                                && let Some(obj) = func.as_object_mut()
475                                            {
476                                                obj.insert(
477                                                    "thought_signature".to_string(),
478                                                    serde_json::Value::String(sig),
479                                                );
480                                            }
481                                        }
482                                    }
483                                    ts_idx += 1;
484
485                                    // Track tool names and ids for Fix 2
486                                    for (i, tc) in tc_arr.iter().enumerate() {
487                                        if let Some(name) = tc
488                                            .get("function")
489                                            .and_then(|f| f.get("name"))
490                                            .and_then(|n| n.as_str())
491                                        {
492                                            prev_tool_names.push(name.to_string());
493                                        }
494                                        // Track tool call ids (synthesize if missing)
495                                        let id = tc
496                                            .get("id")
497                                            .and_then(|v| v.as_str())
498                                            .filter(|s| !s.is_empty())
499                                            .map(|s| s.to_string())
500                                            .unwrap_or_else(|| format!("call_{}", i));
501                                        prev_tool_ids.push(id);
502                                    }
503                                } else {
504                                    ts_idx += 1;
505                                }
506                            }
507                            "tool" => {
508                                // Add "name" field for Ollama/Gemini compatibility
509                                // (only when there's exactly one tool call)
510                                if !msg_map.contains_key("name") && prev_tool_names.len() == 1 {
511                                    msg_map.insert(
512                                        "name".to_string(),
513                                        serde_json::Value::String(prev_tool_names[0].clone()),
514                                    );
515                                }
516                                // Add "tool_call_id" for OpenAI-compatible backends
517                                if !msg_map.contains_key("tool_call_id") && prev_tool_ids.len() == 1
518                                {
519                                    msg_map.insert(
520                                        "tool_call_id".to_string(),
521                                        serde_json::Value::String(prev_tool_ids[0].clone()),
522                                    );
523                                }
524                            }
525                            _ => {}
526                        }
527                    }
528                }
529                for v in arr.iter_mut() {
530                    fix_request_json(v, thought_signatures);
531                }
532            }
533            _ => {}
534        }
535    }
536    fix_request_json(&mut request, thought_signatures);
537
538    let response = client
539        .post(&url)
540        .json(&request)
541        .send()
542        .await
543        .map_err(|e| format!("request failed: {e}"))?;
544
545    if !response.status().is_success() {
546        return Err(format!(
547            "HTTP {}: {}",
548            response.status().as_u16(),
549            response.text().await.unwrap_or_default()
550        ));
551    }
552
553    let mut stream = response.bytes_stream();
554    let mut buffer = String::new();
555
556    while let Some(chunk_result) = stream.next().await {
557        let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
558        buffer.push_str(&String::from_utf8_lossy(&chunk));
559
560        // Process complete SSE lines
561        while let Some(pos) = buffer.find('\n') {
562            let line = buffer[..pos].trim().to_string();
563            buffer = buffer[pos + 1..].to_string();
564
565            if line.is_empty() {
566                continue;
567            }
568
569            match serde_json::from_str::<OllamaChunk>(&line) {
570                Ok(mut chunk) => {
571                    // Capture raw JSON before extracting to preserve Gemini-
572                    // specific fields like `thought_signature`.
573                    if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
574                        chunk.capture_raw_tool_calls(&raw);
575                    }
576                    let done = chunk.done;
577                    let ours = chunk.to_chat_message_response();
578                    if send.send(ours).await.is_err() {
579                        return Ok(()); // receiver dropped
580                    }
581                    if done {
582                        return Ok(());
583                    }
584                }
585                Err(_) => {
586                    // Non-JSON line (e.g. "data: [DONE]" or empty) — skip silently
587                }
588            }
589        }
590    }
591
592    // Process remaining buffer
593    if !buffer.trim().is_empty()
594        && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
595    {
596        if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
597            chunk.capture_raw_tool_calls(&raw);
598        }
599        let done = chunk.done;
600        let ours = chunk.to_chat_message_response();
601        if send.send(ours).await.is_err() {
602            return Ok(());
603        }
604        if done {
605            return Ok(());
606        }
607    }
608
609    // Stream ended without a done marker — send a synthetic done response
610    let _ = send
611        .send(ChatMessageResponse {
612            message: ChatMessage {
613                content: String::new(),
614                tool_calls: vec![],
615                thinking: None,
616            },
617            done: true,
618            is_error: false,
619            usage: None,
620        })
621        .await;
622
623    Ok(())
624}