Skip to main content

tinyharness_lib/provider/
ollama.rs

1use std::future::Future;
2use std::pin::Pin;
3use std::time::Duration;
4
5use ollama_rs::{
6    IntoUrlSealed, Ollama,
7    generation::{
8        chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
9        parameters::ThinkType,
10    },
11};
12use serde::Deserialize;
13use tokio_stream::StreamExt;
14
15use crate::config::OllamaThinkType;
16use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
17
18use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
19
20impl From<Message> for OllamaChatMessage {
21    fn from(msg: Message) -> Self {
22        match msg.role {
23            Role::System => OllamaChatMessage::system(msg.content),
24            Role::User => {
25                let mut m = OllamaChatMessage::user(msg.content);
26                if !msg.images.is_empty() {
27                    let images: Vec<ollama_rs::generation::images::Image> = msg
28                        .images
29                        .iter()
30                        .map(|img| {
31                            ollama_rs::generation::images::Image::from_base64(
32                                img.base64_data.clone(),
33                            )
34                        })
35                        .collect();
36                    m.images = Some(images);
37                }
38                m
39            }
40            Role::Assistant => {
41                let mut m = OllamaChatMessage::assistant(msg.content);
42                if !msg.tool_calls.is_empty() {
43                    m.tool_calls = msg
44                        .tool_calls
45                        .into_iter()
46                        .map(|tc| ollama_rs::generation::tools::ToolCall {
47                            function: ollama_rs::generation::tools::ToolCallFunction {
48                                name: tc.function.name,
49                                arguments: tc.function.arguments,
50                            },
51                        })
52                        .collect();
53                }
54                m
55            }
56            Role::Tool => OllamaChatMessage::tool(msg.content),
57        }
58    }
59}
60
61fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
62    ollama_rs::generation::tools::ToolInfo {
63        tool_type: ollama_rs::generation::tools::ToolType::Function,
64        function: ollama_rs::generation::tools::ToolFunctionInfo {
65            name: ti.name,
66            description: ti.description,
67            parameters: ti.parameters,
68        },
69    }
70}
71
72fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
73    match tt {
74        OllamaThinkType::Off => ThinkType::False,
75        OllamaThinkType::Low => ThinkType::Low,
76        OllamaThinkType::Medium => ThinkType::Medium,
77        OllamaThinkType::High => ThinkType::High,
78    }
79}
80
81pub struct OllamaProvider {
82    client: Ollama,
83    http_client: reqwest::Client,
84    base_url: String,
85    model: Option<String>,
86    timeout_secs: u64,
87    max_retries: u32,
88    think_type: OllamaThinkType,
89}
90
91impl OllamaProvider {
92    pub fn new(
93        base: String,
94        timeout_secs: u64,
95        max_retries: u32,
96        think_type: OllamaThinkType,
97    ) -> Result<Self, String> {
98        // Normalize URL: ensure it ends with '/'
99        let base_url = if base.ends_with('/') {
100            base
101        } else {
102            format!("{base}/")
103        };
104        // Parse the (normalized) URL once, propagating errors instead of panicking
105        // on user-supplied configuration.
106        let parsed_url = base_url
107            .clone()
108            .into_url()
109            .map_err(|e| format!("Invalid Ollama base URL `{base_url}`: {e}"))?;
110        let client = Ollama::from_url(parsed_url);
111        let http_client = reqwest::Client::builder()
112            .connect_timeout(Duration::from_secs(15))
113            .read_timeout(Duration::from_secs(timeout_secs + 60))
114            .build()
115            .unwrap_or_else(|_| reqwest::Client::new());
116        Ok(OllamaProvider {
117            client,
118            http_client,
119            base_url,
120            model: None,
121            timeout_secs,
122            max_retries,
123            think_type,
124        })
125    }
126}
127
128impl Provider for OllamaProvider {
129    fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
130        let client = self.client.clone();
131        Box::pin(async move {
132            match client.list_local_models().await {
133                Ok(_) => Ok(()),
134                Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
135            }
136        })
137    }
138
139    fn list_models(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
140        let client = self.client.clone();
141        Box::pin(async move {
142            match client.list_local_models().await {
143                Ok(models) => models.into_iter().map(|m| m.name).collect(),
144                Err(_) => vec![],
145            }
146        })
147    }
148
149    fn select_model(&mut self, name: String) {
150        self.model = Some(name);
151    }
152
153    fn current_model(&self) -> Option<String> {
154        self.model.clone()
155    }
156
157    fn set_timeout(&mut self, timeout_secs: u64) {
158        self.timeout_secs = timeout_secs;
159    }
160
161    fn set_retries(&mut self, max_retries: u32) {
162        self.max_retries = max_retries;
163    }
164
165    fn set_think_type(&mut self, think_type: OllamaThinkType) {
166        self.think_type = think_type;
167    }
168
169    fn chat(
170        &mut self,
171        messages: Vec<Message>,
172        tools: Vec<ToolDefinition>,
173    ) -> Pin<
174        Box<
175            dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
176                + Send,
177        >,
178    > {
179        let model = match self.model.clone() {
180            Some(m) => m,
181            None => {
182                return Box::pin(async move {
183                    Err("No model selected. Use /model <name> to select one.".to_string())
184                });
185            }
186        };
187        let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
188        let timeout_secs = self.timeout_secs;
189        let max_retries = self.max_retries;
190        let http_client = self.http_client.clone();
191        let base_url = self.base_url.clone();
192        let request = {
193            let chat_messages: Vec<OllamaChatMessage> =
194                messages.iter().map(|m| m.clone().into()).collect();
195
196            // Collect Gemini thought_signatures from incoming messages before
197            // ollama-rs conversion drops them. These must be re-injected into
198            // the serialized request so Gemini accepts multi-turn tool calling.
199            let thought_signatures: Vec<Vec<Option<String>>> = messages
200                .iter()
201                .map(|m| {
202                    m.tool_calls
203                        .iter()
204                        .map(|tc| tc.function.thought_signature.clone())
205                        .collect()
206                })
207                .collect();
208
209            let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
210                tools.into_iter().map(to_ollama_tool_info).collect();
211
212            let mut req = ChatMessageRequest::new(model, chat_messages)
213                .think(to_ollama_think_type(self.think_type));
214
215            if !ollama_tools.is_empty() {
216                req = req.tools(ollama_tools);
217                req.think = None;
218            }
219            (req, thought_signatures)
220        };
221
222        // Spawn the streaming work on a background task.
223        //
224        // We use our own raw SSE parser instead of ollama-rs's streaming to handle
225        // both Ollama-native usage format (flat prompt_eval_count/eval_count) and
226        // cloud proxy / OpenAI-compatible format (nested usage object). The ollama-rs
227        // library only recognises the flat format, so cloud proxies that return
228        // {"usage": {"prompt_tokens": ..., "completion_tokens": ..., "total_tokens": ...}}
229        // would lose their usage data.
230        tokio::spawn(async move {
231            let (request, thought_signatures) = request;
232            // Retry loop with exponential backoff
233            let max_attempts = max_retries.max(1);
234
235            let send_err = |msg: String| {
236                let _ = send.try_send(ChatMessageResponse {
237                    message: ChatMessage {
238                        content: msg,
239                        tool_calls: vec![],
240                        thinking: None,
241                    },
242                    done: true,
243                    is_error: true,
244                    usage: None,
245                });
246            };
247
248            for attempt in 1..=max_attempts {
249                let result = tokio::time::timeout(
250                    Duration::from_secs(timeout_secs),
251                    stream_ollama_chat(
252                        &http_client,
253                        &base_url,
254                        &request,
255                        &send,
256                        &thought_signatures,
257                    ),
258                )
259                .await;
260
261                match result {
262                    Ok(Ok(())) => return, // success
263                    Ok(Err(e)) => {
264                        if attempt >= max_attempts {
265                            send_err(format!("Error after {attempt} attempt(s): {e}"));
266                            return;
267                        }
268                    }
269                    Err(_) => {
270                        if attempt >= max_attempts {
271                            send_err(format!(
272                                "Error: Request timed out after {timeout_secs}s ({attempt} attempt(s))"
273                            ));
274                            return;
275                        }
276                    }
277                }
278
279                // Exponential backoff: 1s, 2s, 4s, ...
280                let backoff = Duration::from_secs(1 << (attempt - 1));
281                tokio::time::sleep(backoff).await;
282            }
283        });
284
285        Box::pin(async move { Ok(recv) })
286    }
287}
288
289// ── Raw Ollama SSE streaming ────────────────────────────────────────────────
290
291/// A raw Ollama SSE chunk that accepts **both** native Ollama and
292/// OpenAI-compatible usage formats.
293///
294/// Ollama-native format has flat fields:
295/// ```json
296/// { "prompt_eval_count": 123, "eval_count": 45, ... }
297/// ```
298///
299/// Some cloud proxies return an OpenAI-compatible nested `usage` object:
300/// ```json
301/// { "usage": { "prompt_tokens": 123, "completion_tokens": 45, "total_tokens": 168 } }
302/// ```
303///
304/// We try both sources and prefer the nested `usage` object when present.
305#[derive(Debug, Clone, Deserialize)]
306struct OllamaChunk {
307    #[serde(default)]
308    message: OllamaChunkMessage,
309    #[serde(default)]
310    done: bool,
311    // Native Ollama flat fields
312    #[serde(default)]
313    prompt_eval_count: Option<u64>,
314    #[serde(default)]
315    eval_count: Option<u64>,
316    // OpenAI-compatible nested usage (cloud proxies)
317    #[serde(default)]
318    usage: Option<OllamaUsage>,
319    /// Raw JSON for the tool_calls array from the message, used to extract
320    /// Gemini-specific fields like `thought_signature` that ollama-rs drops.
321    #[serde(skip, default)]
322    raw_tool_calls_json: Option<serde_json::Value>,
323}
324
325#[derive(Debug, Clone, Deserialize)]
326struct OllamaUsage {
327    #[serde(default)]
328    prompt_tokens: u32,
329    #[serde(default)]
330    completion_tokens: u32,
331    #[serde(default)]
332    total_tokens: u32,
333}
334
335#[derive(Debug, Clone, Default, Deserialize)]
336struct OllamaChunkMessage {
337    #[serde(default)]
338    content: String,
339    #[serde(default)]
340    thinking: Option<String>,
341    #[serde(default)]
342    tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
343}
344
345impl OllamaChunk {
346    /// Set raw tool calls JSON from deserialized chunk line.
347    fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
348        if let Some(msg) = raw_line.get("message")
349            && let Some(tcs) = msg.get("tool_calls")
350        {
351            self.raw_tool_calls_json = Some(tcs.clone());
352        }
353    }
354
355    fn to_chat_message_response(&self) -> ChatMessageResponse {
356        let usage = if let Some(u) = &self.usage {
357            Some(TokenUsage {
358                prompt_tokens: u.prompt_tokens,
359                completion_tokens: u.completion_tokens,
360                total_tokens: u.total_tokens,
361            })
362        } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
363            let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
364            let completion = self.eval_count.unwrap_or(0) as u32;
365            Some(TokenUsage {
366                prompt_tokens: prompt,
367                completion_tokens: completion,
368                total_tokens: prompt.saturating_add(completion),
369            })
370        } else {
371            None
372        };
373
374        // Build tool_calls list. Use raw JSON when available to extract
375        // Gemini `thought_signature` fields that ollama-rs drops.
376        let mut raw_tc_iter = self
377            .raw_tool_calls_json
378            .as_ref()
379            .and_then(|v| v.as_array())
380            .map(|a| a.iter())
381            .into_iter()
382            .flatten();
383        let tool_calls: Vec<ToolCall> = self
384            .message
385            .tool_calls
386            .clone()
387            .into_iter()
388            .map(|tc| {
389                let thought_signature = raw_tc_iter
390                    .next()
391                    .and_then(|raw| {
392                        raw.get("function")
393                            .and_then(|f| f.get("thought_signature"))
394                            .and_then(|ts| ts.as_str())
395                    })
396                    .map(|s| s.to_string());
397                ToolCall {
398                    id: None,
399                    function: ToolCallFunction {
400                        name: tc.function.name,
401                        arguments: tc.function.arguments,
402                        thought_signature,
403                    },
404                }
405            })
406            .collect();
407
408        ChatMessageResponse {
409            message: ChatMessage {
410                content: self.message.content.clone(),
411                thinking: self.message.thinking.clone(),
412                tool_calls,
413            },
414            done: self.done,
415            is_error: false,
416            usage,
417        }
418    }
419}
420
421/// Stream chat completions from the Ollama `/api/chat` endpoint using raw SSE parsing.
422///
423/// This bypasses ollama-rs's `send_chat_messages_stream` so we can recognise both
424/// native Ollama and OpenAI-compatible usage formats in the response chunks.
425async fn stream_ollama_chat(
426    client: &reqwest::Client,
427    base_url: &str,
428    request: &ChatMessageRequest,
429    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
430    thought_signatures: &[Vec<Option<String>>],
431) -> Result<(), String> {
432    let url = format!("{base_url}api/chat");
433    let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
434
435    // Force streaming — ollama-rs sets `stream: true` only inside its own
436    // `send_chat_messages_stream()` method, which we bypass. Without this,
437    // Ollama returns the entire response as a single JSON object instead of
438    // streaming line-by-line, causing the UI to block until the full message
439    // arrives.
440    request["stream"] = serde_json::Value::Bool(true);
441
442    // Fix ollama-rs 0.3.4 serialization quirks for Ollama Cloud / Gemini compatibility:
443    //
444    // 1. ToolType::Function serializes as "Function" (uppercase F), but the
445    //    API spec requires lowercase "function". Ollama Cloud / Gemini rejects
446    //    the uppercase variant with "Invalid tool type". Fix by lowercasing.
447    // Fix 2: "role": "tool" messages need a "name" field (tool_name) so Gemini
448    //    can match results to calls. ollama-rs doesn't add this, so we inject
449    //    it from the preceding assistant message's tool_calls.
450    // 3. Re-inject Gemini `thought_signature` fields captured from tool calls.
451    fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
452        match value {
453            serde_json::Value::Object(map) => {
454                // Fix 1: Lowercase "Function" → "function" in tool definitions
455                if let Some(t) = map.get_mut("type")
456                    && t.as_str() == Some("Function")
457                {
458                    *t = serde_json::Value::String("function".to_string());
459                }
460                for v in map.values_mut() {
461                    fix_request_json(v, thought_signatures);
462                }
463            }
464            serde_json::Value::Array(arr) => {
465                // Fix 2: Add "name" field to tool result messages by tracking
466                // tool_calls from the most recent assistant message.
467                // Also inject "tool_call_id" for OpenAI-compatible backends.
468                // Fix 3: Re-inject thought_signatures into assistant tool_calls.
469                //
470                // `thought_signatures` is indexed by the position of each
471                // message in the original `messages` slice (one entry per
472                // message, regardless of role). We advance `ts_idx` on every
473                // message so the indices stay aligned; otherwise non-assistant
474                // messages (system, user, tool) cause off-by-N misalignment
475                // and signatures are never re-injected, silently breaking
476                // Gemini multi-turn tool calling.
477                let mut prev_tool_names: Vec<String> = Vec::new();
478                let mut prev_tool_ids: Vec<String> = Vec::new();
479                let mut ts_idx: usize = 0;
480                let looks_like_messages_array = arr.iter().any(|v| {
481                    v.as_object()
482                        .and_then(|o| o.get("role"))
483                        .and_then(|r| r.as_str())
484                        .is_some()
485                });
486                if looks_like_messages_array {
487                    for msg in arr.iter_mut() {
488                        if let serde_json::Value::Object(msg_map) = msg {
489                            let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
490                            match role {
491                                "assistant" => {
492                                    prev_tool_names.clear();
493                                    prev_tool_ids.clear();
494                                    if let Some(tcs) = msg_map.get_mut("tool_calls")
495                                        && let Some(tc_arr) = tcs.as_array_mut()
496                                    {
497                                        // Re-inject thought_signatures
498                                        if let Some(sigs) = thought_signatures.get(ts_idx) {
499                                            for (i, tc) in tc_arr.iter_mut().enumerate() {
500                                                if let Some(sig) =
501                                                    sigs.get(i).and_then(|s| s.clone())
502                                                    && let Some(func) = tc.get_mut("function")
503                                                    && let Some(obj) = func.as_object_mut()
504                                                {
505                                                    obj.insert(
506                                                        "thought_signature".to_string(),
507                                                        serde_json::Value::String(sig),
508                                                    );
509                                                }
510                                            }
511                                        }
512
513                                        // Track tool names and ids for Fix 2
514                                        for (i, tc) in tc_arr.iter().enumerate() {
515                                            if let Some(name) = tc
516                                                .get("function")
517                                                .and_then(|f| f.get("name"))
518                                                .and_then(|n| n.as_str())
519                                            {
520                                                prev_tool_names.push(name.to_string());
521                                            }
522                                            // Track tool call ids (synthesize if missing)
523                                            let id = tc
524                                                .get("id")
525                                                .and_then(|v| v.as_str())
526                                                .filter(|s| !s.is_empty())
527                                                .map(|s| s.to_string())
528                                                .unwrap_or_else(|| format!("call_{}", i));
529                                            prev_tool_ids.push(id);
530                                        }
531                                    }
532                                }
533                                "tool" => {
534                                    // Add "name" field for Ollama/Gemini compatibility
535                                    // (only when there's exactly one tool call)
536                                    if !msg_map.contains_key("name") && prev_tool_names.len() == 1 {
537                                        msg_map.insert(
538                                            "name".to_string(),
539                                            serde_json::Value::String(prev_tool_names[0].clone()),
540                                        );
541                                    }
542                                    // Add "tool_call_id" for OpenAI-compatible backends
543                                    if !msg_map.contains_key("tool_call_id")
544                                        && prev_tool_ids.len() == 1
545                                    {
546                                        msg_map.insert(
547                                            "tool_call_id".to_string(),
548                                            serde_json::Value::String(prev_tool_ids[0].clone()),
549                                        );
550                                    }
551                                }
552                                _ => {}
553                            }
554                        }
555                        // Always advance — `thought_signatures` is 1:1 with
556                        // the original `messages` Vec, including non-assistant
557                        // entries.
558                        ts_idx += 1;
559                    }
560                }
561                // Recurse into children for non-message fixes (e.g. lowercasing
562                // tool `type`). We pass an empty slice to avoid re-applying
563                // thought_signatures from index 0 in any nested array.
564                for v in arr.iter_mut() {
565                    fix_request_json(v, &[]);
566                }
567            }
568            _ => {}
569        }
570    }
571    fix_request_json(&mut request, thought_signatures);
572
573    let response = client
574        .post(&url)
575        .json(&request)
576        .send()
577        .await
578        .map_err(|e| format!("request failed: {e}"))?;
579
580    if !response.status().is_success() {
581        return Err(format!(
582            "HTTP {}: {}",
583            response.status().as_u16(),
584            response.text().await.unwrap_or_default()
585        ));
586    }
587
588    let mut stream = response.bytes_stream();
589    let mut buffer = String::new();
590
591    while let Some(chunk_result) = stream.next().await {
592        let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
593        buffer.push_str(&String::from_utf8_lossy(&chunk));
594
595        // Process complete SSE lines
596        while let Some(pos) = buffer.find('\n') {
597            let line = buffer[..pos].trim().to_string();
598            buffer.drain(..=pos);
599
600            if line.is_empty() {
601                continue;
602            }
603
604            match serde_json::from_str::<OllamaChunk>(&line) {
605                Ok(mut chunk) => {
606                    // Capture raw JSON before extracting to preserve Gemini-
607                    // specific fields like `thought_signature`.
608                    if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
609                        chunk.capture_raw_tool_calls(&raw);
610                    }
611                    let done = chunk.done;
612                    let ours = chunk.to_chat_message_response();
613                    if send.send(ours).await.is_err() {
614                        return Ok(()); // receiver dropped
615                    }
616                    if done {
617                        return Ok(());
618                    }
619                }
620                Err(_) => {
621                    // Non-JSON line (e.g. "data: [DONE]" or empty) — skip silently
622                }
623            }
624        }
625    }
626
627    // Process remaining buffer
628    if !buffer.trim().is_empty()
629        && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
630    {
631        if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
632            chunk.capture_raw_tool_calls(&raw);
633        }
634        let done = chunk.done;
635        let ours = chunk.to_chat_message_response();
636        if send.send(ours).await.is_err() {
637            return Ok(());
638        }
639        if done {
640            return Ok(());
641        }
642    }
643
644    // Stream ended without a done marker — send a synthetic done response
645    let _ = send
646        .send(ChatMessageResponse {
647            message: ChatMessage {
648                content: String::new(),
649                tool_calls: vec![],
650                thinking: None,
651            },
652            done: true,
653            is_error: false,
654            usage: None,
655        })
656        .await;
657
658    Ok(())
659}