Skip to main content

tinyharness_lib/provider/
ollama.rs

1use std::time::Duration;
2
3use ollama_rs::{
4    IntoUrlSealed, Ollama,
5    generation::{
6        chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
7        parameters::ThinkType,
8    },
9};
10use serde::Deserialize;
11use tokio_stream::StreamExt;
12
13use crate::config::OllamaThinkType;
14use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
15
16use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
17
18impl From<Message> for OllamaChatMessage {
19    fn from(msg: Message) -> Self {
20        match msg.role {
21            Role::System => OllamaChatMessage::system(msg.content),
22            Role::User => {
23                let mut m = OllamaChatMessage::user(msg.content);
24                if !msg.images.is_empty() {
25                    let images: Vec<ollama_rs::generation::images::Image> = msg
26                        .images
27                        .iter()
28                        .map(|img| {
29                            ollama_rs::generation::images::Image::from_base64(
30                                img.base64_data.clone(),
31                            )
32                        })
33                        .collect();
34                    m.images = Some(images);
35                }
36                m
37            }
38            Role::Assistant => {
39                let mut m = OllamaChatMessage::assistant(msg.content);
40                if !msg.tool_calls.is_empty() {
41                    m.tool_calls = msg
42                        .tool_calls
43                        .into_iter()
44                        .map(|tc| ollama_rs::generation::tools::ToolCall {
45                            function: ollama_rs::generation::tools::ToolCallFunction {
46                                name: tc.function.name,
47                                arguments: tc.function.arguments,
48                            },
49                        })
50                        .collect();
51                }
52                m
53            }
54            Role::Tool => OllamaChatMessage::tool(msg.content),
55        }
56    }
57}
58
59fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
60    ollama_rs::generation::tools::ToolInfo {
61        tool_type: ollama_rs::generation::tools::ToolType::Function,
62        function: ollama_rs::generation::tools::ToolFunctionInfo {
63            name: ti.name,
64            description: ti.description,
65            parameters: ti.parameters,
66        },
67    }
68}
69
70fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
71    match tt {
72        OllamaThinkType::Off => ThinkType::False,
73        OllamaThinkType::Low => ThinkType::Low,
74        OllamaThinkType::Medium => ThinkType::Medium,
75        OllamaThinkType::High => ThinkType::High,
76    }
77}
78
79pub struct OllamaProvider {
80    client: Ollama,
81    http_client: reqwest::Client,
82    base_url: String,
83    model: Option<String>,
84    timeout_secs: u64,
85    max_retries: u32,
86    think_type: OllamaThinkType,
87}
88
89impl OllamaProvider {
90    pub fn new(
91        base: String,
92        timeout_secs: u64,
93        max_retries: u32,
94        think_type: OllamaThinkType,
95    ) -> Result<Self, String> {
96        // Normalize URL: ensure it ends with '/'
97        let base_url = if base.ends_with('/') {
98            base
99        } else {
100            format!("{base}/")
101        };
102        // Parse the (normalized) URL once, propagating errors instead of panicking
103        // on user-supplied configuration.
104        let parsed_url = base_url
105            .clone()
106            .into_url()
107            .map_err(|e| format!("Invalid Ollama base URL `{base_url}`: {e}"))?;
108        let client = Ollama::from_url(parsed_url);
109        let http_client = reqwest::Client::builder()
110            .connect_timeout(Duration::from_secs(15))
111            .read_timeout(Duration::from_secs(timeout_secs + 60))
112            .build()
113            .unwrap_or_else(|_| reqwest::Client::new());
114        Ok(OllamaProvider {
115            client,
116            http_client,
117            base_url,
118            model: None,
119            timeout_secs,
120            max_retries,
121            think_type,
122        })
123    }
124}
125
126impl Provider for OllamaProvider {
127    async fn health_check(&self) -> Result<(), String> {
128        match self.client.list_local_models().await {
129            Ok(_) => Ok(()),
130            Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
131        }
132    }
133
134    async fn list_models(&self) -> Vec<String> {
135        match self.client.list_local_models().await {
136            Ok(models) => models.into_iter().map(|m| m.name).collect(),
137            Err(_) => vec![],
138        }
139    }
140
141    fn select_model(&mut self, name: String) {
142        self.model = Some(name);
143    }
144
145    fn current_model(&self) -> Option<String> {
146        self.model.clone()
147    }
148
149    fn set_timeout(&mut self, timeout_secs: u64) {
150        self.timeout_secs = timeout_secs;
151    }
152
153    fn set_retries(&mut self, max_retries: u32) {
154        self.max_retries = max_retries;
155    }
156
157    fn set_think_type(&mut self, think_type: OllamaThinkType) {
158        self.think_type = think_type;
159    }
160
161    async fn chat(
162        &mut self,
163        messages: Vec<Message>,
164        tools: Vec<ToolDefinition>,
165    ) -> Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String> {
166        let model = match self.model.clone() {
167            Some(m) => m,
168            None => {
169                return Err("No model selected. Use /model <name> to select one.".to_string());
170            }
171        };
172        let (send, recv) =
173            tokio::sync::mpsc::channel::<ChatMessageResponse>(super::STREAM_CHANNEL_CAPACITY);
174        let timeout_secs = self.timeout_secs;
175        let max_retries = self.max_retries;
176        let http_client = self.http_client.clone();
177        let base_url = self.base_url.clone();
178        let request = {
179            let chat_messages: Vec<OllamaChatMessage> =
180                messages.iter().map(|m| m.clone().into()).collect();
181
182            // Collect Gemini thought_signatures from incoming messages before
183            // ollama-rs conversion drops them. These must be re-injected into
184            // the serialized request so Gemini accepts multi-turn tool calling.
185            let thought_signatures: Vec<Vec<Option<String>>> = messages
186                .iter()
187                .map(|m| {
188                    m.tool_calls
189                        .iter()
190                        .map(|tc| tc.function.thought_signature.clone())
191                        .collect()
192                })
193                .collect();
194
195            let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
196                tools.into_iter().map(to_ollama_tool_info).collect();
197
198            let mut req = ChatMessageRequest::new(model, chat_messages)
199                .think(to_ollama_think_type(self.think_type));
200
201            if !ollama_tools.is_empty() {
202                req = req.tools(ollama_tools);
203                req.think = None;
204            }
205            (req, thought_signatures)
206        };
207
208        // Spawn the streaming work on a background task.
209        //
210        // We use our own raw SSE parser instead of ollama-rs's streaming to handle
211        // both Ollama-native usage format (flat prompt_eval_count/eval_count) and
212        // cloud proxy / OpenAI-compatible format (nested usage object). The ollama-rs
213        // library only recognises the flat format, so cloud proxies that return
214        // {"usage": {"prompt_tokens": ..., "completion_tokens": ..., "total_tokens": ...}}
215        // would lose their usage data.
216        tokio::spawn(async move {
217            let (request, thought_signatures) = request;
218            // Retry loop with exponential backoff
219            let max_attempts = max_retries.max(1);
220
221            let send_err = |msg: String| {
222                let _ = send.try_send(ChatMessageResponse {
223                    message: ChatMessage {
224                        content: msg,
225                        tool_calls: vec![],
226                        thinking: None,
227                    },
228                    done: true,
229                    is_error: true,
230                    usage: None,
231                });
232            };
233
234            for attempt in 1..=max_attempts {
235                let result = tokio::time::timeout(
236                    Duration::from_secs(timeout_secs),
237                    stream_ollama_chat(
238                        &http_client,
239                        &base_url,
240                        &request,
241                        &send,
242                        &thought_signatures,
243                    ),
244                )
245                .await;
246
247                match result {
248                    Ok(Ok(())) => return, // success
249                    Ok(Err(e)) => {
250                        if attempt >= max_attempts {
251                            send_err(format!("Error after {attempt} attempt(s): {e}"));
252                            return;
253                        }
254                    }
255                    Err(_) => {
256                        if attempt >= max_attempts {
257                            send_err(format!(
258                                "Error: Request timed out after {timeout_secs}s ({attempt} attempt(s))"
259                            ));
260                            return;
261                        }
262                    }
263                }
264
265                // Exponential backoff: 1s, 2s, 4s, ...
266                let backoff = Duration::from_secs(1 << (attempt - 1));
267                tokio::time::sleep(backoff).await;
268            }
269        });
270
271        Ok(recv)
272    }
273}
274
275// ── Raw Ollama SSE streaming ────────────────────────────────────────────────
276
277/// A raw Ollama SSE chunk that accepts **both** native Ollama and
278/// OpenAI-compatible usage formats.
279///
280/// Ollama-native format has flat fields:
281/// ```json
282/// { "prompt_eval_count": 123, "eval_count": 45, ... }
283/// ```
284///
285/// Some cloud proxies return an OpenAI-compatible nested `usage` object:
286/// ```json
287/// { "usage": { "prompt_tokens": 123, "completion_tokens": 45, "total_tokens": 168 } }
288/// ```
289///
290/// We try both sources and prefer the nested `usage` object when present.
291#[derive(Debug, Clone, Deserialize)]
292struct OllamaChunk {
293    #[serde(default)]
294    message: OllamaChunkMessage,
295    #[serde(default)]
296    done: bool,
297    // Native Ollama flat fields
298    #[serde(default)]
299    prompt_eval_count: Option<u64>,
300    #[serde(default)]
301    eval_count: Option<u64>,
302    // OpenAI-compatible nested usage (cloud proxies)
303    #[serde(default)]
304    usage: Option<OllamaUsage>,
305    /// Raw JSON for the tool_calls array from the message, used to extract
306    /// Gemini-specific fields like `thought_signature` that ollama-rs drops.
307    #[serde(skip, default)]
308    raw_tool_calls_json: Option<serde_json::Value>,
309}
310
311#[derive(Debug, Clone, Deserialize)]
312struct OllamaUsage {
313    #[serde(default)]
314    prompt_tokens: u32,
315    #[serde(default)]
316    completion_tokens: u32,
317    #[serde(default)]
318    total_tokens: u32,
319}
320
321#[derive(Debug, Clone, Default, Deserialize)]
322struct OllamaChunkMessage {
323    #[serde(default)]
324    content: String,
325    #[serde(default)]
326    thinking: Option<String>,
327    #[serde(default)]
328    tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
329}
330
331impl OllamaChunk {
332    /// Set raw tool calls JSON from deserialized chunk line.
333    fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
334        if let Some(msg) = raw_line.get("message")
335            && let Some(tcs) = msg.get("tool_calls")
336        {
337            self.raw_tool_calls_json = Some(tcs.clone());
338        }
339    }
340
341    fn to_chat_message_response(&self) -> ChatMessageResponse {
342        let usage = if let Some(u) = &self.usage {
343            Some(TokenUsage {
344                prompt_tokens: u.prompt_tokens,
345                completion_tokens: u.completion_tokens,
346                total_tokens: u.total_tokens,
347            })
348        } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
349            let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
350            let completion = self.eval_count.unwrap_or(0) as u32;
351            Some(TokenUsage {
352                prompt_tokens: prompt,
353                completion_tokens: completion,
354                total_tokens: prompt.saturating_add(completion),
355            })
356        } else {
357            None
358        };
359
360        // Build tool_calls list. Use raw JSON when available to extract
361        // Gemini `thought_signature` fields that ollama-rs drops.
362        let mut raw_tc_iter = self
363            .raw_tool_calls_json
364            .as_ref()
365            .and_then(|v| v.as_array())
366            .map(|a| a.iter())
367            .into_iter()
368            .flatten();
369        let tool_calls: Vec<ToolCall> = self
370            .message
371            .tool_calls
372            .clone()
373            .into_iter()
374            .map(|tc| {
375                let thought_signature = raw_tc_iter
376                    .next()
377                    .and_then(|raw| {
378                        raw.get("function")
379                            .and_then(|f| f.get("thought_signature"))
380                            .and_then(|ts| ts.as_str())
381                    })
382                    .map(|s| s.to_string());
383                ToolCall {
384                    id: None,
385                    function: ToolCallFunction {
386                        name: tc.function.name,
387                        arguments: tc.function.arguments,
388                        thought_signature,
389                    },
390                }
391            })
392            .collect();
393
394        ChatMessageResponse {
395            message: ChatMessage {
396                content: self.message.content.clone(),
397                thinking: self.message.thinking.clone(),
398                tool_calls,
399            },
400            done: self.done,
401            is_error: false,
402            usage,
403        }
404    }
405}
406
407/// Stream chat completions from the Ollama `/api/chat` endpoint using raw SSE parsing.
408///
409/// This bypasses ollama-rs's `send_chat_messages_stream` so we can recognise both
410/// native Ollama and OpenAI-compatible usage formats in the response chunks.
411async fn stream_ollama_chat(
412    client: &reqwest::Client,
413    base_url: &str,
414    request: &ChatMessageRequest,
415    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
416    thought_signatures: &[Vec<Option<String>>],
417) -> Result<(), String> {
418    let url = format!("{base_url}api/chat");
419    let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
420
421    // Force streaming — ollama-rs sets `stream: true` only inside its own
422    // `send_chat_messages_stream()` method, which we bypass. Without this,
423    // Ollama returns the entire response as a single JSON object instead of
424    // streaming line-by-line, causing the UI to block until the full message
425    // arrives.
426    request["stream"] = serde_json::Value::Bool(true);
427
428    // Fix ollama-rs 0.3.4 serialization quirks for Ollama Cloud / Gemini compatibility:
429    //
430    // 1. ToolType::Function serializes as "Function" (uppercase F), but the
431    //    API spec requires lowercase "function". Ollama Cloud / Gemini rejects
432    //    the uppercase variant with "Invalid tool type". Fix by lowercasing.
433    // Fix 2: "role": "tool" messages need a "name" field (tool_name) so Gemini
434    //    can match results to calls. ollama-rs doesn't add this, so we inject
435    //    it from the preceding assistant message's tool_calls.
436    // 3. Re-inject Gemini `thought_signature` fields captured from tool calls.
437    fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
438        match value {
439            serde_json::Value::Object(map) => {
440                // Fix 1: Lowercase "Function" → "function" in tool definitions
441                if let Some(t) = map.get_mut("type")
442                    && t.as_str() == Some("Function")
443                {
444                    *t = serde_json::Value::String("function".to_string());
445                }
446                for v in map.values_mut() {
447                    fix_request_json(v, thought_signatures);
448                }
449            }
450            serde_json::Value::Array(arr) => {
451                // Fix 2: Add "name" field to tool result messages by tracking
452                // tool_calls from the most recent assistant message.
453                // Also inject "tool_call_id" for OpenAI-compatible backends.
454                // Fix 3: Re-inject thought_signatures into assistant tool_calls.
455                //
456                // `thought_signatures` is indexed by the position of each
457                // message in the original `messages` slice (one entry per
458                // message, regardless of role). We advance `ts_idx` on every
459                // message so the indices stay aligned; otherwise non-assistant
460                // messages (system, user, tool) cause off-by-N misalignment
461                // and signatures are never re-injected, silently breaking
462                // Gemini multi-turn tool calling.
463                let mut prev_tool_names: Vec<String> = Vec::new();
464                let mut prev_tool_ids: Vec<String> = Vec::new();
465                let mut ts_idx: usize = 0;
466                let looks_like_messages_array = arr.iter().any(|v| {
467                    v.as_object()
468                        .and_then(|o| o.get("role"))
469                        .and_then(|r| r.as_str())
470                        .is_some()
471                });
472                if looks_like_messages_array {
473                    for msg in arr.iter_mut() {
474                        if let serde_json::Value::Object(msg_map) = msg {
475                            let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
476                            match role {
477                                "assistant" => {
478                                    prev_tool_names.clear();
479                                    prev_tool_ids.clear();
480                                    if let Some(tcs) = msg_map.get_mut("tool_calls")
481                                        && let Some(tc_arr) = tcs.as_array_mut()
482                                    {
483                                        // Re-inject thought_signatures
484                                        if let Some(sigs) = thought_signatures.get(ts_idx) {
485                                            for (i, tc) in tc_arr.iter_mut().enumerate() {
486                                                if let Some(sig) =
487                                                    sigs.get(i).and_then(|s| s.clone())
488                                                    && let Some(func) = tc.get_mut("function")
489                                                    && let Some(obj) = func.as_object_mut()
490                                                {
491                                                    obj.insert(
492                                                        "thought_signature".to_string(),
493                                                        serde_json::Value::String(sig),
494                                                    );
495                                                }
496                                            }
497                                        }
498
499                                        // Track tool names and ids for Fix 2
500                                        for (i, tc) in tc_arr.iter().enumerate() {
501                                            if let Some(name) = tc
502                                                .get("function")
503                                                .and_then(|f| f.get("name"))
504                                                .and_then(|n| n.as_str())
505                                            {
506                                                prev_tool_names.push(name.to_string());
507                                            }
508                                            // Track tool call ids (synthesize if missing)
509                                            let id = tc
510                                                .get("id")
511                                                .and_then(|v| v.as_str())
512                                                .filter(|s| !s.is_empty())
513                                                .map(|s| s.to_string())
514                                                .unwrap_or_else(|| format!("call_{}", i));
515                                            prev_tool_ids.push(id);
516                                        }
517                                    }
518                                }
519                                "tool" => {
520                                    // Add "name" field for Ollama/Gemini compatibility
521                                    // (only when there's exactly one tool call)
522                                    if !msg_map.contains_key("name") && prev_tool_names.len() == 1 {
523                                        msg_map.insert(
524                                            "name".to_string(),
525                                            serde_json::Value::String(prev_tool_names[0].clone()),
526                                        );
527                                    }
528                                    // Add "tool_call_id" for OpenAI-compatible backends
529                                    if !msg_map.contains_key("tool_call_id")
530                                        && prev_tool_ids.len() == 1
531                                    {
532                                        msg_map.insert(
533                                            "tool_call_id".to_string(),
534                                            serde_json::Value::String(prev_tool_ids[0].clone()),
535                                        );
536                                    }
537                                }
538                                _ => {}
539                            }
540                        }
541                        // Always advance — `thought_signatures` is 1:1 with
542                        // the original `messages` Vec, including non-assistant
543                        // entries.
544                        ts_idx += 1;
545                    }
546                }
547                // Recurse into children for non-message fixes (e.g. lowercasing
548                // tool `type`). We pass an empty slice to avoid re-applying
549                // thought_signatures from index 0 in any nested array.
550                for v in arr.iter_mut() {
551                    fix_request_json(v, &[]);
552                }
553            }
554            _ => {}
555        }
556    }
557    fix_request_json(&mut request, thought_signatures);
558
559    let response = client
560        .post(&url)
561        .json(&request)
562        .send()
563        .await
564        .map_err(|e| format!("request failed: {e}"))?;
565
566    if !response.status().is_success() {
567        return Err(format!(
568            "HTTP {}: {}",
569            response.status().as_u16(),
570            response.text().await.unwrap_or_default()
571        ));
572    }
573
574    let mut stream = response.bytes_stream();
575    let mut buffer = String::new();
576
577    while let Some(chunk_result) = stream.next().await {
578        let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
579        buffer.push_str(&String::from_utf8_lossy(&chunk));
580
581        // Process complete SSE lines
582        while let Some(pos) = buffer.find('\n') {
583            let line = buffer[..pos].trim().to_string();
584            buffer.drain(..=pos);
585
586            if line.is_empty() {
587                continue;
588            }
589
590            match serde_json::from_str::<OllamaChunk>(&line) {
591                Ok(mut chunk) => {
592                    // Capture raw JSON before extracting to preserve Gemini-
593                    // specific fields like `thought_signature`.
594                    if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
595                        chunk.capture_raw_tool_calls(&raw);
596                    }
597                    let done = chunk.done;
598                    let ours = chunk.to_chat_message_response();
599                    if send.send(ours).await.is_err() {
600                        return Ok(()); // receiver dropped
601                    }
602                    if done {
603                        return Ok(());
604                    }
605                }
606                Err(_) => {
607                    // Non-JSON line (e.g. "data: [DONE]" or empty) — skip silently
608                }
609            }
610        }
611    }
612
613    // Process remaining buffer
614    if !buffer.trim().is_empty()
615        && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
616    {
617        if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
618            chunk.capture_raw_tool_calls(&raw);
619        }
620        let done = chunk.done;
621        let ours = chunk.to_chat_message_response();
622        if send.send(ours).await.is_err() {
623            return Ok(());
624        }
625        if done {
626            return Ok(());
627        }
628    }
629
630    // Stream ended without a done marker — send a synthetic done response
631    let _ = send
632        .send(ChatMessageResponse {
633            message: ChatMessage {
634                content: String::new(),
635                tool_calls: vec![],
636                thinking: None,
637            },
638            done: true,
639            is_error: false,
640            usage: None,
641        })
642        .await;
643
644    Ok(())
645}