Skip to main content

tinyharness_lib/provider/
ollama.rs

1use std::future::Future;
2use std::pin::Pin;
3use std::time::Duration;
4
5use ollama_rs::{
6    IntoUrlSealed, Ollama,
7    generation::{
8        chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
9        parameters::ThinkType,
10    },
11};
12use serde::Deserialize;
13use tokio_stream::StreamExt;
14
15use crate::config::OllamaThinkType;
16use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
17
18use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
19
20impl From<Message> for OllamaChatMessage {
21    fn from(msg: Message) -> Self {
22        match msg.role {
23            Role::System => OllamaChatMessage::system(msg.content),
24            Role::User => OllamaChatMessage::user(msg.content),
25            Role::Assistant => {
26                let mut m = OllamaChatMessage::assistant(msg.content);
27                if !msg.tool_calls.is_empty() {
28                    m.tool_calls = msg
29                        .tool_calls
30                        .into_iter()
31                        .map(|tc| ollama_rs::generation::tools::ToolCall {
32                            function: ollama_rs::generation::tools::ToolCallFunction {
33                                name: tc.function.name,
34                                arguments: tc.function.arguments,
35                            },
36                        })
37                        .collect();
38                }
39                m
40            }
41            Role::Tool => OllamaChatMessage::tool(msg.content),
42        }
43    }
44}
45
46fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
47    ollama_rs::generation::tools::ToolInfo {
48        tool_type: ollama_rs::generation::tools::ToolType::Function,
49        function: ollama_rs::generation::tools::ToolFunctionInfo {
50            name: ti.name,
51            description: ti.description,
52            parameters: ti.parameters,
53        },
54    }
55}
56
57fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
58    match tt {
59        OllamaThinkType::Off => ThinkType::False,
60        OllamaThinkType::Low => ThinkType::Low,
61        OllamaThinkType::Medium => ThinkType::Medium,
62        OllamaThinkType::High => ThinkType::High,
63    }
64}
65
66pub struct OllamaProvider {
67    client: Ollama,
68    http_client: reqwest::Client,
69    base_url: String,
70    model: Option<String>,
71    timeout_secs: u64,
72    max_retries: u32,
73    think_type: OllamaThinkType,
74}
75
76impl OllamaProvider {
77    pub fn new(
78        base: String,
79        timeout_secs: u64,
80        max_retries: u32,
81        think_type: OllamaThinkType,
82    ) -> Self {
83        // Normalize URL: ensure it ends with '/'
84        let base_url = if base.ends_with('/') {
85            base.clone()
86        } else {
87            format!("{base}/")
88        };
89        let client = Ollama::from_url(base.into_url().unwrap());
90        let http_client = reqwest::Client::builder()
91            .connect_timeout(Duration::from_secs(15))
92            .read_timeout(Duration::from_secs(timeout_secs + 60))
93            .build()
94            .unwrap_or_else(|_| reqwest::Client::new());
95        OllamaProvider {
96            client,
97            http_client,
98            base_url,
99            model: None,
100            timeout_secs,
101            max_retries,
102            think_type,
103        }
104    }
105}
106
107impl Provider for OllamaProvider {
108    fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
109        let client = self.client.clone();
110        Box::pin(async move {
111            match client.list_local_models().await {
112                Ok(_) => Ok(()),
113                Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
114            }
115        })
116    }
117
118    fn list_models(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
119        let client = self.client.clone();
120        Box::pin(async move {
121            match client.list_local_models().await {
122                Ok(models) => models.into_iter().map(|m| m.name).collect(),
123                Err(_) => vec![],
124            }
125        })
126    }
127
128    fn select_model(&mut self, name: String) {
129        self.model = Some(name);
130    }
131
132    fn current_model(&self) -> Option<String> {
133        self.model.clone()
134    }
135
136    fn set_timeout(&mut self, timeout_secs: u64) {
137        self.timeout_secs = timeout_secs;
138    }
139
140    fn set_retries(&mut self, max_retries: u32) {
141        self.max_retries = max_retries;
142    }
143
144    fn set_think_type(&mut self, think_type: OllamaThinkType) {
145        self.think_type = think_type;
146    }
147
148    fn chat(
149        &mut self,
150        messages: Vec<Message>,
151        tools: Vec<ToolDefinition>,
152    ) -> Pin<
153        Box<
154            dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
155                + Send,
156        >,
157    > {
158        let model = match self.model.clone() {
159            Some(m) => m,
160            None => {
161                return Box::pin(async move {
162                    Err("No model selected. Use /model <name> to select one.".to_string())
163                });
164            }
165        };
166        let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
167        let timeout_secs = self.timeout_secs;
168        let max_retries = self.max_retries;
169        let http_client = self.http_client.clone();
170        let base_url = self.base_url.clone();
171        let request = {
172            let chat_messages: Vec<OllamaChatMessage> =
173                messages.iter().map(|m| m.clone().into()).collect();
174
175            // Collect Gemini thought_signatures from incoming messages before
176            // ollama-rs conversion drops them. These must be re-injected into
177            // the serialized request so Gemini accepts multi-turn tool calling.
178            let thought_signatures: Vec<Vec<Option<String>>> = messages
179                .iter()
180                .map(|m| {
181                    m.tool_calls
182                        .iter()
183                        .map(|tc| tc.function.thought_signature.clone())
184                        .collect()
185                })
186                .collect();
187
188            let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
189                tools.into_iter().map(to_ollama_tool_info).collect();
190
191            let mut req = ChatMessageRequest::new(model, chat_messages)
192                .think(to_ollama_think_type(self.think_type));
193
194            if !ollama_tools.is_empty() {
195                req = req.tools(ollama_tools);
196                req.think = None;
197            }
198            (req, thought_signatures)
199        };
200
201        // Spawn the streaming work on a background task.
202        //
203        // We use our own raw SSE parser instead of ollama-rs's streaming to handle
204        // both Ollama-native usage format (flat prompt_eval_count/eval_count) and
205        // cloud proxy / OpenAI-compatible format (nested usage object). The ollama-rs
206        // library only recognises the flat format, so cloud proxies that return
207        // {"usage": {"prompt_tokens": ..., "completion_tokens": ..., "total_tokens": ...}}
208        // would lose their usage data.
209        tokio::spawn(async move {
210            let (request, thought_signatures) = request;
211            // Retry loop with exponential backoff
212            let max_attempts = max_retries.max(1);
213
214            let send_err = |msg: String| {
215                let _ = send.try_send(ChatMessageResponse {
216                    message: ChatMessage {
217                        content: msg,
218                        tool_calls: vec![],
219                        thinking: None,
220                    },
221                    done: true,
222                    is_error: true,
223                    usage: None,
224                });
225            };
226
227            for attempt in 1..=max_attempts {
228                let result = tokio::time::timeout(
229                    Duration::from_secs(timeout_secs),
230                    stream_ollama_chat(
231                        &http_client,
232                        &base_url,
233                        &request,
234                        &send,
235                        &thought_signatures,
236                    ),
237                )
238                .await;
239
240                match result {
241                    Ok(Ok(())) => return, // success
242                    Ok(Err(e)) => {
243                        if attempt >= max_attempts {
244                            send_err(format!("Error after {attempt} retries: {e}"));
245                            return;
246                        }
247                    }
248                    Err(_) => {
249                        if attempt >= max_attempts {
250                            send_err(format!(
251                                "Error: Request timed out after {timeout_secs}s ({attempt} retries)"
252                            ));
253                            return;
254                        }
255                    }
256                }
257
258                // Exponential backoff: 1s, 2s, 4s, ...
259                let backoff = Duration::from_secs(1 << (attempt - 1));
260                tokio::time::sleep(backoff).await;
261            }
262        });
263
264        Box::pin(async move { Ok(recv) })
265    }
266}
267
268// ── Raw Ollama SSE streaming ────────────────────────────────────────────────
269
270/// A raw Ollama SSE chunk that accepts **both** native Ollama and
271/// OpenAI-compatible usage formats.
272///
273/// Ollama-native format has flat fields:
274/// ```json
275/// { "prompt_eval_count": 123, "eval_count": 45, ... }
276/// ```
277///
278/// Some cloud proxies return an OpenAI-compatible nested `usage` object:
279/// ```json
280/// { "usage": { "prompt_tokens": 123, "completion_tokens": 45, "total_tokens": 168 } }
281/// ```
282///
283/// We try both sources and prefer the nested `usage` object when present.
284#[derive(Debug, Clone, Deserialize)]
285struct OllamaChunk {
286    #[serde(default)]
287    message: OllamaChunkMessage,
288    #[serde(default)]
289    done: bool,
290    // Native Ollama flat fields
291    #[serde(default)]
292    prompt_eval_count: Option<u64>,
293    #[serde(default)]
294    eval_count: Option<u64>,
295    // OpenAI-compatible nested usage (cloud proxies)
296    #[serde(default)]
297    usage: Option<OllamaUsage>,
298    /// Raw JSON for the tool_calls array from the message, used to extract
299    /// Gemini-specific fields like `thought_signature` that ollama-rs drops.
300    #[serde(skip, default)]
301    raw_tool_calls_json: Option<serde_json::Value>,
302}
303
304#[derive(Debug, Clone, Deserialize)]
305struct OllamaUsage {
306    #[serde(default)]
307    prompt_tokens: u32,
308    #[serde(default)]
309    completion_tokens: u32,
310    #[serde(default)]
311    total_tokens: u32,
312}
313
314#[derive(Debug, Clone, Default, Deserialize)]
315struct OllamaChunkMessage {
316    #[serde(default)]
317    content: String,
318    #[serde(default)]
319    thinking: Option<String>,
320    #[serde(default)]
321    tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
322}
323
324impl OllamaChunk {
325    /// Set raw tool calls JSON from deserialized chunk line.
326    fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
327        if let Some(msg) = raw_line.get("message")
328            && let Some(tcs) = msg.get("tool_calls")
329        {
330            self.raw_tool_calls_json = Some(tcs.clone());
331        }
332    }
333
334    fn to_chat_message_response(&self) -> ChatMessageResponse {
335        let usage = if let Some(u) = &self.usage {
336            Some(TokenUsage {
337                prompt_tokens: u.prompt_tokens,
338                completion_tokens: u.completion_tokens,
339                total_tokens: u.total_tokens,
340            })
341        } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
342            let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
343            let completion = self.eval_count.unwrap_or(0) as u32;
344            Some(TokenUsage {
345                prompt_tokens: prompt,
346                completion_tokens: completion,
347                total_tokens: prompt.saturating_add(completion),
348            })
349        } else {
350            None
351        };
352
353        // Build tool_calls list. Use raw JSON when available to extract
354        // Gemini `thought_signature` fields that ollama-rs drops.
355        let mut raw_tc_iter = self
356            .raw_tool_calls_json
357            .as_ref()
358            .and_then(|v| v.as_array())
359            .map(|a| a.iter())
360            .into_iter()
361            .flatten();
362        let tool_calls: Vec<ToolCall> = self
363            .message
364            .tool_calls
365            .clone()
366            .into_iter()
367            .map(|tc| {
368                let thought_signature = raw_tc_iter
369                    .next()
370                    .and_then(|raw| {
371                        raw.get("function")
372                            .and_then(|f| f.get("thought_signature"))
373                            .and_then(|ts| ts.as_str())
374                    })
375                    .map(|s| s.to_string());
376                ToolCall {
377                    function: ToolCallFunction {
378                        name: tc.function.name,
379                        arguments: tc.function.arguments,
380                        thought_signature,
381                    },
382                }
383            })
384            .collect();
385
386        ChatMessageResponse {
387            message: ChatMessage {
388                content: self.message.content.clone(),
389                thinking: self.message.thinking.clone(),
390                tool_calls,
391            },
392            done: self.done,
393            is_error: false,
394            usage,
395        }
396    }
397}
398
399/// Stream chat completions from the Ollama `/api/chat` endpoint using raw SSE parsing.
400///
401/// This bypasses ollama-rs's `send_chat_messages_stream` so we can recognise both
402/// native Ollama and OpenAI-compatible usage formats in the response chunks.
403async fn stream_ollama_chat(
404    client: &reqwest::Client,
405    base_url: &str,
406    request: &ChatMessageRequest,
407    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
408    thought_signatures: &[Vec<Option<String>>],
409) -> Result<(), String> {
410    let url = format!("{base_url}api/chat");
411    let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
412
413    // Fix ollama-rs 0.3.4 serialization quirks for Ollama Cloud / Gemini compatibility:
414    //
415    // 1. ToolType::Function serializes as "Function" (uppercase F), but the
416    //    API spec requires lowercase "function". Ollama Cloud / Gemini rejects
417    //    the uppercase variant with "Invalid tool type". Fix by lowercasing.
418    // Fix 2: "role": "tool" messages need a "name" field (tool_name) so Gemini
419    //    can match results to calls. ollama-rs doesn't add this, so we inject
420    //    it from the preceding assistant message's tool_calls.
421    // 3. Re-inject Gemini `thought_signature` fields captured from tool calls.
422    fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
423        match value {
424            serde_json::Value::Object(map) => {
425                // Fix 1: Lowercase "Function" → "function" in tool definitions
426                if let Some(t) = map.get_mut("type")
427                    && t.as_str() == Some("Function")
428                {
429                    *t = serde_json::Value::String("function".to_string());
430                }
431                for v in map.values_mut() {
432                    fix_request_json(v, thought_signatures);
433                }
434            }
435            serde_json::Value::Array(arr) => {
436                // Fix 2: Add "name" field to tool result messages by tracking
437                // tool_calls from the most recent assistant message.
438                // Fix 3: Re-inject thought_signatures into assistant tool_calls.
439                let mut prev_tool_names: Vec<String> = Vec::new();
440                let mut ts_idx: usize = 0;
441                for msg in arr.iter_mut() {
442                    if let serde_json::Value::Object(msg_map) = msg {
443                        let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
444                        match role {
445                            "assistant" => {
446                                prev_tool_names.clear();
447                                if let Some(tcs) = msg_map.get_mut("tool_calls")
448                                    && let Some(tc_arr) = tcs.as_array_mut()
449                                {
450                                    // Re-inject thought_signatures
451                                    if let Some(sigs) = thought_signatures.get(ts_idx) {
452                                        for (i, tc) in tc_arr.iter_mut().enumerate() {
453                                            if let Some(sig) = sigs.get(i).and_then(|s| s.clone())
454                                                && let Some(func) = tc.get_mut("function")
455                                                && let Some(obj) = func.as_object_mut()
456                                            {
457                                                obj.insert(
458                                                    "thought_signature".to_string(),
459                                                    serde_json::Value::String(sig),
460                                                );
461                                            }
462                                        }
463                                    }
464                                    ts_idx += 1;
465
466                                    // Track tool names for Fix 2
467                                    for tc in tc_arr.iter() {
468                                        if let Some(name) = tc
469                                            .get("function")
470                                            .and_then(|f| f.get("name"))
471                                            .and_then(|n| n.as_str())
472                                        {
473                                            prev_tool_names.push(name.to_string());
474                                        }
475                                    }
476                                } else {
477                                    ts_idx += 1;
478                                }
479                            }
480                            "tool"
481                                if !msg_map.contains_key("name") && prev_tool_names.len() == 1 =>
482                            {
483                                msg_map.insert(
484                                    "name".to_string(),
485                                    serde_json::Value::String(prev_tool_names[0].clone()),
486                                );
487                            }
488                            _ => {}
489                        }
490                    }
491                }
492                for v in arr.iter_mut() {
493                    fix_request_json(v, thought_signatures);
494                }
495            }
496            _ => {}
497        }
498    }
499    fix_request_json(&mut request, thought_signatures);
500
501    let response = client
502        .post(&url)
503        .json(&request)
504        .send()
505        .await
506        .map_err(|e| format!("request failed: {e}"))?;
507
508    if !response.status().is_success() {
509        return Err(format!(
510            "HTTP {}: {}",
511            response.status().as_u16(),
512            response.text().await.unwrap_or_default()
513        ));
514    }
515
516    let mut stream = response.bytes_stream();
517    let mut buffer = String::new();
518
519    while let Some(chunk_result) = stream.next().await {
520        let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
521        buffer.push_str(&String::from_utf8_lossy(&chunk));
522
523        // Process complete SSE lines
524        while let Some(pos) = buffer.find('\n') {
525            let line = buffer[..pos].trim().to_string();
526            buffer = buffer[pos + 1..].to_string();
527
528            if line.is_empty() {
529                continue;
530            }
531
532            match serde_json::from_str::<OllamaChunk>(&line) {
533                Ok(mut chunk) => {
534                    // Capture raw JSON before extracting to preserve Gemini-
535                    // specific fields like `thought_signature`.
536                    if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
537                        chunk.capture_raw_tool_calls(&raw);
538                    }
539                    let done = chunk.done;
540                    let ours = chunk.to_chat_message_response();
541                    if send.send(ours).await.is_err() {
542                        return Ok(()); // receiver dropped
543                    }
544                    if done {
545                        return Ok(());
546                    }
547                }
548                Err(_) => {
549                    // Non-JSON line (e.g. "data: [DONE]" or empty) — skip silently
550                }
551            }
552        }
553    }
554
555    // Process remaining buffer
556    if !buffer.trim().is_empty()
557        && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
558    {
559        if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
560            chunk.capture_raw_tool_calls(&raw);
561        }
562        let done = chunk.done;
563        let ours = chunk.to_chat_message_response();
564        if send.send(ours).await.is_err() {
565            return Ok(());
566        }
567        if done {
568            return Ok(());
569        }
570    }
571
572    // Stream ended without a done marker — send a synthetic done response
573    let _ = send
574        .send(ChatMessageResponse {
575            message: ChatMessage {
576                content: String::new(),
577                tool_calls: vec![],
578                thinking: None,
579            },
580            done: true,
581            is_error: false,
582            usage: None,
583        })
584        .await;
585
586    Ok(())
587}