tinyharness_lib/provider/
ollama.rs1use std::future::Future;
2use std::pin::Pin;
3use std::time::Duration;
4
5use ollama_rs::{
6 IntoUrlSealed, Ollama,
7 generation::{
8 chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
9 parameters::ThinkType,
10 },
11};
12use serde::Deserialize;
13use tokio_stream::StreamExt;
14
15use crate::config::OllamaThinkType;
16use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
17
18use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
19
20impl From<Message> for OllamaChatMessage {
21 fn from(msg: Message) -> Self {
22 match msg.role {
23 Role::System => OllamaChatMessage::system(msg.content),
24 Role::User => {
25 let mut m = OllamaChatMessage::user(msg.content);
26 if !msg.images.is_empty() {
27 let images: Vec<ollama_rs::generation::images::Image> = msg
28 .images
29 .iter()
30 .map(|img| {
31 ollama_rs::generation::images::Image::from_base64(
32 img.base64_data.clone(),
33 )
34 })
35 .collect();
36 m.images = Some(images);
37 }
38 m
39 }
40 Role::Assistant => {
41 let mut m = OllamaChatMessage::assistant(msg.content);
42 if !msg.tool_calls.is_empty() {
43 m.tool_calls = msg
44 .tool_calls
45 .into_iter()
46 .map(|tc| ollama_rs::generation::tools::ToolCall {
47 function: ollama_rs::generation::tools::ToolCallFunction {
48 name: tc.function.name,
49 arguments: tc.function.arguments,
50 },
51 })
52 .collect();
53 }
54 m
55 }
56 Role::Tool => OllamaChatMessage::tool(msg.content),
57 }
58 }
59}
60
61fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
62 ollama_rs::generation::tools::ToolInfo {
63 tool_type: ollama_rs::generation::tools::ToolType::Function,
64 function: ollama_rs::generation::tools::ToolFunctionInfo {
65 name: ti.name,
66 description: ti.description,
67 parameters: ti.parameters,
68 },
69 }
70}
71
72fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
73 match tt {
74 OllamaThinkType::Off => ThinkType::False,
75 OllamaThinkType::Low => ThinkType::Low,
76 OllamaThinkType::Medium => ThinkType::Medium,
77 OllamaThinkType::High => ThinkType::High,
78 }
79}
80
81pub struct OllamaProvider {
82 client: Ollama,
83 http_client: reqwest::Client,
84 base_url: String,
85 model: Option<String>,
86 timeout_secs: u64,
87 max_retries: u32,
88 think_type: OllamaThinkType,
89}
90
91impl OllamaProvider {
92 pub fn new(
93 base: String,
94 timeout_secs: u64,
95 max_retries: u32,
96 think_type: OllamaThinkType,
97 ) -> Self {
98 let base_url = if base.ends_with('/') {
100 base.clone()
101 } else {
102 format!("{base}/")
103 };
104 let client = Ollama::from_url(base.into_url().unwrap());
105 let http_client = reqwest::Client::builder()
106 .connect_timeout(Duration::from_secs(15))
107 .read_timeout(Duration::from_secs(timeout_secs + 60))
108 .build()
109 .unwrap_or_else(|_| reqwest::Client::new());
110 OllamaProvider {
111 client,
112 http_client,
113 base_url,
114 model: None,
115 timeout_secs,
116 max_retries,
117 think_type,
118 }
119 }
120}
121
122impl Provider for OllamaProvider {
123 fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
124 let client = self.client.clone();
125 Box::pin(async move {
126 match client.list_local_models().await {
127 Ok(_) => Ok(()),
128 Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
129 }
130 })
131 }
132
133 fn list_models(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
134 let client = self.client.clone();
135 Box::pin(async move {
136 match client.list_local_models().await {
137 Ok(models) => models.into_iter().map(|m| m.name).collect(),
138 Err(_) => vec![],
139 }
140 })
141 }
142
143 fn select_model(&mut self, name: String) {
144 self.model = Some(name);
145 }
146
147 fn current_model(&self) -> Option<String> {
148 self.model.clone()
149 }
150
151 fn set_timeout(&mut self, timeout_secs: u64) {
152 self.timeout_secs = timeout_secs;
153 }
154
155 fn set_retries(&mut self, max_retries: u32) {
156 self.max_retries = max_retries;
157 }
158
159 fn set_think_type(&mut self, think_type: OllamaThinkType) {
160 self.think_type = think_type;
161 }
162
163 fn chat(
164 &mut self,
165 messages: Vec<Message>,
166 tools: Vec<ToolDefinition>,
167 ) -> Pin<
168 Box<
169 dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
170 + Send,
171 >,
172 > {
173 let model = match self.model.clone() {
174 Some(m) => m,
175 None => {
176 return Box::pin(async move {
177 Err("No model selected. Use /model <name> to select one.".to_string())
178 });
179 }
180 };
181 let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
182 let timeout_secs = self.timeout_secs;
183 let max_retries = self.max_retries;
184 let http_client = self.http_client.clone();
185 let base_url = self.base_url.clone();
186 let request = {
187 let chat_messages: Vec<OllamaChatMessage> =
188 messages.iter().map(|m| m.clone().into()).collect();
189
190 let thought_signatures: Vec<Vec<Option<String>>> = messages
194 .iter()
195 .map(|m| {
196 m.tool_calls
197 .iter()
198 .map(|tc| tc.function.thought_signature.clone())
199 .collect()
200 })
201 .collect();
202
203 let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
204 tools.into_iter().map(to_ollama_tool_info).collect();
205
206 let mut req = ChatMessageRequest::new(model, chat_messages)
207 .think(to_ollama_think_type(self.think_type));
208
209 if !ollama_tools.is_empty() {
210 req = req.tools(ollama_tools);
211 req.think = None;
212 }
213 (req, thought_signatures)
214 };
215
216 tokio::spawn(async move {
225 let (request, thought_signatures) = request;
226 let max_attempts = max_retries.max(1);
228
229 let send_err = |msg: String| {
230 let _ = send.try_send(ChatMessageResponse {
231 message: ChatMessage {
232 content: msg,
233 tool_calls: vec![],
234 thinking: None,
235 },
236 done: true,
237 is_error: true,
238 usage: None,
239 });
240 };
241
242 for attempt in 1..=max_attempts {
243 let result = tokio::time::timeout(
244 Duration::from_secs(timeout_secs),
245 stream_ollama_chat(
246 &http_client,
247 &base_url,
248 &request,
249 &send,
250 &thought_signatures,
251 ),
252 )
253 .await;
254
255 match result {
256 Ok(Ok(())) => return, Ok(Err(e)) => {
258 if attempt >= max_attempts {
259 send_err(format!("Error after {attempt} retries: {e}"));
260 return;
261 }
262 }
263 Err(_) => {
264 if attempt >= max_attempts {
265 send_err(format!(
266 "Error: Request timed out after {timeout_secs}s ({attempt} retries)"
267 ));
268 return;
269 }
270 }
271 }
272
273 let backoff = Duration::from_secs(1 << (attempt - 1));
275 tokio::time::sleep(backoff).await;
276 }
277 });
278
279 Box::pin(async move { Ok(recv) })
280 }
281}
282
283#[derive(Debug, Clone, Deserialize)]
300struct OllamaChunk {
301 #[serde(default)]
302 message: OllamaChunkMessage,
303 #[serde(default)]
304 done: bool,
305 #[serde(default)]
307 prompt_eval_count: Option<u64>,
308 #[serde(default)]
309 eval_count: Option<u64>,
310 #[serde(default)]
312 usage: Option<OllamaUsage>,
313 #[serde(skip, default)]
316 raw_tool_calls_json: Option<serde_json::Value>,
317}
318
319#[derive(Debug, Clone, Deserialize)]
320struct OllamaUsage {
321 #[serde(default)]
322 prompt_tokens: u32,
323 #[serde(default)]
324 completion_tokens: u32,
325 #[serde(default)]
326 total_tokens: u32,
327}
328
329#[derive(Debug, Clone, Default, Deserialize)]
330struct OllamaChunkMessage {
331 #[serde(default)]
332 content: String,
333 #[serde(default)]
334 thinking: Option<String>,
335 #[serde(default)]
336 tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
337}
338
339impl OllamaChunk {
340 fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
342 if let Some(msg) = raw_line.get("message")
343 && let Some(tcs) = msg.get("tool_calls")
344 {
345 self.raw_tool_calls_json = Some(tcs.clone());
346 }
347 }
348
349 fn to_chat_message_response(&self) -> ChatMessageResponse {
350 let usage = if let Some(u) = &self.usage {
351 Some(TokenUsage {
352 prompt_tokens: u.prompt_tokens,
353 completion_tokens: u.completion_tokens,
354 total_tokens: u.total_tokens,
355 })
356 } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
357 let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
358 let completion = self.eval_count.unwrap_or(0) as u32;
359 Some(TokenUsage {
360 prompt_tokens: prompt,
361 completion_tokens: completion,
362 total_tokens: prompt.saturating_add(completion),
363 })
364 } else {
365 None
366 };
367
368 let mut raw_tc_iter = self
371 .raw_tool_calls_json
372 .as_ref()
373 .and_then(|v| v.as_array())
374 .map(|a| a.iter())
375 .into_iter()
376 .flatten();
377 let tool_calls: Vec<ToolCall> = self
378 .message
379 .tool_calls
380 .clone()
381 .into_iter()
382 .map(|tc| {
383 let thought_signature = raw_tc_iter
384 .next()
385 .and_then(|raw| {
386 raw.get("function")
387 .and_then(|f| f.get("thought_signature"))
388 .and_then(|ts| ts.as_str())
389 })
390 .map(|s| s.to_string());
391 ToolCall {
392 id: None,
393 function: ToolCallFunction {
394 name: tc.function.name,
395 arguments: tc.function.arguments,
396 thought_signature,
397 },
398 }
399 })
400 .collect();
401
402 ChatMessageResponse {
403 message: ChatMessage {
404 content: self.message.content.clone(),
405 thinking: self.message.thinking.clone(),
406 tool_calls,
407 },
408 done: self.done,
409 is_error: false,
410 usage,
411 }
412 }
413}
414
415async fn stream_ollama_chat(
420 client: &reqwest::Client,
421 base_url: &str,
422 request: &ChatMessageRequest,
423 send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
424 thought_signatures: &[Vec<Option<String>>],
425) -> Result<(), String> {
426 let url = format!("{base_url}api/chat");
427 let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
428
429 fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
439 match value {
440 serde_json::Value::Object(map) => {
441 if let Some(t) = map.get_mut("type")
443 && t.as_str() == Some("Function")
444 {
445 *t = serde_json::Value::String("function".to_string());
446 }
447 for v in map.values_mut() {
448 fix_request_json(v, thought_signatures);
449 }
450 }
451 serde_json::Value::Array(arr) => {
452 let mut prev_tool_names: Vec<String> = Vec::new();
457 let mut prev_tool_ids: Vec<String> = Vec::new();
458 let mut ts_idx: usize = 0;
459 for msg in arr.iter_mut() {
460 if let serde_json::Value::Object(msg_map) = msg {
461 let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
462 match role {
463 "assistant" => {
464 prev_tool_names.clear();
465 prev_tool_ids.clear();
466 if let Some(tcs) = msg_map.get_mut("tool_calls")
467 && let Some(tc_arr) = tcs.as_array_mut()
468 {
469 if let Some(sigs) = thought_signatures.get(ts_idx) {
471 for (i, tc) in tc_arr.iter_mut().enumerate() {
472 if let Some(sig) = sigs.get(i).and_then(|s| s.clone())
473 && let Some(func) = tc.get_mut("function")
474 && let Some(obj) = func.as_object_mut()
475 {
476 obj.insert(
477 "thought_signature".to_string(),
478 serde_json::Value::String(sig),
479 );
480 }
481 }
482 }
483 ts_idx += 1;
484
485 for (i, tc) in tc_arr.iter().enumerate() {
487 if let Some(name) = tc
488 .get("function")
489 .and_then(|f| f.get("name"))
490 .and_then(|n| n.as_str())
491 {
492 prev_tool_names.push(name.to_string());
493 }
494 let id = tc
496 .get("id")
497 .and_then(|v| v.as_str())
498 .filter(|s| !s.is_empty())
499 .map(|s| s.to_string())
500 .unwrap_or_else(|| format!("call_{}", i));
501 prev_tool_ids.push(id);
502 }
503 } else {
504 ts_idx += 1;
505 }
506 }
507 "tool" => {
508 if !msg_map.contains_key("name") && prev_tool_names.len() == 1 {
511 msg_map.insert(
512 "name".to_string(),
513 serde_json::Value::String(prev_tool_names[0].clone()),
514 );
515 }
516 if !msg_map.contains_key("tool_call_id") && prev_tool_ids.len() == 1
518 {
519 msg_map.insert(
520 "tool_call_id".to_string(),
521 serde_json::Value::String(prev_tool_ids[0].clone()),
522 );
523 }
524 }
525 _ => {}
526 }
527 }
528 }
529 for v in arr.iter_mut() {
530 fix_request_json(v, thought_signatures);
531 }
532 }
533 _ => {}
534 }
535 }
536 fix_request_json(&mut request, thought_signatures);
537
538 let response = client
539 .post(&url)
540 .json(&request)
541 .send()
542 .await
543 .map_err(|e| format!("request failed: {e}"))?;
544
545 if !response.status().is_success() {
546 return Err(format!(
547 "HTTP {}: {}",
548 response.status().as_u16(),
549 response.text().await.unwrap_or_default()
550 ));
551 }
552
553 let mut stream = response.bytes_stream();
554 let mut buffer = String::new();
555
556 while let Some(chunk_result) = stream.next().await {
557 let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
558 buffer.push_str(&String::from_utf8_lossy(&chunk));
559
560 while let Some(pos) = buffer.find('\n') {
562 let line = buffer[..pos].trim().to_string();
563 buffer = buffer[pos + 1..].to_string();
564
565 if line.is_empty() {
566 continue;
567 }
568
569 match serde_json::from_str::<OllamaChunk>(&line) {
570 Ok(mut chunk) => {
571 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
574 chunk.capture_raw_tool_calls(&raw);
575 }
576 let done = chunk.done;
577 let ours = chunk.to_chat_message_response();
578 if send.send(ours).await.is_err() {
579 return Ok(()); }
581 if done {
582 return Ok(());
583 }
584 }
585 Err(_) => {
586 }
588 }
589 }
590 }
591
592 if !buffer.trim().is_empty()
594 && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
595 {
596 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
597 chunk.capture_raw_tool_calls(&raw);
598 }
599 let done = chunk.done;
600 let ours = chunk.to_chat_message_response();
601 if send.send(ours).await.is_err() {
602 return Ok(());
603 }
604 if done {
605 return Ok(());
606 }
607 }
608
609 let _ = send
611 .send(ChatMessageResponse {
612 message: ChatMessage {
613 content: String::new(),
614 tool_calls: vec![],
615 thinking: None,
616 },
617 done: true,
618 is_error: false,
619 usage: None,
620 })
621 .await;
622
623 Ok(())
624}