tinyharness_lib/provider/
ollama.rs1use std::future::Future;
2use std::pin::Pin;
3use std::time::Duration;
4
5use ollama_rs::{
6 IntoUrlSealed, Ollama,
7 generation::{
8 chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
9 parameters::ThinkType,
10 },
11};
12use serde::Deserialize;
13use tokio_stream::StreamExt;
14
15use crate::config::OllamaThinkType;
16use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
17
18use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
19
20impl From<Message> for OllamaChatMessage {
21 fn from(msg: Message) -> Self {
22 match msg.role {
23 Role::System => OllamaChatMessage::system(msg.content),
24 Role::User => {
25 let mut m = OllamaChatMessage::user(msg.content);
26 if !msg.images.is_empty() {
27 let images: Vec<ollama_rs::generation::images::Image> = msg
28 .images
29 .iter()
30 .map(|img| {
31 ollama_rs::generation::images::Image::from_base64(
32 img.base64_data.clone(),
33 )
34 })
35 .collect();
36 m.images = Some(images);
37 }
38 m
39 }
40 Role::Assistant => {
41 let mut m = OllamaChatMessage::assistant(msg.content);
42 if !msg.tool_calls.is_empty() {
43 m.tool_calls = msg
44 .tool_calls
45 .into_iter()
46 .map(|tc| ollama_rs::generation::tools::ToolCall {
47 function: ollama_rs::generation::tools::ToolCallFunction {
48 name: tc.function.name,
49 arguments: tc.function.arguments,
50 },
51 })
52 .collect();
53 }
54 m
55 }
56 Role::Tool => OllamaChatMessage::tool(msg.content),
57 }
58 }
59}
60
61fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
62 ollama_rs::generation::tools::ToolInfo {
63 tool_type: ollama_rs::generation::tools::ToolType::Function,
64 function: ollama_rs::generation::tools::ToolFunctionInfo {
65 name: ti.name,
66 description: ti.description,
67 parameters: ti.parameters,
68 },
69 }
70}
71
72fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
73 match tt {
74 OllamaThinkType::Off => ThinkType::False,
75 OllamaThinkType::Low => ThinkType::Low,
76 OllamaThinkType::Medium => ThinkType::Medium,
77 OllamaThinkType::High => ThinkType::High,
78 }
79}
80
81pub struct OllamaProvider {
82 client: Ollama,
83 http_client: reqwest::Client,
84 base_url: String,
85 model: Option<String>,
86 timeout_secs: u64,
87 max_retries: u32,
88 think_type: OllamaThinkType,
89}
90
91impl OllamaProvider {
92 pub fn new(
93 base: String,
94 timeout_secs: u64,
95 max_retries: u32,
96 think_type: OllamaThinkType,
97 ) -> Result<Self, String> {
98 let base_url = if base.ends_with('/') {
100 base
101 } else {
102 format!("{base}/")
103 };
104 let parsed_url = base_url
107 .clone()
108 .into_url()
109 .map_err(|e| format!("Invalid Ollama base URL `{base_url}`: {e}"))?;
110 let client = Ollama::from_url(parsed_url);
111 let http_client = reqwest::Client::builder()
112 .connect_timeout(Duration::from_secs(15))
113 .read_timeout(Duration::from_secs(timeout_secs + 60))
114 .build()
115 .unwrap_or_else(|_| reqwest::Client::new());
116 Ok(OllamaProvider {
117 client,
118 http_client,
119 base_url,
120 model: None,
121 timeout_secs,
122 max_retries,
123 think_type,
124 })
125 }
126}
127
128impl Provider for OllamaProvider {
129 fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
130 let client = self.client.clone();
131 Box::pin(async move {
132 match client.list_local_models().await {
133 Ok(_) => Ok(()),
134 Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
135 }
136 })
137 }
138
139 fn list_models(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
140 let client = self.client.clone();
141 Box::pin(async move {
142 match client.list_local_models().await {
143 Ok(models) => models.into_iter().map(|m| m.name).collect(),
144 Err(_) => vec![],
145 }
146 })
147 }
148
149 fn select_model(&mut self, name: String) {
150 self.model = Some(name);
151 }
152
153 fn current_model(&self) -> Option<String> {
154 self.model.clone()
155 }
156
157 fn set_timeout(&mut self, timeout_secs: u64) {
158 self.timeout_secs = timeout_secs;
159 }
160
161 fn set_retries(&mut self, max_retries: u32) {
162 self.max_retries = max_retries;
163 }
164
165 fn set_think_type(&mut self, think_type: OllamaThinkType) {
166 self.think_type = think_type;
167 }
168
169 fn chat(
170 &mut self,
171 messages: Vec<Message>,
172 tools: Vec<ToolDefinition>,
173 ) -> Pin<
174 Box<
175 dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
176 + Send,
177 >,
178 > {
179 let model = match self.model.clone() {
180 Some(m) => m,
181 None => {
182 return Box::pin(async move {
183 Err("No model selected. Use /model <name> to select one.".to_string())
184 });
185 }
186 };
187 let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
188 let timeout_secs = self.timeout_secs;
189 let max_retries = self.max_retries;
190 let http_client = self.http_client.clone();
191 let base_url = self.base_url.clone();
192 let request = {
193 let chat_messages: Vec<OllamaChatMessage> =
194 messages.iter().map(|m| m.clone().into()).collect();
195
196 let thought_signatures: Vec<Vec<Option<String>>> = messages
200 .iter()
201 .map(|m| {
202 m.tool_calls
203 .iter()
204 .map(|tc| tc.function.thought_signature.clone())
205 .collect()
206 })
207 .collect();
208
209 let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
210 tools.into_iter().map(to_ollama_tool_info).collect();
211
212 let mut req = ChatMessageRequest::new(model, chat_messages)
213 .think(to_ollama_think_type(self.think_type));
214
215 if !ollama_tools.is_empty() {
216 req = req.tools(ollama_tools);
217 req.think = None;
218 }
219 (req, thought_signatures)
220 };
221
222 tokio::spawn(async move {
231 let (request, thought_signatures) = request;
232 let max_attempts = max_retries.max(1);
234
235 let send_err = |msg: String| {
236 let _ = send.try_send(ChatMessageResponse {
237 message: ChatMessage {
238 content: msg,
239 tool_calls: vec![],
240 thinking: None,
241 },
242 done: true,
243 is_error: true,
244 usage: None,
245 });
246 };
247
248 for attempt in 1..=max_attempts {
249 let result = tokio::time::timeout(
250 Duration::from_secs(timeout_secs),
251 stream_ollama_chat(
252 &http_client,
253 &base_url,
254 &request,
255 &send,
256 &thought_signatures,
257 ),
258 )
259 .await;
260
261 match result {
262 Ok(Ok(())) => return, Ok(Err(e)) => {
264 if attempt >= max_attempts {
265 send_err(format!("Error after {attempt} attempt(s): {e}"));
266 return;
267 }
268 }
269 Err(_) => {
270 if attempt >= max_attempts {
271 send_err(format!(
272 "Error: Request timed out after {timeout_secs}s ({attempt} attempt(s))"
273 ));
274 return;
275 }
276 }
277 }
278
279 let backoff = Duration::from_secs(1 << (attempt - 1));
281 tokio::time::sleep(backoff).await;
282 }
283 });
284
285 Box::pin(async move { Ok(recv) })
286 }
287}
288
289#[derive(Debug, Clone, Deserialize)]
306struct OllamaChunk {
307 #[serde(default)]
308 message: OllamaChunkMessage,
309 #[serde(default)]
310 done: bool,
311 #[serde(default)]
313 prompt_eval_count: Option<u64>,
314 #[serde(default)]
315 eval_count: Option<u64>,
316 #[serde(default)]
318 usage: Option<OllamaUsage>,
319 #[serde(skip, default)]
322 raw_tool_calls_json: Option<serde_json::Value>,
323}
324
325#[derive(Debug, Clone, Deserialize)]
326struct OllamaUsage {
327 #[serde(default)]
328 prompt_tokens: u32,
329 #[serde(default)]
330 completion_tokens: u32,
331 #[serde(default)]
332 total_tokens: u32,
333}
334
335#[derive(Debug, Clone, Default, Deserialize)]
336struct OllamaChunkMessage {
337 #[serde(default)]
338 content: String,
339 #[serde(default)]
340 thinking: Option<String>,
341 #[serde(default)]
342 tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
343}
344
345impl OllamaChunk {
346 fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
348 if let Some(msg) = raw_line.get("message")
349 && let Some(tcs) = msg.get("tool_calls")
350 {
351 self.raw_tool_calls_json = Some(tcs.clone());
352 }
353 }
354
355 fn to_chat_message_response(&self) -> ChatMessageResponse {
356 let usage = if let Some(u) = &self.usage {
357 Some(TokenUsage {
358 prompt_tokens: u.prompt_tokens,
359 completion_tokens: u.completion_tokens,
360 total_tokens: u.total_tokens,
361 })
362 } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
363 let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
364 let completion = self.eval_count.unwrap_or(0) as u32;
365 Some(TokenUsage {
366 prompt_tokens: prompt,
367 completion_tokens: completion,
368 total_tokens: prompt.saturating_add(completion),
369 })
370 } else {
371 None
372 };
373
374 let mut raw_tc_iter = self
377 .raw_tool_calls_json
378 .as_ref()
379 .and_then(|v| v.as_array())
380 .map(|a| a.iter())
381 .into_iter()
382 .flatten();
383 let tool_calls: Vec<ToolCall> = self
384 .message
385 .tool_calls
386 .clone()
387 .into_iter()
388 .map(|tc| {
389 let thought_signature = raw_tc_iter
390 .next()
391 .and_then(|raw| {
392 raw.get("function")
393 .and_then(|f| f.get("thought_signature"))
394 .and_then(|ts| ts.as_str())
395 })
396 .map(|s| s.to_string());
397 ToolCall {
398 id: None,
399 function: ToolCallFunction {
400 name: tc.function.name,
401 arguments: tc.function.arguments,
402 thought_signature,
403 },
404 }
405 })
406 .collect();
407
408 ChatMessageResponse {
409 message: ChatMessage {
410 content: self.message.content.clone(),
411 thinking: self.message.thinking.clone(),
412 tool_calls,
413 },
414 done: self.done,
415 is_error: false,
416 usage,
417 }
418 }
419}
420
421async fn stream_ollama_chat(
426 client: &reqwest::Client,
427 base_url: &str,
428 request: &ChatMessageRequest,
429 send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
430 thought_signatures: &[Vec<Option<String>>],
431) -> Result<(), String> {
432 let url = format!("{base_url}api/chat");
433 let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
434
435 request["stream"] = serde_json::Value::Bool(true);
441
442 fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
452 match value {
453 serde_json::Value::Object(map) => {
454 if let Some(t) = map.get_mut("type")
456 && t.as_str() == Some("Function")
457 {
458 *t = serde_json::Value::String("function".to_string());
459 }
460 for v in map.values_mut() {
461 fix_request_json(v, thought_signatures);
462 }
463 }
464 serde_json::Value::Array(arr) => {
465 let mut prev_tool_names: Vec<String> = Vec::new();
478 let mut prev_tool_ids: Vec<String> = Vec::new();
479 let mut ts_idx: usize = 0;
480 let looks_like_messages_array = arr.iter().any(|v| {
481 v.as_object()
482 .and_then(|o| o.get("role"))
483 .and_then(|r| r.as_str())
484 .is_some()
485 });
486 if looks_like_messages_array {
487 for msg in arr.iter_mut() {
488 if let serde_json::Value::Object(msg_map) = msg {
489 let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
490 match role {
491 "assistant" => {
492 prev_tool_names.clear();
493 prev_tool_ids.clear();
494 if let Some(tcs) = msg_map.get_mut("tool_calls")
495 && let Some(tc_arr) = tcs.as_array_mut()
496 {
497 if let Some(sigs) = thought_signatures.get(ts_idx) {
499 for (i, tc) in tc_arr.iter_mut().enumerate() {
500 if let Some(sig) =
501 sigs.get(i).and_then(|s| s.clone())
502 && let Some(func) = tc.get_mut("function")
503 && let Some(obj) = func.as_object_mut()
504 {
505 obj.insert(
506 "thought_signature".to_string(),
507 serde_json::Value::String(sig),
508 );
509 }
510 }
511 }
512
513 for (i, tc) in tc_arr.iter().enumerate() {
515 if let Some(name) = tc
516 .get("function")
517 .and_then(|f| f.get("name"))
518 .and_then(|n| n.as_str())
519 {
520 prev_tool_names.push(name.to_string());
521 }
522 let id = tc
524 .get("id")
525 .and_then(|v| v.as_str())
526 .filter(|s| !s.is_empty())
527 .map(|s| s.to_string())
528 .unwrap_or_else(|| format!("call_{}", i));
529 prev_tool_ids.push(id);
530 }
531 }
532 }
533 "tool" => {
534 if !msg_map.contains_key("name") && prev_tool_names.len() == 1 {
537 msg_map.insert(
538 "name".to_string(),
539 serde_json::Value::String(prev_tool_names[0].clone()),
540 );
541 }
542 if !msg_map.contains_key("tool_call_id")
544 && prev_tool_ids.len() == 1
545 {
546 msg_map.insert(
547 "tool_call_id".to_string(),
548 serde_json::Value::String(prev_tool_ids[0].clone()),
549 );
550 }
551 }
552 _ => {}
553 }
554 }
555 ts_idx += 1;
559 }
560 }
561 for v in arr.iter_mut() {
565 fix_request_json(v, &[]);
566 }
567 }
568 _ => {}
569 }
570 }
571 fix_request_json(&mut request, thought_signatures);
572
573 let response = client
574 .post(&url)
575 .json(&request)
576 .send()
577 .await
578 .map_err(|e| format!("request failed: {e}"))?;
579
580 if !response.status().is_success() {
581 return Err(format!(
582 "HTTP {}: {}",
583 response.status().as_u16(),
584 response.text().await.unwrap_or_default()
585 ));
586 }
587
588 let mut stream = response.bytes_stream();
589 let mut buffer = String::new();
590
591 while let Some(chunk_result) = stream.next().await {
592 let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
593 buffer.push_str(&String::from_utf8_lossy(&chunk));
594
595 while let Some(pos) = buffer.find('\n') {
597 let line = buffer[..pos].trim().to_string();
598 buffer.drain(..=pos);
599
600 if line.is_empty() {
601 continue;
602 }
603
604 match serde_json::from_str::<OllamaChunk>(&line) {
605 Ok(mut chunk) => {
606 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
609 chunk.capture_raw_tool_calls(&raw);
610 }
611 let done = chunk.done;
612 let ours = chunk.to_chat_message_response();
613 if send.send(ours).await.is_err() {
614 return Ok(()); }
616 if done {
617 return Ok(());
618 }
619 }
620 Err(_) => {
621 }
623 }
624 }
625 }
626
627 if !buffer.trim().is_empty()
629 && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
630 {
631 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
632 chunk.capture_raw_tool_calls(&raw);
633 }
634 let done = chunk.done;
635 let ours = chunk.to_chat_message_response();
636 if send.send(ours).await.is_err() {
637 return Ok(());
638 }
639 if done {
640 return Ok(());
641 }
642 }
643
644 let _ = send
646 .send(ChatMessageResponse {
647 message: ChatMessage {
648 content: String::new(),
649 tool_calls: vec![],
650 thinking: None,
651 },
652 done: true,
653 is_error: false,
654 usage: None,
655 })
656 .await;
657
658 Ok(())
659}