tinyharness_lib/provider/
ollama.rs1use std::future::Future;
2use std::pin::Pin;
3use std::time::Duration;
4
5use ollama_rs::{
6 IntoUrlSealed, Ollama,
7 generation::{
8 chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
9 parameters::ThinkType,
10 },
11};
12use serde::Deserialize;
13use tokio_stream::StreamExt;
14
15use crate::config::OllamaThinkType;
16use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
17
18use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
19
20impl From<Message> for OllamaChatMessage {
21 fn from(msg: Message) -> Self {
22 match msg.role {
23 Role::System => OllamaChatMessage::system(msg.content),
24 Role::User => {
25 let mut m = OllamaChatMessage::user(msg.content);
26 if !msg.images.is_empty() {
27 let images: Vec<ollama_rs::generation::images::Image> = msg
28 .images
29 .iter()
30 .map(|img| {
31 ollama_rs::generation::images::Image::from_base64(
32 img.base64_data.clone(),
33 )
34 })
35 .collect();
36 m.images = Some(images);
37 }
38 m
39 }
40 Role::Assistant => {
41 let mut m = OllamaChatMessage::assistant(msg.content);
42 if !msg.tool_calls.is_empty() {
43 m.tool_calls = msg
44 .tool_calls
45 .into_iter()
46 .map(|tc| ollama_rs::generation::tools::ToolCall {
47 function: ollama_rs::generation::tools::ToolCallFunction {
48 name: tc.function.name,
49 arguments: tc.function.arguments,
50 },
51 })
52 .collect();
53 }
54 m
55 }
56 Role::Tool => OllamaChatMessage::tool(msg.content),
57 }
58 }
59}
60
61fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
62 ollama_rs::generation::tools::ToolInfo {
63 tool_type: ollama_rs::generation::tools::ToolType::Function,
64 function: ollama_rs::generation::tools::ToolFunctionInfo {
65 name: ti.name,
66 description: ti.description,
67 parameters: ti.parameters,
68 },
69 }
70}
71
72fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
73 match tt {
74 OllamaThinkType::Off => ThinkType::False,
75 OllamaThinkType::Low => ThinkType::Low,
76 OllamaThinkType::Medium => ThinkType::Medium,
77 OllamaThinkType::High => ThinkType::High,
78 }
79}
80
81pub struct OllamaProvider {
82 client: Ollama,
83 http_client: reqwest::Client,
84 base_url: String,
85 model: Option<String>,
86 timeout_secs: u64,
87 max_retries: u32,
88 think_type: OllamaThinkType,
89}
90
91impl OllamaProvider {
92 pub fn new(
93 base: String,
94 timeout_secs: u64,
95 max_retries: u32,
96 think_type: OllamaThinkType,
97 ) -> Self {
98 let base_url = if base.ends_with('/') {
100 base.clone()
101 } else {
102 format!("{base}/")
103 };
104 let client = Ollama::from_url(base.into_url().unwrap());
105 let http_client = reqwest::Client::builder()
106 .connect_timeout(Duration::from_secs(15))
107 .read_timeout(Duration::from_secs(timeout_secs + 60))
108 .build()
109 .unwrap_or_else(|_| reqwest::Client::new());
110 OllamaProvider {
111 client,
112 http_client,
113 base_url,
114 model: None,
115 timeout_secs,
116 max_retries,
117 think_type,
118 }
119 }
120}
121
122impl Provider for OllamaProvider {
123 fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
124 let client = self.client.clone();
125 Box::pin(async move {
126 match client.list_local_models().await {
127 Ok(_) => Ok(()),
128 Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
129 }
130 })
131 }
132
133 fn list_models(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
134 let client = self.client.clone();
135 Box::pin(async move {
136 match client.list_local_models().await {
137 Ok(models) => models.into_iter().map(|m| m.name).collect(),
138 Err(_) => vec![],
139 }
140 })
141 }
142
143 fn select_model(&mut self, name: String) {
144 self.model = Some(name);
145 }
146
147 fn current_model(&self) -> Option<String> {
148 self.model.clone()
149 }
150
151 fn set_timeout(&mut self, timeout_secs: u64) {
152 self.timeout_secs = timeout_secs;
153 }
154
155 fn set_retries(&mut self, max_retries: u32) {
156 self.max_retries = max_retries;
157 }
158
159 fn set_think_type(&mut self, think_type: OllamaThinkType) {
160 self.think_type = think_type;
161 }
162
163 fn chat(
164 &mut self,
165 messages: Vec<Message>,
166 tools: Vec<ToolDefinition>,
167 ) -> Pin<
168 Box<
169 dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
170 + Send,
171 >,
172 > {
173 let model = match self.model.clone() {
174 Some(m) => m,
175 None => {
176 return Box::pin(async move {
177 Err("No model selected. Use /model <name> to select one.".to_string())
178 });
179 }
180 };
181 let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
182 let timeout_secs = self.timeout_secs;
183 let max_retries = self.max_retries;
184 let http_client = self.http_client.clone();
185 let base_url = self.base_url.clone();
186 let request = {
187 let chat_messages: Vec<OllamaChatMessage> =
188 messages.iter().map(|m| m.clone().into()).collect();
189
190 let thought_signatures: Vec<Vec<Option<String>>> = messages
194 .iter()
195 .map(|m| {
196 m.tool_calls
197 .iter()
198 .map(|tc| tc.function.thought_signature.clone())
199 .collect()
200 })
201 .collect();
202
203 let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
204 tools.into_iter().map(to_ollama_tool_info).collect();
205
206 let mut req = ChatMessageRequest::new(model, chat_messages)
207 .think(to_ollama_think_type(self.think_type));
208
209 if !ollama_tools.is_empty() {
210 req = req.tools(ollama_tools);
211 req.think = None;
212 }
213 (req, thought_signatures)
214 };
215
216 tokio::spawn(async move {
225 let (request, thought_signatures) = request;
226 let max_attempts = max_retries.max(1);
228
229 let send_err = |msg: String| {
230 let _ = send.try_send(ChatMessageResponse {
231 message: ChatMessage {
232 content: msg,
233 tool_calls: vec![],
234 thinking: None,
235 },
236 done: true,
237 is_error: true,
238 usage: None,
239 });
240 };
241
242 for attempt in 1..=max_attempts {
243 let result = tokio::time::timeout(
244 Duration::from_secs(timeout_secs),
245 stream_ollama_chat(
246 &http_client,
247 &base_url,
248 &request,
249 &send,
250 &thought_signatures,
251 ),
252 )
253 .await;
254
255 match result {
256 Ok(Ok(())) => return, Ok(Err(e)) => {
258 if attempt >= max_attempts {
259 send_err(format!("Error after {attempt} retries: {e}"));
260 return;
261 }
262 }
263 Err(_) => {
264 if attempt >= max_attempts {
265 send_err(format!(
266 "Error: Request timed out after {timeout_secs}s ({attempt} retries)"
267 ));
268 return;
269 }
270 }
271 }
272
273 let backoff = Duration::from_secs(1 << (attempt - 1));
275 tokio::time::sleep(backoff).await;
276 }
277 });
278
279 Box::pin(async move { Ok(recv) })
280 }
281}
282
283#[derive(Debug, Clone, Deserialize)]
300struct OllamaChunk {
301 #[serde(default)]
302 message: OllamaChunkMessage,
303 #[serde(default)]
304 done: bool,
305 #[serde(default)]
307 prompt_eval_count: Option<u64>,
308 #[serde(default)]
309 eval_count: Option<u64>,
310 #[serde(default)]
312 usage: Option<OllamaUsage>,
313 #[serde(skip, default)]
316 raw_tool_calls_json: Option<serde_json::Value>,
317}
318
319#[derive(Debug, Clone, Deserialize)]
320struct OllamaUsage {
321 #[serde(default)]
322 prompt_tokens: u32,
323 #[serde(default)]
324 completion_tokens: u32,
325 #[serde(default)]
326 total_tokens: u32,
327}
328
329#[derive(Debug, Clone, Default, Deserialize)]
330struct OllamaChunkMessage {
331 #[serde(default)]
332 content: String,
333 #[serde(default)]
334 thinking: Option<String>,
335 #[serde(default)]
336 tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
337}
338
339impl OllamaChunk {
340 fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
342 if let Some(msg) = raw_line.get("message")
343 && let Some(tcs) = msg.get("tool_calls")
344 {
345 self.raw_tool_calls_json = Some(tcs.clone());
346 }
347 }
348
349 fn to_chat_message_response(&self) -> ChatMessageResponse {
350 let usage = if let Some(u) = &self.usage {
351 Some(TokenUsage {
352 prompt_tokens: u.prompt_tokens,
353 completion_tokens: u.completion_tokens,
354 total_tokens: u.total_tokens,
355 })
356 } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
357 let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
358 let completion = self.eval_count.unwrap_or(0) as u32;
359 Some(TokenUsage {
360 prompt_tokens: prompt,
361 completion_tokens: completion,
362 total_tokens: prompt.saturating_add(completion),
363 })
364 } else {
365 None
366 };
367
368 let mut raw_tc_iter = self
371 .raw_tool_calls_json
372 .as_ref()
373 .and_then(|v| v.as_array())
374 .map(|a| a.iter())
375 .into_iter()
376 .flatten();
377 let tool_calls: Vec<ToolCall> = self
378 .message
379 .tool_calls
380 .clone()
381 .into_iter()
382 .map(|tc| {
383 let thought_signature = raw_tc_iter
384 .next()
385 .and_then(|raw| {
386 raw.get("function")
387 .and_then(|f| f.get("thought_signature"))
388 .and_then(|ts| ts.as_str())
389 })
390 .map(|s| s.to_string());
391 ToolCall {
392 function: ToolCallFunction {
393 name: tc.function.name,
394 arguments: tc.function.arguments,
395 thought_signature,
396 },
397 }
398 })
399 .collect();
400
401 ChatMessageResponse {
402 message: ChatMessage {
403 content: self.message.content.clone(),
404 thinking: self.message.thinking.clone(),
405 tool_calls,
406 },
407 done: self.done,
408 is_error: false,
409 usage,
410 }
411 }
412}
413
414async fn stream_ollama_chat(
419 client: &reqwest::Client,
420 base_url: &str,
421 request: &ChatMessageRequest,
422 send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
423 thought_signatures: &[Vec<Option<String>>],
424) -> Result<(), String> {
425 let url = format!("{base_url}api/chat");
426 let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
427
428 fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
438 match value {
439 serde_json::Value::Object(map) => {
440 if let Some(t) = map.get_mut("type")
442 && t.as_str() == Some("Function")
443 {
444 *t = serde_json::Value::String("function".to_string());
445 }
446 for v in map.values_mut() {
447 fix_request_json(v, thought_signatures);
448 }
449 }
450 serde_json::Value::Array(arr) => {
451 let mut prev_tool_names: Vec<String> = Vec::new();
455 let mut ts_idx: usize = 0;
456 for msg in arr.iter_mut() {
457 if let serde_json::Value::Object(msg_map) = msg {
458 let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
459 match role {
460 "assistant" => {
461 prev_tool_names.clear();
462 if let Some(tcs) = msg_map.get_mut("tool_calls")
463 && let Some(tc_arr) = tcs.as_array_mut()
464 {
465 if let Some(sigs) = thought_signatures.get(ts_idx) {
467 for (i, tc) in tc_arr.iter_mut().enumerate() {
468 if let Some(sig) = sigs.get(i).and_then(|s| s.clone())
469 && let Some(func) = tc.get_mut("function")
470 && let Some(obj) = func.as_object_mut()
471 {
472 obj.insert(
473 "thought_signature".to_string(),
474 serde_json::Value::String(sig),
475 );
476 }
477 }
478 }
479 ts_idx += 1;
480
481 for tc in tc_arr.iter() {
483 if let Some(name) = tc
484 .get("function")
485 .and_then(|f| f.get("name"))
486 .and_then(|n| n.as_str())
487 {
488 prev_tool_names.push(name.to_string());
489 }
490 }
491 } else {
492 ts_idx += 1;
493 }
494 }
495 "tool"
496 if !msg_map.contains_key("name") && prev_tool_names.len() == 1 =>
497 {
498 msg_map.insert(
499 "name".to_string(),
500 serde_json::Value::String(prev_tool_names[0].clone()),
501 );
502 }
503 _ => {}
504 }
505 }
506 }
507 for v in arr.iter_mut() {
508 fix_request_json(v, thought_signatures);
509 }
510 }
511 _ => {}
512 }
513 }
514 fix_request_json(&mut request, thought_signatures);
515
516 let response = client
517 .post(&url)
518 .json(&request)
519 .send()
520 .await
521 .map_err(|e| format!("request failed: {e}"))?;
522
523 if !response.status().is_success() {
524 return Err(format!(
525 "HTTP {}: {}",
526 response.status().as_u16(),
527 response.text().await.unwrap_or_default()
528 ));
529 }
530
531 let mut stream = response.bytes_stream();
532 let mut buffer = String::new();
533
534 while let Some(chunk_result) = stream.next().await {
535 let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
536 buffer.push_str(&String::from_utf8_lossy(&chunk));
537
538 while let Some(pos) = buffer.find('\n') {
540 let line = buffer[..pos].trim().to_string();
541 buffer = buffer[pos + 1..].to_string();
542
543 if line.is_empty() {
544 continue;
545 }
546
547 match serde_json::from_str::<OllamaChunk>(&line) {
548 Ok(mut chunk) => {
549 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
552 chunk.capture_raw_tool_calls(&raw);
553 }
554 let done = chunk.done;
555 let ours = chunk.to_chat_message_response();
556 if send.send(ours).await.is_err() {
557 return Ok(()); }
559 if done {
560 return Ok(());
561 }
562 }
563 Err(_) => {
564 }
566 }
567 }
568 }
569
570 if !buffer.trim().is_empty()
572 && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
573 {
574 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
575 chunk.capture_raw_tool_calls(&raw);
576 }
577 let done = chunk.done;
578 let ours = chunk.to_chat_message_response();
579 if send.send(ours).await.is_err() {
580 return Ok(());
581 }
582 if done {
583 return Ok(());
584 }
585 }
586
587 let _ = send
589 .send(ChatMessageResponse {
590 message: ChatMessage {
591 content: String::new(),
592 tool_calls: vec![],
593 thinking: None,
594 },
595 done: true,
596 is_error: false,
597 usage: None,
598 })
599 .await;
600
601 Ok(())
602}