tinyharness_lib/provider/
ollama.rs1use std::time::Duration;
2
3use ollama_rs::{
4 IntoUrlSealed, Ollama,
5 generation::{
6 chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
7 parameters::ThinkType,
8 },
9};
10use serde::Deserialize;
11use tokio_stream::StreamExt;
12
13use crate::config::OllamaThinkType;
14use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
15
16use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
17
18impl From<Message> for OllamaChatMessage {
19 fn from(msg: Message) -> Self {
20 match msg.role {
21 Role::System => OllamaChatMessage::system(msg.content),
22 Role::User => {
23 let mut m = OllamaChatMessage::user(msg.content);
24 if !msg.images.is_empty() {
25 let images: Vec<ollama_rs::generation::images::Image> = msg
26 .images
27 .iter()
28 .map(|img| {
29 ollama_rs::generation::images::Image::from_base64(
30 img.base64_data.clone(),
31 )
32 })
33 .collect();
34 m.images = Some(images);
35 }
36 m
37 }
38 Role::Assistant => {
39 let mut m = OllamaChatMessage::assistant(msg.content);
40 if !msg.tool_calls.is_empty() {
41 m.tool_calls = msg
42 .tool_calls
43 .into_iter()
44 .map(|tc| ollama_rs::generation::tools::ToolCall {
45 function: ollama_rs::generation::tools::ToolCallFunction {
46 name: tc.function.name,
47 arguments: tc.function.arguments,
48 },
49 })
50 .collect();
51 }
52 m
53 }
54 Role::Tool => OllamaChatMessage::tool(msg.content),
55 }
56 }
57}
58
59fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
60 ollama_rs::generation::tools::ToolInfo {
61 tool_type: ollama_rs::generation::tools::ToolType::Function,
62 function: ollama_rs::generation::tools::ToolFunctionInfo {
63 name: ti.name,
64 description: ti.description,
65 parameters: ti.parameters,
66 },
67 }
68}
69
70fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
71 match tt {
72 OllamaThinkType::Off => ThinkType::False,
73 OllamaThinkType::Low => ThinkType::Low,
74 OllamaThinkType::Medium => ThinkType::Medium,
75 OllamaThinkType::High => ThinkType::High,
76 }
77}
78
79pub struct OllamaProvider {
80 client: Ollama,
81 http_client: reqwest::Client,
82 base_url: String,
83 model: Option<String>,
84 timeout_secs: u64,
85 max_retries: u32,
86 think_type: OllamaThinkType,
87}
88
89impl OllamaProvider {
90 pub fn new(
91 base: String,
92 timeout_secs: u64,
93 max_retries: u32,
94 think_type: OllamaThinkType,
95 ) -> Result<Self, String> {
96 let base_url = if base.ends_with('/') {
98 base
99 } else {
100 format!("{base}/")
101 };
102 let parsed_url = base_url
105 .clone()
106 .into_url()
107 .map_err(|e| format!("Invalid Ollama base URL `{base_url}`: {e}"))?;
108 let client = Ollama::from_url(parsed_url);
109 let http_client = reqwest::Client::builder()
110 .connect_timeout(Duration::from_secs(15))
111 .read_timeout(Duration::from_secs(timeout_secs + 60))
112 .build()
113 .unwrap_or_else(|_| reqwest::Client::new());
114 Ok(OllamaProvider {
115 client,
116 http_client,
117 base_url,
118 model: None,
119 timeout_secs,
120 max_retries,
121 think_type,
122 })
123 }
124}
125
126impl Provider for OllamaProvider {
127 async fn health_check(&self) -> Result<(), String> {
128 match self.client.list_local_models().await {
129 Ok(_) => Ok(()),
130 Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
131 }
132 }
133
134 async fn list_models(&self) -> Vec<String> {
135 match self.client.list_local_models().await {
136 Ok(models) => models.into_iter().map(|m| m.name).collect(),
137 Err(_) => vec![],
138 }
139 }
140
141 fn select_model(&mut self, name: String) {
142 self.model = Some(name);
143 }
144
145 fn current_model(&self) -> Option<String> {
146 self.model.clone()
147 }
148
149 fn set_timeout(&mut self, timeout_secs: u64) {
150 self.timeout_secs = timeout_secs;
151 }
152
153 fn set_retries(&mut self, max_retries: u32) {
154 self.max_retries = max_retries;
155 }
156
157 fn set_think_type(&mut self, think_type: OllamaThinkType) {
158 self.think_type = think_type;
159 }
160
161 async fn chat(
162 &mut self,
163 messages: Vec<Message>,
164 tools: Vec<ToolDefinition>,
165 ) -> Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String> {
166 let model = match self.model.clone() {
167 Some(m) => m,
168 None => {
169 return Err("No model selected. Use /model <name> to select one.".to_string());
170 }
171 };
172 let (send, recv) =
173 tokio::sync::mpsc::channel::<ChatMessageResponse>(super::STREAM_CHANNEL_CAPACITY);
174 let timeout_secs = self.timeout_secs;
175 let max_retries = self.max_retries;
176 let http_client = self.http_client.clone();
177 let base_url = self.base_url.clone();
178 let request = {
179 let chat_messages: Vec<OllamaChatMessage> =
180 messages.iter().map(|m| m.clone().into()).collect();
181
182 let thought_signatures: Vec<Vec<Option<String>>> = messages
186 .iter()
187 .map(|m| {
188 m.tool_calls
189 .iter()
190 .map(|tc| tc.function.thought_signature.clone())
191 .collect()
192 })
193 .collect();
194
195 let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
196 tools.into_iter().map(to_ollama_tool_info).collect();
197
198 let mut req = ChatMessageRequest::new(model, chat_messages)
199 .think(to_ollama_think_type(self.think_type));
200
201 if !ollama_tools.is_empty() {
202 req = req.tools(ollama_tools);
203 req.think = None;
204 }
205 (req, thought_signatures)
206 };
207
208 tokio::spawn(async move {
217 let (request, thought_signatures) = request;
218 let max_attempts = max_retries.max(1);
220
221 let send_err = |msg: String| {
222 let _ = send.try_send(ChatMessageResponse {
223 message: ChatMessage {
224 content: msg,
225 tool_calls: vec![],
226 thinking: None,
227 },
228 done: true,
229 is_error: true,
230 usage: None,
231 });
232 };
233
234 for attempt in 1..=max_attempts {
235 let result = tokio::time::timeout(
236 Duration::from_secs(timeout_secs),
237 stream_ollama_chat(
238 &http_client,
239 &base_url,
240 &request,
241 &send,
242 &thought_signatures,
243 ),
244 )
245 .await;
246
247 match result {
248 Ok(Ok(())) => return, Ok(Err(e)) => {
250 if attempt >= max_attempts {
251 send_err(format!("Error after {attempt} attempt(s): {e}"));
252 return;
253 }
254 }
255 Err(_) => {
256 if attempt >= max_attempts {
257 send_err(format!(
258 "Error: Request timed out after {timeout_secs}s ({attempt} attempt(s))"
259 ));
260 return;
261 }
262 }
263 }
264
265 let backoff = Duration::from_secs(1 << (attempt - 1));
267 tokio::time::sleep(backoff).await;
268 }
269 });
270
271 Ok(recv)
272 }
273}
274
275#[derive(Debug, Clone, Deserialize)]
292struct OllamaChunk {
293 #[serde(default)]
294 message: OllamaChunkMessage,
295 #[serde(default)]
296 done: bool,
297 #[serde(default)]
299 prompt_eval_count: Option<u64>,
300 #[serde(default)]
301 eval_count: Option<u64>,
302 #[serde(default)]
304 usage: Option<OllamaUsage>,
305 #[serde(skip, default)]
308 raw_tool_calls_json: Option<serde_json::Value>,
309}
310
311#[derive(Debug, Clone, Deserialize)]
312struct OllamaUsage {
313 #[serde(default)]
314 prompt_tokens: u32,
315 #[serde(default)]
316 completion_tokens: u32,
317 #[serde(default)]
318 total_tokens: u32,
319}
320
321#[derive(Debug, Clone, Default, Deserialize)]
322struct OllamaChunkMessage {
323 #[serde(default)]
324 content: String,
325 #[serde(default)]
326 thinking: Option<String>,
327 #[serde(default)]
328 tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
329}
330
331impl OllamaChunk {
332 fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
334 if let Some(msg) = raw_line.get("message")
335 && let Some(tcs) = msg.get("tool_calls")
336 {
337 self.raw_tool_calls_json = Some(tcs.clone());
338 }
339 }
340
341 fn to_chat_message_response(&self) -> ChatMessageResponse {
342 let usage = if let Some(u) = &self.usage {
343 Some(TokenUsage {
344 prompt_tokens: u.prompt_tokens,
345 completion_tokens: u.completion_tokens,
346 total_tokens: u.total_tokens,
347 })
348 } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
349 let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
350 let completion = self.eval_count.unwrap_or(0) as u32;
351 Some(TokenUsage {
352 prompt_tokens: prompt,
353 completion_tokens: completion,
354 total_tokens: prompt.saturating_add(completion),
355 })
356 } else {
357 None
358 };
359
360 let mut raw_tc_iter = self
363 .raw_tool_calls_json
364 .as_ref()
365 .and_then(|v| v.as_array())
366 .map(|a| a.iter())
367 .into_iter()
368 .flatten();
369 let tool_calls: Vec<ToolCall> = self
370 .message
371 .tool_calls
372 .clone()
373 .into_iter()
374 .map(|tc| {
375 let thought_signature = raw_tc_iter
376 .next()
377 .and_then(|raw| {
378 raw.get("function")
379 .and_then(|f| f.get("thought_signature"))
380 .and_then(|ts| ts.as_str())
381 })
382 .map(|s| s.to_string());
383 ToolCall {
384 id: None,
385 function: ToolCallFunction {
386 name: tc.function.name,
387 arguments: tc.function.arguments,
388 thought_signature,
389 },
390 }
391 })
392 .collect();
393
394 ChatMessageResponse {
395 message: ChatMessage {
396 content: self.message.content.clone(),
397 thinking: self.message.thinking.clone(),
398 tool_calls,
399 },
400 done: self.done,
401 is_error: false,
402 usage,
403 }
404 }
405}
406
407async fn stream_ollama_chat(
412 client: &reqwest::Client,
413 base_url: &str,
414 request: &ChatMessageRequest,
415 send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
416 thought_signatures: &[Vec<Option<String>>],
417) -> Result<(), String> {
418 let url = format!("{base_url}api/chat");
419 let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
420
421 request["stream"] = serde_json::Value::Bool(true);
427
428 fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
438 match value {
439 serde_json::Value::Object(map) => {
440 if let Some(t) = map.get_mut("type")
442 && t.as_str() == Some("Function")
443 {
444 *t = serde_json::Value::String("function".to_string());
445 }
446 for v in map.values_mut() {
447 fix_request_json(v, thought_signatures);
448 }
449 }
450 serde_json::Value::Array(arr) => {
451 let mut prev_tool_names: Vec<String> = Vec::new();
464 let mut prev_tool_ids: Vec<String> = Vec::new();
465 let mut ts_idx: usize = 0;
466 let looks_like_messages_array = arr.iter().any(|v| {
467 v.as_object()
468 .and_then(|o| o.get("role"))
469 .and_then(|r| r.as_str())
470 .is_some()
471 });
472 if looks_like_messages_array {
473 for msg in arr.iter_mut() {
474 if let serde_json::Value::Object(msg_map) = msg {
475 let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
476 match role {
477 "assistant" => {
478 prev_tool_names.clear();
479 prev_tool_ids.clear();
480 if let Some(tcs) = msg_map.get_mut("tool_calls")
481 && let Some(tc_arr) = tcs.as_array_mut()
482 {
483 if let Some(sigs) = thought_signatures.get(ts_idx) {
485 for (i, tc) in tc_arr.iter_mut().enumerate() {
486 if let Some(sig) =
487 sigs.get(i).and_then(|s| s.clone())
488 && let Some(func) = tc.get_mut("function")
489 && let Some(obj) = func.as_object_mut()
490 {
491 obj.insert(
492 "thought_signature".to_string(),
493 serde_json::Value::String(sig),
494 );
495 }
496 }
497 }
498
499 for (i, tc) in tc_arr.iter().enumerate() {
501 if let Some(name) = tc
502 .get("function")
503 .and_then(|f| f.get("name"))
504 .and_then(|n| n.as_str())
505 {
506 prev_tool_names.push(name.to_string());
507 }
508 let id = tc
510 .get("id")
511 .and_then(|v| v.as_str())
512 .filter(|s| !s.is_empty())
513 .map(|s| s.to_string())
514 .unwrap_or_else(|| format!("call_{}", i));
515 prev_tool_ids.push(id);
516 }
517 }
518 }
519 "tool" => {
520 if !msg_map.contains_key("name") && prev_tool_names.len() == 1 {
523 msg_map.insert(
524 "name".to_string(),
525 serde_json::Value::String(prev_tool_names[0].clone()),
526 );
527 }
528 if !msg_map.contains_key("tool_call_id")
530 && prev_tool_ids.len() == 1
531 {
532 msg_map.insert(
533 "tool_call_id".to_string(),
534 serde_json::Value::String(prev_tool_ids[0].clone()),
535 );
536 }
537 }
538 _ => {}
539 }
540 }
541 ts_idx += 1;
545 }
546 }
547 for v in arr.iter_mut() {
551 fix_request_json(v, &[]);
552 }
553 }
554 _ => {}
555 }
556 }
557 fix_request_json(&mut request, thought_signatures);
558
559 let response = client
560 .post(&url)
561 .json(&request)
562 .send()
563 .await
564 .map_err(|e| format!("request failed: {e}"))?;
565
566 if !response.status().is_success() {
567 return Err(format!(
568 "HTTP {}: {}",
569 response.status().as_u16(),
570 response.text().await.unwrap_or_default()
571 ));
572 }
573
574 let mut stream = response.bytes_stream();
575 let mut buffer = String::new();
576
577 while let Some(chunk_result) = stream.next().await {
578 let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
579 buffer.push_str(&String::from_utf8_lossy(&chunk));
580
581 while let Some(pos) = buffer.find('\n') {
583 let line = buffer[..pos].trim().to_string();
584 buffer.drain(..=pos);
585
586 if line.is_empty() {
587 continue;
588 }
589
590 match serde_json::from_str::<OllamaChunk>(&line) {
591 Ok(mut chunk) => {
592 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
595 chunk.capture_raw_tool_calls(&raw);
596 }
597 let done = chunk.done;
598 let ours = chunk.to_chat_message_response();
599 if send.send(ours).await.is_err() {
600 return Ok(()); }
602 if done {
603 return Ok(());
604 }
605 }
606 Err(_) => {
607 }
609 }
610 }
611 }
612
613 if !buffer.trim().is_empty()
615 && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
616 {
617 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
618 chunk.capture_raw_tool_calls(&raw);
619 }
620 let done = chunk.done;
621 let ours = chunk.to_chat_message_response();
622 if send.send(ours).await.is_err() {
623 return Ok(());
624 }
625 if done {
626 return Ok(());
627 }
628 }
629
630 let _ = send
632 .send(ChatMessageResponse {
633 message: ChatMessage {
634 content: String::new(),
635 tool_calls: vec![],
636 thinking: None,
637 },
638 done: true,
639 is_error: false,
640 usage: None,
641 })
642 .await;
643
644 Ok(())
645}