tinyharness_lib/provider/
ollama.rs1use std::future::Future;
2use std::pin::Pin;
3use std::time::Duration;
4
5use ollama_rs::{
6 IntoUrlSealed, Ollama,
7 generation::{
8 chat::{ChatMessage as OllamaChatMessage, request::ChatMessageRequest},
9 parameters::ThinkType,
10 },
11};
12use serde::Deserialize;
13use tokio_stream::StreamExt;
14
15use crate::config::OllamaThinkType;
16use crate::provider::{ChatMessage, ChatMessageResponse, Message, Provider, ToolDefinition};
17
18use super::{Role, TokenUsage, ToolCall, ToolCallFunction};
19
20impl From<Message> for OllamaChatMessage {
21 fn from(msg: Message) -> Self {
22 match msg.role {
23 Role::System => OllamaChatMessage::system(msg.content),
24 Role::User => OllamaChatMessage::user(msg.content),
25 Role::Assistant => {
26 let mut m = OllamaChatMessage::assistant(msg.content);
27 if !msg.tool_calls.is_empty() {
28 m.tool_calls = msg
29 .tool_calls
30 .into_iter()
31 .map(|tc| ollama_rs::generation::tools::ToolCall {
32 function: ollama_rs::generation::tools::ToolCallFunction {
33 name: tc.function.name,
34 arguments: tc.function.arguments,
35 },
36 })
37 .collect();
38 }
39 m
40 }
41 Role::Tool => OllamaChatMessage::tool(msg.content),
42 }
43 }
44}
45
46fn to_ollama_tool_info(ti: ToolDefinition) -> ollama_rs::generation::tools::ToolInfo {
47 ollama_rs::generation::tools::ToolInfo {
48 tool_type: ollama_rs::generation::tools::ToolType::Function,
49 function: ollama_rs::generation::tools::ToolFunctionInfo {
50 name: ti.name,
51 description: ti.description,
52 parameters: ti.parameters,
53 },
54 }
55}
56
57fn to_ollama_think_type(tt: OllamaThinkType) -> ThinkType {
58 match tt {
59 OllamaThinkType::Off => ThinkType::False,
60 OllamaThinkType::Low => ThinkType::Low,
61 OllamaThinkType::Medium => ThinkType::Medium,
62 OllamaThinkType::High => ThinkType::High,
63 }
64}
65
66pub struct OllamaProvider {
67 client: Ollama,
68 http_client: reqwest::Client,
69 base_url: String,
70 model: Option<String>,
71 timeout_secs: u64,
72 max_retries: u32,
73 think_type: OllamaThinkType,
74}
75
76impl OllamaProvider {
77 pub fn new(
78 base: String,
79 timeout_secs: u64,
80 max_retries: u32,
81 think_type: OllamaThinkType,
82 ) -> Self {
83 let base_url = if base.ends_with('/') {
85 base.clone()
86 } else {
87 format!("{base}/")
88 };
89 let client = Ollama::from_url(base.into_url().unwrap());
90 let http_client = reqwest::Client::builder()
91 .connect_timeout(Duration::from_secs(15))
92 .read_timeout(Duration::from_secs(timeout_secs + 60))
93 .build()
94 .unwrap_or_else(|_| reqwest::Client::new());
95 OllamaProvider {
96 client,
97 http_client,
98 base_url,
99 model: None,
100 timeout_secs,
101 max_retries,
102 think_type,
103 }
104 }
105}
106
107impl Provider for OllamaProvider {
108 fn health_check(&self) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> {
109 let client = self.client.clone();
110 Box::pin(async move {
111 match client.list_local_models().await {
112 Ok(_) => Ok(()),
113 Err(e) => Err(format!("Cannot reach Ollama: {}", e)),
114 }
115 })
116 }
117
118 fn list_models(&self) -> Pin<Box<dyn Future<Output = Vec<String>> + Send>> {
119 let client = self.client.clone();
120 Box::pin(async move {
121 match client.list_local_models().await {
122 Ok(models) => models.into_iter().map(|m| m.name).collect(),
123 Err(_) => vec![],
124 }
125 })
126 }
127
128 fn select_model(&mut self, name: String) {
129 self.model = Some(name);
130 }
131
132 fn current_model(&self) -> Option<String> {
133 self.model.clone()
134 }
135
136 fn set_timeout(&mut self, timeout_secs: u64) {
137 self.timeout_secs = timeout_secs;
138 }
139
140 fn set_retries(&mut self, max_retries: u32) {
141 self.max_retries = max_retries;
142 }
143
144 fn set_think_type(&mut self, think_type: OllamaThinkType) {
145 self.think_type = think_type;
146 }
147
148 fn chat(
149 &mut self,
150 messages: Vec<Message>,
151 tools: Vec<ToolDefinition>,
152 ) -> Pin<
153 Box<
154 dyn Future<Output = Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String>>
155 + Send,
156 >,
157 > {
158 let model = match self.model.clone() {
159 Some(m) => m,
160 None => {
161 return Box::pin(async move {
162 Err("No model selected. Use /model <name> to select one.".to_string())
163 });
164 }
165 };
166 let (send, recv) = tokio::sync::mpsc::channel::<ChatMessageResponse>(1024);
167 let timeout_secs = self.timeout_secs;
168 let max_retries = self.max_retries;
169 let http_client = self.http_client.clone();
170 let base_url = self.base_url.clone();
171 let request = {
172 let chat_messages: Vec<OllamaChatMessage> =
173 messages.iter().map(|m| m.clone().into()).collect();
174
175 let thought_signatures: Vec<Vec<Option<String>>> = messages
179 .iter()
180 .map(|m| {
181 m.tool_calls
182 .iter()
183 .map(|tc| tc.function.thought_signature.clone())
184 .collect()
185 })
186 .collect();
187
188 let ollama_tools: Vec<ollama_rs::generation::tools::ToolInfo> =
189 tools.into_iter().map(to_ollama_tool_info).collect();
190
191 let mut req = ChatMessageRequest::new(model, chat_messages)
192 .think(to_ollama_think_type(self.think_type));
193
194 if !ollama_tools.is_empty() {
195 req = req.tools(ollama_tools);
196 req.think = None;
197 }
198 (req, thought_signatures)
199 };
200
201 tokio::spawn(async move {
210 let (request, thought_signatures) = request;
211 let max_attempts = max_retries.max(1);
213
214 let send_err = |msg: String| {
215 let _ = send.try_send(ChatMessageResponse {
216 message: ChatMessage {
217 content: msg,
218 tool_calls: vec![],
219 thinking: None,
220 },
221 done: true,
222 is_error: true,
223 usage: None,
224 });
225 };
226
227 for attempt in 1..=max_attempts {
228 let result = tokio::time::timeout(
229 Duration::from_secs(timeout_secs),
230 stream_ollama_chat(
231 &http_client,
232 &base_url,
233 &request,
234 &send,
235 &thought_signatures,
236 ),
237 )
238 .await;
239
240 match result {
241 Ok(Ok(())) => return, Ok(Err(e)) => {
243 if attempt >= max_attempts {
244 send_err(format!("Error after {attempt} retries: {e}"));
245 return;
246 }
247 }
248 Err(_) => {
249 if attempt >= max_attempts {
250 send_err(format!(
251 "Error: Request timed out after {timeout_secs}s ({attempt} retries)"
252 ));
253 return;
254 }
255 }
256 }
257
258 let backoff = Duration::from_secs(1 << (attempt - 1));
260 tokio::time::sleep(backoff).await;
261 }
262 });
263
264 Box::pin(async move { Ok(recv) })
265 }
266}
267
268#[derive(Debug, Clone, Deserialize)]
285struct OllamaChunk {
286 #[serde(default)]
287 message: OllamaChunkMessage,
288 #[serde(default)]
289 done: bool,
290 #[serde(default)]
292 prompt_eval_count: Option<u64>,
293 #[serde(default)]
294 eval_count: Option<u64>,
295 #[serde(default)]
297 usage: Option<OllamaUsage>,
298 #[serde(skip, default)]
301 raw_tool_calls_json: Option<serde_json::Value>,
302}
303
304#[derive(Debug, Clone, Deserialize)]
305struct OllamaUsage {
306 #[serde(default)]
307 prompt_tokens: u32,
308 #[serde(default)]
309 completion_tokens: u32,
310 #[serde(default)]
311 total_tokens: u32,
312}
313
314#[derive(Debug, Clone, Default, Deserialize)]
315struct OllamaChunkMessage {
316 #[serde(default)]
317 content: String,
318 #[serde(default)]
319 thinking: Option<String>,
320 #[serde(default)]
321 tool_calls: Vec<ollama_rs::generation::tools::ToolCall>,
322}
323
324impl OllamaChunk {
325 fn capture_raw_tool_calls(&mut self, raw_line: &serde_json::Value) {
327 if let Some(msg) = raw_line.get("message")
328 && let Some(tcs) = msg.get("tool_calls")
329 {
330 self.raw_tool_calls_json = Some(tcs.clone());
331 }
332 }
333
334 fn to_chat_message_response(&self) -> ChatMessageResponse {
335 let usage = if let Some(u) = &self.usage {
336 Some(TokenUsage {
337 prompt_tokens: u.prompt_tokens,
338 completion_tokens: u.completion_tokens,
339 total_tokens: u.total_tokens,
340 })
341 } else if self.prompt_eval_count.is_some() || self.eval_count.is_some() {
342 let prompt = self.prompt_eval_count.unwrap_or(0) as u32;
343 let completion = self.eval_count.unwrap_or(0) as u32;
344 Some(TokenUsage {
345 prompt_tokens: prompt,
346 completion_tokens: completion,
347 total_tokens: prompt.saturating_add(completion),
348 })
349 } else {
350 None
351 };
352
353 let mut raw_tc_iter = self
356 .raw_tool_calls_json
357 .as_ref()
358 .and_then(|v| v.as_array())
359 .map(|a| a.iter())
360 .into_iter()
361 .flatten();
362 let tool_calls: Vec<ToolCall> = self
363 .message
364 .tool_calls
365 .clone()
366 .into_iter()
367 .map(|tc| {
368 let thought_signature = raw_tc_iter
369 .next()
370 .and_then(|raw| {
371 raw.get("function")
372 .and_then(|f| f.get("thought_signature"))
373 .and_then(|ts| ts.as_str())
374 })
375 .map(|s| s.to_string());
376 ToolCall {
377 function: ToolCallFunction {
378 name: tc.function.name,
379 arguments: tc.function.arguments,
380 thought_signature,
381 },
382 }
383 })
384 .collect();
385
386 ChatMessageResponse {
387 message: ChatMessage {
388 content: self.message.content.clone(),
389 thinking: self.message.thinking.clone(),
390 tool_calls,
391 },
392 done: self.done,
393 is_error: false,
394 usage,
395 }
396 }
397}
398
399async fn stream_ollama_chat(
404 client: &reqwest::Client,
405 base_url: &str,
406 request: &ChatMessageRequest,
407 send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
408 thought_signatures: &[Vec<Option<String>>],
409) -> Result<(), String> {
410 let url = format!("{base_url}api/chat");
411 let mut request = serde_json::to_value(request).map_err(|e| format!("serialize: {e}"))?;
412
413 fn fix_request_json(value: &mut serde_json::Value, thought_signatures: &[Vec<Option<String>>]) {
423 match value {
424 serde_json::Value::Object(map) => {
425 if let Some(t) = map.get_mut("type")
427 && t.as_str() == Some("Function")
428 {
429 *t = serde_json::Value::String("function".to_string());
430 }
431 for v in map.values_mut() {
432 fix_request_json(v, thought_signatures);
433 }
434 }
435 serde_json::Value::Array(arr) => {
436 let mut prev_tool_names: Vec<String> = Vec::new();
440 let mut ts_idx: usize = 0;
441 for msg in arr.iter_mut() {
442 if let serde_json::Value::Object(msg_map) = msg {
443 let role = msg_map.get("role").and_then(|v| v.as_str()).unwrap_or("");
444 match role {
445 "assistant" => {
446 prev_tool_names.clear();
447 if let Some(tcs) = msg_map.get_mut("tool_calls")
448 && let Some(tc_arr) = tcs.as_array_mut()
449 {
450 if let Some(sigs) = thought_signatures.get(ts_idx) {
452 for (i, tc) in tc_arr.iter_mut().enumerate() {
453 if let Some(sig) = sigs.get(i).and_then(|s| s.clone())
454 && let Some(func) = tc.get_mut("function")
455 && let Some(obj) = func.as_object_mut()
456 {
457 obj.insert(
458 "thought_signature".to_string(),
459 serde_json::Value::String(sig),
460 );
461 }
462 }
463 }
464 ts_idx += 1;
465
466 for tc in tc_arr.iter() {
468 if let Some(name) = tc
469 .get("function")
470 .and_then(|f| f.get("name"))
471 .and_then(|n| n.as_str())
472 {
473 prev_tool_names.push(name.to_string());
474 }
475 }
476 } else {
477 ts_idx += 1;
478 }
479 }
480 "tool"
481 if !msg_map.contains_key("name") && prev_tool_names.len() == 1 =>
482 {
483 msg_map.insert(
484 "name".to_string(),
485 serde_json::Value::String(prev_tool_names[0].clone()),
486 );
487 }
488 _ => {}
489 }
490 }
491 }
492 for v in arr.iter_mut() {
493 fix_request_json(v, thought_signatures);
494 }
495 }
496 _ => {}
497 }
498 }
499 fix_request_json(&mut request, thought_signatures);
500
501 let response = client
502 .post(&url)
503 .json(&request)
504 .send()
505 .await
506 .map_err(|e| format!("request failed: {e}"))?;
507
508 if !response.status().is_success() {
509 return Err(format!(
510 "HTTP {}: {}",
511 response.status().as_u16(),
512 response.text().await.unwrap_or_default()
513 ));
514 }
515
516 let mut stream = response.bytes_stream();
517 let mut buffer = String::new();
518
519 while let Some(chunk_result) = stream.next().await {
520 let chunk = chunk_result.map_err(|e| format!("stream read: {e}"))?;
521 buffer.push_str(&String::from_utf8_lossy(&chunk));
522
523 while let Some(pos) = buffer.find('\n') {
525 let line = buffer[..pos].trim().to_string();
526 buffer = buffer[pos + 1..].to_string();
527
528 if line.is_empty() {
529 continue;
530 }
531
532 match serde_json::from_str::<OllamaChunk>(&line) {
533 Ok(mut chunk) => {
534 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(&line) {
537 chunk.capture_raw_tool_calls(&raw);
538 }
539 let done = chunk.done;
540 let ours = chunk.to_chat_message_response();
541 if send.send(ours).await.is_err() {
542 return Ok(()); }
544 if done {
545 return Ok(());
546 }
547 }
548 Err(_) => {
549 }
551 }
552 }
553 }
554
555 if !buffer.trim().is_empty()
557 && let Ok(mut chunk) = serde_json::from_str::<OllamaChunk>(buffer.trim())
558 {
559 if let Ok(raw) = serde_json::from_str::<serde_json::Value>(buffer.trim()) {
560 chunk.capture_raw_tool_calls(&raw);
561 }
562 let done = chunk.done;
563 let ours = chunk.to_chat_message_response();
564 if send.send(ours).await.is_err() {
565 return Ok(());
566 }
567 if done {
568 return Ok(());
569 }
570 }
571
572 let _ = send
574 .send(ChatMessageResponse {
575 message: ChatMessage {
576 content: String::new(),
577 tool_calls: vec![],
578 thinking: None,
579 },
580 done: true,
581 is_error: false,
582 usage: None,
583 })
584 .await;
585
586 Ok(())
587}