1use crate::{
2 http::{HttpError, Request, Transport, read_json},
3 identity::{ProcessIdentity, ProcessProbe},
4};
5use serde::{Deserialize, Serialize};
6use serde_json::{Value, json};
7use std::{collections::BTreeMap, time::Duration};
8use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
9use tokio_util::sync::CancellationToken;
10
11#[derive(Debug, Clone, Serialize, Deserialize)]
12#[serde(rename_all = "camelCase", deny_unknown_fields)]
13pub struct ToolCall {
14 pub name: String,
15 #[serde(default)]
16 pub arguments: BTreeMap<String, Value>,
17}
18#[derive(Debug, Clone, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase", deny_unknown_fields)]
20pub struct ChatMessage {
21 pub role: String,
22 pub content: String,
23 #[serde(default)]
24 pub tool_calls: Vec<ToolCall>,
25 pub tool_name: Option<String>,
26}
27#[derive(Debug, Clone, Serialize, Deserialize)]
28#[serde(deny_unknown_fields)]
29pub struct Tool {
30 pub name: String,
31 pub description: String,
32 pub parameters: BTreeMap<String, Value>,
33}
34#[derive(Debug, Clone, Serialize, Deserialize)]
35#[serde(deny_unknown_fields)]
36pub struct ChatInput {
37 pub model: String,
38 pub messages: Vec<ChatMessage>,
39 #[serde(default)]
40 pub tools: Vec<Tool>,
41 pub temperature: Option<f64>,
42 #[serde(default, skip_serializing_if = "Option::is_none")]
45 pub slot: Option<u32>,
46 #[serde(default, skip_serializing_if = "Option::is_none")]
49 pub max_tokens: Option<u32>,
50}
51#[derive(Debug, Clone, Serialize)]
52#[serde(rename_all = "camelCase")]
53pub struct ChatResult {
54 pub content: String,
55 #[serde(skip_serializing_if = "Vec::is_empty")]
56 pub tool_calls: Vec<ToolCall>,
57 #[serde(default, skip_serializing_if = "Option::is_none")]
59 pub timings: Option<Timings>,
60}
61#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
63#[serde(rename_all = "camelCase")]
64pub struct Timings {
65 pub prompt_tokens: u64,
67 pub prompt_ms: f64,
68 pub generated_tokens: u64,
70 pub generated_ms: f64,
71 #[serde(default, skip_serializing_if = "Option::is_none")]
73 pub cached_tokens: Option<u64>,
74 #[serde(default, skip_serializing_if = "Option::is_none")]
76 pub draft_tokens: Option<u64>,
77 #[serde(default, skip_serializing_if = "Option::is_none")]
78 pub draft_accepted: Option<u64>,
79}
80impl Timings {
81 pub fn from_llama_cpp(value: &Value) -> Option<Self> {
84 let timings = value.as_object()?;
85 let count = |key: &str| timings.get(key).and_then(Value::as_u64);
86 let millis = |key: &str| {
87 timings
88 .get(key)
89 .and_then(Value::as_f64)
90 .filter(|ms| ms.is_finite() && *ms >= 0.0)
91 };
92 let draft_tokens = count("draft_n");
93 let draft_accepted = count("draft_n_accepted");
94 if let (Some(drafted), Some(accepted)) = (draft_tokens, draft_accepted)
95 && accepted > drafted
96 {
97 return None;
98 }
99 Some(Self {
100 prompt_tokens: count("prompt_n")?,
101 prompt_ms: millis("prompt_ms")?,
102 generated_tokens: count("predicted_n")?,
103 generated_ms: millis("predicted_ms")?,
104 cached_tokens: count("cache_n"),
105 draft_tokens,
106 draft_accepted,
107 })
108 }
109 pub fn prompt_tokens_per_second(&self) -> Option<f64> {
111 rate(self.prompt_tokens, self.prompt_ms)
112 }
113 pub fn generated_tokens_per_second(&self) -> Option<f64> {
115 rate(self.generated_tokens, self.generated_ms)
116 }
117}
118fn rate(tokens: u64, ms: f64) -> Option<f64> {
119 (tokens > 0 && ms > 0.0).then(|| tokens as f64 * 1000.0 / ms)
120}
121#[derive(Debug, Serialize)]
122pub struct EmbedResult {
123 pub vectors: Vec<Vec<f64>>,
124 pub dimension: usize,
125}
126fn model_id(value: &str) -> Result<(), HttpError> {
127 if value.is_empty()
128 || value.len() > 8192
129 || !value.as_bytes()[0].is_ascii_lowercase() && !value.as_bytes()[0].is_ascii_digit()
130 || !value.bytes().all(|byte| {
131 byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
132 })
133 || value.split('/').any(|part| part == "..")
134 {
135 return Err(HttpError::Invalid);
136 }
137 Ok(())
138}
139fn chat_body(input: &ChatInput, stream: bool) -> Result<Value, HttpError> {
140 model_id(&input.model)?;
141 if input.messages.len() > 10000
142 || input.tools.len() > 1024
143 || input.temperature.is_some_and(|value| !value.is_finite())
144 {
145 return Err(HttpError::Invalid);
146 }
147 let mut messages = Vec::new();
148 for message in &input.messages {
149 if !["system", "user", "assistant", "tool"].contains(&message.role.as_str()) {
150 return Err(HttpError::Invalid);
151 }
152 let mut value = json!({"role":message.role,"content":message.content});
153 if !message.tool_calls.is_empty() {
154 value["tool_calls"] = json!(
155 message
156 .tool_calls
157 .iter()
158 .map(|call| json!({"function":{"name":call.name,"arguments":call.arguments}}))
159 .collect::<Vec<_>>()
160 );
161 }
162 if let Some(name) = &message.tool_name {
163 value["tool_name"] = json!(name);
164 }
165 messages.push(value);
166 }
167 let mut body = json!({"model":input.model,"messages":messages,"stream":stream});
168 if !input.tools.is_empty() {
169 body["tools"] = json!(
170 input
171 .tools
172 .iter()
173 .map(|tool| json!({"type":"function","function":tool}))
174 .collect::<Vec<_>>()
175 );
176 }
177 let mut options = serde_json::Map::new();
178 if let Some(temperature) = input.temperature {
179 options.insert("temperature".into(), json!(temperature));
180 }
181 if let Some(max_tokens) = input.max_tokens {
182 options.insert("num_predict".into(), json!(max_tokens));
183 }
184 if !options.is_empty() {
185 body["options"] = Value::Object(options);
186 }
187 Ok(body)
188}
189#[derive(Deserialize)]
191struct NativeFunction {
192 name: String,
193 #[serde(default)]
194 arguments: BTreeMap<String, Value>,
195}
196#[derive(Deserialize)]
197struct NativeCall {
198 function: NativeFunction,
199}
200impl From<NativeCall> for ToolCall {
201 fn from(call: NativeCall) -> Self {
202 Self {
203 name: call.function.name,
204 arguments: call.function.arguments,
205 }
206 }
207}
208#[derive(Deserialize)]
209struct NativeMessage {
210 #[serde(default)]
211 content: String,
212 #[serde(default)]
213 tool_calls: Vec<NativeCall>,
214}
215#[derive(Deserialize)]
216struct NativeChunk {
217 message: Option<NativeMessage>,
218 #[serde(rename = "done")]
219 _done: Option<bool>,
220}
221fn decode_result(value: Value) -> Result<ChatResult, HttpError> {
222 crate::usage::record(&value, crate::usage::Provider::Ollama);
223 #[derive(Deserialize)]
224 struct Reply {
225 message: NativeMessage,
226 }
227 if value.get("error").is_some()
228 || !value
229 .get("message")
230 .and_then(|message| message.get("content"))
231 .is_some_and(Value::is_string)
232 {
233 return Err(HttpError::Response);
234 }
235 let response: Reply = serde_json::from_value(value).map_err(|_| HttpError::Response)?;
236 Ok(ChatResult {
237 content: response.message.content,
238 tool_calls: response
239 .message
240 .tool_calls
241 .into_iter()
242 .map(ToolCall::from)
243 .collect(),
244 timings: None,
245 })
246}
247pub struct OllamaInference<'runtime> {
248 transport: &'runtime dyn Transport,
249 probe: &'runtime dyn ProcessProbe,
250 executable: &'runtime str,
251}
252impl<'runtime> OllamaInference<'runtime> {
253 pub fn new(
254 transport: &'runtime dyn Transport,
255 probe: &'runtime dyn ProcessProbe,
256 executable: &'runtime str,
257 ) -> Self {
258 Self {
259 transport,
260 probe,
261 executable,
262 }
263 }
264 async fn check(
265 &self,
266 endpoint: &str,
267 expected: &ProcessIdentity,
268 cancel: &CancellationToken,
269 ) -> Result<(), HttpError> {
270 let url = crate::state::loopback(endpoint).map_err(|_| HttpError::Invalid)?;
271 if expected.pid == 0
272 || expected.executable != self.executable
273 || expected.started.is_empty()
274 || !std::path::Path::new(self.executable).is_absolute()
275 {
276 return Err(HttpError::Invalid);
277 }
278 let port = url.port_or_known_default().ok_or(HttpError::Invalid)?;
279 let host = url.host_str().ok_or(HttpError::Invalid)?;
280 let listener = crate::process_control::listener(self.probe, endpoint, cancel)
281 .await
282 .map_err(|_| {
283 if cancel.is_cancelled() {
284 HttpError::Cancelled
285 } else {
286 HttpError::Invalid
287 }
288 })?;
289 crate::identity::choose_listener(std::slice::from_ref(&listener), port, host)
290 .map_err(|_| HttpError::Invalid)?;
291 if listener.identity.pid != expected.pid
292 || listener.identity.executable != expected.executable
293 || listener.identity.started != expected.started
294 {
295 return Err(HttpError::Invalid);
296 }
297 Ok(())
298 }
299 async fn trusted(
300 &self,
301 endpoint: &str,
302 expected: &ProcessIdentity,
303 cancel: &CancellationToken,
304 ) -> Result<(), HttpError> {
305 if cancel.is_cancelled() {
306 return Err(HttpError::Cancelled);
307 }
308 self.check(endpoint, expected, cancel).await?;
309 let version = read_json(
310 self.transport,
311 Request::new(endpoint, "/api/version", None, None)?,
312 cancel,
313 4096,
314 )
315 .await?;
316 if !version
317 .get("version")
318 .and_then(Value::as_str)
319 .is_some_and(|value| {
320 !value.is_empty()
321 && value.len() <= 100
322 && value.bytes().all(|byte| !byte.is_ascii_control())
323 })
324 {
325 return Err(HttpError::Response);
326 }
327 self.check(endpoint, expected, cancel).await
328 }
329 pub async fn chat(
330 &self,
331 endpoint: &str,
332 expected: &ProcessIdentity,
333 input: &ChatInput,
334 cancel: &CancellationToken,
335 ) -> Result<ChatResult, HttpError> {
336 let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, false)?), None)?;
337 crate::usage::begin();
338 self.trusted(endpoint, expected, cancel).await?;
339 let result =
340 decode_result(read_json(self.transport, request, cancel, 16 * 1024 * 1024).await?)?;
341 self.check(endpoint, expected, cancel).await?;
342 Ok(result)
343 }
344 pub async fn chat_stream(
345 &self,
346 endpoint: &str,
347 expected: &ProcessIdentity,
348 input: &ChatInput,
349 cancel: &CancellationToken,
350 on_delta: &mut (dyn FnMut(&str) + Send),
351 ) -> Result<ChatResult, HttpError> {
352 let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, true)?), None)?;
353 crate::usage::begin();
354 self.trusted(endpoint, expected, cancel).await?;
355 let operation = async {
356 let response = self.transport.send(request).await?;
357 if !(200..300).contains(&response.status) {
358 return Err(HttpError::Status(response.status));
359 }
360 let mut reader = BufReader::new(response.body);
361 let mut result = ChatResult {
362 content: String::new(),
363 tool_calls: Vec::new(),
364 timings: None,
365 };
366 let mut total = 0usize;
367 loop {
368 let mut line = Vec::new();
369 let count = (&mut reader)
370 .take(65537)
371 .read_until(b'\n', &mut line)
372 .await
373 .map_err(|_| HttpError::Transport)?;
374 if count == 0 {
375 break;
376 }
377 total += count;
378 if count > 65536 || total > 16 * 1024 * 1024 {
379 return Err(HttpError::Limit);
380 }
381 let terminated = line.last() == Some(&b'\n');
382 let records: Vec<_> = line
383 .split(|byte| *byte == b'\r' || *byte == b'\n')
384 .collect();
385 for record in records
386 .iter()
387 .take(records.len().saturating_sub(usize::from(!terminated)))
388 {
389 let Ok(chunk) = serde_json::from_slice::<NativeChunk>(record) else {
390 continue;
391 };
392 if let Ok(value) = serde_json::from_slice::<Value>(record) {
393 crate::usage::record(&value, crate::usage::Provider::Ollama);
394 }
395 if let Some(message) = chunk.message {
396 if !message.content.is_empty() {
397 on_delta(&message.content);
398 result.content.push_str(&message.content);
399 }
400 result
401 .tool_calls
402 .extend(message.tool_calls.into_iter().map(ToolCall::from));
403 }
404 }
405 }
406 Ok(result)
407 };
408 let result = tokio::select! {biased;_=cancel.cancelled()=>Err(HttpError::Cancelled),result=tokio::time::timeout(Duration::from_secs(1800),operation)=>result.map_err(|_|HttpError::Timeout)?}?;
409 self.check(endpoint, expected, cancel).await?;
410 Ok(result)
411 }
412 pub async fn embed(
413 &self,
414 endpoint: &str,
415 expected: &ProcessIdentity,
416 model: &str,
417 inputs: &[String],
418 cancel: &CancellationToken,
419 ) -> Result<EmbedResult, HttpError> {
420 model_id(model)?;
421 if inputs.is_empty()
422 || inputs.len() > 1024
423 || inputs
424 .iter()
425 .any(|input| input.is_empty() || input.len() > 1024 * 1024)
426 || inputs.iter().map(String::len).sum::<usize>() > 4 * 1024 * 1024
427 {
428 return Err(HttpError::Invalid);
429 }
430 let request = Request::new(
431 endpoint,
432 "/api/embed",
433 Some(json!({"model":model,"input":inputs})),
434 None,
435 )?;
436 self.trusted(endpoint, expected, cancel).await?;
437 let value = tokio::time::timeout(
438 Duration::from_secs(30),
439 read_json(self.transport, request, cancel, 16 * 1024 * 1024),
440 )
441 .await
442 .map_err(|_| HttpError::Timeout)??;
443 #[derive(Deserialize)]
444 struct Embeddings {
445 embeddings: Vec<Vec<f64>>,
446 }
447 let vectors = serde_json::from_value::<Embeddings>(value)
448 .map_err(|_| HttpError::Response)?
449 .embeddings;
450 let dimension = vectors.first().map(Vec::len).ok_or(HttpError::Response)?;
451 if vectors.len() != inputs.len()
452 || dimension == 0
453 || dimension > 8192
454 || vectors.len() * dimension > 1000000
455 || vectors.iter().any(|vector| {
456 vector.len() != dimension || vector.iter().any(|value| !value.is_finite())
457 })
458 {
459 return Err(HttpError::Response);
460 }
461 self.check(endpoint, expected, cancel).await?;
462 Ok(EmbedResult { vectors, dimension })
463 }
464}
465
466pub async fn preload(
470 transport: &dyn Transport,
471 endpoint: &str,
472 model: &str,
473 cancel: &CancellationToken,
474) -> Result<(), String> {
475 let request = Request::new(
476 endpoint,
477 "/api/generate",
478 Some(serde_json::json!({
479 "model": model,
480 "prompt": "Hi",
481 "stream": false,
482 "options": {"num_predict": 1},
483 "keep_alive": "30s",
484 })),
485 None,
486 )
487 .map_err(|error| error.to_string())?;
488 let operation = async {
489 let response = transport
490 .send(request)
491 .await
492 .map_err(|error| error.to_string())?;
493 let mut bytes = Vec::new();
494 response
495 .body
496 .take(64 * 1024)
497 .read_to_end(&mut bytes)
498 .await
499 .map_err(|_| "unreadable Ollama response".to_owned())?;
500 if (200..300).contains(&response.status) {
501 return Ok(());
502 }
503 let reason = serde_json::from_slice::<Value>(&bytes)
504 .ok()
505 .and_then(|value| {
506 value
507 .get("error")
508 .and_then(Value::as_str)
509 .map(str::to_owned)
510 })
511 .map(|text| {
512 rigspark_core::reports::strip_control(&text)
513 .chars()
514 .take(400)
515 .collect::<String>()
516 })
517 .filter(|text| !text.trim().is_empty())
518 .unwrap_or_else(|| format!("Ollama returned HTTP {}", response.status));
519 Err(reason)
520 };
521 tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(600),operation)=>result.map_err(|_|"loading the model timed out".to_owned())?}
522}
523
524#[cfg(test)]
525mod timing_tests {
526 use super::*;
527
528 #[test]
529 fn llama_cpp_timings_parse_only_when_complete_and_sane() {
530 let full = json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0});
531 let parsed = Timings::from_llama_cpp(&full).unwrap();
532 assert_eq!(parsed.prompt_tokens_per_second(), Some(500.0));
533 assert_eq!(parsed.generated_tokens_per_second(), Some(500.0));
534 assert_eq!(parsed.cached_tokens, None);
535 let idle = json!({"prompt_n": 0, "prompt_ms": 0.0, "predicted_n": 0, "predicted_ms": 0.0});
536 let idle = Timings::from_llama_cpp(&idle).unwrap();
537 assert_eq!(idle.prompt_tokens_per_second(), None);
538 assert_eq!(idle.generated_tokens_per_second(), None);
539 let all_accepted = json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4,
541 "predicted_ms": 8.0, "draft_n": 2, "draft_n_accepted": 2});
542 assert_eq!(
543 Timings::from_llama_cpp(&all_accepted)
544 .unwrap()
545 .draft_accepted,
546 Some(2)
547 );
548 let instant =
549 json!({"prompt_n": 3, "prompt_ms": 0.0, "predicted_n": 0, "predicted_ms": 5.0});
550 let instant = Timings::from_llama_cpp(&instant).unwrap();
551 assert_eq!(instant.prompt_tokens_per_second(), None);
552 assert_eq!(instant.generated_tokens_per_second(), None);
553 for bad in [
554 json!(null),
555 json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4}),
556 json!({"prompt_n": -1, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0}),
557 json!({"prompt_n": 3, "prompt_ms": -6.0, "predicted_n": 4, "predicted_ms": 8.0}),
558 json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0,
559 "draft_n": 2, "draft_n_accepted": 3}),
560 ] {
561 assert_eq!(Timings::from_llama_cpp(&bad), None, "{bad}");
562 }
563 }
564}
565
566#[cfg(test)]
567mod max_tokens_tests {
568 use super::*;
569
570 #[test]
571 fn ollama_gets_max_tokens_as_num_predict_beside_temperature() {
572 let input: ChatInput = serde_json::from_value(json!({
573 "model": "qwen3:8b",
574 "messages": [{"role": "user", "content": "hi"}],
575 "temperature": 0.2,
576 "max_tokens": 64
577 }))
578 .unwrap();
579 let body = chat_body(&input, false).unwrap();
580 assert_eq!(
581 body["options"],
582 json!({"temperature": 0.2, "num_predict": 64})
583 );
584 let plain = ChatInput {
585 temperature: None,
586 max_tokens: None,
587 ..input
588 };
589 assert!(chat_body(&plain, false).unwrap().get("options").is_none());
590 }
591}