1use crate::{
2 http::{HttpError, Request, Transport, read_json},
3 identity::{ProcessIdentity, ProcessProbe},
4};
5use serde::{Deserialize, Serialize};
6use serde_json::{Value, json};
7use std::{collections::BTreeMap, time::Duration};
8use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
9use tokio_util::sync::CancellationToken;
10
11#[derive(Debug, Clone, Serialize, Deserialize)]
12#[serde(rename_all = "camelCase", deny_unknown_fields)]
13pub struct ToolCall {
14 pub name: String,
15 #[serde(default)]
16 pub arguments: BTreeMap<String, Value>,
17}
18#[derive(Debug, Clone, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase", deny_unknown_fields)]
20pub struct ChatMessage {
21 pub role: String,
22 pub content: String,
23 #[serde(default)]
24 pub tool_calls: Vec<ToolCall>,
25 pub tool_name: Option<String>,
26}
27#[derive(Debug, Clone, Serialize, Deserialize)]
28#[serde(deny_unknown_fields)]
29pub struct Tool {
30 pub name: String,
31 pub description: String,
32 pub parameters: BTreeMap<String, Value>,
33}
34#[derive(Debug, Clone, Serialize, Deserialize)]
35#[serde(deny_unknown_fields)]
36pub struct ChatInput {
37 pub model: String,
38 pub messages: Vec<ChatMessage>,
39 #[serde(default)]
40 pub tools: Vec<Tool>,
41 pub temperature: Option<f64>,
42 #[serde(default, skip_serializing_if = "Option::is_none")]
45 pub slot: Option<u32>,
46}
47#[derive(Debug, Clone, Serialize)]
48#[serde(rename_all = "camelCase")]
49pub struct ChatResult {
50 pub content: String,
51 #[serde(skip_serializing_if = "Vec::is_empty")]
52 pub tool_calls: Vec<ToolCall>,
53 #[serde(default, skip_serializing_if = "Option::is_none")]
55 pub timings: Option<Timings>,
56}
57#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
59#[serde(rename_all = "camelCase")]
60pub struct Timings {
61 pub prompt_tokens: u64,
63 pub prompt_ms: f64,
64 pub generated_tokens: u64,
66 pub generated_ms: f64,
67 #[serde(default, skip_serializing_if = "Option::is_none")]
69 pub cached_tokens: Option<u64>,
70 #[serde(default, skip_serializing_if = "Option::is_none")]
72 pub draft_tokens: Option<u64>,
73 #[serde(default, skip_serializing_if = "Option::is_none")]
74 pub draft_accepted: Option<u64>,
75}
76impl Timings {
77 pub fn from_llama_cpp(value: &Value) -> Option<Self> {
80 let timings = value.as_object()?;
81 let count = |key: &str| timings.get(key).and_then(Value::as_u64);
82 let millis = |key: &str| {
83 timings
84 .get(key)
85 .and_then(Value::as_f64)
86 .filter(|ms| ms.is_finite() && *ms >= 0.0)
87 };
88 let draft_tokens = count("draft_n");
89 let draft_accepted = count("draft_n_accepted");
90 if let (Some(drafted), Some(accepted)) = (draft_tokens, draft_accepted)
91 && accepted > drafted
92 {
93 return None;
94 }
95 Some(Self {
96 prompt_tokens: count("prompt_n")?,
97 prompt_ms: millis("prompt_ms")?,
98 generated_tokens: count("predicted_n")?,
99 generated_ms: millis("predicted_ms")?,
100 cached_tokens: count("cache_n"),
101 draft_tokens,
102 draft_accepted,
103 })
104 }
105 pub fn prompt_tokens_per_second(&self) -> Option<f64> {
107 rate(self.prompt_tokens, self.prompt_ms)
108 }
109 pub fn generated_tokens_per_second(&self) -> Option<f64> {
111 rate(self.generated_tokens, self.generated_ms)
112 }
113}
114fn rate(tokens: u64, ms: f64) -> Option<f64> {
115 (tokens > 0 && ms > 0.0).then(|| tokens as f64 * 1000.0 / ms)
116}
117#[derive(Debug, Serialize)]
118pub struct EmbedResult {
119 pub vectors: Vec<Vec<f64>>,
120 pub dimension: usize,
121}
122fn model_id(value: &str) -> Result<(), HttpError> {
123 if value.is_empty()
124 || value.len() > 8192
125 || !value.as_bytes()[0].is_ascii_lowercase() && !value.as_bytes()[0].is_ascii_digit()
126 || !value.bytes().all(|byte| {
127 byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
128 })
129 || value.split('/').any(|part| part == "..")
130 {
131 return Err(HttpError::Invalid);
132 }
133 Ok(())
134}
135fn chat_body(input: &ChatInput, stream: bool) -> Result<Value, HttpError> {
136 model_id(&input.model)?;
137 if input.messages.len() > 10000
138 || input.tools.len() > 1024
139 || input.temperature.is_some_and(|value| !value.is_finite())
140 {
141 return Err(HttpError::Invalid);
142 }
143 let mut messages = Vec::new();
144 for message in &input.messages {
145 if !["system", "user", "assistant", "tool"].contains(&message.role.as_str()) {
146 return Err(HttpError::Invalid);
147 }
148 let mut value = json!({"role":message.role,"content":message.content});
149 if !message.tool_calls.is_empty() {
150 value["tool_calls"] = json!(
151 message
152 .tool_calls
153 .iter()
154 .map(|call| json!({"function":{"name":call.name,"arguments":call.arguments}}))
155 .collect::<Vec<_>>()
156 );
157 }
158 if let Some(name) = &message.tool_name {
159 value["tool_name"] = json!(name);
160 }
161 messages.push(value);
162 }
163 let mut body = json!({"model":input.model,"messages":messages,"stream":stream});
164 if !input.tools.is_empty() {
165 body["tools"] = json!(
166 input
167 .tools
168 .iter()
169 .map(|tool| json!({"type":"function","function":tool}))
170 .collect::<Vec<_>>()
171 );
172 }
173 if let Some(temperature) = input.temperature {
174 body["options"] = json!({"temperature":temperature});
175 }
176 Ok(body)
177}
178#[derive(Deserialize)]
180struct NativeFunction {
181 name: String,
182 #[serde(default)]
183 arguments: BTreeMap<String, Value>,
184}
185#[derive(Deserialize)]
186struct NativeCall {
187 function: NativeFunction,
188}
189impl From<NativeCall> for ToolCall {
190 fn from(call: NativeCall) -> Self {
191 Self {
192 name: call.function.name,
193 arguments: call.function.arguments,
194 }
195 }
196}
197#[derive(Deserialize)]
198struct NativeMessage {
199 #[serde(default)]
200 content: String,
201 #[serde(default)]
202 tool_calls: Vec<NativeCall>,
203}
204#[derive(Deserialize)]
205struct NativeChunk {
206 message: Option<NativeMessage>,
207 #[serde(rename = "done")]
208 _done: Option<bool>,
209}
210fn decode_result(value: Value) -> Result<ChatResult, HttpError> {
211 crate::usage::record(&value, crate::usage::Provider::Ollama);
212 #[derive(Deserialize)]
213 struct Reply {
214 message: NativeMessage,
215 }
216 if value.get("error").is_some()
217 || !value
218 .get("message")
219 .and_then(|message| message.get("content"))
220 .is_some_and(Value::is_string)
221 {
222 return Err(HttpError::Response);
223 }
224 let response: Reply = serde_json::from_value(value).map_err(|_| HttpError::Response)?;
225 Ok(ChatResult {
226 content: response.message.content,
227 tool_calls: response
228 .message
229 .tool_calls
230 .into_iter()
231 .map(ToolCall::from)
232 .collect(),
233 timings: None,
234 })
235}
236pub struct OllamaInference<'runtime> {
237 transport: &'runtime dyn Transport,
238 probe: &'runtime dyn ProcessProbe,
239 executable: &'runtime str,
240}
241impl<'runtime> OllamaInference<'runtime> {
242 pub fn new(
243 transport: &'runtime dyn Transport,
244 probe: &'runtime dyn ProcessProbe,
245 executable: &'runtime str,
246 ) -> Self {
247 Self {
248 transport,
249 probe,
250 executable,
251 }
252 }
253 async fn check(
254 &self,
255 endpoint: &str,
256 expected: &ProcessIdentity,
257 cancel: &CancellationToken,
258 ) -> Result<(), HttpError> {
259 let url = crate::state::loopback(endpoint).map_err(|_| HttpError::Invalid)?;
260 if expected.pid == 0
261 || expected.executable != self.executable
262 || expected.started.is_empty()
263 || !std::path::Path::new(self.executable).is_absolute()
264 {
265 return Err(HttpError::Invalid);
266 }
267 let port = url.port_or_known_default().ok_or(HttpError::Invalid)?;
268 let host = url.host_str().ok_or(HttpError::Invalid)?;
269 let listener = crate::process_control::listener(self.probe, endpoint, cancel)
270 .await
271 .map_err(|_| {
272 if cancel.is_cancelled() {
273 HttpError::Cancelled
274 } else {
275 HttpError::Invalid
276 }
277 })?;
278 crate::identity::choose_listener(std::slice::from_ref(&listener), port, host)
279 .map_err(|_| HttpError::Invalid)?;
280 if listener.identity.pid != expected.pid
281 || listener.identity.executable != expected.executable
282 || listener.identity.started != expected.started
283 {
284 return Err(HttpError::Invalid);
285 }
286 Ok(())
287 }
288 async fn trusted(
289 &self,
290 endpoint: &str,
291 expected: &ProcessIdentity,
292 cancel: &CancellationToken,
293 ) -> Result<(), HttpError> {
294 if cancel.is_cancelled() {
295 return Err(HttpError::Cancelled);
296 }
297 self.check(endpoint, expected, cancel).await?;
298 let version = read_json(
299 self.transport,
300 Request::new(endpoint, "/api/version", None, None)?,
301 cancel,
302 4096,
303 )
304 .await?;
305 if !version
306 .get("version")
307 .and_then(Value::as_str)
308 .is_some_and(|value| {
309 !value.is_empty()
310 && value.len() <= 100
311 && value.bytes().all(|byte| !byte.is_ascii_control())
312 })
313 {
314 return Err(HttpError::Response);
315 }
316 self.check(endpoint, expected, cancel).await
317 }
318 pub async fn chat(
319 &self,
320 endpoint: &str,
321 expected: &ProcessIdentity,
322 input: &ChatInput,
323 cancel: &CancellationToken,
324 ) -> Result<ChatResult, HttpError> {
325 let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, false)?), None)?;
326 crate::usage::begin();
327 self.trusted(endpoint, expected, cancel).await?;
328 let result =
329 decode_result(read_json(self.transport, request, cancel, 16 * 1024 * 1024).await?)?;
330 self.check(endpoint, expected, cancel).await?;
331 Ok(result)
332 }
333 pub async fn chat_stream(
334 &self,
335 endpoint: &str,
336 expected: &ProcessIdentity,
337 input: &ChatInput,
338 cancel: &CancellationToken,
339 on_delta: &mut (dyn FnMut(&str) + Send),
340 ) -> Result<ChatResult, HttpError> {
341 let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, true)?), None)?;
342 crate::usage::begin();
343 self.trusted(endpoint, expected, cancel).await?;
344 let operation = async {
345 let response = self.transport.send(request).await?;
346 if !(200..300).contains(&response.status) {
347 return Err(HttpError::Status(response.status));
348 }
349 let mut reader = BufReader::new(response.body);
350 let mut result = ChatResult {
351 content: String::new(),
352 tool_calls: Vec::new(),
353 timings: None,
354 };
355 let mut total = 0usize;
356 loop {
357 let mut line = Vec::new();
358 let count = (&mut reader)
359 .take(65537)
360 .read_until(b'\n', &mut line)
361 .await
362 .map_err(|_| HttpError::Transport)?;
363 if count == 0 {
364 break;
365 }
366 total += count;
367 if count > 65536 || total > 16 * 1024 * 1024 {
368 return Err(HttpError::Limit);
369 }
370 let terminated = line.last() == Some(&b'\n');
371 let records: Vec<_> = line
372 .split(|byte| *byte == b'\r' || *byte == b'\n')
373 .collect();
374 for record in records
375 .iter()
376 .take(records.len().saturating_sub(usize::from(!terminated)))
377 {
378 let Ok(chunk) = serde_json::from_slice::<NativeChunk>(record) else {
379 continue;
380 };
381 if let Ok(value) = serde_json::from_slice::<Value>(record) {
382 crate::usage::record(&value, crate::usage::Provider::Ollama);
383 }
384 if let Some(message) = chunk.message {
385 if !message.content.is_empty() {
386 on_delta(&message.content);
387 result.content.push_str(&message.content);
388 }
389 result
390 .tool_calls
391 .extend(message.tool_calls.into_iter().map(ToolCall::from));
392 }
393 }
394 }
395 Ok(result)
396 };
397 let result = tokio::select! {biased;_=cancel.cancelled()=>Err(HttpError::Cancelled),result=tokio::time::timeout(Duration::from_secs(1800),operation)=>result.map_err(|_|HttpError::Timeout)?}?;
398 self.check(endpoint, expected, cancel).await?;
399 Ok(result)
400 }
401 pub async fn embed(
402 &self,
403 endpoint: &str,
404 expected: &ProcessIdentity,
405 model: &str,
406 inputs: &[String],
407 cancel: &CancellationToken,
408 ) -> Result<EmbedResult, HttpError> {
409 model_id(model)?;
410 if inputs.is_empty()
411 || inputs.len() > 1024
412 || inputs
413 .iter()
414 .any(|input| input.is_empty() || input.len() > 1024 * 1024)
415 || inputs.iter().map(String::len).sum::<usize>() > 4 * 1024 * 1024
416 {
417 return Err(HttpError::Invalid);
418 }
419 let request = Request::new(
420 endpoint,
421 "/api/embed",
422 Some(json!({"model":model,"input":inputs})),
423 None,
424 )?;
425 self.trusted(endpoint, expected, cancel).await?;
426 let value = tokio::time::timeout(
427 Duration::from_secs(30),
428 read_json(self.transport, request, cancel, 16 * 1024 * 1024),
429 )
430 .await
431 .map_err(|_| HttpError::Timeout)??;
432 #[derive(Deserialize)]
433 struct Embeddings {
434 embeddings: Vec<Vec<f64>>,
435 }
436 let vectors = serde_json::from_value::<Embeddings>(value)
437 .map_err(|_| HttpError::Response)?
438 .embeddings;
439 let dimension = vectors.first().map(Vec::len).ok_or(HttpError::Response)?;
440 if vectors.len() != inputs.len()
441 || dimension == 0
442 || dimension > 8192
443 || vectors.len() * dimension > 1000000
444 || vectors.iter().any(|vector| {
445 vector.len() != dimension || vector.iter().any(|value| !value.is_finite())
446 })
447 {
448 return Err(HttpError::Response);
449 }
450 self.check(endpoint, expected, cancel).await?;
451 Ok(EmbedResult { vectors, dimension })
452 }
453}
454
455pub async fn preload(
459 transport: &dyn Transport,
460 endpoint: &str,
461 model: &str,
462 cancel: &CancellationToken,
463) -> Result<(), String> {
464 let request = Request::new(
465 endpoint,
466 "/api/generate",
467 Some(serde_json::json!({
468 "model": model,
469 "prompt": "Hi",
470 "stream": false,
471 "options": {"num_predict": 1},
472 "keep_alive": "30s",
473 })),
474 None,
475 )
476 .map_err(|error| error.to_string())?;
477 let operation = async {
478 let response = transport
479 .send(request)
480 .await
481 .map_err(|error| error.to_string())?;
482 let mut bytes = Vec::new();
483 response
484 .body
485 .take(64 * 1024)
486 .read_to_end(&mut bytes)
487 .await
488 .map_err(|_| "unreadable Ollama response".to_owned())?;
489 if (200..300).contains(&response.status) {
490 return Ok(());
491 }
492 let reason = serde_json::from_slice::<Value>(&bytes)
493 .ok()
494 .and_then(|value| {
495 value
496 .get("error")
497 .and_then(Value::as_str)
498 .map(str::to_owned)
499 })
500 .map(|text| {
501 rigspark_core::reports::strip_control(&text)
502 .chars()
503 .take(400)
504 .collect::<String>()
505 })
506 .filter(|text| !text.trim().is_empty())
507 .unwrap_or_else(|| format!("Ollama returned HTTP {}", response.status));
508 Err(reason)
509 };
510 tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(600),operation)=>result.map_err(|_|"loading the model timed out".to_owned())?}
511}
512
513#[cfg(test)]
514mod timing_tests {
515 use super::*;
516
517 #[test]
518 fn llama_cpp_timings_parse_only_when_complete_and_sane() {
519 let full = json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0});
520 let parsed = Timings::from_llama_cpp(&full).unwrap();
521 assert_eq!(parsed.prompt_tokens_per_second(), Some(500.0));
522 assert_eq!(parsed.generated_tokens_per_second(), Some(500.0));
523 assert_eq!(parsed.cached_tokens, None);
524 let idle = json!({"prompt_n": 0, "prompt_ms": 0.0, "predicted_n": 0, "predicted_ms": 0.0});
525 let idle = Timings::from_llama_cpp(&idle).unwrap();
526 assert_eq!(idle.prompt_tokens_per_second(), None);
527 assert_eq!(idle.generated_tokens_per_second(), None);
528 for bad in [
529 json!(null),
530 json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4}),
531 json!({"prompt_n": -1, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0}),
532 json!({"prompt_n": 3, "prompt_ms": -6.0, "predicted_n": 4, "predicted_ms": 8.0}),
533 json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0,
534 "draft_n": 2, "draft_n_accepted": 3}),
535 ] {
536 assert_eq!(Timings::from_llama_cpp(&bad), None, "{bad}");
537 }
538 }
539}