Skip to main content

rigspark_runtime/
ollama_inference.rs

1use crate::{
2    http::{HttpError, Request, Transport, read_json},
3    identity::{ProcessIdentity, ProcessProbe},
4};
5use serde::{Deserialize, Serialize};
6use serde_json::{Value, json};
7use std::{collections::BTreeMap, time::Duration};
8use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
9use tokio_util::sync::CancellationToken;
10
11#[derive(Debug, Clone, Serialize, Deserialize)]
12#[serde(rename_all = "camelCase", deny_unknown_fields)]
13pub struct ToolCall {
14    pub name: String,
15    #[serde(default)]
16    pub arguments: BTreeMap<String, Value>,
17}
18#[derive(Debug, Clone, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase", deny_unknown_fields)]
20pub struct ChatMessage {
21    pub role: String,
22    pub content: String,
23    #[serde(default)]
24    pub tool_calls: Vec<ToolCall>,
25    pub tool_name: Option<String>,
26}
27#[derive(Debug, Clone, Serialize, Deserialize)]
28#[serde(deny_unknown_fields)]
29pub struct Tool {
30    pub name: String,
31    pub description: String,
32    pub parameters: BTreeMap<String, Value>,
33}
34#[derive(Debug, Clone, Serialize, Deserialize)]
35#[serde(deny_unknown_fields)]
36pub struct ChatInput {
37    pub model: String,
38    pub messages: Vec<ChatMessage>,
39    #[serde(default)]
40    pub tools: Vec<Tool>,
41    pub temperature: Option<f64>,
42    /// llama.cpp request slot to run in (`id_slot`), so its KV cache can be saved and
43    /// restored later. Other backends ignore it.
44    #[serde(default, skip_serializing_if = "Option::is_none")]
45    pub slot: Option<u32>,
46    /// Most tokens to generate for this reply (llama.cpp `max_tokens`, Ollama `num_predict`).
47    /// `None`: the server's default, which may run to the end of the context.
48    #[serde(default, skip_serializing_if = "Option::is_none")]
49    pub max_tokens: Option<u32>,
50}
51#[derive(Debug, Clone, Serialize)]
52#[serde(rename_all = "camelCase")]
53pub struct ChatResult {
54    pub content: String,
55    #[serde(skip_serializing_if = "Vec::is_empty")]
56    pub tool_calls: Vec<ToolCall>,
57    /// The server's own measurements for this reply (llama.cpp), when it reports them.
58    #[serde(default, skip_serializing_if = "Option::is_none")]
59    pub timings: Option<Timings>,
60}
61/// Prompt processing and generation as measured by the model server for one reply.
62#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
63#[serde(rename_all = "camelCase")]
64pub struct Timings {
65    /// Prompt tokens evaluated for this reply (cached prefix tokens excluded).
66    pub prompt_tokens: u64,
67    pub prompt_ms: f64,
68    /// Tokens generated.
69    pub generated_tokens: u64,
70    pub generated_ms: f64,
71    /// Prompt tokens reused from the KV cache.
72    #[serde(default, skip_serializing_if = "Option::is_none")]
73    pub cached_tokens: Option<u64>,
74    /// Speculative tokens drafted and accepted.
75    #[serde(default, skip_serializing_if = "Option::is_none")]
76    pub draft_tokens: Option<u64>,
77    #[serde(default, skip_serializing_if = "Option::is_none")]
78    pub draft_accepted: Option<u64>,
79}
80impl Timings {
81    /// Parses llama.cpp's `timings` object; `None` when it is absent or malformed, so a
82    /// reply never fails over its measurements.
83    pub fn from_llama_cpp(value: &Value) -> Option<Self> {
84        let timings = value.as_object()?;
85        let count = |key: &str| timings.get(key).and_then(Value::as_u64);
86        let millis = |key: &str| {
87            timings
88                .get(key)
89                .and_then(Value::as_f64)
90                .filter(|ms| ms.is_finite() && *ms >= 0.0)
91        };
92        let draft_tokens = count("draft_n");
93        let draft_accepted = count("draft_n_accepted");
94        if let (Some(drafted), Some(accepted)) = (draft_tokens, draft_accepted)
95            && accepted > drafted
96        {
97            return None;
98        }
99        Some(Self {
100            prompt_tokens: count("prompt_n")?,
101            prompt_ms: millis("prompt_ms")?,
102            generated_tokens: count("predicted_n")?,
103            generated_ms: millis("predicted_ms")?,
104            cached_tokens: count("cache_n"),
105            draft_tokens,
106            draft_accepted,
107        })
108    }
109    /// Prompt tokens per second, when any prompt was processed.
110    pub fn prompt_tokens_per_second(&self) -> Option<f64> {
111        rate(self.prompt_tokens, self.prompt_ms)
112    }
113    /// Generated tokens per second, when anything was generated.
114    pub fn generated_tokens_per_second(&self) -> Option<f64> {
115        rate(self.generated_tokens, self.generated_ms)
116    }
117}
118fn rate(tokens: u64, ms: f64) -> Option<f64> {
119    (tokens > 0 && ms > 0.0).then(|| tokens as f64 * 1000.0 / ms)
120}
121#[derive(Debug, Serialize)]
122pub struct EmbedResult {
123    pub vectors: Vec<Vec<f64>>,
124    pub dimension: usize,
125}
126fn model_id(value: &str) -> Result<(), HttpError> {
127    if value.is_empty()
128        || value.len() > 8192
129        || !value.as_bytes()[0].is_ascii_lowercase() && !value.as_bytes()[0].is_ascii_digit()
130        || !value.bytes().all(|byte| {
131            byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
132        })
133        || value.split('/').any(|part| part == "..")
134    {
135        return Err(HttpError::Invalid);
136    }
137    Ok(())
138}
139fn chat_body(input: &ChatInput, stream: bool) -> Result<Value, HttpError> {
140    model_id(&input.model)?;
141    if input.messages.len() > 10000
142        || input.tools.len() > 1024
143        || input.temperature.is_some_and(|value| !value.is_finite())
144    {
145        return Err(HttpError::Invalid);
146    }
147    let mut messages = Vec::new();
148    for message in &input.messages {
149        if !["system", "user", "assistant", "tool"].contains(&message.role.as_str()) {
150            return Err(HttpError::Invalid);
151        }
152        let mut value = json!({"role":message.role,"content":message.content});
153        if !message.tool_calls.is_empty() {
154            value["tool_calls"] = json!(
155                message
156                    .tool_calls
157                    .iter()
158                    .map(|call| json!({"function":{"name":call.name,"arguments":call.arguments}}))
159                    .collect::<Vec<_>>()
160            );
161        }
162        if let Some(name) = &message.tool_name {
163            value["tool_name"] = json!(name);
164        }
165        messages.push(value);
166    }
167    let mut body = json!({"model":input.model,"messages":messages,"stream":stream});
168    if !input.tools.is_empty() {
169        body["tools"] = json!(
170            input
171                .tools
172                .iter()
173                .map(|tool| json!({"type":"function","function":tool}))
174                .collect::<Vec<_>>()
175        );
176    }
177    let mut options = serde_json::Map::new();
178    if let Some(temperature) = input.temperature {
179        options.insert("temperature".into(), json!(temperature));
180    }
181    if let Some(max_tokens) = input.max_tokens {
182        options.insert("num_predict".into(), json!(max_tokens));
183    }
184    if !options.is_empty() {
185        body["options"] = Value::Object(options);
186    }
187    Ok(body)
188}
189/// Ollama's wire shape: newer releases add `id` and `function.index`, which are ignored.
190#[derive(Deserialize)]
191struct NativeFunction {
192    name: String,
193    #[serde(default)]
194    arguments: BTreeMap<String, Value>,
195}
196#[derive(Deserialize)]
197struct NativeCall {
198    function: NativeFunction,
199}
200impl From<NativeCall> for ToolCall {
201    fn from(call: NativeCall) -> Self {
202        Self {
203            name: call.function.name,
204            arguments: call.function.arguments,
205        }
206    }
207}
208#[derive(Deserialize)]
209struct NativeMessage {
210    #[serde(default)]
211    content: String,
212    #[serde(default)]
213    tool_calls: Vec<NativeCall>,
214}
215#[derive(Deserialize)]
216struct NativeChunk {
217    message: Option<NativeMessage>,
218    #[serde(rename = "done")]
219    _done: Option<bool>,
220}
221fn decode_result(value: Value) -> Result<ChatResult, HttpError> {
222    crate::usage::record(&value, crate::usage::Provider::Ollama);
223    #[derive(Deserialize)]
224    struct Reply {
225        message: NativeMessage,
226    }
227    if value.get("error").is_some()
228        || !value
229            .get("message")
230            .and_then(|message| message.get("content"))
231            .is_some_and(Value::is_string)
232    {
233        return Err(HttpError::Response);
234    }
235    let response: Reply = serde_json::from_value(value).map_err(|_| HttpError::Response)?;
236    Ok(ChatResult {
237        content: response.message.content,
238        tool_calls: response
239            .message
240            .tool_calls
241            .into_iter()
242            .map(ToolCall::from)
243            .collect(),
244        timings: None,
245    })
246}
247pub struct OllamaInference<'runtime> {
248    transport: &'runtime dyn Transport,
249    probe: &'runtime dyn ProcessProbe,
250    executable: &'runtime str,
251}
252impl<'runtime> OllamaInference<'runtime> {
253    pub fn new(
254        transport: &'runtime dyn Transport,
255        probe: &'runtime dyn ProcessProbe,
256        executable: &'runtime str,
257    ) -> Self {
258        Self {
259            transport,
260            probe,
261            executable,
262        }
263    }
264    async fn check(
265        &self,
266        endpoint: &str,
267        expected: &ProcessIdentity,
268        cancel: &CancellationToken,
269    ) -> Result<(), HttpError> {
270        let url = crate::state::loopback(endpoint).map_err(|_| HttpError::Invalid)?;
271        if expected.pid == 0
272            || expected.executable != self.executable
273            || expected.started.is_empty()
274            || !std::path::Path::new(self.executable).is_absolute()
275        {
276            return Err(HttpError::Invalid);
277        }
278        let port = url.port_or_known_default().ok_or(HttpError::Invalid)?;
279        let host = url.host_str().ok_or(HttpError::Invalid)?;
280        let listener = crate::process_control::listener(self.probe, endpoint, cancel)
281            .await
282            .map_err(|_| {
283                if cancel.is_cancelled() {
284                    HttpError::Cancelled
285                } else {
286                    HttpError::Invalid
287                }
288            })?;
289        crate::identity::choose_listener(std::slice::from_ref(&listener), port, host)
290            .map_err(|_| HttpError::Invalid)?;
291        if listener.identity.pid != expected.pid
292            || listener.identity.executable != expected.executable
293            || listener.identity.started != expected.started
294        {
295            return Err(HttpError::Invalid);
296        }
297        Ok(())
298    }
299    async fn trusted(
300        &self,
301        endpoint: &str,
302        expected: &ProcessIdentity,
303        cancel: &CancellationToken,
304    ) -> Result<(), HttpError> {
305        if cancel.is_cancelled() {
306            return Err(HttpError::Cancelled);
307        }
308        self.check(endpoint, expected, cancel).await?;
309        let version = read_json(
310            self.transport,
311            Request::new(endpoint, "/api/version", None, None)?,
312            cancel,
313            4096,
314        )
315        .await?;
316        if !version
317            .get("version")
318            .and_then(Value::as_str)
319            .is_some_and(|value| {
320                !value.is_empty()
321                    && value.len() <= 100
322                    && value.bytes().all(|byte| !byte.is_ascii_control())
323            })
324        {
325            return Err(HttpError::Response);
326        }
327        self.check(endpoint, expected, cancel).await
328    }
329    pub async fn chat(
330        &self,
331        endpoint: &str,
332        expected: &ProcessIdentity,
333        input: &ChatInput,
334        cancel: &CancellationToken,
335    ) -> Result<ChatResult, HttpError> {
336        let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, false)?), None)?;
337        crate::usage::begin();
338        self.trusted(endpoint, expected, cancel).await?;
339        let result =
340            decode_result(read_json(self.transport, request, cancel, 16 * 1024 * 1024).await?)?;
341        self.check(endpoint, expected, cancel).await?;
342        Ok(result)
343    }
344    pub async fn chat_stream(
345        &self,
346        endpoint: &str,
347        expected: &ProcessIdentity,
348        input: &ChatInput,
349        cancel: &CancellationToken,
350        on_delta: &mut (dyn FnMut(&str) + Send),
351    ) -> Result<ChatResult, HttpError> {
352        let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, true)?), None)?;
353        crate::usage::begin();
354        self.trusted(endpoint, expected, cancel).await?;
355        let operation = async {
356            let response = self.transport.send(request).await?;
357            if !(200..300).contains(&response.status) {
358                return Err(HttpError::Status(response.status));
359            }
360            let mut reader = BufReader::new(response.body);
361            let mut result = ChatResult {
362                content: String::new(),
363                tool_calls: Vec::new(),
364                timings: None,
365            };
366            let mut total = 0usize;
367            loop {
368                let mut line = Vec::new();
369                let count = (&mut reader)
370                    .take(65537)
371                    .read_until(b'\n', &mut line)
372                    .await
373                    .map_err(|_| HttpError::Transport)?;
374                if count == 0 {
375                    break;
376                }
377                total += count;
378                if count > 65536 || total > 16 * 1024 * 1024 {
379                    return Err(HttpError::Limit);
380                }
381                let terminated = line.last() == Some(&b'\n');
382                let records: Vec<_> = line
383                    .split(|byte| *byte == b'\r' || *byte == b'\n')
384                    .collect();
385                for record in records
386                    .iter()
387                    .take(records.len().saturating_sub(usize::from(!terminated)))
388                {
389                    let Ok(chunk) = serde_json::from_slice::<NativeChunk>(record) else {
390                        continue;
391                    };
392                    if let Ok(value) = serde_json::from_slice::<Value>(record) {
393                        crate::usage::record(&value, crate::usage::Provider::Ollama);
394                    }
395                    if let Some(message) = chunk.message {
396                        if !message.content.is_empty() {
397                            on_delta(&message.content);
398                            result.content.push_str(&message.content);
399                        }
400                        result
401                            .tool_calls
402                            .extend(message.tool_calls.into_iter().map(ToolCall::from));
403                    }
404                }
405            }
406            Ok(result)
407        };
408        let result = tokio::select! {biased;_=cancel.cancelled()=>Err(HttpError::Cancelled),result=tokio::time::timeout(Duration::from_secs(1800),operation)=>result.map_err(|_|HttpError::Timeout)?}?;
409        self.check(endpoint, expected, cancel).await?;
410        Ok(result)
411    }
412    pub async fn embed(
413        &self,
414        endpoint: &str,
415        expected: &ProcessIdentity,
416        model: &str,
417        inputs: &[String],
418        cancel: &CancellationToken,
419    ) -> Result<EmbedResult, HttpError> {
420        model_id(model)?;
421        if inputs.is_empty()
422            || inputs.len() > 1024
423            || inputs
424                .iter()
425                .any(|input| input.is_empty() || input.len() > 1024 * 1024)
426            || inputs.iter().map(String::len).sum::<usize>() > 4 * 1024 * 1024
427        {
428            return Err(HttpError::Invalid);
429        }
430        let request = Request::new(
431            endpoint,
432            "/api/embed",
433            Some(json!({"model":model,"input":inputs})),
434            None,
435        )?;
436        self.trusted(endpoint, expected, cancel).await?;
437        let value = tokio::time::timeout(
438            Duration::from_secs(30),
439            read_json(self.transport, request, cancel, 16 * 1024 * 1024),
440        )
441        .await
442        .map_err(|_| HttpError::Timeout)??;
443        #[derive(Deserialize)]
444        struct Embeddings {
445            embeddings: Vec<Vec<f64>>,
446        }
447        let vectors = serde_json::from_value::<Embeddings>(value)
448            .map_err(|_| HttpError::Response)?
449            .embeddings;
450        let dimension = vectors.first().map(Vec::len).ok_or(HttpError::Response)?;
451        if vectors.len() != inputs.len()
452            || dimension == 0
453            || dimension > 8192
454            || vectors.len() * dimension > 1000000
455            || vectors.iter().any(|vector| {
456                vector.len() != dimension || vector.iter().any(|value| !value.is_finite())
457            })
458        {
459            return Err(HttpError::Response);
460        }
461        self.check(endpoint, expected, cancel).await?;
462        Ok(EmbedResult { vectors, dimension })
463    }
464}
465
466/// Loads `model` in Ollama with a one-token generation so a runtime that cannot run it
467/// (for example an unknown architecture) fails now, with Ollama's reason, rather than on
468/// the first chat. The reason is sanitized and bounded.
469pub async fn preload(
470    transport: &dyn Transport,
471    endpoint: &str,
472    model: &str,
473    cancel: &CancellationToken,
474) -> Result<(), String> {
475    let request = Request::new(
476        endpoint,
477        "/api/generate",
478        Some(serde_json::json!({
479            "model": model,
480            "prompt": "Hi",
481            "stream": false,
482            "options": {"num_predict": 1},
483            "keep_alive": "30s",
484        })),
485        None,
486    )
487    .map_err(|error| error.to_string())?;
488    let operation = async {
489        let response = transport
490            .send(request)
491            .await
492            .map_err(|error| error.to_string())?;
493        let mut bytes = Vec::new();
494        response
495            .body
496            .take(64 * 1024)
497            .read_to_end(&mut bytes)
498            .await
499            .map_err(|_| "unreadable Ollama response".to_owned())?;
500        if (200..300).contains(&response.status) {
501            return Ok(());
502        }
503        let reason = serde_json::from_slice::<Value>(&bytes)
504            .ok()
505            .and_then(|value| {
506                value
507                    .get("error")
508                    .and_then(Value::as_str)
509                    .map(str::to_owned)
510            })
511            .map(|text| {
512                rigspark_core::reports::strip_control(&text)
513                    .chars()
514                    .take(400)
515                    .collect::<String>()
516            })
517            .filter(|text| !text.trim().is_empty())
518            .unwrap_or_else(|| format!("Ollama returned HTTP {}", response.status));
519        Err(reason)
520    };
521    tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(600),operation)=>result.map_err(|_|"loading the model timed out".to_owned())?}
522}
523
524#[cfg(test)]
525mod timing_tests {
526    use super::*;
527
528    #[test]
529    fn llama_cpp_timings_parse_only_when_complete_and_sane() {
530        let full = json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0});
531        let parsed = Timings::from_llama_cpp(&full).unwrap();
532        assert_eq!(parsed.prompt_tokens_per_second(), Some(500.0));
533        assert_eq!(parsed.generated_tokens_per_second(), Some(500.0));
534        assert_eq!(parsed.cached_tokens, None);
535        let idle = json!({"prompt_n": 0, "prompt_ms": 0.0, "predicted_n": 0, "predicted_ms": 0.0});
536        let idle = Timings::from_llama_cpp(&idle).unwrap();
537        assert_eq!(idle.prompt_tokens_per_second(), None);
538        assert_eq!(idle.generated_tokens_per_second(), None);
539        // Every draft accepted is valid; zero time or zero tokens give no rate.
540        let all_accepted = json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4,
541            "predicted_ms": 8.0, "draft_n": 2, "draft_n_accepted": 2});
542        assert_eq!(
543            Timings::from_llama_cpp(&all_accepted)
544                .unwrap()
545                .draft_accepted,
546            Some(2)
547        );
548        let instant =
549            json!({"prompt_n": 3, "prompt_ms": 0.0, "predicted_n": 0, "predicted_ms": 5.0});
550        let instant = Timings::from_llama_cpp(&instant).unwrap();
551        assert_eq!(instant.prompt_tokens_per_second(), None);
552        assert_eq!(instant.generated_tokens_per_second(), None);
553        for bad in [
554            json!(null),
555            json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4}),
556            json!({"prompt_n": -1, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0}),
557            json!({"prompt_n": 3, "prompt_ms": -6.0, "predicted_n": 4, "predicted_ms": 8.0}),
558            json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0,
559                "draft_n": 2, "draft_n_accepted": 3}),
560        ] {
561            assert_eq!(Timings::from_llama_cpp(&bad), None, "{bad}");
562        }
563    }
564}
565
566#[cfg(test)]
567mod max_tokens_tests {
568    use super::*;
569
570    #[test]
571    fn ollama_gets_max_tokens_as_num_predict_beside_temperature() {
572        let input: ChatInput = serde_json::from_value(json!({
573            "model": "qwen3:8b",
574            "messages": [{"role": "user", "content": "hi"}],
575            "temperature": 0.2,
576            "max_tokens": 64
577        }))
578        .unwrap();
579        let body = chat_body(&input, false).unwrap();
580        assert_eq!(
581            body["options"],
582            json!({"temperature": 0.2, "num_predict": 64})
583        );
584        let plain = ChatInput {
585            temperature: None,
586            max_tokens: None,
587            ..input
588        };
589        assert!(chat_body(&plain, false).unwrap().get("options").is_none());
590    }
591}