Skip to main content

rigspark_runtime/
ollama_inference.rs

1use crate::{
2    http::{HttpError, Request, Transport, read_json},
3    identity::{ProcessIdentity, ProcessProbe},
4};
5use serde::{Deserialize, Serialize};
6use serde_json::{Value, json};
7use std::{collections::BTreeMap, time::Duration};
8use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
9use tokio_util::sync::CancellationToken;
10
11#[derive(Debug, Clone, Serialize, Deserialize)]
12#[serde(rename_all = "camelCase", deny_unknown_fields)]
13pub struct ToolCall {
14    pub name: String,
15    #[serde(default)]
16    pub arguments: BTreeMap<String, Value>,
17}
18#[derive(Debug, Clone, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase", deny_unknown_fields)]
20pub struct ChatMessage {
21    pub role: String,
22    pub content: String,
23    #[serde(default)]
24    pub tool_calls: Vec<ToolCall>,
25    pub tool_name: Option<String>,
26}
27#[derive(Debug, Clone, Serialize, Deserialize)]
28#[serde(deny_unknown_fields)]
29pub struct Tool {
30    pub name: String,
31    pub description: String,
32    pub parameters: BTreeMap<String, Value>,
33}
34#[derive(Debug, Clone, Serialize, Deserialize)]
35#[serde(deny_unknown_fields)]
36pub struct ChatInput {
37    pub model: String,
38    pub messages: Vec<ChatMessage>,
39    #[serde(default)]
40    pub tools: Vec<Tool>,
41    pub temperature: Option<f64>,
42    /// llama.cpp request slot to run in (`id_slot`), so its KV cache can be saved and
43    /// restored later. Other backends ignore it.
44    #[serde(default, skip_serializing_if = "Option::is_none")]
45    pub slot: Option<u32>,
46}
47#[derive(Debug, Clone, Serialize)]
48#[serde(rename_all = "camelCase")]
49pub struct ChatResult {
50    pub content: String,
51    #[serde(skip_serializing_if = "Vec::is_empty")]
52    pub tool_calls: Vec<ToolCall>,
53    /// The server's own measurements for this reply (llama.cpp), when it reports them.
54    #[serde(default, skip_serializing_if = "Option::is_none")]
55    pub timings: Option<Timings>,
56}
57/// Prompt processing and generation as measured by the model server for one reply.
58#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
59#[serde(rename_all = "camelCase")]
60pub struct Timings {
61    /// Prompt tokens evaluated for this reply (cached prefix tokens excluded).
62    pub prompt_tokens: u64,
63    pub prompt_ms: f64,
64    /// Tokens generated.
65    pub generated_tokens: u64,
66    pub generated_ms: f64,
67    /// Prompt tokens reused from the KV cache.
68    #[serde(default, skip_serializing_if = "Option::is_none")]
69    pub cached_tokens: Option<u64>,
70    /// Speculative tokens drafted and accepted.
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub draft_tokens: Option<u64>,
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    pub draft_accepted: Option<u64>,
75}
76impl Timings {
77    /// Parses llama.cpp's `timings` object; `None` when it is absent or malformed, so a
78    /// reply never fails over its measurements.
79    pub fn from_llama_cpp(value: &Value) -> Option<Self> {
80        let timings = value.as_object()?;
81        let count = |key: &str| timings.get(key).and_then(Value::as_u64);
82        let millis = |key: &str| {
83            timings
84                .get(key)
85                .and_then(Value::as_f64)
86                .filter(|ms| ms.is_finite() && *ms >= 0.0)
87        };
88        let draft_tokens = count("draft_n");
89        let draft_accepted = count("draft_n_accepted");
90        if let (Some(drafted), Some(accepted)) = (draft_tokens, draft_accepted)
91            && accepted > drafted
92        {
93            return None;
94        }
95        Some(Self {
96            prompt_tokens: count("prompt_n")?,
97            prompt_ms: millis("prompt_ms")?,
98            generated_tokens: count("predicted_n")?,
99            generated_ms: millis("predicted_ms")?,
100            cached_tokens: count("cache_n"),
101            draft_tokens,
102            draft_accepted,
103        })
104    }
105    /// Prompt tokens per second, when any prompt was processed.
106    pub fn prompt_tokens_per_second(&self) -> Option<f64> {
107        rate(self.prompt_tokens, self.prompt_ms)
108    }
109    /// Generated tokens per second, when anything was generated.
110    pub fn generated_tokens_per_second(&self) -> Option<f64> {
111        rate(self.generated_tokens, self.generated_ms)
112    }
113}
114fn rate(tokens: u64, ms: f64) -> Option<f64> {
115    (tokens > 0 && ms > 0.0).then(|| tokens as f64 * 1000.0 / ms)
116}
117#[derive(Debug, Serialize)]
118pub struct EmbedResult {
119    pub vectors: Vec<Vec<f64>>,
120    pub dimension: usize,
121}
122fn model_id(value: &str) -> Result<(), HttpError> {
123    if value.is_empty()
124        || value.len() > 8192
125        || !value.as_bytes()[0].is_ascii_lowercase() && !value.as_bytes()[0].is_ascii_digit()
126        || !value.bytes().all(|byte| {
127            byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
128        })
129        || value.split('/').any(|part| part == "..")
130    {
131        return Err(HttpError::Invalid);
132    }
133    Ok(())
134}
135fn chat_body(input: &ChatInput, stream: bool) -> Result<Value, HttpError> {
136    model_id(&input.model)?;
137    if input.messages.len() > 10000
138        || input.tools.len() > 1024
139        || input.temperature.is_some_and(|value| !value.is_finite())
140    {
141        return Err(HttpError::Invalid);
142    }
143    let mut messages = Vec::new();
144    for message in &input.messages {
145        if !["system", "user", "assistant", "tool"].contains(&message.role.as_str()) {
146            return Err(HttpError::Invalid);
147        }
148        let mut value = json!({"role":message.role,"content":message.content});
149        if !message.tool_calls.is_empty() {
150            value["tool_calls"] = json!(
151                message
152                    .tool_calls
153                    .iter()
154                    .map(|call| json!({"function":{"name":call.name,"arguments":call.arguments}}))
155                    .collect::<Vec<_>>()
156            );
157        }
158        if let Some(name) = &message.tool_name {
159            value["tool_name"] = json!(name);
160        }
161        messages.push(value);
162    }
163    let mut body = json!({"model":input.model,"messages":messages,"stream":stream});
164    if !input.tools.is_empty() {
165        body["tools"] = json!(
166            input
167                .tools
168                .iter()
169                .map(|tool| json!({"type":"function","function":tool}))
170                .collect::<Vec<_>>()
171        );
172    }
173    if let Some(temperature) = input.temperature {
174        body["options"] = json!({"temperature":temperature});
175    }
176    Ok(body)
177}
178/// Ollama's wire shape: newer releases add `id` and `function.index`, which are ignored.
179#[derive(Deserialize)]
180struct NativeFunction {
181    name: String,
182    #[serde(default)]
183    arguments: BTreeMap<String, Value>,
184}
185#[derive(Deserialize)]
186struct NativeCall {
187    function: NativeFunction,
188}
189impl From<NativeCall> for ToolCall {
190    fn from(call: NativeCall) -> Self {
191        Self {
192            name: call.function.name,
193            arguments: call.function.arguments,
194        }
195    }
196}
197#[derive(Deserialize)]
198struct NativeMessage {
199    #[serde(default)]
200    content: String,
201    #[serde(default)]
202    tool_calls: Vec<NativeCall>,
203}
204#[derive(Deserialize)]
205struct NativeChunk {
206    message: Option<NativeMessage>,
207    #[serde(rename = "done")]
208    _done: Option<bool>,
209}
210fn decode_result(value: Value) -> Result<ChatResult, HttpError> {
211    crate::usage::record(&value, crate::usage::Provider::Ollama);
212    #[derive(Deserialize)]
213    struct Reply {
214        message: NativeMessage,
215    }
216    if value.get("error").is_some()
217        || !value
218            .get("message")
219            .and_then(|message| message.get("content"))
220            .is_some_and(Value::is_string)
221    {
222        return Err(HttpError::Response);
223    }
224    let response: Reply = serde_json::from_value(value).map_err(|_| HttpError::Response)?;
225    Ok(ChatResult {
226        content: response.message.content,
227        tool_calls: response
228            .message
229            .tool_calls
230            .into_iter()
231            .map(ToolCall::from)
232            .collect(),
233        timings: None,
234    })
235}
236pub struct OllamaInference<'runtime> {
237    transport: &'runtime dyn Transport,
238    probe: &'runtime dyn ProcessProbe,
239    executable: &'runtime str,
240}
241impl<'runtime> OllamaInference<'runtime> {
242    pub fn new(
243        transport: &'runtime dyn Transport,
244        probe: &'runtime dyn ProcessProbe,
245        executable: &'runtime str,
246    ) -> Self {
247        Self {
248            transport,
249            probe,
250            executable,
251        }
252    }
253    async fn check(
254        &self,
255        endpoint: &str,
256        expected: &ProcessIdentity,
257        cancel: &CancellationToken,
258    ) -> Result<(), HttpError> {
259        let url = crate::state::loopback(endpoint).map_err(|_| HttpError::Invalid)?;
260        if expected.pid == 0
261            || expected.executable != self.executable
262            || expected.started.is_empty()
263            || !std::path::Path::new(self.executable).is_absolute()
264        {
265            return Err(HttpError::Invalid);
266        }
267        let port = url.port_or_known_default().ok_or(HttpError::Invalid)?;
268        let host = url.host_str().ok_or(HttpError::Invalid)?;
269        let listener = crate::process_control::listener(self.probe, endpoint, cancel)
270            .await
271            .map_err(|_| {
272                if cancel.is_cancelled() {
273                    HttpError::Cancelled
274                } else {
275                    HttpError::Invalid
276                }
277            })?;
278        crate::identity::choose_listener(std::slice::from_ref(&listener), port, host)
279            .map_err(|_| HttpError::Invalid)?;
280        if listener.identity.pid != expected.pid
281            || listener.identity.executable != expected.executable
282            || listener.identity.started != expected.started
283        {
284            return Err(HttpError::Invalid);
285        }
286        Ok(())
287    }
288    async fn trusted(
289        &self,
290        endpoint: &str,
291        expected: &ProcessIdentity,
292        cancel: &CancellationToken,
293    ) -> Result<(), HttpError> {
294        if cancel.is_cancelled() {
295            return Err(HttpError::Cancelled);
296        }
297        self.check(endpoint, expected, cancel).await?;
298        let version = read_json(
299            self.transport,
300            Request::new(endpoint, "/api/version", None, None)?,
301            cancel,
302            4096,
303        )
304        .await?;
305        if !version
306            .get("version")
307            .and_then(Value::as_str)
308            .is_some_and(|value| {
309                !value.is_empty()
310                    && value.len() <= 100
311                    && value.bytes().all(|byte| !byte.is_ascii_control())
312            })
313        {
314            return Err(HttpError::Response);
315        }
316        self.check(endpoint, expected, cancel).await
317    }
318    pub async fn chat(
319        &self,
320        endpoint: &str,
321        expected: &ProcessIdentity,
322        input: &ChatInput,
323        cancel: &CancellationToken,
324    ) -> Result<ChatResult, HttpError> {
325        let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, false)?), None)?;
326        crate::usage::begin();
327        self.trusted(endpoint, expected, cancel).await?;
328        let result =
329            decode_result(read_json(self.transport, request, cancel, 16 * 1024 * 1024).await?)?;
330        self.check(endpoint, expected, cancel).await?;
331        Ok(result)
332    }
333    pub async fn chat_stream(
334        &self,
335        endpoint: &str,
336        expected: &ProcessIdentity,
337        input: &ChatInput,
338        cancel: &CancellationToken,
339        on_delta: &mut (dyn FnMut(&str) + Send),
340    ) -> Result<ChatResult, HttpError> {
341        let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, true)?), None)?;
342        crate::usage::begin();
343        self.trusted(endpoint, expected, cancel).await?;
344        let operation = async {
345            let response = self.transport.send(request).await?;
346            if !(200..300).contains(&response.status) {
347                return Err(HttpError::Status(response.status));
348            }
349            let mut reader = BufReader::new(response.body);
350            let mut result = ChatResult {
351                content: String::new(),
352                tool_calls: Vec::new(),
353                timings: None,
354            };
355            let mut total = 0usize;
356            loop {
357                let mut line = Vec::new();
358                let count = (&mut reader)
359                    .take(65537)
360                    .read_until(b'\n', &mut line)
361                    .await
362                    .map_err(|_| HttpError::Transport)?;
363                if count == 0 {
364                    break;
365                }
366                total += count;
367                if count > 65536 || total > 16 * 1024 * 1024 {
368                    return Err(HttpError::Limit);
369                }
370                let terminated = line.last() == Some(&b'\n');
371                let records: Vec<_> = line
372                    .split(|byte| *byte == b'\r' || *byte == b'\n')
373                    .collect();
374                for record in records
375                    .iter()
376                    .take(records.len().saturating_sub(usize::from(!terminated)))
377                {
378                    let Ok(chunk) = serde_json::from_slice::<NativeChunk>(record) else {
379                        continue;
380                    };
381                    if let Ok(value) = serde_json::from_slice::<Value>(record) {
382                        crate::usage::record(&value, crate::usage::Provider::Ollama);
383                    }
384                    if let Some(message) = chunk.message {
385                        if !message.content.is_empty() {
386                            on_delta(&message.content);
387                            result.content.push_str(&message.content);
388                        }
389                        result
390                            .tool_calls
391                            .extend(message.tool_calls.into_iter().map(ToolCall::from));
392                    }
393                }
394            }
395            Ok(result)
396        };
397        let result = tokio::select! {biased;_=cancel.cancelled()=>Err(HttpError::Cancelled),result=tokio::time::timeout(Duration::from_secs(1800),operation)=>result.map_err(|_|HttpError::Timeout)?}?;
398        self.check(endpoint, expected, cancel).await?;
399        Ok(result)
400    }
401    pub async fn embed(
402        &self,
403        endpoint: &str,
404        expected: &ProcessIdentity,
405        model: &str,
406        inputs: &[String],
407        cancel: &CancellationToken,
408    ) -> Result<EmbedResult, HttpError> {
409        model_id(model)?;
410        if inputs.is_empty()
411            || inputs.len() > 1024
412            || inputs
413                .iter()
414                .any(|input| input.is_empty() || input.len() > 1024 * 1024)
415            || inputs.iter().map(String::len).sum::<usize>() > 4 * 1024 * 1024
416        {
417            return Err(HttpError::Invalid);
418        }
419        let request = Request::new(
420            endpoint,
421            "/api/embed",
422            Some(json!({"model":model,"input":inputs})),
423            None,
424        )?;
425        self.trusted(endpoint, expected, cancel).await?;
426        let value = tokio::time::timeout(
427            Duration::from_secs(30),
428            read_json(self.transport, request, cancel, 16 * 1024 * 1024),
429        )
430        .await
431        .map_err(|_| HttpError::Timeout)??;
432        #[derive(Deserialize)]
433        struct Embeddings {
434            embeddings: Vec<Vec<f64>>,
435        }
436        let vectors = serde_json::from_value::<Embeddings>(value)
437            .map_err(|_| HttpError::Response)?
438            .embeddings;
439        let dimension = vectors.first().map(Vec::len).ok_or(HttpError::Response)?;
440        if vectors.len() != inputs.len()
441            || dimension == 0
442            || dimension > 8192
443            || vectors.len() * dimension > 1000000
444            || vectors.iter().any(|vector| {
445                vector.len() != dimension || vector.iter().any(|value| !value.is_finite())
446            })
447        {
448            return Err(HttpError::Response);
449        }
450        self.check(endpoint, expected, cancel).await?;
451        Ok(EmbedResult { vectors, dimension })
452    }
453}
454
455/// Loads `model` in Ollama with a one-token generation so a runtime that cannot run it
456/// (for example an unknown architecture) fails now, with Ollama's reason, rather than on
457/// the first chat. The reason is sanitized and bounded.
458pub async fn preload(
459    transport: &dyn Transport,
460    endpoint: &str,
461    model: &str,
462    cancel: &CancellationToken,
463) -> Result<(), String> {
464    let request = Request::new(
465        endpoint,
466        "/api/generate",
467        Some(serde_json::json!({
468            "model": model,
469            "prompt": "Hi",
470            "stream": false,
471            "options": {"num_predict": 1},
472            "keep_alive": "30s",
473        })),
474        None,
475    )
476    .map_err(|error| error.to_string())?;
477    let operation = async {
478        let response = transport
479            .send(request)
480            .await
481            .map_err(|error| error.to_string())?;
482        let mut bytes = Vec::new();
483        response
484            .body
485            .take(64 * 1024)
486            .read_to_end(&mut bytes)
487            .await
488            .map_err(|_| "unreadable Ollama response".to_owned())?;
489        if (200..300).contains(&response.status) {
490            return Ok(());
491        }
492        let reason = serde_json::from_slice::<Value>(&bytes)
493            .ok()
494            .and_then(|value| {
495                value
496                    .get("error")
497                    .and_then(Value::as_str)
498                    .map(str::to_owned)
499            })
500            .map(|text| {
501                rigspark_core::reports::strip_control(&text)
502                    .chars()
503                    .take(400)
504                    .collect::<String>()
505            })
506            .filter(|text| !text.trim().is_empty())
507            .unwrap_or_else(|| format!("Ollama returned HTTP {}", response.status));
508        Err(reason)
509    };
510    tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(600),operation)=>result.map_err(|_|"loading the model timed out".to_owned())?}
511}
512
513#[cfg(test)]
514mod timing_tests {
515    use super::*;
516
517    #[test]
518    fn llama_cpp_timings_parse_only_when_complete_and_sane() {
519        let full = json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0});
520        let parsed = Timings::from_llama_cpp(&full).unwrap();
521        assert_eq!(parsed.prompt_tokens_per_second(), Some(500.0));
522        assert_eq!(parsed.generated_tokens_per_second(), Some(500.0));
523        assert_eq!(parsed.cached_tokens, None);
524        let idle = json!({"prompt_n": 0, "prompt_ms": 0.0, "predicted_n": 0, "predicted_ms": 0.0});
525        let idle = Timings::from_llama_cpp(&idle).unwrap();
526        assert_eq!(idle.prompt_tokens_per_second(), None);
527        assert_eq!(idle.generated_tokens_per_second(), None);
528        for bad in [
529            json!(null),
530            json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4}),
531            json!({"prompt_n": -1, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0}),
532            json!({"prompt_n": 3, "prompt_ms": -6.0, "predicted_n": 4, "predicted_ms": 8.0}),
533            json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0,
534                "draft_n": 2, "draft_n_accepted": 3}),
535        ] {
536            assert_eq!(Timings::from_llama_cpp(&bad), None, "{bad}");
537        }
538    }
539}