Skip to main content

rigspark_runtime/
ollama_inference.rs

1use crate::{
2    http::{HttpError, Request, Transport, read_json},
3    identity::{ProcessIdentity, ProcessProbe},
4};
5use serde::{Deserialize, Serialize};
6use serde_json::{Value, json};
7use std::{collections::BTreeMap, time::Duration};
8use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
9use tokio_util::sync::CancellationToken;
10
11#[derive(Debug, Clone, Serialize, Deserialize)]
12#[serde(rename_all = "camelCase", deny_unknown_fields)]
13pub struct ToolCall {
14    pub name: String,
15    #[serde(default)]
16    pub arguments: BTreeMap<String, Value>,
17}
18#[derive(Debug, Clone, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase", deny_unknown_fields)]
20pub struct ChatMessage {
21    pub role: String,
22    pub content: String,
23    #[serde(default)]
24    pub tool_calls: Vec<ToolCall>,
25    pub tool_name: Option<String>,
26}
27#[derive(Debug, Clone, Serialize, Deserialize)]
28#[serde(deny_unknown_fields)]
29pub struct Tool {
30    pub name: String,
31    pub description: String,
32    pub parameters: BTreeMap<String, Value>,
33}
34#[derive(Debug, Clone, Serialize, Deserialize)]
35#[serde(deny_unknown_fields)]
36pub struct ChatInput {
37    pub model: String,
38    pub messages: Vec<ChatMessage>,
39    #[serde(default)]
40    pub tools: Vec<Tool>,
41    pub temperature: Option<f64>,
42}
43#[derive(Debug, Clone, Serialize)]
44#[serde(rename_all = "camelCase")]
45pub struct ChatResult {
46    pub content: String,
47    #[serde(skip_serializing_if = "Vec::is_empty")]
48    pub tool_calls: Vec<ToolCall>,
49}
50#[derive(Debug, Serialize)]
51pub struct EmbedResult {
52    pub vectors: Vec<Vec<f64>>,
53    pub dimension: usize,
54}
55fn model_id(value: &str) -> Result<(), HttpError> {
56    if value.is_empty()
57        || value.len() > 8192
58        || !value.as_bytes()[0].is_ascii_lowercase() && !value.as_bytes()[0].is_ascii_digit()
59        || !value.bytes().all(|byte| {
60            byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
61        })
62        || value.split('/').any(|part| part == "..")
63    {
64        return Err(HttpError::Invalid);
65    }
66    Ok(())
67}
68fn chat_body(input: &ChatInput, stream: bool) -> Result<Value, HttpError> {
69    model_id(&input.model)?;
70    if input.messages.len() > 10000
71        || input.tools.len() > 1024
72        || input.temperature.is_some_and(|value| !value.is_finite())
73    {
74        return Err(HttpError::Invalid);
75    }
76    let mut messages = Vec::new();
77    for message in &input.messages {
78        if !["system", "user", "assistant", "tool"].contains(&message.role.as_str()) {
79            return Err(HttpError::Invalid);
80        }
81        let mut value = json!({"role":message.role,"content":message.content});
82        if !message.tool_calls.is_empty() {
83            value["tool_calls"] = json!(
84                message
85                    .tool_calls
86                    .iter()
87                    .map(|call| json!({"function":{"name":call.name,"arguments":call.arguments}}))
88                    .collect::<Vec<_>>()
89            );
90        }
91        if let Some(name) = &message.tool_name {
92            value["tool_name"] = json!(name);
93        }
94        messages.push(value);
95    }
96    let mut body = json!({"model":input.model,"messages":messages,"stream":stream});
97    if !input.tools.is_empty() {
98        body["tools"] = json!(
99            input
100                .tools
101                .iter()
102                .map(|tool| json!({"type":"function","function":tool}))
103                .collect::<Vec<_>>()
104        );
105    }
106    if let Some(temperature) = input.temperature {
107        body["options"] = json!({"temperature":temperature});
108    }
109    Ok(body)
110}
111/// Ollama's wire shape: newer releases add `id` and `function.index`, which are ignored.
112#[derive(Deserialize)]
113struct NativeFunction {
114    name: String,
115    #[serde(default)]
116    arguments: BTreeMap<String, Value>,
117}
118#[derive(Deserialize)]
119struct NativeCall {
120    function: NativeFunction,
121}
122impl From<NativeCall> for ToolCall {
123    fn from(call: NativeCall) -> Self {
124        Self {
125            name: call.function.name,
126            arguments: call.function.arguments,
127        }
128    }
129}
130#[derive(Deserialize)]
131struct NativeMessage {
132    #[serde(default)]
133    content: String,
134    #[serde(default)]
135    tool_calls: Vec<NativeCall>,
136}
137#[derive(Deserialize)]
138struct NativeChunk {
139    message: Option<NativeMessage>,
140    #[serde(rename = "done")]
141    _done: Option<bool>,
142}
143fn decode_result(value: Value) -> Result<ChatResult, HttpError> {
144    crate::usage::record(&value, crate::usage::Provider::Ollama);
145    #[derive(Deserialize)]
146    struct Reply {
147        message: NativeMessage,
148    }
149    if value.get("error").is_some()
150        || !value
151            .get("message")
152            .and_then(|message| message.get("content"))
153            .is_some_and(Value::is_string)
154    {
155        return Err(HttpError::Response);
156    }
157    let response: Reply = serde_json::from_value(value).map_err(|_| HttpError::Response)?;
158    Ok(ChatResult {
159        content: response.message.content,
160        tool_calls: response
161            .message
162            .tool_calls
163            .into_iter()
164            .map(ToolCall::from)
165            .collect(),
166    })
167}
168pub struct OllamaInference<'runtime> {
169    transport: &'runtime dyn Transport,
170    probe: &'runtime dyn ProcessProbe,
171    executable: &'runtime str,
172}
173impl<'runtime> OllamaInference<'runtime> {
174    pub fn new(
175        transport: &'runtime dyn Transport,
176        probe: &'runtime dyn ProcessProbe,
177        executable: &'runtime str,
178    ) -> Self {
179        Self {
180            transport,
181            probe,
182            executable,
183        }
184    }
185    async fn check(
186        &self,
187        endpoint: &str,
188        expected: &ProcessIdentity,
189        cancel: &CancellationToken,
190    ) -> Result<(), HttpError> {
191        let url = crate::state::loopback(endpoint).map_err(|_| HttpError::Invalid)?;
192        if expected.pid == 0
193            || expected.executable != self.executable
194            || expected.started.is_empty()
195            || !std::path::Path::new(self.executable).is_absolute()
196        {
197            return Err(HttpError::Invalid);
198        }
199        let port = url.port_or_known_default().ok_or(HttpError::Invalid)?;
200        let host = url.host_str().ok_or(HttpError::Invalid)?;
201        let listener = crate::process_control::listener(self.probe, endpoint, cancel)
202            .await
203            .map_err(|_| {
204                if cancel.is_cancelled() {
205                    HttpError::Cancelled
206                } else {
207                    HttpError::Invalid
208                }
209            })?;
210        crate::identity::choose_listener(std::slice::from_ref(&listener), port, host)
211            .map_err(|_| HttpError::Invalid)?;
212        if listener.identity.pid != expected.pid
213            || listener.identity.executable != expected.executable
214            || listener.identity.started != expected.started
215        {
216            return Err(HttpError::Invalid);
217        }
218        Ok(())
219    }
220    async fn trusted(
221        &self,
222        endpoint: &str,
223        expected: &ProcessIdentity,
224        cancel: &CancellationToken,
225    ) -> Result<(), HttpError> {
226        if cancel.is_cancelled() {
227            return Err(HttpError::Cancelled);
228        }
229        self.check(endpoint, expected, cancel).await?;
230        let version = read_json(
231            self.transport,
232            Request::new(endpoint, "/api/version", None, None)?,
233            cancel,
234            4096,
235        )
236        .await?;
237        if !version
238            .get("version")
239            .and_then(Value::as_str)
240            .is_some_and(|value| {
241                !value.is_empty()
242                    && value.len() <= 100
243                    && value.bytes().all(|byte| !byte.is_ascii_control())
244            })
245        {
246            return Err(HttpError::Response);
247        }
248        self.check(endpoint, expected, cancel).await
249    }
250    pub async fn chat(
251        &self,
252        endpoint: &str,
253        expected: &ProcessIdentity,
254        input: &ChatInput,
255        cancel: &CancellationToken,
256    ) -> Result<ChatResult, HttpError> {
257        let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, false)?), None)?;
258        crate::usage::begin();
259        self.trusted(endpoint, expected, cancel).await?;
260        let result =
261            decode_result(read_json(self.transport, request, cancel, 16 * 1024 * 1024).await?)?;
262        self.check(endpoint, expected, cancel).await?;
263        Ok(result)
264    }
265    pub async fn chat_stream(
266        &self,
267        endpoint: &str,
268        expected: &ProcessIdentity,
269        input: &ChatInput,
270        cancel: &CancellationToken,
271        on_delta: &mut (dyn FnMut(&str) + Send),
272    ) -> Result<ChatResult, HttpError> {
273        let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, true)?), None)?;
274        crate::usage::begin();
275        self.trusted(endpoint, expected, cancel).await?;
276        let operation = async {
277            let response = self.transport.send(request).await?;
278            if !(200..300).contains(&response.status) {
279                return Err(HttpError::Status(response.status));
280            }
281            let mut reader = BufReader::new(response.body);
282            let mut result = ChatResult {
283                content: String::new(),
284                tool_calls: Vec::new(),
285            };
286            let mut total = 0usize;
287            loop {
288                let mut line = Vec::new();
289                let count = (&mut reader)
290                    .take(65537)
291                    .read_until(b'\n', &mut line)
292                    .await
293                    .map_err(|_| HttpError::Transport)?;
294                if count == 0 {
295                    break;
296                }
297                total += count;
298                if count > 65536 || total > 16 * 1024 * 1024 {
299                    return Err(HttpError::Limit);
300                }
301                let terminated = line.last() == Some(&b'\n');
302                let records: Vec<_> = line
303                    .split(|byte| *byte == b'\r' || *byte == b'\n')
304                    .collect();
305                for record in records
306                    .iter()
307                    .take(records.len().saturating_sub(usize::from(!terminated)))
308                {
309                    let Ok(chunk) = serde_json::from_slice::<NativeChunk>(record) else {
310                        continue;
311                    };
312                    if let Ok(value) = serde_json::from_slice::<Value>(record) {
313                        crate::usage::record(&value, crate::usage::Provider::Ollama);
314                    }
315                    if let Some(message) = chunk.message {
316                        if !message.content.is_empty() {
317                            on_delta(&message.content);
318                            result.content.push_str(&message.content);
319                        }
320                        result
321                            .tool_calls
322                            .extend(message.tool_calls.into_iter().map(ToolCall::from));
323                    }
324                }
325            }
326            Ok(result)
327        };
328        let result = tokio::select! {biased;_=cancel.cancelled()=>Err(HttpError::Cancelled),result=tokio::time::timeout(Duration::from_secs(1800),operation)=>result.map_err(|_|HttpError::Timeout)?}?;
329        self.check(endpoint, expected, cancel).await?;
330        Ok(result)
331    }
332    pub async fn embed(
333        &self,
334        endpoint: &str,
335        expected: &ProcessIdentity,
336        model: &str,
337        inputs: &[String],
338        cancel: &CancellationToken,
339    ) -> Result<EmbedResult, HttpError> {
340        model_id(model)?;
341        if inputs.is_empty()
342            || inputs.len() > 1024
343            || inputs
344                .iter()
345                .any(|input| input.is_empty() || input.len() > 1024 * 1024)
346            || inputs.iter().map(String::len).sum::<usize>() > 4 * 1024 * 1024
347        {
348            return Err(HttpError::Invalid);
349        }
350        let request = Request::new(
351            endpoint,
352            "/api/embed",
353            Some(json!({"model":model,"input":inputs})),
354            None,
355        )?;
356        self.trusted(endpoint, expected, cancel).await?;
357        let value = tokio::time::timeout(
358            Duration::from_secs(30),
359            read_json(self.transport, request, cancel, 16 * 1024 * 1024),
360        )
361        .await
362        .map_err(|_| HttpError::Timeout)??;
363        #[derive(Deserialize)]
364        struct Embeddings {
365            embeddings: Vec<Vec<f64>>,
366        }
367        let vectors = serde_json::from_value::<Embeddings>(value)
368            .map_err(|_| HttpError::Response)?
369            .embeddings;
370        let dimension = vectors.first().map(Vec::len).ok_or(HttpError::Response)?;
371        if vectors.len() != inputs.len()
372            || dimension == 0
373            || dimension > 8192
374            || vectors.len() * dimension > 1000000
375            || vectors.iter().any(|vector| {
376                vector.len() != dimension || vector.iter().any(|value| !value.is_finite())
377            })
378        {
379            return Err(HttpError::Response);
380        }
381        self.check(endpoint, expected, cancel).await?;
382        Ok(EmbedResult { vectors, dimension })
383    }
384}