1use crate::{
2 http::{HttpError, Request, Transport, read_json},
3 identity::{ProcessIdentity, ProcessProbe},
4};
5use serde::{Deserialize, Serialize};
6use serde_json::{Value, json};
7use std::{collections::BTreeMap, time::Duration};
8use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
9use tokio_util::sync::CancellationToken;
10
11#[derive(Debug, Clone, Serialize, Deserialize)]
12#[serde(rename_all = "camelCase", deny_unknown_fields)]
13pub struct ToolCall {
14 pub name: String,
15 #[serde(default)]
16 pub arguments: BTreeMap<String, Value>,
17}
18#[derive(Debug, Clone, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase", deny_unknown_fields)]
20pub struct ChatMessage {
21 pub role: String,
22 pub content: String,
23 #[serde(default)]
24 pub tool_calls: Vec<ToolCall>,
25 pub tool_name: Option<String>,
26}
27#[derive(Debug, Clone, Serialize, Deserialize)]
28#[serde(deny_unknown_fields)]
29pub struct Tool {
30 pub name: String,
31 pub description: String,
32 pub parameters: BTreeMap<String, Value>,
33}
34#[derive(Debug, Clone, Serialize, Deserialize)]
35#[serde(deny_unknown_fields)]
36pub struct ChatInput {
37 pub model: String,
38 pub messages: Vec<ChatMessage>,
39 #[serde(default)]
40 pub tools: Vec<Tool>,
41 pub temperature: Option<f64>,
42}
43#[derive(Debug, Clone, Serialize)]
44#[serde(rename_all = "camelCase")]
45pub struct ChatResult {
46 pub content: String,
47 #[serde(skip_serializing_if = "Vec::is_empty")]
48 pub tool_calls: Vec<ToolCall>,
49}
50#[derive(Debug, Serialize)]
51pub struct EmbedResult {
52 pub vectors: Vec<Vec<f64>>,
53 pub dimension: usize,
54}
55fn model_id(value: &str) -> Result<(), HttpError> {
56 if value.is_empty()
57 || value.len() > 8192
58 || !value.as_bytes()[0].is_ascii_lowercase() && !value.as_bytes()[0].is_ascii_digit()
59 || !value.bytes().all(|byte| {
60 byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
61 })
62 || value.split('/').any(|part| part == "..")
63 {
64 return Err(HttpError::Invalid);
65 }
66 Ok(())
67}
68fn chat_body(input: &ChatInput, stream: bool) -> Result<Value, HttpError> {
69 model_id(&input.model)?;
70 if input.messages.len() > 10000
71 || input.tools.len() > 1024
72 || input.temperature.is_some_and(|value| !value.is_finite())
73 {
74 return Err(HttpError::Invalid);
75 }
76 let mut messages = Vec::new();
77 for message in &input.messages {
78 if !["system", "user", "assistant", "tool"].contains(&message.role.as_str()) {
79 return Err(HttpError::Invalid);
80 }
81 let mut value = json!({"role":message.role,"content":message.content});
82 if !message.tool_calls.is_empty() {
83 value["tool_calls"] = json!(
84 message
85 .tool_calls
86 .iter()
87 .map(|call| json!({"function":{"name":call.name,"arguments":call.arguments}}))
88 .collect::<Vec<_>>()
89 );
90 }
91 if let Some(name) = &message.tool_name {
92 value["tool_name"] = json!(name);
93 }
94 messages.push(value);
95 }
96 let mut body = json!({"model":input.model,"messages":messages,"stream":stream});
97 if !input.tools.is_empty() {
98 body["tools"] = json!(
99 input
100 .tools
101 .iter()
102 .map(|tool| json!({"type":"function","function":tool}))
103 .collect::<Vec<_>>()
104 );
105 }
106 if let Some(temperature) = input.temperature {
107 body["options"] = json!({"temperature":temperature});
108 }
109 Ok(body)
110}
111#[derive(Deserialize)]
113struct NativeFunction {
114 name: String,
115 #[serde(default)]
116 arguments: BTreeMap<String, Value>,
117}
118#[derive(Deserialize)]
119struct NativeCall {
120 function: NativeFunction,
121}
122impl From<NativeCall> for ToolCall {
123 fn from(call: NativeCall) -> Self {
124 Self {
125 name: call.function.name,
126 arguments: call.function.arguments,
127 }
128 }
129}
130#[derive(Deserialize)]
131struct NativeMessage {
132 #[serde(default)]
133 content: String,
134 #[serde(default)]
135 tool_calls: Vec<NativeCall>,
136}
137#[derive(Deserialize)]
138struct NativeChunk {
139 message: Option<NativeMessage>,
140 #[serde(rename = "done")]
141 _done: Option<bool>,
142}
143fn decode_result(value: Value) -> Result<ChatResult, HttpError> {
144 crate::usage::record(&value, crate::usage::Provider::Ollama);
145 #[derive(Deserialize)]
146 struct Reply {
147 message: NativeMessage,
148 }
149 if value.get("error").is_some()
150 || !value
151 .get("message")
152 .and_then(|message| message.get("content"))
153 .is_some_and(Value::is_string)
154 {
155 return Err(HttpError::Response);
156 }
157 let response: Reply = serde_json::from_value(value).map_err(|_| HttpError::Response)?;
158 Ok(ChatResult {
159 content: response.message.content,
160 tool_calls: response
161 .message
162 .tool_calls
163 .into_iter()
164 .map(ToolCall::from)
165 .collect(),
166 })
167}
168pub struct OllamaInference<'runtime> {
169 transport: &'runtime dyn Transport,
170 probe: &'runtime dyn ProcessProbe,
171 executable: &'runtime str,
172}
173impl<'runtime> OllamaInference<'runtime> {
174 pub fn new(
175 transport: &'runtime dyn Transport,
176 probe: &'runtime dyn ProcessProbe,
177 executable: &'runtime str,
178 ) -> Self {
179 Self {
180 transport,
181 probe,
182 executable,
183 }
184 }
185 async fn check(
186 &self,
187 endpoint: &str,
188 expected: &ProcessIdentity,
189 cancel: &CancellationToken,
190 ) -> Result<(), HttpError> {
191 let url = crate::state::loopback(endpoint).map_err(|_| HttpError::Invalid)?;
192 if expected.pid == 0
193 || expected.executable != self.executable
194 || expected.started.is_empty()
195 || !std::path::Path::new(self.executable).is_absolute()
196 {
197 return Err(HttpError::Invalid);
198 }
199 let port = url.port_or_known_default().ok_or(HttpError::Invalid)?;
200 let host = url.host_str().ok_or(HttpError::Invalid)?;
201 let listener = crate::process_control::listener(self.probe, endpoint, cancel)
202 .await
203 .map_err(|_| {
204 if cancel.is_cancelled() {
205 HttpError::Cancelled
206 } else {
207 HttpError::Invalid
208 }
209 })?;
210 crate::identity::choose_listener(std::slice::from_ref(&listener), port, host)
211 .map_err(|_| HttpError::Invalid)?;
212 if listener.identity.pid != expected.pid
213 || listener.identity.executable != expected.executable
214 || listener.identity.started != expected.started
215 {
216 return Err(HttpError::Invalid);
217 }
218 Ok(())
219 }
220 async fn trusted(
221 &self,
222 endpoint: &str,
223 expected: &ProcessIdentity,
224 cancel: &CancellationToken,
225 ) -> Result<(), HttpError> {
226 if cancel.is_cancelled() {
227 return Err(HttpError::Cancelled);
228 }
229 self.check(endpoint, expected, cancel).await?;
230 let version = read_json(
231 self.transport,
232 Request::new(endpoint, "/api/version", None, None)?,
233 cancel,
234 4096,
235 )
236 .await?;
237 if !version
238 .get("version")
239 .and_then(Value::as_str)
240 .is_some_and(|value| {
241 !value.is_empty()
242 && value.len() <= 100
243 && value.bytes().all(|byte| !byte.is_ascii_control())
244 })
245 {
246 return Err(HttpError::Response);
247 }
248 self.check(endpoint, expected, cancel).await
249 }
250 pub async fn chat(
251 &self,
252 endpoint: &str,
253 expected: &ProcessIdentity,
254 input: &ChatInput,
255 cancel: &CancellationToken,
256 ) -> Result<ChatResult, HttpError> {
257 let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, false)?), None)?;
258 crate::usage::begin();
259 self.trusted(endpoint, expected, cancel).await?;
260 let result =
261 decode_result(read_json(self.transport, request, cancel, 16 * 1024 * 1024).await?)?;
262 self.check(endpoint, expected, cancel).await?;
263 Ok(result)
264 }
265 pub async fn chat_stream(
266 &self,
267 endpoint: &str,
268 expected: &ProcessIdentity,
269 input: &ChatInput,
270 cancel: &CancellationToken,
271 on_delta: &mut (dyn FnMut(&str) + Send),
272 ) -> Result<ChatResult, HttpError> {
273 let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, true)?), None)?;
274 crate::usage::begin();
275 self.trusted(endpoint, expected, cancel).await?;
276 let operation = async {
277 let response = self.transport.send(request).await?;
278 if !(200..300).contains(&response.status) {
279 return Err(HttpError::Status(response.status));
280 }
281 let mut reader = BufReader::new(response.body);
282 let mut result = ChatResult {
283 content: String::new(),
284 tool_calls: Vec::new(),
285 };
286 let mut total = 0usize;
287 loop {
288 let mut line = Vec::new();
289 let count = (&mut reader)
290 .take(65537)
291 .read_until(b'\n', &mut line)
292 .await
293 .map_err(|_| HttpError::Transport)?;
294 if count == 0 {
295 break;
296 }
297 total += count;
298 if count > 65536 || total > 16 * 1024 * 1024 {
299 return Err(HttpError::Limit);
300 }
301 let terminated = line.last() == Some(&b'\n');
302 let records: Vec<_> = line
303 .split(|byte| *byte == b'\r' || *byte == b'\n')
304 .collect();
305 for record in records
306 .iter()
307 .take(records.len().saturating_sub(usize::from(!terminated)))
308 {
309 let Ok(chunk) = serde_json::from_slice::<NativeChunk>(record) else {
310 continue;
311 };
312 if let Ok(value) = serde_json::from_slice::<Value>(record) {
313 crate::usage::record(&value, crate::usage::Provider::Ollama);
314 }
315 if let Some(message) = chunk.message {
316 if !message.content.is_empty() {
317 on_delta(&message.content);
318 result.content.push_str(&message.content);
319 }
320 result
321 .tool_calls
322 .extend(message.tool_calls.into_iter().map(ToolCall::from));
323 }
324 }
325 }
326 Ok(result)
327 };
328 let result = tokio::select! {biased;_=cancel.cancelled()=>Err(HttpError::Cancelled),result=tokio::time::timeout(Duration::from_secs(1800),operation)=>result.map_err(|_|HttpError::Timeout)?}?;
329 self.check(endpoint, expected, cancel).await?;
330 Ok(result)
331 }
332 pub async fn embed(
333 &self,
334 endpoint: &str,
335 expected: &ProcessIdentity,
336 model: &str,
337 inputs: &[String],
338 cancel: &CancellationToken,
339 ) -> Result<EmbedResult, HttpError> {
340 model_id(model)?;
341 if inputs.is_empty()
342 || inputs.len() > 1024
343 || inputs
344 .iter()
345 .any(|input| input.is_empty() || input.len() > 1024 * 1024)
346 || inputs.iter().map(String::len).sum::<usize>() > 4 * 1024 * 1024
347 {
348 return Err(HttpError::Invalid);
349 }
350 let request = Request::new(
351 endpoint,
352 "/api/embed",
353 Some(json!({"model":model,"input":inputs})),
354 None,
355 )?;
356 self.trusted(endpoint, expected, cancel).await?;
357 let value = tokio::time::timeout(
358 Duration::from_secs(30),
359 read_json(self.transport, request, cancel, 16 * 1024 * 1024),
360 )
361 .await
362 .map_err(|_| HttpError::Timeout)??;
363 #[derive(Deserialize)]
364 struct Embeddings {
365 embeddings: Vec<Vec<f64>>,
366 }
367 let vectors = serde_json::from_value::<Embeddings>(value)
368 .map_err(|_| HttpError::Response)?
369 .embeddings;
370 let dimension = vectors.first().map(Vec::len).ok_or(HttpError::Response)?;
371 if vectors.len() != inputs.len()
372 || dimension == 0
373 || dimension > 8192
374 || vectors.len() * dimension > 1000000
375 || vectors.iter().any(|vector| {
376 vector.len() != dimension || vector.iter().any(|value| !value.is_finite())
377 })
378 {
379 return Err(HttpError::Response);
380 }
381 self.check(endpoint, expected, cancel).await?;
382 Ok(EmbedResult { vectors, dimension })
383 }
384}
385
386pub async fn preload(
390 transport: &dyn Transport,
391 endpoint: &str,
392 model: &str,
393 cancel: &CancellationToken,
394) -> Result<(), String> {
395 let request = Request::new(
396 endpoint,
397 "/api/generate",
398 Some(serde_json::json!({
399 "model": model,
400 "prompt": "Hi",
401 "stream": false,
402 "options": {"num_predict": 1},
403 "keep_alive": "30s",
404 })),
405 None,
406 )
407 .map_err(|error| error.to_string())?;
408 let operation = async {
409 let response = transport
410 .send(request)
411 .await
412 .map_err(|error| error.to_string())?;
413 let mut bytes = Vec::new();
414 response
415 .body
416 .take(64 * 1024)
417 .read_to_end(&mut bytes)
418 .await
419 .map_err(|_| "unreadable Ollama response".to_owned())?;
420 if (200..300).contains(&response.status) {
421 return Ok(());
422 }
423 let reason = serde_json::from_slice::<Value>(&bytes)
424 .ok()
425 .and_then(|value| {
426 value
427 .get("error")
428 .and_then(Value::as_str)
429 .map(str::to_owned)
430 })
431 .map(|text| {
432 rigspark_core::reports::strip_control(&text)
433 .chars()
434 .take(400)
435 .collect::<String>()
436 })
437 .filter(|text| !text.trim().is_empty())
438 .unwrap_or_else(|| format!("Ollama returned HTTP {}", response.status));
439 Err(reason)
440 };
441 tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(600),operation)=>result.map_err(|_|"loading the model timed out".to_owned())?}
442}