use crate::{
http::{HttpError, Request, Transport, read_json},
identity::{ProcessIdentity, ProcessProbe},
};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::{collections::BTreeMap, time::Duration};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
use tokio_util::sync::CancellationToken;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ToolCall {
pub name: String,
#[serde(default)]
pub arguments: BTreeMap<String, Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ChatMessage {
pub role: String,
pub content: String,
#[serde(default)]
pub tool_calls: Vec<ToolCall>,
pub tool_name: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Tool {
pub name: String,
pub description: String,
pub parameters: BTreeMap<String, Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ChatInput {
pub model: String,
pub messages: Vec<ChatMessage>,
#[serde(default)]
pub tools: Vec<Tool>,
pub temperature: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub slot: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u32>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ChatResult {
pub content: String,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub tool_calls: Vec<ToolCall>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub timings: Option<Timings>,
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Timings {
pub prompt_tokens: u64,
pub prompt_ms: f64,
pub generated_tokens: u64,
pub generated_ms: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cached_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub draft_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub draft_accepted: Option<u64>,
}
impl Timings {
pub fn from_llama_cpp(value: &Value) -> Option<Self> {
let timings = value.as_object()?;
let count = |key: &str| timings.get(key).and_then(Value::as_u64);
let millis = |key: &str| {
timings
.get(key)
.and_then(Value::as_f64)
.filter(|ms| ms.is_finite() && *ms >= 0.0)
};
let draft_tokens = count("draft_n");
let draft_accepted = count("draft_n_accepted");
if let (Some(drafted), Some(accepted)) = (draft_tokens, draft_accepted)
&& accepted > drafted
{
return None;
}
Some(Self {
prompt_tokens: count("prompt_n")?,
prompt_ms: millis("prompt_ms")?,
generated_tokens: count("predicted_n")?,
generated_ms: millis("predicted_ms")?,
cached_tokens: count("cache_n"),
draft_tokens,
draft_accepted,
})
}
pub fn prompt_tokens_per_second(&self) -> Option<f64> {
rate(self.prompt_tokens, self.prompt_ms)
}
pub fn generated_tokens_per_second(&self) -> Option<f64> {
rate(self.generated_tokens, self.generated_ms)
}
}
fn rate(tokens: u64, ms: f64) -> Option<f64> {
(tokens > 0 && ms > 0.0).then(|| tokens as f64 * 1000.0 / ms)
}
#[derive(Debug, Serialize)]
pub struct EmbedResult {
pub vectors: Vec<Vec<f64>>,
pub dimension: usize,
}
fn model_id(value: &str) -> Result<(), HttpError> {
if value.is_empty()
|| value.len() > 8192
|| !value.as_bytes()[0].is_ascii_lowercase() && !value.as_bytes()[0].is_ascii_digit()
|| !value.bytes().all(|byte| {
byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._:/-".contains(&byte)
})
|| value.split('/').any(|part| part == "..")
{
return Err(HttpError::Invalid);
}
Ok(())
}
fn chat_body(input: &ChatInput, stream: bool) -> Result<Value, HttpError> {
model_id(&input.model)?;
if input.messages.len() > 10000
|| input.tools.len() > 1024
|| input.temperature.is_some_and(|value| !value.is_finite())
{
return Err(HttpError::Invalid);
}
let mut messages = Vec::new();
for message in &input.messages {
if !["system", "user", "assistant", "tool"].contains(&message.role.as_str()) {
return Err(HttpError::Invalid);
}
let mut value = json!({"role":message.role,"content":message.content});
if !message.tool_calls.is_empty() {
value["tool_calls"] = json!(
message
.tool_calls
.iter()
.map(|call| json!({"function":{"name":call.name,"arguments":call.arguments}}))
.collect::<Vec<_>>()
);
}
if let Some(name) = &message.tool_name {
value["tool_name"] = json!(name);
}
messages.push(value);
}
let mut body = json!({"model":input.model,"messages":messages,"stream":stream});
if !input.tools.is_empty() {
body["tools"] = json!(
input
.tools
.iter()
.map(|tool| json!({"type":"function","function":tool}))
.collect::<Vec<_>>()
);
}
let mut options = serde_json::Map::new();
if let Some(temperature) = input.temperature {
options.insert("temperature".into(), json!(temperature));
}
if let Some(max_tokens) = input.max_tokens {
options.insert("num_predict".into(), json!(max_tokens));
}
if !options.is_empty() {
body["options"] = Value::Object(options);
}
Ok(body)
}
#[derive(Deserialize)]
struct NativeFunction {
name: String,
#[serde(default)]
arguments: BTreeMap<String, Value>,
}
#[derive(Deserialize)]
struct NativeCall {
function: NativeFunction,
}
impl From<NativeCall> for ToolCall {
fn from(call: NativeCall) -> Self {
Self {
name: call.function.name,
arguments: call.function.arguments,
}
}
}
#[derive(Deserialize)]
struct NativeMessage {
#[serde(default)]
content: String,
#[serde(default)]
tool_calls: Vec<NativeCall>,
}
#[derive(Deserialize)]
struct NativeChunk {
message: Option<NativeMessage>,
#[serde(rename = "done")]
_done: Option<bool>,
}
fn decode_result(value: Value) -> Result<ChatResult, HttpError> {
crate::usage::record(&value, crate::usage::Provider::Ollama);
#[derive(Deserialize)]
struct Reply {
message: NativeMessage,
}
if value.get("error").is_some()
|| !value
.get("message")
.and_then(|message| message.get("content"))
.is_some_and(Value::is_string)
{
return Err(HttpError::Response);
}
let response: Reply = serde_json::from_value(value).map_err(|_| HttpError::Response)?;
Ok(ChatResult {
content: response.message.content,
tool_calls: response
.message
.tool_calls
.into_iter()
.map(ToolCall::from)
.collect(),
timings: None,
})
}
pub struct OllamaInference<'runtime> {
transport: &'runtime dyn Transport,
probe: &'runtime dyn ProcessProbe,
executable: &'runtime str,
}
impl<'runtime> OllamaInference<'runtime> {
pub fn new(
transport: &'runtime dyn Transport,
probe: &'runtime dyn ProcessProbe,
executable: &'runtime str,
) -> Self {
Self {
transport,
probe,
executable,
}
}
async fn check(
&self,
endpoint: &str,
expected: &ProcessIdentity,
cancel: &CancellationToken,
) -> Result<(), HttpError> {
let url = crate::state::loopback(endpoint).map_err(|_| HttpError::Invalid)?;
if expected.pid == 0
|| expected.executable != self.executable
|| expected.started.is_empty()
|| !std::path::Path::new(self.executable).is_absolute()
{
return Err(HttpError::Invalid);
}
let port = url.port_or_known_default().ok_or(HttpError::Invalid)?;
let host = url.host_str().ok_or(HttpError::Invalid)?;
let listener = crate::process_control::listener(self.probe, endpoint, cancel)
.await
.map_err(|_| {
if cancel.is_cancelled() {
HttpError::Cancelled
} else {
HttpError::Invalid
}
})?;
crate::identity::choose_listener(std::slice::from_ref(&listener), port, host)
.map_err(|_| HttpError::Invalid)?;
if listener.identity.pid != expected.pid
|| listener.identity.executable != expected.executable
|| listener.identity.started != expected.started
{
return Err(HttpError::Invalid);
}
Ok(())
}
async fn trusted(
&self,
endpoint: &str,
expected: &ProcessIdentity,
cancel: &CancellationToken,
) -> Result<(), HttpError> {
if cancel.is_cancelled() {
return Err(HttpError::Cancelled);
}
self.check(endpoint, expected, cancel).await?;
let version = read_json(
self.transport,
Request::new(endpoint, "/api/version", None, None)?,
cancel,
4096,
)
.await?;
if !version
.get("version")
.and_then(Value::as_str)
.is_some_and(|value| {
!value.is_empty()
&& value.len() <= 100
&& value.bytes().all(|byte| !byte.is_ascii_control())
})
{
return Err(HttpError::Response);
}
self.check(endpoint, expected, cancel).await
}
pub async fn chat(
&self,
endpoint: &str,
expected: &ProcessIdentity,
input: &ChatInput,
cancel: &CancellationToken,
) -> Result<ChatResult, HttpError> {
let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, false)?), None)?;
crate::usage::begin();
self.trusted(endpoint, expected, cancel).await?;
let result =
decode_result(read_json(self.transport, request, cancel, 16 * 1024 * 1024).await?)?;
self.check(endpoint, expected, cancel).await?;
Ok(result)
}
pub async fn chat_stream(
&self,
endpoint: &str,
expected: &ProcessIdentity,
input: &ChatInput,
cancel: &CancellationToken,
on_delta: &mut (dyn FnMut(&str) + Send),
) -> Result<ChatResult, HttpError> {
let request = Request::new(endpoint, "/api/chat", Some(chat_body(input, true)?), None)?;
crate::usage::begin();
self.trusted(endpoint, expected, cancel).await?;
let operation = async {
let response = self.transport.send(request).await?;
if !(200..300).contains(&response.status) {
return Err(HttpError::Status(response.status));
}
let mut reader = BufReader::new(response.body);
let mut result = ChatResult {
content: String::new(),
tool_calls: Vec::new(),
timings: None,
};
let mut total = 0usize;
loop {
let mut line = Vec::new();
let count = (&mut reader)
.take(65537)
.read_until(b'\n', &mut line)
.await
.map_err(|_| HttpError::Transport)?;
if count == 0 {
break;
}
total += count;
if count > 65536 || total > 16 * 1024 * 1024 {
return Err(HttpError::Limit);
}
let terminated = line.last() == Some(&b'\n');
let records: Vec<_> = line
.split(|byte| *byte == b'\r' || *byte == b'\n')
.collect();
for record in records
.iter()
.take(records.len().saturating_sub(usize::from(!terminated)))
{
let Ok(chunk) = serde_json::from_slice::<NativeChunk>(record) else {
continue;
};
if let Ok(value) = serde_json::from_slice::<Value>(record) {
crate::usage::record(&value, crate::usage::Provider::Ollama);
}
if let Some(message) = chunk.message {
if !message.content.is_empty() {
on_delta(&message.content);
result.content.push_str(&message.content);
}
result
.tool_calls
.extend(message.tool_calls.into_iter().map(ToolCall::from));
}
}
}
Ok(result)
};
let result = tokio::select! {biased;_=cancel.cancelled()=>Err(HttpError::Cancelled),result=tokio::time::timeout(Duration::from_secs(1800),operation)=>result.map_err(|_|HttpError::Timeout)?}?;
self.check(endpoint, expected, cancel).await?;
Ok(result)
}
pub async fn embed(
&self,
endpoint: &str,
expected: &ProcessIdentity,
model: &str,
inputs: &[String],
cancel: &CancellationToken,
) -> Result<EmbedResult, HttpError> {
model_id(model)?;
if inputs.is_empty()
|| inputs.len() > 1024
|| inputs
.iter()
.any(|input| input.is_empty() || input.len() > 1024 * 1024)
|| inputs.iter().map(String::len).sum::<usize>() > 4 * 1024 * 1024
{
return Err(HttpError::Invalid);
}
let request = Request::new(
endpoint,
"/api/embed",
Some(json!({"model":model,"input":inputs})),
None,
)?;
self.trusted(endpoint, expected, cancel).await?;
let value = tokio::time::timeout(
Duration::from_secs(30),
read_json(self.transport, request, cancel, 16 * 1024 * 1024),
)
.await
.map_err(|_| HttpError::Timeout)??;
#[derive(Deserialize)]
struct Embeddings {
embeddings: Vec<Vec<f64>>,
}
let vectors = serde_json::from_value::<Embeddings>(value)
.map_err(|_| HttpError::Response)?
.embeddings;
let dimension = vectors.first().map(Vec::len).ok_or(HttpError::Response)?;
if vectors.len() != inputs.len()
|| dimension == 0
|| dimension > 8192
|| vectors.len() * dimension > 1000000
|| vectors.iter().any(|vector| {
vector.len() != dimension || vector.iter().any(|value| !value.is_finite())
})
{
return Err(HttpError::Response);
}
self.check(endpoint, expected, cancel).await?;
Ok(EmbedResult { vectors, dimension })
}
}
pub async fn preload(
transport: &dyn Transport,
endpoint: &str,
model: &str,
cancel: &CancellationToken,
) -> Result<(), String> {
let request = Request::new(
endpoint,
"/api/generate",
Some(serde_json::json!({
"model": model,
"prompt": "Hi",
"stream": false,
"options": {"num_predict": 1},
"keep_alive": "30s",
})),
None,
)
.map_err(|error| error.to_string())?;
let operation = async {
let response = transport
.send(request)
.await
.map_err(|error| error.to_string())?;
let mut bytes = Vec::new();
response
.body
.take(64 * 1024)
.read_to_end(&mut bytes)
.await
.map_err(|_| "unreadable Ollama response".to_owned())?;
if (200..300).contains(&response.status) {
return Ok(());
}
let reason = serde_json::from_slice::<Value>(&bytes)
.ok()
.and_then(|value| {
value
.get("error")
.and_then(Value::as_str)
.map(str::to_owned)
})
.map(|text| {
rigspark_core::reports::strip_control(&text)
.chars()
.take(400)
.collect::<String>()
})
.filter(|text| !text.trim().is_empty())
.unwrap_or_else(|| format!("Ollama returned HTTP {}", response.status));
Err(reason)
};
tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(600),operation)=>result.map_err(|_|"loading the model timed out".to_owned())?}
}
#[cfg(test)]
mod timing_tests {
use super::*;
#[test]
fn llama_cpp_timings_parse_only_when_complete_and_sane() {
let full = json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0});
let parsed = Timings::from_llama_cpp(&full).unwrap();
assert_eq!(parsed.prompt_tokens_per_second(), Some(500.0));
assert_eq!(parsed.generated_tokens_per_second(), Some(500.0));
assert_eq!(parsed.cached_tokens, None);
let idle = json!({"prompt_n": 0, "prompt_ms": 0.0, "predicted_n": 0, "predicted_ms": 0.0});
let idle = Timings::from_llama_cpp(&idle).unwrap();
assert_eq!(idle.prompt_tokens_per_second(), None);
assert_eq!(idle.generated_tokens_per_second(), None);
let all_accepted = json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4,
"predicted_ms": 8.0, "draft_n": 2, "draft_n_accepted": 2});
assert_eq!(
Timings::from_llama_cpp(&all_accepted)
.unwrap()
.draft_accepted,
Some(2)
);
let instant =
json!({"prompt_n": 3, "prompt_ms": 0.0, "predicted_n": 0, "predicted_ms": 5.0});
let instant = Timings::from_llama_cpp(&instant).unwrap();
assert_eq!(instant.prompt_tokens_per_second(), None);
assert_eq!(instant.generated_tokens_per_second(), None);
for bad in [
json!(null),
json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4}),
json!({"prompt_n": -1, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0}),
json!({"prompt_n": 3, "prompt_ms": -6.0, "predicted_n": 4, "predicted_ms": 8.0}),
json!({"prompt_n": 3, "prompt_ms": 6.0, "predicted_n": 4, "predicted_ms": 8.0,
"draft_n": 2, "draft_n_accepted": 3}),
] {
assert_eq!(Timings::from_llama_cpp(&bad), None, "{bad}");
}
}
}
#[cfg(test)]
mod max_tokens_tests {
use super::*;
#[test]
fn ollama_gets_max_tokens_as_num_predict_beside_temperature() {
let input: ChatInput = serde_json::from_value(json!({
"model": "qwen3:8b",
"messages": [{"role": "user", "content": "hi"}],
"temperature": 0.2,
"max_tokens": 64
}))
.unwrap();
let body = chat_body(&input, false).unwrap();
assert_eq!(
body["options"],
json!({"temperature": 0.2, "num_predict": 64})
);
let plain = ChatInput {
temperature: None,
max_tokens: None,
..input
};
assert!(chat_body(&plain, false).unwrap().get("options").is_none());
}
}