1use crate::{
2 adapters::{BackendAdapter, BackendError, ServeRequest},
3 http::{Request, Transport, read_json},
4 identity::{ProcessProbe, capture},
5 ollama_inference::{ChatInput, ChatMessage, ChatResult, EmbedResult, ToolCall},
6 process_control::listener,
7 state::ServerState,
8};
9use serde::Deserialize;
10use serde_json::{Value, json};
11use std::{collections::BTreeMap, path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13pub fn chat_result(value: Value) -> Result<ChatResult, BackendError> {
14 crate::usage::begin();
15 crate::usage::record(&value, crate::usage::Provider::OpenAi);
16 #[derive(Deserialize)]
17 struct Function {
18 name: String,
19 #[serde(default)]
21 arguments: Value,
22 }
23 #[derive(Deserialize)]
24 struct Call {
25 function: Function,
26 }
27 #[derive(Deserialize)]
28 struct Message {
29 #[serde(default)]
31 content: Option<String>,
32 #[serde(default)]
33 tool_calls: Vec<Call>,
34 }
35 #[derive(Deserialize)]
36 struct Choice {
37 message: Message,
38 }
39 #[derive(Deserialize)]
40 struct Reply {
41 choices: Vec<Choice>,
42 }
43 let timings = value
44 .get("timings")
45 .and_then(crate::ollama_inference::Timings::from_llama_cpp);
46 let reply: Reply =
47 serde_json::from_value(value).map_err(|_| BackendError("invalid chat response".into()))?;
48 if reply.choices.is_empty() || reply.choices.len() > 8 {
49 return Err(BackendError("invalid chat choice count".into()));
50 }
51 let message = reply
52 .choices
53 .into_iter()
54 .next()
55 .ok_or_else(|| BackendError("no chat choice".into()))?
56 .message;
57 let content = message.content.unwrap_or_default();
58 if content.len() > 1048576 {
59 return Err(BackendError("generated content exceeds limit".into()));
60 }
61 if message.tool_calls.len() > 64 {
62 return Err(BackendError("too many tool calls".into()));
63 }
64 let mut tool_calls = Vec::with_capacity(message.tool_calls.len());
65 for call in message.tool_calls {
66 let arguments = match call.function.arguments {
67 Value::String(text) if text.trim().is_empty() => BTreeMap::new(),
68 Value::String(text) => serde_json::from_str(&text)
69 .map_err(|_| BackendError("tool call arguments are not a JSON object".into()))?,
70 Value::Object(map) => map.into_iter().collect(),
71 Value::Null => BTreeMap::new(),
72 _ => {
73 return Err(BackendError(
74 "tool call arguments are not a JSON object".into(),
75 ));
76 }
77 };
78 if call.function.name.is_empty() || call.function.name.len() > 128 {
79 return Err(BackendError("invalid tool call name".into()));
80 }
81 tool_calls.push(ToolCall {
82 name: call.function.name,
83 arguments,
84 });
85 }
86 Ok(ChatResult {
87 content,
88 tool_calls,
89 timings,
90 })
91}
92pub fn embedding_result(value: Value, count: usize) -> Result<EmbedResult, BackendError> {
93 #[derive(Deserialize)]
94 struct Vector {
95 index: usize,
96 embedding: Vec<f64>,
97 }
98 #[derive(Deserialize)]
99 struct Reply {
100 data: Vec<Vector>,
101 }
102 let mut reply: Reply = serde_json::from_value(value)
103 .map_err(|_| BackendError("invalid embedding response".into()))?;
104 if count == 0 || count > 1024 || reply.data.len() != count {
105 return Err(BackendError("embedding count mismatch".into()));
106 }
107 reply.data.sort_by_key(|entry| entry.index);
108 let dimension = reply.data[0].embedding.len();
109 if dimension == 0
110 || dimension > 65536
111 || reply.data.iter().enumerate().any(|(index, entry)| {
112 entry.index != index
113 || entry.embedding.len() != dimension
114 || entry.embedding.iter().any(|value| !value.is_finite())
115 })
116 {
117 return Err(BackendError("inconsistent embedding vectors".into()));
118 }
119 Ok(EmbedResult {
120 vectors: reply
121 .data
122 .into_iter()
123 .map(|entry| entry.embedding)
124 .collect(),
125 dimension,
126 })
127}
128pub struct OpenAiInference<'runtime> {
129 pub http: &'runtime dyn Transport,
130 pub probe: &'runtime dyn ProcessProbe,
131 pub adapter: &'runtime dyn BackendAdapter,
132 pub token: Option<&'runtime str>,
133}
134impl OpenAiInference<'_> {
135 async fn checked(
136 &self,
137 handle: &ServerState,
138 cancel: &CancellationToken,
139 ) -> Result<(), BackendError> {
140 if handle.backend != self.adapter.name() {
141 return Err(BackendError("inference backend mismatch".into()));
142 }
143 let observed = listener(self.probe, &handle.endpoint, cancel).await?;
144 capture(handle, &observed, self.adapter.trusts(&observed.identity))
145 .map_err(|error| BackendError(error.to_string()))?;
146 if handle.backend == "lmstudio" {
147 self.adapter
148 .ready_handle(
149 &ServeRequest {
150 draft_model_path: None,
151 model_id: handle.model_id.clone(),
152 endpoint: handle.endpoint.clone(),
153 model_path: handle.model_path.as_ref().map(PathBuf::from),
154 context: None,
155 cache: None,
156 slot_save_path: None,
157 },
158 handle,
159 cancel,
160 )
161 .await?;
162 }
163 Ok(())
164 }
165 async fn json(
166 &self,
167 handle: &ServerState,
168 path: &str,
169 body: Value,
170 timeout: Duration,
171 cancel: &CancellationToken,
172 ) -> Result<Value, BackendError> {
173 if serde_json::to_vec(&body)
174 .map_err(|error| BackendError(error.to_string()))?
175 .len()
176 > 4 * 1024 * 1024
177 {
178 return Err(BackendError("inference request exceeds limit".into()));
179 }
180 let token = if handle.backend == "mlx" {
181 handle.auth_token.clone()
182 } else {
183 self.token.map(str::to_owned)
184 };
185 let request = Request::new(&handle.endpoint, path, Some(body), token)
186 .map_err(|error| BackendError(error.to_string()))?;
187 tokio::time::timeout(
188 timeout,
189 read_json(self.http, request, cancel, 2 * 1024 * 1024),
190 )
191 .await
192 .map_err(|_| BackendError("inference timed out".into()))?
193 .map_err(|error| BackendError(error.to_string()))
194 }
195 pub async fn chat(
196 &self,
197 handle: &ServerState,
198 input: &ChatInput,
199 cancel: &CancellationToken,
200 ) -> Result<ChatResult, BackendError> {
201 crate::adapters::model_id(&input.model)?;
202 if input.model != handle.model_id
203 || input.messages.len() > 10000
204 || input.messages.iter().any(|message| {
205 !["user", "assistant", "system", "tool"].contains(&message.role.as_str())
206 })
207 {
208 return Err(BackendError("invalid chat target or messages".into()));
209 }
210 self.checked(handle, cancel).await?;
211 let model = if handle.backend == "mlx" {
212 "default_model"
213 } else {
214 &input.model
215 };
216 if input.tools.len() > 1024 {
217 return Err(BackendError("too many tools".into()));
218 }
219 let messages = openai_messages(&input.messages);
220 let mut body = json!({"model":model,"messages":messages,"stream":false});
221 if !input.tools.is_empty() && handle.backend == "llamacpp" {
224 body["tools"] = json!(
225 input
226 .tools
227 .iter()
228 .map(|tool| json!({"type":"function","function":tool}))
229 .collect::<Vec<_>>()
230 );
231 }
232 if let Some(temperature) = input.temperature {
233 if !temperature.is_finite() {
234 return Err(BackendError("invalid temperature".into()));
235 }
236 body["temperature"] = json!(temperature);
237 }
238 if let Some(max_tokens) = input.max_tokens {
239 if max_tokens == 0 {
240 return Err(BackendError("max_tokens must be at least 1".into()));
241 }
242 body["max_tokens"] = json!(max_tokens);
243 }
244 if let Some(slot) = input.slot {
245 if handle.backend != "llamacpp" || slot >= MAX_SLOTS {
246 return Err(BackendError(
247 "a request slot applies only to llama.cpp, 0 to 15".into(),
248 ));
249 }
250 body["id_slot"] = json!(slot);
251 body["cache_prompt"] = json!(true);
252 }
253 let result = chat_result(
254 self.json(handle, "/v1/chat/completions", body, CHAT_TIMEOUT, cancel)
255 .await?,
256 )?;
257 self.checked(handle, cancel).await?;
258 Ok(result)
259 }
260 pub async fn embed(
261 &self,
262 handle: &ServerState,
263 model: &str,
264 inputs: &[String],
265 cancel: &CancellationToken,
266 ) -> Result<EmbedResult, BackendError> {
267 if handle.backend != "lmstudio" {
268 return Err(BackendError("backend does not support embeddings".into()));
269 }
270 crate::adapters::model_id(model)?;
271 if model != handle.model_id || inputs.is_empty() || inputs.len() > 1024 {
272 return Err(BackendError(
273 "invalid embedding target or input count".into(),
274 ));
275 }
276 self.checked(handle, cancel).await?;
277 let result = embedding_result(
278 self.json(
279 handle,
280 "/v1/embeddings",
281 json!({"model":model,"input":inputs}),
282 EMBED_TIMEOUT,
283 cancel,
284 )
285 .await?,
286 inputs.len(),
287 )?;
288 self.checked(handle, cancel).await?;
289 Ok(result)
290 }
291}
292
293pub fn openai_messages(messages: &[ChatMessage]) -> Vec<Value> {
297 let mut pending: std::collections::VecDeque<String> = std::collections::VecDeque::new();
298 messages
299 .iter()
300 .enumerate()
301 .map(|(index, message)| {
302 let mut value = json!({"role":message.role,"content":message.content});
303 if !message.tool_calls.is_empty() {
304 pending.clear();
305 value["tool_calls"] = json!(
306 message
307 .tool_calls
308 .iter()
309 .enumerate()
310 .map(|(n, call)| {
311 let id = format!("call_{index}_{n}");
312 pending.push_back(id.clone());
313 json!({
314 "id": id,
315 "type": "function",
316 "function": {
317 "name": call.name,
318 "arguments": serde_json::to_string(&call.arguments)
319 .unwrap_or_else(|_| "{}".into()),
320 },
321 })
322 })
323 .collect::<Vec<_>>()
324 );
325 }
326 if message.role == "tool" {
327 if let Some(id) = pending.pop_front() {
328 value["tool_call_id"] = json!(id);
329 }
330 if let Some(name) = &message.tool_name {
331 value["name"] = json!(name);
332 }
333 }
334 value
335 })
336 .collect()
337}
338
339pub const CHAT_TIMEOUT: Duration = Duration::from_secs(120);
342pub const EMBED_TIMEOUT: Duration = Duration::from_secs(30);
344
345pub const MAX_SLOTS: u32 = 16;
347
348#[derive(Debug, Clone, PartialEq, Eq)]
350pub enum KvSlotAction {
351 Save { name: String },
353 Restore { name: String },
355 Erase,
357}
358
359#[derive(Debug, Clone, PartialEq, serde::Serialize)]
361#[serde(rename_all = "camelCase")]
362pub struct KvSlotReport {
363 pub slot: u32,
364 pub tokens: u64,
366 #[serde(skip_serializing_if = "Option::is_none")]
368 pub bytes: Option<u64>,
369 #[serde(skip_serializing_if = "Option::is_none")]
371 pub ms: Option<f64>,
372}
373
374pub fn kv_slot_file(name: &str) -> Result<String, BackendError> {
376 if name.is_empty()
377 || name.len() > 64
378 || !name
379 .bytes()
380 .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')
381 {
382 return Err(BackendError(
383 "KV slot names are 1 to 64 letters, digits, - or _".into(),
384 ));
385 }
386 Ok(format!("{name}.bin"))
387}
388
389pub fn kv_slot_report(
390 slot: u32,
391 action: &KvSlotAction,
392 value: &Value,
393) -> Result<KvSlotReport, BackendError> {
394 let invalid = || BackendError("invalid KV slot response".into());
395 let count = |key: &str| value.get(key).and_then(Value::as_u64);
396 let (tokens, bytes, ms) = match action {
397 KvSlotAction::Save { .. } => (
398 count("n_saved").ok_or_else(invalid)?,
399 count("n_written"),
400 value.pointer("/timings/save_ms").and_then(Value::as_f64),
401 ),
402 KvSlotAction::Restore { .. } => (
403 count("n_restored").ok_or_else(invalid)?,
404 count("n_read"),
405 value.pointer("/timings/restore_ms").and_then(Value::as_f64),
406 ),
407 KvSlotAction::Erase => (count("n_erased").ok_or_else(invalid)?, None, None),
408 };
409 if value.get("id_slot").and_then(Value::as_u64) != Some(u64::from(slot))
410 || ms.is_some_and(|ms| !ms.is_finite() || ms < 0.0)
411 {
412 return Err(invalid());
413 }
414 Ok(KvSlotReport {
415 slot,
416 tokens,
417 bytes,
418 ms,
419 })
420}
421
422impl OpenAiInference<'_> {
423 pub async fn kv_slot(
427 &self,
428 handle: &ServerState,
429 slot: u32,
430 action: &KvSlotAction,
431 cancel: &CancellationToken,
432 ) -> Result<KvSlotReport, BackendError> {
433 if handle.backend != "llamacpp" || slot >= MAX_SLOTS {
434 return Err(BackendError(
435 "KV slots apply only to llama.cpp, slots 0 to 15".into(),
436 ));
437 }
438 let (query, body) = match action {
439 KvSlotAction::Save { name } => ("save", json!({"filename": kv_slot_file(name)?})),
440 KvSlotAction::Restore { name } => ("restore", json!({"filename": kv_slot_file(name)?})),
441 KvSlotAction::Erase => ("erase", json!({})),
442 };
443 self.checked(handle, cancel).await?;
444 let request = Request::new(
445 &handle.endpoint,
446 &format!("/slots/{slot}"),
447 Some(body),
448 None,
449 )
450 .and_then(|request| request.with_query(&[("action", query)]))
451 .map_err(|error| BackendError(error.to_string()))?;
452 let value = tokio::time::timeout(
453 Duration::from_secs(120),
454 read_json(self.http, request, cancel, 64 * 1024),
455 )
456 .await
457 .map_err(|_| BackendError("KV slot request timed out".into()))?
458 .map_err(|error| BackendError(error.to_string()))?;
459 let report = kv_slot_report(slot, action, &value)?;
460 self.checked(handle, cancel).await?;
461 Ok(report)
462 }
463}