1use crate::{
2 adapters::{BackendAdapter, BackendError, ServeRequest},
3 http::{Request, Transport, read_json},
4 identity::{ProcessProbe, capture},
5 ollama_inference::{ChatInput, ChatResult, EmbedResult},
6 process_control::listener,
7 state::ServerState,
8};
9use serde::Deserialize;
10use serde_json::{Value, json};
11use std::{path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13pub fn chat_result(value: Value) -> Result<ChatResult, BackendError> {
14 crate::usage::begin();
15 crate::usage::record(&value, crate::usage::Provider::OpenAi);
16 #[derive(Deserialize)]
17 struct Message {
18 content: String,
19 }
20 #[derive(Deserialize)]
21 struct Choice {
22 message: Message,
23 }
24 #[derive(Deserialize)]
25 struct Reply {
26 choices: Vec<Choice>,
27 }
28 let timings = value
29 .get("timings")
30 .and_then(crate::ollama_inference::Timings::from_llama_cpp);
31 let reply: Reply =
32 serde_json::from_value(value).map_err(|_| BackendError("invalid chat response".into()))?;
33 if reply.choices.is_empty() || reply.choices.len() > 8 {
34 return Err(BackendError("invalid chat choice count".into()));
35 }
36 let content = reply
37 .choices
38 .into_iter()
39 .next()
40 .ok_or_else(|| BackendError("no chat choice".into()))?
41 .message
42 .content;
43 if content.len() > 1048576 {
44 return Err(BackendError("generated content exceeds limit".into()));
45 }
46 Ok(ChatResult {
47 content,
48 tool_calls: Vec::new(),
49 timings,
50 })
51}
52pub fn embedding_result(value: Value, count: usize) -> Result<EmbedResult, BackendError> {
53 #[derive(Deserialize)]
54 struct Vector {
55 index: usize,
56 embedding: Vec<f64>,
57 }
58 #[derive(Deserialize)]
59 struct Reply {
60 data: Vec<Vector>,
61 }
62 let mut reply: Reply = serde_json::from_value(value)
63 .map_err(|_| BackendError("invalid embedding response".into()))?;
64 if count == 0 || count > 1024 || reply.data.len() != count {
65 return Err(BackendError("embedding count mismatch".into()));
66 }
67 reply.data.sort_by_key(|entry| entry.index);
68 let dimension = reply.data[0].embedding.len();
69 if dimension == 0
70 || dimension > 65536
71 || reply.data.iter().enumerate().any(|(index, entry)| {
72 entry.index != index
73 || entry.embedding.len() != dimension
74 || entry.embedding.iter().any(|value| !value.is_finite())
75 })
76 {
77 return Err(BackendError("inconsistent embedding vectors".into()));
78 }
79 Ok(EmbedResult {
80 vectors: reply
81 .data
82 .into_iter()
83 .map(|entry| entry.embedding)
84 .collect(),
85 dimension,
86 })
87}
88pub struct OpenAiInference<'runtime> {
89 pub http: &'runtime dyn Transport,
90 pub probe: &'runtime dyn ProcessProbe,
91 pub adapter: &'runtime dyn BackendAdapter,
92 pub token: Option<&'runtime str>,
93}
94impl OpenAiInference<'_> {
95 async fn checked(
96 &self,
97 handle: &ServerState,
98 cancel: &CancellationToken,
99 ) -> Result<(), BackendError> {
100 if handle.backend != self.adapter.name() {
101 return Err(BackendError("inference backend mismatch".into()));
102 }
103 let observed = listener(self.probe, &handle.endpoint, cancel).await?;
104 capture(handle, &observed, self.adapter.trusts(&observed.identity))
105 .map_err(|error| BackendError(error.to_string()))?;
106 if handle.backend == "lmstudio" {
107 self.adapter
108 .ready_handle(
109 &ServeRequest {
110 draft_model_path: None,
111 model_id: handle.model_id.clone(),
112 endpoint: handle.endpoint.clone(),
113 model_path: handle.model_path.as_ref().map(PathBuf::from),
114 context: None,
115 cache: None,
116 slot_save_path: None,
117 },
118 handle,
119 cancel,
120 )
121 .await?;
122 }
123 Ok(())
124 }
125 async fn json(
126 &self,
127 handle: &ServerState,
128 path: &str,
129 body: Value,
130 cancel: &CancellationToken,
131 ) -> Result<Value, BackendError> {
132 if serde_json::to_vec(&body)
133 .map_err(|error| BackendError(error.to_string()))?
134 .len()
135 > 4 * 1024 * 1024
136 {
137 return Err(BackendError("inference request exceeds limit".into()));
138 }
139 let token = if handle.backend == "mlx" {
140 handle.auth_token.clone()
141 } else {
142 self.token.map(str::to_owned)
143 };
144 let request = Request::new(&handle.endpoint, path, Some(body), token)
145 .map_err(|error| BackendError(error.to_string()))?;
146 tokio::time::timeout(
147 Duration::from_secs(30),
148 read_json(self.http, request, cancel, 2 * 1024 * 1024),
149 )
150 .await
151 .map_err(|_| BackendError("inference timed out".into()))?
152 .map_err(|error| BackendError(error.to_string()))
153 }
154 pub async fn chat(
155 &self,
156 handle: &ServerState,
157 input: &ChatInput,
158 cancel: &CancellationToken,
159 ) -> Result<ChatResult, BackendError> {
160 crate::adapters::model_id(&input.model)?;
161 if input.model != handle.model_id
162 || input.messages.len() > 10000
163 || input.messages.iter().any(|message| {
164 !["user", "assistant", "system", "tool"].contains(&message.role.as_str())
165 })
166 {
167 return Err(BackendError("invalid chat target or messages".into()));
168 }
169 self.checked(handle, cancel).await?;
170 let model = if handle.backend == "mlx" {
171 "default_model"
172 } else {
173 &input.model
174 };
175 let messages: Vec<_> = input
176 .messages
177 .iter()
178 .map(|message| json!({"role":message.role,"content":message.content}))
179 .collect();
180 let mut body = json!({"model":model,"messages":messages,"stream":false});
181 if let Some(slot) = input.slot {
182 if handle.backend != "llamacpp" || slot >= MAX_SLOTS {
183 return Err(BackendError(
184 "a request slot applies only to llama.cpp, 0 to 15".into(),
185 ));
186 }
187 body["id_slot"] = json!(slot);
188 body["cache_prompt"] = json!(true);
189 }
190 let result = chat_result(
191 self.json(handle, "/v1/chat/completions", body, cancel)
192 .await?,
193 )?;
194 self.checked(handle, cancel).await?;
195 Ok(result)
196 }
197 pub async fn embed(
198 &self,
199 handle: &ServerState,
200 model: &str,
201 inputs: &[String],
202 cancel: &CancellationToken,
203 ) -> Result<EmbedResult, BackendError> {
204 if handle.backend != "lmstudio" {
205 return Err(BackendError("backend does not support embeddings".into()));
206 }
207 crate::adapters::model_id(model)?;
208 if model != handle.model_id || inputs.is_empty() || inputs.len() > 1024 {
209 return Err(BackendError(
210 "invalid embedding target or input count".into(),
211 ));
212 }
213 self.checked(handle, cancel).await?;
214 let result = embedding_result(
215 self.json(
216 handle,
217 "/v1/embeddings",
218 json!({"model":model,"input":inputs}),
219 cancel,
220 )
221 .await?,
222 inputs.len(),
223 )?;
224 self.checked(handle, cancel).await?;
225 Ok(result)
226 }
227}
228
229pub const MAX_SLOTS: u32 = 16;
231
232#[derive(Debug, Clone, PartialEq, Eq)]
234pub enum KvSlotAction {
235 Save { name: String },
237 Restore { name: String },
239 Erase,
241}
242
243#[derive(Debug, Clone, PartialEq, serde::Serialize)]
245#[serde(rename_all = "camelCase")]
246pub struct KvSlotReport {
247 pub slot: u32,
248 pub tokens: u64,
250 #[serde(skip_serializing_if = "Option::is_none")]
252 pub bytes: Option<u64>,
253 #[serde(skip_serializing_if = "Option::is_none")]
255 pub ms: Option<f64>,
256}
257
258pub fn kv_slot_file(name: &str) -> Result<String, BackendError> {
260 if name.is_empty()
261 || name.len() > 64
262 || !name
263 .bytes()
264 .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')
265 {
266 return Err(BackendError(
267 "KV slot names are 1 to 64 letters, digits, - or _".into(),
268 ));
269 }
270 Ok(format!("{name}.bin"))
271}
272
273pub fn kv_slot_report(
274 slot: u32,
275 action: &KvSlotAction,
276 value: &Value,
277) -> Result<KvSlotReport, BackendError> {
278 let invalid = || BackendError("invalid KV slot response".into());
279 let count = |key: &str| value.get(key).and_then(Value::as_u64);
280 let (tokens, bytes, ms) = match action {
281 KvSlotAction::Save { .. } => (
282 count("n_saved").ok_or_else(invalid)?,
283 count("n_written"),
284 value.pointer("/timings/save_ms").and_then(Value::as_f64),
285 ),
286 KvSlotAction::Restore { .. } => (
287 count("n_restored").ok_or_else(invalid)?,
288 count("n_read"),
289 value.pointer("/timings/restore_ms").and_then(Value::as_f64),
290 ),
291 KvSlotAction::Erase => (count("n_erased").ok_or_else(invalid)?, None, None),
292 };
293 if value.get("id_slot").and_then(Value::as_u64) != Some(u64::from(slot))
294 || ms.is_some_and(|ms| !ms.is_finite() || ms < 0.0)
295 {
296 return Err(invalid());
297 }
298 Ok(KvSlotReport {
299 slot,
300 tokens,
301 bytes,
302 ms,
303 })
304}
305
306impl OpenAiInference<'_> {
307 pub async fn kv_slot(
311 &self,
312 handle: &ServerState,
313 slot: u32,
314 action: &KvSlotAction,
315 cancel: &CancellationToken,
316 ) -> Result<KvSlotReport, BackendError> {
317 if handle.backend != "llamacpp" || slot >= MAX_SLOTS {
318 return Err(BackendError(
319 "KV slots apply only to llama.cpp, slots 0 to 15".into(),
320 ));
321 }
322 let (query, body) = match action {
323 KvSlotAction::Save { name } => ("save", json!({"filename": kv_slot_file(name)?})),
324 KvSlotAction::Restore { name } => ("restore", json!({"filename": kv_slot_file(name)?})),
325 KvSlotAction::Erase => ("erase", json!({})),
326 };
327 self.checked(handle, cancel).await?;
328 let request = Request::new(
329 &handle.endpoint,
330 &format!("/slots/{slot}"),
331 Some(body),
332 None,
333 )
334 .and_then(|request| request.with_query(&[("action", query)]))
335 .map_err(|error| BackendError(error.to_string()))?;
336 let value = tokio::time::timeout(
337 Duration::from_secs(120),
338 read_json(self.http, request, cancel, 64 * 1024),
339 )
340 .await
341 .map_err(|_| BackendError("KV slot request timed out".into()))?
342 .map_err(|error| BackendError(error.to_string()))?;
343 let report = kv_slot_report(slot, action, &value)?;
344 self.checked(handle, cancel).await?;
345 Ok(report)
346 }
347}