Skip to main content

rigspark_runtime/
openai.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest},
3    http::{Request, Transport, read_json},
4    identity::{ProcessProbe, capture},
5    ollama_inference::{ChatInput, ChatMessage, ChatResult, EmbedResult, ToolCall},
6    process_control::listener,
7    state::ServerState,
8};
9use serde::Deserialize;
10use serde_json::{Value, json};
11use std::{collections::BTreeMap, path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13pub fn chat_result(value: Value) -> Result<ChatResult, BackendError> {
14    crate::usage::begin();
15    crate::usage::record(&value, crate::usage::Provider::OpenAi);
16    #[derive(Deserialize)]
17    struct Function {
18        name: String,
19        /// OpenAI sends the arguments as a JSON string; some servers send an object.
20        #[serde(default)]
21        arguments: Value,
22    }
23    #[derive(Deserialize)]
24    struct Call {
25        function: Function,
26    }
27    #[derive(Deserialize)]
28    struct Message {
29        /// `null` when the reply is only tool calls.
30        #[serde(default)]
31        content: Option<String>,
32        #[serde(default)]
33        tool_calls: Vec<Call>,
34    }
35    #[derive(Deserialize)]
36    struct Choice {
37        message: Message,
38    }
39    #[derive(Deserialize)]
40    struct Reply {
41        choices: Vec<Choice>,
42    }
43    let timings = value
44        .get("timings")
45        .and_then(crate::ollama_inference::Timings::from_llama_cpp);
46    let reply: Reply =
47        serde_json::from_value(value).map_err(|_| BackendError("invalid chat response".into()))?;
48    if reply.choices.is_empty() || reply.choices.len() > 8 {
49        return Err(BackendError("invalid chat choice count".into()));
50    }
51    let message = reply
52        .choices
53        .into_iter()
54        .next()
55        .ok_or_else(|| BackendError("no chat choice".into()))?
56        .message;
57    let content = message.content.unwrap_or_default();
58    if content.len() > 1048576 {
59        return Err(BackendError("generated content exceeds limit".into()));
60    }
61    if message.tool_calls.len() > 64 {
62        return Err(BackendError("too many tool calls".into()));
63    }
64    let mut tool_calls = Vec::with_capacity(message.tool_calls.len());
65    for call in message.tool_calls {
66        let arguments = match call.function.arguments {
67            Value::String(text) if text.trim().is_empty() => BTreeMap::new(),
68            Value::String(text) => serde_json::from_str(&text)
69                .map_err(|_| BackendError("tool call arguments are not a JSON object".into()))?,
70            Value::Object(map) => map.into_iter().collect(),
71            Value::Null => BTreeMap::new(),
72            _ => {
73                return Err(BackendError(
74                    "tool call arguments are not a JSON object".into(),
75                ));
76            }
77        };
78        if call.function.name.is_empty() || call.function.name.len() > 128 {
79            return Err(BackendError("invalid tool call name".into()));
80        }
81        tool_calls.push(ToolCall {
82            name: call.function.name,
83            arguments,
84        });
85    }
86    Ok(ChatResult {
87        content,
88        tool_calls,
89        timings,
90    })
91}
92pub fn embedding_result(value: Value, count: usize) -> Result<EmbedResult, BackendError> {
93    #[derive(Deserialize)]
94    struct Vector {
95        index: usize,
96        embedding: Vec<f64>,
97    }
98    #[derive(Deserialize)]
99    struct Reply {
100        data: Vec<Vector>,
101    }
102    let mut reply: Reply = serde_json::from_value(value)
103        .map_err(|_| BackendError("invalid embedding response".into()))?;
104    if count == 0 || count > 1024 || reply.data.len() != count {
105        return Err(BackendError("embedding count mismatch".into()));
106    }
107    reply.data.sort_by_key(|entry| entry.index);
108    let dimension = reply.data[0].embedding.len();
109    if dimension == 0
110        || dimension > 65536
111        || reply.data.iter().enumerate().any(|(index, entry)| {
112            entry.index != index
113                || entry.embedding.len() != dimension
114                || entry.embedding.iter().any(|value| !value.is_finite())
115        })
116    {
117        return Err(BackendError("inconsistent embedding vectors".into()));
118    }
119    Ok(EmbedResult {
120        vectors: reply
121            .data
122            .into_iter()
123            .map(|entry| entry.embedding)
124            .collect(),
125        dimension,
126    })
127}
128pub struct OpenAiInference<'runtime> {
129    pub http: &'runtime dyn Transport,
130    pub probe: &'runtime dyn ProcessProbe,
131    pub adapter: &'runtime dyn BackendAdapter,
132    pub token: Option<&'runtime str>,
133}
134impl OpenAiInference<'_> {
135    async fn checked(
136        &self,
137        handle: &ServerState,
138        cancel: &CancellationToken,
139    ) -> Result<(), BackendError> {
140        if handle.backend != self.adapter.name() {
141            return Err(BackendError("inference backend mismatch".into()));
142        }
143        let observed = listener(self.probe, &handle.endpoint, cancel).await?;
144        capture(handle, &observed, self.adapter.trusts(&observed.identity))
145            .map_err(|error| BackendError(error.to_string()))?;
146        if handle.backend == "lmstudio" {
147            self.adapter
148                .ready_handle(
149                    &ServeRequest {
150                        draft_model_path: None,
151                        model_id: handle.model_id.clone(),
152                        endpoint: handle.endpoint.clone(),
153                        model_path: handle.model_path.as_ref().map(PathBuf::from),
154                        context: None,
155                        cache: None,
156                        slot_save_path: None,
157                    },
158                    handle,
159                    cancel,
160                )
161                .await?;
162        }
163        Ok(())
164    }
165    async fn json(
166        &self,
167        handle: &ServerState,
168        path: &str,
169        body: Value,
170        timeout: Duration,
171        cancel: &CancellationToken,
172    ) -> Result<Value, BackendError> {
173        if serde_json::to_vec(&body)
174            .map_err(|error| BackendError(error.to_string()))?
175            .len()
176            > 4 * 1024 * 1024
177        {
178            return Err(BackendError("inference request exceeds limit".into()));
179        }
180        let token = if handle.backend == "mlx" {
181            handle.auth_token.clone()
182        } else {
183            self.token.map(str::to_owned)
184        };
185        let request = Request::new(&handle.endpoint, path, Some(body), token)
186            .map_err(|error| BackendError(error.to_string()))?;
187        tokio::time::timeout(
188            timeout,
189            read_json(self.http, request, cancel, 2 * 1024 * 1024),
190        )
191        .await
192        .map_err(|_| BackendError("inference timed out".into()))?
193        .map_err(|error| BackendError(error.to_string()))
194    }
195    pub async fn chat(
196        &self,
197        handle: &ServerState,
198        input: &ChatInput,
199        cancel: &CancellationToken,
200    ) -> Result<ChatResult, BackendError> {
201        crate::adapters::model_id(&input.model)?;
202        if input.model != handle.model_id
203            || input.messages.len() > 10000
204            || input.messages.iter().any(|message| {
205                !["user", "assistant", "system", "tool"].contains(&message.role.as_str())
206            })
207        {
208            return Err(BackendError("invalid chat target or messages".into()));
209        }
210        self.checked(handle, cancel).await?;
211        let model = if handle.backend == "mlx" {
212            "default_model"
213        } else {
214            &input.model
215        };
216        if input.tools.len() > 1024 {
217            return Err(BackendError("too many tools".into()));
218        }
219        let messages = openai_messages(&input.messages);
220        let mut body = json!({"model":model,"messages":messages,"stream":false});
221        // Tools go to llama.cpp (its server handles them with the model's chat template); other
222        // OpenAI-style servers here keep the plain chat they had.
223        if !input.tools.is_empty() && handle.backend == "llamacpp" {
224            body["tools"] = json!(
225                input
226                    .tools
227                    .iter()
228                    .map(|tool| json!({"type":"function","function":tool}))
229                    .collect::<Vec<_>>()
230            );
231        }
232        if let Some(temperature) = input.temperature {
233            if !temperature.is_finite() {
234                return Err(BackendError("invalid temperature".into()));
235            }
236            body["temperature"] = json!(temperature);
237        }
238        if let Some(max_tokens) = input.max_tokens {
239            if max_tokens == 0 {
240                return Err(BackendError("max_tokens must be at least 1".into()));
241            }
242            body["max_tokens"] = json!(max_tokens);
243        }
244        if let Some(slot) = input.slot {
245            if handle.backend != "llamacpp" || slot >= MAX_SLOTS {
246                return Err(BackendError(
247                    "a request slot applies only to llama.cpp, 0 to 15".into(),
248                ));
249            }
250            body["id_slot"] = json!(slot);
251            body["cache_prompt"] = json!(true);
252        }
253        let result = chat_result(
254            self.json(handle, "/v1/chat/completions", body, CHAT_TIMEOUT, cancel)
255                .await?,
256        )?;
257        self.checked(handle, cancel).await?;
258        Ok(result)
259    }
260    pub async fn embed(
261        &self,
262        handle: &ServerState,
263        model: &str,
264        inputs: &[String],
265        cancel: &CancellationToken,
266    ) -> Result<EmbedResult, BackendError> {
267        if handle.backend != "lmstudio" {
268            return Err(BackendError("backend does not support embeddings".into()));
269        }
270        crate::adapters::model_id(model)?;
271        if model != handle.model_id || inputs.is_empty() || inputs.len() > 1024 {
272            return Err(BackendError(
273                "invalid embedding target or input count".into(),
274            ));
275        }
276        self.checked(handle, cancel).await?;
277        let result = embedding_result(
278            self.json(
279                handle,
280                "/v1/embeddings",
281                json!({"model":model,"input":inputs}),
282                EMBED_TIMEOUT,
283                cancel,
284            )
285            .await?,
286            inputs.len(),
287        )?;
288        self.checked(handle, cancel).await?;
289        Ok(result)
290    }
291}
292
293/// Messages in OpenAI's shape. An assistant's tool calls get ids (`call_<message>_<n>`) and
294/// each following `tool` message answers the next unanswered one, in order, so servers that
295/// require `tool_call_id` accept the history.
296pub fn openai_messages(messages: &[ChatMessage]) -> Vec<Value> {
297    let mut pending: std::collections::VecDeque<String> = std::collections::VecDeque::new();
298    messages
299        .iter()
300        .enumerate()
301        .map(|(index, message)| {
302            let mut value = json!({"role":message.role,"content":message.content});
303            if !message.tool_calls.is_empty() {
304                pending.clear();
305                value["tool_calls"] = json!(
306                    message
307                        .tool_calls
308                        .iter()
309                        .enumerate()
310                        .map(|(n, call)| {
311                            let id = format!("call_{index}_{n}");
312                            pending.push_back(id.clone());
313                            json!({
314                                "id": id,
315                                "type": "function",
316                                "function": {
317                                    "name": call.name,
318                                    "arguments": serde_json::to_string(&call.arguments)
319                                        .unwrap_or_else(|_| "{}".into()),
320                                },
321                            })
322                        })
323                        .collect::<Vec<_>>()
324                );
325            }
326            if message.role == "tool" {
327                if let Some(id) = pending.pop_front() {
328                    value["tool_call_id"] = json!(id);
329                }
330                if let Some(name) = &message.tool_name {
331                    value["name"] = json!(name);
332                }
333            }
334            value
335        })
336        .collect()
337}
338
339/// A chat reply may need to read a long prompt first, several requests at once sharing the
340/// server, so it gets as long as the local harness allows a whole call (120 s).
341pub const CHAT_TIMEOUT: Duration = Duration::from_secs(120);
342/// Embeddings are short reads.
343pub const EMBED_TIMEOUT: Duration = Duration::from_secs(30);
344
345/// llama.cpp request slots (`--parallel`) never exceed 16.
346pub const MAX_SLOTS: u32 = 16;
347
348/// What to do with one llama.cpp request slot's KV cache.
349#[derive(Debug, Clone, PartialEq, Eq)]
350pub enum KvSlotAction {
351    /// Write the slot's KV cache to `<name>.bin` in the server's slot directory.
352    Save { name: String },
353    /// Load `<name>.bin` into the slot.
354    Restore { name: String },
355    /// Drop the slot's KV cache, freeing it for other requests.
356    Erase,
357}
358
359/// What llama.cpp reported for a slot action.
360#[derive(Debug, Clone, PartialEq, serde::Serialize)]
361#[serde(rename_all = "camelCase")]
362pub struct KvSlotReport {
363    pub slot: u32,
364    /// Tokens saved, restored or erased.
365    pub tokens: u64,
366    /// Bytes written or read (save and restore only).
367    #[serde(skip_serializing_if = "Option::is_none")]
368    pub bytes: Option<u64>,
369    /// How long the server took (save and restore only).
370    #[serde(skip_serializing_if = "Option::is_none")]
371    pub ms: Option<f64>,
372}
373
374/// A slot file name: 1 to 64 ASCII letters, digits, `-` or `_`; `.bin` is added.
375pub fn kv_slot_file(name: &str) -> Result<String, BackendError> {
376    if name.is_empty()
377        || name.len() > 64
378        || !name
379            .bytes()
380            .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')
381    {
382        return Err(BackendError(
383            "KV slot names are 1 to 64 letters, digits, - or _".into(),
384        ));
385    }
386    Ok(format!("{name}.bin"))
387}
388
389pub fn kv_slot_report(
390    slot: u32,
391    action: &KvSlotAction,
392    value: &Value,
393) -> Result<KvSlotReport, BackendError> {
394    let invalid = || BackendError("invalid KV slot response".into());
395    let count = |key: &str| value.get(key).and_then(Value::as_u64);
396    let (tokens, bytes, ms) = match action {
397        KvSlotAction::Save { .. } => (
398            count("n_saved").ok_or_else(invalid)?,
399            count("n_written"),
400            value.pointer("/timings/save_ms").and_then(Value::as_f64),
401        ),
402        KvSlotAction::Restore { .. } => (
403            count("n_restored").ok_or_else(invalid)?,
404            count("n_read"),
405            value.pointer("/timings/restore_ms").and_then(Value::as_f64),
406        ),
407        KvSlotAction::Erase => (count("n_erased").ok_or_else(invalid)?, None, None),
408    };
409    if value.get("id_slot").and_then(Value::as_u64) != Some(u64::from(slot))
410        || ms.is_some_and(|ms| !ms.is_finite() || ms < 0.0)
411    {
412        return Err(invalid());
413    }
414    Ok(KvSlotReport {
415        slot,
416        tokens,
417        bytes,
418        ms,
419    })
420}
421
422impl OpenAiInference<'_> {
423    /// Saves, restores or erases one request slot's KV cache on a llama.cpp server that
424    /// rigspark started with a slot directory. The listener is re-identified before and
425    /// after, as for chat.
426    pub async fn kv_slot(
427        &self,
428        handle: &ServerState,
429        slot: u32,
430        action: &KvSlotAction,
431        cancel: &CancellationToken,
432    ) -> Result<KvSlotReport, BackendError> {
433        if handle.backend != "llamacpp" || slot >= MAX_SLOTS {
434            return Err(BackendError(
435                "KV slots apply only to llama.cpp, slots 0 to 15".into(),
436            ));
437        }
438        let (query, body) = match action {
439            KvSlotAction::Save { name } => ("save", json!({"filename": kv_slot_file(name)?})),
440            KvSlotAction::Restore { name } => ("restore", json!({"filename": kv_slot_file(name)?})),
441            KvSlotAction::Erase => ("erase", json!({})),
442        };
443        self.checked(handle, cancel).await?;
444        let request = Request::new(
445            &handle.endpoint,
446            &format!("/slots/{slot}"),
447            Some(body),
448            None,
449        )
450        .and_then(|request| request.with_query(&[("action", query)]))
451        .map_err(|error| BackendError(error.to_string()))?;
452        let value = tokio::time::timeout(
453            Duration::from_secs(120),
454            read_json(self.http, request, cancel, 64 * 1024),
455        )
456        .await
457        .map_err(|_| BackendError("KV slot request timed out".into()))?
458        .map_err(|error| BackendError(error.to_string()))?;
459        let report = kv_slot_report(slot, action, &value)?;
460        self.checked(handle, cancel).await?;
461        Ok(report)
462    }
463}