Skip to main content

rigspark_runtime/
openai.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest},
3    http::{Request, Transport, read_json},
4    identity::{ProcessProbe, capture},
5    ollama_inference::{ChatInput, ChatResult, EmbedResult},
6    process_control::listener,
7    state::ServerState,
8};
9use serde::Deserialize;
10use serde_json::{Value, json};
11use std::{path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13pub fn chat_result(value: Value) -> Result<ChatResult, BackendError> {
14    crate::usage::begin();
15    crate::usage::record(&value, crate::usage::Provider::OpenAi);
16    #[derive(Deserialize)]
17    struct Message {
18        content: String,
19    }
20    #[derive(Deserialize)]
21    struct Choice {
22        message: Message,
23    }
24    #[derive(Deserialize)]
25    struct Reply {
26        choices: Vec<Choice>,
27    }
28    let timings = value
29        .get("timings")
30        .and_then(crate::ollama_inference::Timings::from_llama_cpp);
31    let reply: Reply =
32        serde_json::from_value(value).map_err(|_| BackendError("invalid chat response".into()))?;
33    if reply.choices.is_empty() || reply.choices.len() > 8 {
34        return Err(BackendError("invalid chat choice count".into()));
35    }
36    let content = reply
37        .choices
38        .into_iter()
39        .next()
40        .ok_or_else(|| BackendError("no chat choice".into()))?
41        .message
42        .content;
43    if content.len() > 1048576 {
44        return Err(BackendError("generated content exceeds limit".into()));
45    }
46    Ok(ChatResult {
47        content,
48        tool_calls: Vec::new(),
49        timings,
50    })
51}
52pub fn embedding_result(value: Value, count: usize) -> Result<EmbedResult, BackendError> {
53    #[derive(Deserialize)]
54    struct Vector {
55        index: usize,
56        embedding: Vec<f64>,
57    }
58    #[derive(Deserialize)]
59    struct Reply {
60        data: Vec<Vector>,
61    }
62    let mut reply: Reply = serde_json::from_value(value)
63        .map_err(|_| BackendError("invalid embedding response".into()))?;
64    if count == 0 || count > 1024 || reply.data.len() != count {
65        return Err(BackendError("embedding count mismatch".into()));
66    }
67    reply.data.sort_by_key(|entry| entry.index);
68    let dimension = reply.data[0].embedding.len();
69    if dimension == 0
70        || dimension > 65536
71        || reply.data.iter().enumerate().any(|(index, entry)| {
72            entry.index != index
73                || entry.embedding.len() != dimension
74                || entry.embedding.iter().any(|value| !value.is_finite())
75        })
76    {
77        return Err(BackendError("inconsistent embedding vectors".into()));
78    }
79    Ok(EmbedResult {
80        vectors: reply
81            .data
82            .into_iter()
83            .map(|entry| entry.embedding)
84            .collect(),
85        dimension,
86    })
87}
88pub struct OpenAiInference<'runtime> {
89    pub http: &'runtime dyn Transport,
90    pub probe: &'runtime dyn ProcessProbe,
91    pub adapter: &'runtime dyn BackendAdapter,
92    pub token: Option<&'runtime str>,
93}
94impl OpenAiInference<'_> {
95    async fn checked(
96        &self,
97        handle: &ServerState,
98        cancel: &CancellationToken,
99    ) -> Result<(), BackendError> {
100        if handle.backend != self.adapter.name() {
101            return Err(BackendError("inference backend mismatch".into()));
102        }
103        let observed = listener(self.probe, &handle.endpoint, cancel).await?;
104        capture(handle, &observed, self.adapter.trusts(&observed.identity))
105            .map_err(|error| BackendError(error.to_string()))?;
106        if handle.backend == "lmstudio" {
107            self.adapter
108                .ready_handle(
109                    &ServeRequest {
110                        draft_model_path: None,
111                        model_id: handle.model_id.clone(),
112                        endpoint: handle.endpoint.clone(),
113                        model_path: handle.model_path.as_ref().map(PathBuf::from),
114                        context: None,
115                        cache: None,
116                        slot_save_path: None,
117                    },
118                    handle,
119                    cancel,
120                )
121                .await?;
122        }
123        Ok(())
124    }
125    async fn json(
126        &self,
127        handle: &ServerState,
128        path: &str,
129        body: Value,
130        cancel: &CancellationToken,
131    ) -> Result<Value, BackendError> {
132        if serde_json::to_vec(&body)
133            .map_err(|error| BackendError(error.to_string()))?
134            .len()
135            > 4 * 1024 * 1024
136        {
137            return Err(BackendError("inference request exceeds limit".into()));
138        }
139        let token = if handle.backend == "mlx" {
140            handle.auth_token.clone()
141        } else {
142            self.token.map(str::to_owned)
143        };
144        let request = Request::new(&handle.endpoint, path, Some(body), token)
145            .map_err(|error| BackendError(error.to_string()))?;
146        tokio::time::timeout(
147            Duration::from_secs(30),
148            read_json(self.http, request, cancel, 2 * 1024 * 1024),
149        )
150        .await
151        .map_err(|_| BackendError("inference timed out".into()))?
152        .map_err(|error| BackendError(error.to_string()))
153    }
154    pub async fn chat(
155        &self,
156        handle: &ServerState,
157        input: &ChatInput,
158        cancel: &CancellationToken,
159    ) -> Result<ChatResult, BackendError> {
160        crate::adapters::model_id(&input.model)?;
161        if input.model != handle.model_id
162            || input.messages.len() > 10000
163            || input.messages.iter().any(|message| {
164                !["user", "assistant", "system", "tool"].contains(&message.role.as_str())
165            })
166        {
167            return Err(BackendError("invalid chat target or messages".into()));
168        }
169        self.checked(handle, cancel).await?;
170        let model = if handle.backend == "mlx" {
171            "default_model"
172        } else {
173            &input.model
174        };
175        let messages: Vec<_> = input
176            .messages
177            .iter()
178            .map(|message| json!({"role":message.role,"content":message.content}))
179            .collect();
180        let mut body = json!({"model":model,"messages":messages,"stream":false});
181        if let Some(slot) = input.slot {
182            if handle.backend != "llamacpp" || slot >= MAX_SLOTS {
183                return Err(BackendError(
184                    "a request slot applies only to llama.cpp, 0 to 15".into(),
185                ));
186            }
187            body["id_slot"] = json!(slot);
188            body["cache_prompt"] = json!(true);
189        }
190        let result = chat_result(
191            self.json(handle, "/v1/chat/completions", body, cancel)
192                .await?,
193        )?;
194        self.checked(handle, cancel).await?;
195        Ok(result)
196    }
197    pub async fn embed(
198        &self,
199        handle: &ServerState,
200        model: &str,
201        inputs: &[String],
202        cancel: &CancellationToken,
203    ) -> Result<EmbedResult, BackendError> {
204        if handle.backend != "lmstudio" {
205            return Err(BackendError("backend does not support embeddings".into()));
206        }
207        crate::adapters::model_id(model)?;
208        if model != handle.model_id || inputs.is_empty() || inputs.len() > 1024 {
209            return Err(BackendError(
210                "invalid embedding target or input count".into(),
211            ));
212        }
213        self.checked(handle, cancel).await?;
214        let result = embedding_result(
215            self.json(
216                handle,
217                "/v1/embeddings",
218                json!({"model":model,"input":inputs}),
219                cancel,
220            )
221            .await?,
222            inputs.len(),
223        )?;
224        self.checked(handle, cancel).await?;
225        Ok(result)
226    }
227}
228
229/// llama.cpp request slots (`--parallel`) never exceed 16.
230pub const MAX_SLOTS: u32 = 16;
231
232/// What to do with one llama.cpp request slot's KV cache.
233#[derive(Debug, Clone, PartialEq, Eq)]
234pub enum KvSlotAction {
235    /// Write the slot's KV cache to `<name>.bin` in the server's slot directory.
236    Save { name: String },
237    /// Load `<name>.bin` into the slot.
238    Restore { name: String },
239    /// Drop the slot's KV cache, freeing it for other requests.
240    Erase,
241}
242
243/// What llama.cpp reported for a slot action.
244#[derive(Debug, Clone, PartialEq, serde::Serialize)]
245#[serde(rename_all = "camelCase")]
246pub struct KvSlotReport {
247    pub slot: u32,
248    /// Tokens saved, restored or erased.
249    pub tokens: u64,
250    /// Bytes written or read (save and restore only).
251    #[serde(skip_serializing_if = "Option::is_none")]
252    pub bytes: Option<u64>,
253    /// How long the server took (save and restore only).
254    #[serde(skip_serializing_if = "Option::is_none")]
255    pub ms: Option<f64>,
256}
257
258/// A slot file name: 1 to 64 ASCII letters, digits, `-` or `_`; `.bin` is added.
259pub fn kv_slot_file(name: &str) -> Result<String, BackendError> {
260    if name.is_empty()
261        || name.len() > 64
262        || !name
263            .bytes()
264            .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')
265    {
266        return Err(BackendError(
267            "KV slot names are 1 to 64 letters, digits, - or _".into(),
268        ));
269    }
270    Ok(format!("{name}.bin"))
271}
272
273pub fn kv_slot_report(
274    slot: u32,
275    action: &KvSlotAction,
276    value: &Value,
277) -> Result<KvSlotReport, BackendError> {
278    let invalid = || BackendError("invalid KV slot response".into());
279    let count = |key: &str| value.get(key).and_then(Value::as_u64);
280    let (tokens, bytes, ms) = match action {
281        KvSlotAction::Save { .. } => (
282            count("n_saved").ok_or_else(invalid)?,
283            count("n_written"),
284            value.pointer("/timings/save_ms").and_then(Value::as_f64),
285        ),
286        KvSlotAction::Restore { .. } => (
287            count("n_restored").ok_or_else(invalid)?,
288            count("n_read"),
289            value.pointer("/timings/restore_ms").and_then(Value::as_f64),
290        ),
291        KvSlotAction::Erase => (count("n_erased").ok_or_else(invalid)?, None, None),
292    };
293    if value.get("id_slot").and_then(Value::as_u64) != Some(u64::from(slot))
294        || ms.is_some_and(|ms| !ms.is_finite() || ms < 0.0)
295    {
296        return Err(invalid());
297    }
298    Ok(KvSlotReport {
299        slot,
300        tokens,
301        bytes,
302        ms,
303    })
304}
305
306impl OpenAiInference<'_> {
307    /// Saves, restores or erases one request slot's KV cache on a llama.cpp server that
308    /// rigspark started with a slot directory. The listener is re-identified before and
309    /// after, as for chat.
310    pub async fn kv_slot(
311        &self,
312        handle: &ServerState,
313        slot: u32,
314        action: &KvSlotAction,
315        cancel: &CancellationToken,
316    ) -> Result<KvSlotReport, BackendError> {
317        if handle.backend != "llamacpp" || slot >= MAX_SLOTS {
318            return Err(BackendError(
319                "KV slots apply only to llama.cpp, slots 0 to 15".into(),
320            ));
321        }
322        let (query, body) = match action {
323            KvSlotAction::Save { name } => ("save", json!({"filename": kv_slot_file(name)?})),
324            KvSlotAction::Restore { name } => ("restore", json!({"filename": kv_slot_file(name)?})),
325            KvSlotAction::Erase => ("erase", json!({})),
326        };
327        self.checked(handle, cancel).await?;
328        let request = Request::new(
329            &handle.endpoint,
330            &format!("/slots/{slot}"),
331            Some(body),
332            None,
333        )
334        .and_then(|request| request.with_query(&[("action", query)]))
335        .map_err(|error| BackendError(error.to_string()))?;
336        let value = tokio::time::timeout(
337            Duration::from_secs(120),
338            read_json(self.http, request, cancel, 64 * 1024),
339        )
340        .await
341        .map_err(|_| BackendError("KV slot request timed out".into()))?
342        .map_err(|error| BackendError(error.to_string()))?;
343        let report = kv_slot_report(slot, action, &value)?;
344        self.checked(handle, cancel).await?;
345        Ok(report)
346    }
347}