Skip to main content

rigspark_runtime/
openai.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest},
3    http::{Request, Transport, read_json},
4    identity::{ProcessProbe, capture},
5    ollama_inference::{ChatInput, ChatResult, EmbedResult},
6    process_control::listener,
7    state::ServerState,
8};
9use serde::Deserialize;
10use serde_json::{Value, json};
11use std::{path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13pub fn chat_result(value: Value) -> Result<ChatResult, BackendError> {
14    crate::usage::begin();
15    crate::usage::record(&value, crate::usage::Provider::OpenAi);
16    #[derive(Deserialize)]
17    struct Message {
18        content: String,
19    }
20    #[derive(Deserialize)]
21    struct Choice {
22        message: Message,
23    }
24    #[derive(Deserialize)]
25    struct Reply {
26        choices: Vec<Choice>,
27    }
28    let reply: Reply =
29        serde_json::from_value(value).map_err(|_| BackendError("invalid chat response".into()))?;
30    if reply.choices.is_empty() || reply.choices.len() > 8 {
31        return Err(BackendError("invalid chat choice count".into()));
32    }
33    let content = reply
34        .choices
35        .into_iter()
36        .next()
37        .ok_or_else(|| BackendError("no chat choice".into()))?
38        .message
39        .content;
40    if content.len() > 1048576 {
41        return Err(BackendError("generated content exceeds limit".into()));
42    }
43    Ok(ChatResult {
44        content,
45        tool_calls: Vec::new(),
46    })
47}
48pub fn embedding_result(value: Value, count: usize) -> Result<EmbedResult, BackendError> {
49    #[derive(Deserialize)]
50    struct Vector {
51        index: usize,
52        embedding: Vec<f64>,
53    }
54    #[derive(Deserialize)]
55    struct Reply {
56        data: Vec<Vector>,
57    }
58    let mut reply: Reply = serde_json::from_value(value)
59        .map_err(|_| BackendError("invalid embedding response".into()))?;
60    if count == 0 || count > 1024 || reply.data.len() != count {
61        return Err(BackendError("embedding count mismatch".into()));
62    }
63    reply.data.sort_by_key(|entry| entry.index);
64    let dimension = reply.data[0].embedding.len();
65    if dimension == 0
66        || dimension > 65536
67        || reply.data.iter().enumerate().any(|(index, entry)| {
68            entry.index != index
69                || entry.embedding.len() != dimension
70                || entry.embedding.iter().any(|value| !value.is_finite())
71        })
72    {
73        return Err(BackendError("inconsistent embedding vectors".into()));
74    }
75    Ok(EmbedResult {
76        vectors: reply
77            .data
78            .into_iter()
79            .map(|entry| entry.embedding)
80            .collect(),
81        dimension,
82    })
83}
84pub struct OpenAiInference<'runtime> {
85    pub http: &'runtime dyn Transport,
86    pub probe: &'runtime dyn ProcessProbe,
87    pub adapter: &'runtime dyn BackendAdapter,
88    pub token: Option<&'runtime str>,
89}
90impl OpenAiInference<'_> {
91    async fn checked(
92        &self,
93        handle: &ServerState,
94        cancel: &CancellationToken,
95    ) -> Result<(), BackendError> {
96        if handle.backend != self.adapter.name() {
97            return Err(BackendError("inference backend mismatch".into()));
98        }
99        let observed = listener(self.probe, &handle.endpoint, cancel).await?;
100        capture(handle, &observed, self.adapter.trusts(&observed.identity))
101            .map_err(|error| BackendError(error.to_string()))?;
102        if handle.backend == "lmstudio" {
103            self.adapter
104                .ready_handle(
105                    &ServeRequest {
106                        model_id: handle.model_id.clone(),
107                        endpoint: handle.endpoint.clone(),
108                        model_path: handle.model_path.as_ref().map(PathBuf::from),
109                        context: None,
110                        cache: None,
111                    },
112                    handle,
113                    cancel,
114                )
115                .await?;
116        }
117        Ok(())
118    }
119    async fn json(
120        &self,
121        handle: &ServerState,
122        path: &str,
123        body: Value,
124        cancel: &CancellationToken,
125    ) -> Result<Value, BackendError> {
126        if serde_json::to_vec(&body)
127            .map_err(|error| BackendError(error.to_string()))?
128            .len()
129            > 4 * 1024 * 1024
130        {
131            return Err(BackendError("inference request exceeds limit".into()));
132        }
133        let token = if handle.backend == "mlx" {
134            handle.auth_token.clone()
135        } else {
136            self.token.map(str::to_owned)
137        };
138        let request = Request::new(&handle.endpoint, path, Some(body), token)
139            .map_err(|error| BackendError(error.to_string()))?;
140        tokio::time::timeout(
141            Duration::from_secs(30),
142            read_json(self.http, request, cancel, 2 * 1024 * 1024),
143        )
144        .await
145        .map_err(|_| BackendError("inference timed out".into()))?
146        .map_err(|error| BackendError(error.to_string()))
147    }
148    pub async fn chat(
149        &self,
150        handle: &ServerState,
151        input: &ChatInput,
152        cancel: &CancellationToken,
153    ) -> Result<ChatResult, BackendError> {
154        crate::adapters::model_id(&input.model)?;
155        if input.model != handle.model_id
156            || input.messages.len() > 10000
157            || input.messages.iter().any(|message| {
158                !["user", "assistant", "system", "tool"].contains(&message.role.as_str())
159            })
160        {
161            return Err(BackendError("invalid chat target or messages".into()));
162        }
163        self.checked(handle, cancel).await?;
164        let model = if handle.backend == "mlx" {
165            "default_model"
166        } else {
167            &input.model
168        };
169        let messages: Vec<_> = input
170            .messages
171            .iter()
172            .map(|message| json!({"role":message.role,"content":message.content}))
173            .collect();
174        let result = chat_result(
175            self.json(
176                handle,
177                "/v1/chat/completions",
178                json!({"model":model,"messages":messages,"stream":false}),
179                cancel,
180            )
181            .await?,
182        )?;
183        self.checked(handle, cancel).await?;
184        Ok(result)
185    }
186    pub async fn embed(
187        &self,
188        handle: &ServerState,
189        model: &str,
190        inputs: &[String],
191        cancel: &CancellationToken,
192    ) -> Result<EmbedResult, BackendError> {
193        if handle.backend != "lmstudio" {
194            return Err(BackendError("backend does not support embeddings".into()));
195        }
196        crate::adapters::model_id(model)?;
197        if model != handle.model_id || inputs.is_empty() || inputs.len() > 1024 {
198            return Err(BackendError(
199                "invalid embedding target or input count".into(),
200            ));
201        }
202        self.checked(handle, cancel).await?;
203        let result = embedding_result(
204            self.json(
205                handle,
206                "/v1/embeddings",
207                json!({"model":model,"input":inputs}),
208                cancel,
209            )
210            .await?,
211            inputs.len(),
212        )?;
213        self.checked(handle, cancel).await?;
214        Ok(result)
215    }
216}