Skip to main content

rigspark_runtime/
openai.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest},
3    http::{Request, Transport, read_json},
4    identity::{ProcessProbe, capture},
5    ollama_inference::{ChatInput, ChatResult, EmbedResult},
6    process_control::listener,
7    state::ServerState,
8};
9use serde::Deserialize;
10use serde_json::{Value, json};
11use std::{path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13pub fn chat_result(value: Value) -> Result<ChatResult, BackendError> {
14    crate::usage::begin();
15    crate::usage::record(&value, crate::usage::Provider::OpenAi);
16    #[derive(Deserialize)]
17    struct Message {
18        content: String,
19    }
20    #[derive(Deserialize)]
21    struct Choice {
22        message: Message,
23    }
24    #[derive(Deserialize)]
25    struct Reply {
26        choices: Vec<Choice>,
27    }
28    let reply: Reply =
29        serde_json::from_value(value).map_err(|_| BackendError("invalid chat response".into()))?;
30    if reply.choices.is_empty() || reply.choices.len() > 8 {
31        return Err(BackendError("invalid chat choice count".into()));
32    }
33    let content = reply
34        .choices
35        .into_iter()
36        .next()
37        .ok_or_else(|| BackendError("no chat choice".into()))?
38        .message
39        .content;
40    if content.len() > 1048576 {
41        return Err(BackendError("generated content exceeds limit".into()));
42    }
43    Ok(ChatResult {
44        content,
45        tool_calls: Vec::new(),
46    })
47}
48pub fn embedding_result(value: Value, count: usize) -> Result<EmbedResult, BackendError> {
49    #[derive(Deserialize)]
50    struct Vector {
51        index: usize,
52        embedding: Vec<f64>,
53    }
54    #[derive(Deserialize)]
55    struct Reply {
56        data: Vec<Vector>,
57    }
58    let mut reply: Reply = serde_json::from_value(value)
59        .map_err(|_| BackendError("invalid embedding response".into()))?;
60    if count == 0 || count > 1024 || reply.data.len() != count {
61        return Err(BackendError("embedding count mismatch".into()));
62    }
63    reply.data.sort_by_key(|entry| entry.index);
64    let dimension = reply.data[0].embedding.len();
65    if dimension == 0
66        || dimension > 65536
67        || reply.data.iter().enumerate().any(|(index, entry)| {
68            entry.index != index
69                || entry.embedding.len() != dimension
70                || entry.embedding.iter().any(|value| !value.is_finite())
71        })
72    {
73        return Err(BackendError("inconsistent embedding vectors".into()));
74    }
75    Ok(EmbedResult {
76        vectors: reply
77            .data
78            .into_iter()
79            .map(|entry| entry.embedding)
80            .collect(),
81        dimension,
82    })
83}
84pub struct OpenAiInference<'runtime> {
85    pub http: &'runtime dyn Transport,
86    pub probe: &'runtime dyn ProcessProbe,
87    pub adapter: &'runtime dyn BackendAdapter,
88    pub token: Option<&'runtime str>,
89}
90impl OpenAiInference<'_> {
91    async fn checked(
92        &self,
93        handle: &ServerState,
94        cancel: &CancellationToken,
95    ) -> Result<(), BackendError> {
96        if handle.backend != self.adapter.name() {
97            return Err(BackendError("inference backend mismatch".into()));
98        }
99        let observed = listener(self.probe, &handle.endpoint, cancel).await?;
100        capture(handle, &observed, self.adapter.trusts(&observed.identity))
101            .map_err(|error| BackendError(error.to_string()))?;
102        if handle.backend == "lmstudio" {
103            self.adapter
104                .ready_handle(
105                    &ServeRequest {
106                        draft_model_path: None,
107                        model_id: handle.model_id.clone(),
108                        endpoint: handle.endpoint.clone(),
109                        model_path: handle.model_path.as_ref().map(PathBuf::from),
110                        context: None,
111                        cache: None,
112                    },
113                    handle,
114                    cancel,
115                )
116                .await?;
117        }
118        Ok(())
119    }
120    async fn json(
121        &self,
122        handle: &ServerState,
123        path: &str,
124        body: Value,
125        cancel: &CancellationToken,
126    ) -> Result<Value, BackendError> {
127        if serde_json::to_vec(&body)
128            .map_err(|error| BackendError(error.to_string()))?
129            .len()
130            > 4 * 1024 * 1024
131        {
132            return Err(BackendError("inference request exceeds limit".into()));
133        }
134        let token = if handle.backend == "mlx" {
135            handle.auth_token.clone()
136        } else {
137            self.token.map(str::to_owned)
138        };
139        let request = Request::new(&handle.endpoint, path, Some(body), token)
140            .map_err(|error| BackendError(error.to_string()))?;
141        tokio::time::timeout(
142            Duration::from_secs(30),
143            read_json(self.http, request, cancel, 2 * 1024 * 1024),
144        )
145        .await
146        .map_err(|_| BackendError("inference timed out".into()))?
147        .map_err(|error| BackendError(error.to_string()))
148    }
149    pub async fn chat(
150        &self,
151        handle: &ServerState,
152        input: &ChatInput,
153        cancel: &CancellationToken,
154    ) -> Result<ChatResult, BackendError> {
155        crate::adapters::model_id(&input.model)?;
156        if input.model != handle.model_id
157            || input.messages.len() > 10000
158            || input.messages.iter().any(|message| {
159                !["user", "assistant", "system", "tool"].contains(&message.role.as_str())
160            })
161        {
162            return Err(BackendError("invalid chat target or messages".into()));
163        }
164        self.checked(handle, cancel).await?;
165        let model = if handle.backend == "mlx" {
166            "default_model"
167        } else {
168            &input.model
169        };
170        let messages: Vec<_> = input
171            .messages
172            .iter()
173            .map(|message| json!({"role":message.role,"content":message.content}))
174            .collect();
175        let result = chat_result(
176            self.json(
177                handle,
178                "/v1/chat/completions",
179                json!({"model":model,"messages":messages,"stream":false}),
180                cancel,
181            )
182            .await?,
183        )?;
184        self.checked(handle, cancel).await?;
185        Ok(result)
186    }
187    pub async fn embed(
188        &self,
189        handle: &ServerState,
190        model: &str,
191        inputs: &[String],
192        cancel: &CancellationToken,
193    ) -> Result<EmbedResult, BackendError> {
194        if handle.backend != "lmstudio" {
195            return Err(BackendError("backend does not support embeddings".into()));
196        }
197        crate::adapters::model_id(model)?;
198        if model != handle.model_id || inputs.is_empty() || inputs.len() > 1024 {
199            return Err(BackendError(
200                "invalid embedding target or input count".into(),
201            ));
202        }
203        self.checked(handle, cancel).await?;
204        let result = embedding_result(
205            self.json(
206                handle,
207                "/v1/embeddings",
208                json!({"model":model,"input":inputs}),
209                cancel,
210            )
211            .await?,
212            inputs.len(),
213        )?;
214        self.checked(handle, cancel).await?;
215        Ok(result)
216    }
217}