1use crate::{
2 adapters::{BackendAdapter, BackendError, ServeRequest},
3 http::{Request, Transport, read_json},
4 identity::{ProcessProbe, capture},
5 ollama_inference::{ChatInput, ChatResult, EmbedResult},
6 process_control::listener,
7 state::ServerState,
8};
9use serde::Deserialize;
10use serde_json::{Value, json};
11use std::{path::PathBuf, time::Duration};
12use tokio_util::sync::CancellationToken;
13pub fn chat_result(value: Value) -> Result<ChatResult, BackendError> {
14 crate::usage::begin();
15 crate::usage::record(&value, crate::usage::Provider::OpenAi);
16 #[derive(Deserialize)]
17 struct Message {
18 content: String,
19 }
20 #[derive(Deserialize)]
21 struct Choice {
22 message: Message,
23 }
24 #[derive(Deserialize)]
25 struct Reply {
26 choices: Vec<Choice>,
27 }
28 let reply: Reply =
29 serde_json::from_value(value).map_err(|_| BackendError("invalid chat response".into()))?;
30 if reply.choices.is_empty() || reply.choices.len() > 8 {
31 return Err(BackendError("invalid chat choice count".into()));
32 }
33 let content = reply
34 .choices
35 .into_iter()
36 .next()
37 .ok_or_else(|| BackendError("no chat choice".into()))?
38 .message
39 .content;
40 if content.len() > 1048576 {
41 return Err(BackendError("generated content exceeds limit".into()));
42 }
43 Ok(ChatResult {
44 content,
45 tool_calls: Vec::new(),
46 })
47}
48pub fn embedding_result(value: Value, count: usize) -> Result<EmbedResult, BackendError> {
49 #[derive(Deserialize)]
50 struct Vector {
51 index: usize,
52 embedding: Vec<f64>,
53 }
54 #[derive(Deserialize)]
55 struct Reply {
56 data: Vec<Vector>,
57 }
58 let mut reply: Reply = serde_json::from_value(value)
59 .map_err(|_| BackendError("invalid embedding response".into()))?;
60 if count == 0 || count > 1024 || reply.data.len() != count {
61 return Err(BackendError("embedding count mismatch".into()));
62 }
63 reply.data.sort_by_key(|entry| entry.index);
64 let dimension = reply.data[0].embedding.len();
65 if dimension == 0
66 || dimension > 65536
67 || reply.data.iter().enumerate().any(|(index, entry)| {
68 entry.index != index
69 || entry.embedding.len() != dimension
70 || entry.embedding.iter().any(|value| !value.is_finite())
71 })
72 {
73 return Err(BackendError("inconsistent embedding vectors".into()));
74 }
75 Ok(EmbedResult {
76 vectors: reply
77 .data
78 .into_iter()
79 .map(|entry| entry.embedding)
80 .collect(),
81 dimension,
82 })
83}
84pub struct OpenAiInference<'runtime> {
85 pub http: &'runtime dyn Transport,
86 pub probe: &'runtime dyn ProcessProbe,
87 pub adapter: &'runtime dyn BackendAdapter,
88 pub token: Option<&'runtime str>,
89}
90impl OpenAiInference<'_> {
91 async fn checked(
92 &self,
93 handle: &ServerState,
94 cancel: &CancellationToken,
95 ) -> Result<(), BackendError> {
96 if handle.backend != self.adapter.name() {
97 return Err(BackendError("inference backend mismatch".into()));
98 }
99 let observed = listener(self.probe, &handle.endpoint, cancel).await?;
100 capture(handle, &observed, self.adapter.trusts(&observed.identity))
101 .map_err(|error| BackendError(error.to_string()))?;
102 if handle.backend == "lmstudio" {
103 self.adapter
104 .ready_handle(
105 &ServeRequest {
106 draft_model_path: None,
107 model_id: handle.model_id.clone(),
108 endpoint: handle.endpoint.clone(),
109 model_path: handle.model_path.as_ref().map(PathBuf::from),
110 context: None,
111 cache: None,
112 },
113 handle,
114 cancel,
115 )
116 .await?;
117 }
118 Ok(())
119 }
120 async fn json(
121 &self,
122 handle: &ServerState,
123 path: &str,
124 body: Value,
125 cancel: &CancellationToken,
126 ) -> Result<Value, BackendError> {
127 if serde_json::to_vec(&body)
128 .map_err(|error| BackendError(error.to_string()))?
129 .len()
130 > 4 * 1024 * 1024
131 {
132 return Err(BackendError("inference request exceeds limit".into()));
133 }
134 let token = if handle.backend == "mlx" {
135 handle.auth_token.clone()
136 } else {
137 self.token.map(str::to_owned)
138 };
139 let request = Request::new(&handle.endpoint, path, Some(body), token)
140 .map_err(|error| BackendError(error.to_string()))?;
141 tokio::time::timeout(
142 Duration::from_secs(30),
143 read_json(self.http, request, cancel, 2 * 1024 * 1024),
144 )
145 .await
146 .map_err(|_| BackendError("inference timed out".into()))?
147 .map_err(|error| BackendError(error.to_string()))
148 }
149 pub async fn chat(
150 &self,
151 handle: &ServerState,
152 input: &ChatInput,
153 cancel: &CancellationToken,
154 ) -> Result<ChatResult, BackendError> {
155 crate::adapters::model_id(&input.model)?;
156 if input.model != handle.model_id
157 || input.messages.len() > 10000
158 || input.messages.iter().any(|message| {
159 !["user", "assistant", "system", "tool"].contains(&message.role.as_str())
160 })
161 {
162 return Err(BackendError("invalid chat target or messages".into()));
163 }
164 self.checked(handle, cancel).await?;
165 let model = if handle.backend == "mlx" {
166 "default_model"
167 } else {
168 &input.model
169 };
170 let messages: Vec<_> = input
171 .messages
172 .iter()
173 .map(|message| json!({"role":message.role,"content":message.content}))
174 .collect();
175 let result = chat_result(
176 self.json(
177 handle,
178 "/v1/chat/completions",
179 json!({"model":model,"messages":messages,"stream":false}),
180 cancel,
181 )
182 .await?,
183 )?;
184 self.checked(handle, cancel).await?;
185 Ok(result)
186 }
187 pub async fn embed(
188 &self,
189 handle: &ServerState,
190 model: &str,
191 inputs: &[String],
192 cancel: &CancellationToken,
193 ) -> Result<EmbedResult, BackendError> {
194 if handle.backend != "lmstudio" {
195 return Err(BackendError("backend does not support embeddings".into()));
196 }
197 crate::adapters::model_id(model)?;
198 if model != handle.model_id || inputs.is_empty() || inputs.len() > 1024 {
199 return Err(BackendError(
200 "invalid embedding target or input count".into(),
201 ));
202 }
203 self.checked(handle, cancel).await?;
204 let result = embedding_result(
205 self.json(
206 handle,
207 "/v1/embeddings",
208 json!({"model":model,"input":inputs}),
209 cancel,
210 )
211 .await?,
212 inputs.len(),
213 )?;
214 self.checked(handle, cancel).await?;
215 Ok(result)
216 }
217}