1use std::collections::HashMap;
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::sync::{Arc, Mutex};
7
8use bytes::Bytes;
9use futures_util::{StreamExt, future::BoxFuture, stream::BoxStream};
10use tokio_util::sync::CancellationToken;
11
12use crate::callbacks::{Caller, RequestScope};
13use crate::client::CallbackError;
14use crate::gojson::{self, Header};
15use crate::host::{Host, lock};
16use crate::{abi, go_struct};
17
18go_struct! {
19 pub struct ModelRequest("pluginhost.rpcHostModelExecutionRequest") {
20 "entry_protocol" => entry_protocol: String,
21 "exit_protocol" => exit_protocol: String,
22 "model" => model: String,
23 "stream" => stream: bool,
24 "body" => body: Bytes,
25 "headers" => headers: Header,
26 "query" => query: Header,
27 "alt" => alt: String,
28 "forced_provider" omitempty => forced_provider: String,
29 "auth_id" omitempty => auth_id: String,
30 "proxy_url" omitempty => proxy_url: String,
31 "path" omitempty => path: String,
32 "host_callback_id" omitempty => host_callback_id: String,
33 }
34}
35
36go_struct! {
37 pub struct ModelResponse("pluginapi.HostModelExecutionResponse") {
38 "status_code" => status_code: i64,
39 "headers" => headers: Header,
40 "body" => body: Bytes,
41 }
42}
43
44go_struct! {
45 pub struct ModelStreamResponse("pluginapi.HostModelStreamResponse") {
46 "status_code" => status_code: i64,
47 "headers" => headers: Header,
48 "stream_id" => stream_id: String,
49 }
50}
51
52go_struct! {
53 pub struct StreamRequest("pluginapi.HostModelStreamReadRequest") {
54 "stream_id" => stream_id: String,
55 }
56}
57
58go_struct! {
59 pub struct StreamResponse("pluginapi.HostModelStreamReadResponse") {
60 "payload" => payload: Bytes,
61 "error" => error: String,
62 "done" => done: bool,
63 }
64}
65
66pub trait ModelExecutor: Send + Sync {
69 fn execute(
70 &self,
71 request: ModelRequest,
72 skip_plugin: String,
73 scope: RequestScope,
74 ) -> BoxFuture<'_, Result<ModelOutput, CallbackError>>;
75}
76
77pub enum ModelOutput {
78 Buffered(ModelResponse),
79 Stream {
80 status_code: i64,
81 headers: Header,
82 chunks: BoxStream<'static, crate::streams::Chunk>,
83 },
84}
85
86struct Entry {
87 chunks: tokio::sync::Mutex<BoxStream<'static, crate::streams::Chunk>>,
88 cancel: CancellationToken,
89}
90
91impl Drop for Entry {
92 fn drop(&mut self) {
93 self.cancel.cancel();
94 }
95}
96
97#[derive(Default)]
98pub(crate) struct ModelBridge {
99 next: AtomicU64,
100 streams: Mutex<HashMap<String, Arc<Entry>>>,
101}
102
103impl ModelBridge {
104 fn close(&self, id: &str) {
105 let entry = lock(&self.streams).remove(id);
108 if let Some(entry) = entry {
109 entry.cancel.cancel();
110 }
111 }
112}
113
114impl Host {
115 pub fn set_model_executor(&self, executor: Option<Arc<dyn ModelExecutor>>) {
116 let previous = std::mem::replace(
117 &mut *self
118 .inner
119 .callbacks
120 .model_executor
121 .write()
122 .unwrap_or_else(std::sync::PoisonError::into_inner),
123 executor,
124 );
125 drop(previous);
126 }
127
128 pub(crate) fn host_model_execute(&self, caller: &Caller, raw: &[u8], stream: bool) -> Result<Bytes, CallbackError> {
129 let label = if stream {
130 "host model execution stream request"
131 } else {
132 "host model execution request"
133 };
134 let mut req: ModelRequest =
135 gojson::from_slice(raw).map_err(|e| CallbackError::new(format!("decode {label}: {e}")))?;
136 if req.stream != stream {
137 return Err(CallbackError::new(if stream {
138 "host.model.execute_stream requires stream=true"
139 } else {
140 "host.model.execute requires stream=false"
141 }));
142 }
143 req.proxy_url = req.proxy_url.trim().to_owned();
144 if !req.proxy_url.is_empty() && !valid_proxy(&req.proxy_url) {
145 return Err(CallbackError {
146 message: "invalid proxy_url".into(),
147 status: 400,
148 });
149 }
150 req.path = req.path.trim().to_owned();
151 let executor = self
152 .inner
153 .callbacks
154 .model_executor
155 .read()
156 .unwrap_or_else(std::sync::PoisonError::into_inner)
157 .clone()
158 .ok_or_else(|| CallbackError::new("host model executor is unavailable"))?;
159 let runtime = self
160 .inner
161 .callbacks
162 .runtime_handle()
163 .ok_or_else(|| CallbackError::new("host model executor is unavailable"))?;
164 let mut scope = self
165 .inner
166 .callbacks
167 .lookup(&req.host_callback_id)
168 .map(|(_, _, scope)| scope)
169 .unwrap_or_default();
170 if stream {
171 scope.cancel = CancellationToken::new();
174 }
175 let cancel = scope.cancel.clone();
176 let callback_id = req.host_callback_id.clone();
177 let output = runtime.block_on(async {
178 tokio::select! {
179 biased;
180 () = cancel.cancelled() => Err(CallbackError::new("context canceled")),
181 result = executor.execute(req, caller.plugin_id.trim().to_owned(), scope) => result,
182 }
183 });
184 match output {
185 Ok(ModelOutput::Buffered(resp)) => Ok(abi::ok_envelope(&resp)),
186 Ok(ModelOutput::Stream {
187 status_code,
188 headers,
189 chunks,
190 }) => {
191 let bridge = &self.inner.callbacks.model;
192 let stream_id = (bridge.next.fetch_add(1, Ordering::SeqCst) + 1).to_string();
193 lock(&bridge.streams).insert(
194 stream_id.clone(),
195 Arc::new(Entry {
196 chunks: tokio::sync::Mutex::new(chunks),
197 cancel,
198 }),
199 );
200 if !callback_id.is_empty() {
201 let weak = Arc::downgrade(&self.inner);
202 let id = stream_id.clone();
203 if self
204 .inner
205 .callbacks
206 .add_cleanup(
207 &callback_id,
208 Box::new(move || {
209 if let Some(inner) = weak.upgrade() {
210 inner.callbacks.model.close(&id);
211 }
212 }),
213 )
214 .is_none()
215 {
216 bridge.close(&stream_id);
217 }
218 }
219 Ok(abi::ok_envelope(&ModelStreamResponse {
220 status_code,
221 headers,
222 stream_id,
223 }))
224 }
225 Err(e) => {
226 if stream {
227 cancel.cancel();
228 }
229 Err(e)
230 }
231 }
232 }
233
234 pub(crate) fn host_model_stream_read(&self, raw: &[u8]) -> Result<Bytes, CallbackError> {
235 let req: StreamRequest = gojson::from_slice(raw)
236 .map_err(|e| CallbackError::new(format!("decode host model stream read request: {e}")))?;
237 if req.stream_id.is_empty() {
238 return Err(CallbackError::new("model stream id is required"));
239 }
240 let bridge = &self.inner.callbacks.model;
241 let entry = lock(&bridge.streams).get(&req.stream_id).cloned();
242 let next = match entry {
243 None => None,
244 Some(entry) => {
245 let runtime = self
246 .inner
247 .callbacks
248 .runtime_handle()
249 .ok_or_else(|| CallbackError::new("host model stream bridge is unavailable"))?;
250 runtime.block_on(async {
251 tokio::select! {
252 biased;
253 () = entry.cancel.cancelled() => None,
254 chunk = async { entry.chunks.lock().await.next().await } => chunk,
255 }
256 })
257 }
258 };
259 let resp = match next {
260 Some(chunk) => StreamResponse {
261 payload: chunk.payload,
262 done: chunk.error.is_some(),
263 error: chunk.error.unwrap_or_default(),
264 },
265 None => StreamResponse {
266 done: true,
267 ..Default::default()
268 },
269 };
270 if resp.done {
271 bridge.close(&req.stream_id);
272 }
273 Ok(abi::ok_envelope(&resp))
274 }
275
276 pub(crate) fn host_model_stream_close(&self, raw: &[u8]) -> Result<Bytes, CallbackError> {
277 let req: StreamRequest = gojson::from_slice(raw)
278 .map_err(|e| CallbackError::new(format!("decode host model stream close request: {e}")))?;
279 self.inner.callbacks.model.close(&req.stream_id);
280 Ok(abi::ok_envelope(&crate::rpc::Empty {}))
281 }
282}
283
284fn valid_proxy(raw: &str) -> bool {
285 cpa_exec::xai_url::parse(raw).is_ok_and(|u| {
286 matches!(u.scheme.as_str(), "http" | "https" | "socks5" | "socks5h")
287 && !u.hostname.trim().is_empty()
288 && (u.port.is_empty() || u.port.parse::<u16>().is_ok_and(|p| p != 0))
289 })
290}