Skip to main content

cpa_plugin/
hostmodel.rs

1//! `host.model.*` callbacks (v8.0.35 host_callbacks.go,
2//! host_model_stream_callbacks.go and model_stream_bridge.go).
3
4use std::collections::HashMap;
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::sync::{Arc, Mutex};
7
8use bytes::Bytes;
9use futures_util::{StreamExt, future::BoxFuture, stream::BoxStream};
10use tokio_util::sync::CancellationToken;
11
12use crate::callbacks::{Caller, RequestScope};
13use crate::client::CallbackError;
14use crate::gojson::{self, Header};
15use crate::host::{Host, lock};
16use crate::{abi, go_struct};
17
18go_struct! {
19    pub struct ModelRequest("pluginhost.rpcHostModelExecutionRequest") {
20        "entry_protocol" => entry_protocol: String,
21        "exit_protocol" => exit_protocol: String,
22        "model" => model: String,
23        "stream" => stream: bool,
24        "body" => body: Bytes,
25        "headers" => headers: Header,
26        "query" => query: Header,
27        "alt" => alt: String,
28        "forced_provider" omitempty => forced_provider: String,
29        "auth_id" omitempty => auth_id: String,
30        "proxy_url" omitempty => proxy_url: String,
31        "path" omitempty => path: String,
32        "host_callback_id" omitempty => host_callback_id: String,
33    }
34}
35
36go_struct! {
37    pub struct ModelResponse("pluginapi.HostModelExecutionResponse") {
38        "status_code" => status_code: i64,
39        "headers" => headers: Header,
40        "body" => body: Bytes,
41    }
42}
43
44go_struct! {
45    pub struct ModelStreamResponse("pluginapi.HostModelStreamResponse") {
46        "status_code" => status_code: i64,
47        "headers" => headers: Header,
48        "stream_id" => stream_id: String,
49    }
50}
51
52go_struct! {
53    pub struct StreamRequest("pluginapi.HostModelStreamReadRequest") {
54        "stream_id" => stream_id: String,
55    }
56}
57
58go_struct! {
59    pub struct StreamResponse("pluginapi.HostModelStreamReadResponse") {
60        "payload" => payload: Bytes,
61        "error" => error: String,
62        "done" => done: bool,
63    }
64}
65
66/// The server's model execution boundary (Go `HostModelExecutor`). The caller is
67/// skipped only by routers and interceptors; other plugins still participate.
68pub trait ModelExecutor: Send + Sync {
69    fn execute(
70        &self,
71        request: ModelRequest,
72        skip_plugin: String,
73        scope: RequestScope,
74    ) -> BoxFuture<'_, Result<ModelOutput, CallbackError>>;
75}
76
77pub enum ModelOutput {
78    Buffered(ModelResponse),
79    Stream {
80        status_code: i64,
81        headers: Header,
82        chunks: BoxStream<'static, crate::streams::Chunk>,
83    },
84}
85
86struct Entry {
87    chunks: tokio::sync::Mutex<BoxStream<'static, crate::streams::Chunk>>,
88    cancel: CancellationToken,
89}
90
91impl Drop for Entry {
92    fn drop(&mut self) {
93        self.cancel.cancel();
94    }
95}
96
97#[derive(Default)]
98pub(crate) struct ModelBridge {
99    next: AtomicU64,
100    streams: Mutex<HashMap<String, Arc<Entry>>>,
101}
102
103impl ModelBridge {
104    fn close(&self, id: &str) {
105        // Dropping a plugin stream can close its callback-owned child streams.
106        // Release the registry lock before either cancellation or destruction.
107        let entry = lock(&self.streams).remove(id);
108        if let Some(entry) = entry {
109            entry.cancel.cancel();
110        }
111    }
112}
113
114impl Host {
115    pub fn set_model_executor(&self, executor: Option<Arc<dyn ModelExecutor>>) {
116        let previous = std::mem::replace(
117            &mut *self
118                .inner
119                .callbacks
120                .model_executor
121                .write()
122                .unwrap_or_else(std::sync::PoisonError::into_inner),
123            executor,
124        );
125        drop(previous);
126    }
127
128    pub(crate) fn host_model_execute(&self, caller: &Caller, raw: &[u8], stream: bool) -> Result<Bytes, CallbackError> {
129        let label = if stream {
130            "host model execution stream request"
131        } else {
132            "host model execution request"
133        };
134        let mut req: ModelRequest =
135            gojson::from_slice(raw).map_err(|e| CallbackError::new(format!("decode {label}: {e}")))?;
136        if req.stream != stream {
137            return Err(CallbackError::new(if stream {
138                "host.model.execute_stream requires stream=true"
139            } else {
140                "host.model.execute requires stream=false"
141            }));
142        }
143        req.proxy_url = req.proxy_url.trim().to_owned();
144        if !req.proxy_url.is_empty() && !valid_proxy(&req.proxy_url) {
145            return Err(CallbackError {
146                message: "invalid proxy_url".into(),
147                status: 400,
148            });
149        }
150        req.path = req.path.trim().to_owned();
151        let executor = self
152            .inner
153            .callbacks
154            .model_executor
155            .read()
156            .unwrap_or_else(std::sync::PoisonError::into_inner)
157            .clone()
158            .ok_or_else(|| CallbackError::new("host model executor is unavailable"))?;
159        let runtime = self
160            .inner
161            .callbacks
162            .runtime_handle()
163            .ok_or_else(|| CallbackError::new("host model executor is unavailable"))?;
164        let mut scope = self
165            .inner
166            .callbacks
167            .lookup(&req.host_callback_id)
168            .map(|(_, _, scope)| scope)
169            .unwrap_or_default();
170        if stream {
171            // v8.0.35 detaches cancellation, not request values. Callback cleanup owns
172            // the stream even when the parent request has already been cancelled.
173            scope.cancel = CancellationToken::new();
174        }
175        let cancel = scope.cancel.clone();
176        let callback_id = req.host_callback_id.clone();
177        let output = runtime.block_on(async {
178            tokio::select! {
179                biased;
180                () = cancel.cancelled() => Err(CallbackError::new("context canceled")),
181                result = executor.execute(req, caller.plugin_id.trim().to_owned(), scope) => result,
182            }
183        });
184        match output {
185            Ok(ModelOutput::Buffered(resp)) => Ok(abi::ok_envelope(&resp)),
186            Ok(ModelOutput::Stream {
187                status_code,
188                headers,
189                chunks,
190            }) => {
191                let bridge = &self.inner.callbacks.model;
192                let stream_id = (bridge.next.fetch_add(1, Ordering::SeqCst) + 1).to_string();
193                lock(&bridge.streams).insert(
194                    stream_id.clone(),
195                    Arc::new(Entry {
196                        chunks: tokio::sync::Mutex::new(chunks),
197                        cancel,
198                    }),
199                );
200                if !callback_id.is_empty() {
201                    let weak = Arc::downgrade(&self.inner);
202                    let id = stream_id.clone();
203                    if self
204                        .inner
205                        .callbacks
206                        .add_cleanup(
207                            &callback_id,
208                            Box::new(move || {
209                                if let Some(inner) = weak.upgrade() {
210                                    inner.callbacks.model.close(&id);
211                                }
212                            }),
213                        )
214                        .is_none()
215                    {
216                        bridge.close(&stream_id);
217                    }
218                }
219                Ok(abi::ok_envelope(&ModelStreamResponse {
220                    status_code,
221                    headers,
222                    stream_id,
223                }))
224            }
225            Err(e) => {
226                if stream {
227                    cancel.cancel();
228                }
229                Err(e)
230            }
231        }
232    }
233
234    pub(crate) fn host_model_stream_read(&self, raw: &[u8]) -> Result<Bytes, CallbackError> {
235        let req: StreamRequest = gojson::from_slice(raw)
236            .map_err(|e| CallbackError::new(format!("decode host model stream read request: {e}")))?;
237        if req.stream_id.is_empty() {
238            return Err(CallbackError::new("model stream id is required"));
239        }
240        let bridge = &self.inner.callbacks.model;
241        let entry = lock(&bridge.streams).get(&req.stream_id).cloned();
242        let next = match entry {
243            None => None,
244            Some(entry) => {
245                let runtime = self
246                    .inner
247                    .callbacks
248                    .runtime_handle()
249                    .ok_or_else(|| CallbackError::new("host model stream bridge is unavailable"))?;
250                runtime.block_on(async {
251                    tokio::select! {
252                        biased;
253                        () = entry.cancel.cancelled() => None,
254                        chunk = async { entry.chunks.lock().await.next().await } => chunk,
255                    }
256                })
257            }
258        };
259        let resp = match next {
260            Some(chunk) => StreamResponse {
261                payload: chunk.payload,
262                done: chunk.error.is_some(),
263                error: chunk.error.unwrap_or_default(),
264            },
265            None => StreamResponse {
266                done: true,
267                ..Default::default()
268            },
269        };
270        if resp.done {
271            bridge.close(&req.stream_id);
272        }
273        Ok(abi::ok_envelope(&resp))
274    }
275
276    pub(crate) fn host_model_stream_close(&self, raw: &[u8]) -> Result<Bytes, CallbackError> {
277        let req: StreamRequest = gojson::from_slice(raw)
278            .map_err(|e| CallbackError::new(format!("decode host model stream close request: {e}")))?;
279        self.inner.callbacks.model.close(&req.stream_id);
280        Ok(abi::ok_envelope(&crate::rpc::Empty {}))
281    }
282}
283
284fn valid_proxy(raw: &str) -> bool {
285    cpa_exec::xai_url::parse(raw).is_ok_and(|u| {
286        matches!(u.scheme.as_str(), "http" | "https" | "socks5" | "socks5h")
287            && !u.hostname.trim().is_empty()
288            && (u.port.is_empty() || u.port.parse::<u16>().is_ok_and(|p| p != 0))
289    })
290}