Skip to main content

dekopon_model_proxy/
lib.rs

1#![forbid(unsafe_code)]
2#![cfg_attr(test, allow(clippy::unwrap_used))]
3#![cfg_attr(
4    test,
5    allow(
6        clippy::disallowed_methods,
7        reason = "tests spawn freely; production sites carry their own expectation"
8    )
9)]
10
11mod dialect;
12mod tee;
13pub mod tls;
14
15#[cfg(test)]
16mod tests;
17
18use std::{
19    collections::{BTreeSet, HashMap},
20    sync::Arc,
21    time::Duration,
22};
23
24use axum::{
25    Router,
26    body::Body,
27    extract::State,
28    http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header},
29    response::Response,
30    routing::post,
31};
32use bytes::Bytes;
33use dekopon_core::{AgentId, Redacted, VM_SESSION_HEADER, VmSession};
34use dekopon_model::{
35    chatgpt::{CredentialFile, RefreshClient},
36    wire::{Field, PeekError, RequestPeek},
37};
38use dekopon_model_token_governor::{
39    Call, Estimate, Metering, ModelUsage, Outcome, Sizes, Tokens, Via,
40};
41use tracing::{Instrument as _, Span, field::Empty};
42
43use dialect::Problem;
44pub use dialect::{Dialect, SANDBOX};
45use tee::{Shape, Timing};
46
47pub const MAX_BODY_BYTES: usize = 8 * 1024 * 1024;
48pub const SUBJECT_HEADER: &str = "x-dekopon-vm-subject";
49const MAX_LABEL_BYTES: usize = 128;
50pub const PING_INTERVAL: Duration = Duration::from_secs(20);
51pub const UPSTREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
52const MAX_ERROR_BYTES: usize = 64 * 1024;
53const PASSED_HEADERS: [&str; 3] = ["anthropic-version", "anthropic-beta", "x-request-id"];
54const RETURNED_HEADERS: [&str; 5] = [
55    "content-type",
56    "x-request-id",
57    "request-id",
58    "cache-control",
59    "retry-after",
60];
61
62pub const ANTHROPIC_ENDPOINT: &str = "https://api.anthropic.com";
63pub const OPENROUTER_ENDPOINT: &str = "https://openrouter.ai/api";
64pub const CODEX_ENDPOINT: &str = "https://chatgpt.com/backend-api/codex";
65
66#[derive(Clone)]
67pub enum Upstream {
68    Codex {
69        endpoint: String,
70        credential: Arc<CredentialFile>,
71        refresh: RefreshClient,
72    },
73    OpenRouter {
74        endpoint: String,
75        api_key: Redacted<String>,
76        settings: Vec<(Field, String)>,
77    },
78    Anthropic {
79        endpoint: String,
80        api_key: Redacted<String>,
81    },
82}
83
84impl Upstream {
85    const fn serves(&self, dialect: Dialect) -> bool {
86        match (self, dialect) {
87            (Self::Anthropic { .. }, Dialect::Messages | Dialect::CountTokens)
88            | (Self::Codex { .. }, Dialect::Responses)
89            | (Self::OpenRouter { .. }, Dialect::Chat) => true,
90            (Self::Anthropic { .. }, Dialect::Responses | Dialect::Chat)
91            | (Self::Codex { .. }, Dialect::Messages | Dialect::CountTokens | Dialect::Chat)
92            | (
93                Self::OpenRouter { .. },
94                Dialect::Messages | Dialect::CountTokens | Dialect::Responses,
95            ) => false,
96        }
97    }
98
99    const fn path(&self) -> &'static str {
100        match self {
101            Self::Anthropic { .. } => "/v1/messages",
102            Self::Codex { .. } => "/v1/responses",
103            Self::OpenRouter { .. } => "/v1/chat/completions",
104        }
105    }
106
107    fn url(&self, dialect: Dialect) -> String {
108        let (endpoint, path) = match (self, dialect) {
109            (Self::Anthropic { endpoint, .. }, Dialect::CountTokens) => {
110                (endpoint, "/v1/messages/count_tokens")
111            }
112            (Self::Anthropic { endpoint, .. }, _) => (endpoint, "/v1/messages"),
113            (Self::Codex { endpoint, .. }, _) => (endpoint, "/responses"),
114            (Self::OpenRouter { endpoint, .. }, _) => (endpoint, "/v1/chat/completions"),
115        };
116        format!("{}{path}", endpoint.trim_end_matches('/'))
117    }
118}
119
120pub struct ProxyModel {
121    pub name: String,
122    pub wire_model: String,
123    pub backend: &'static str,
124    pub reserve: Tokens,
125    pub upstream: Upstream,
126}
127
128pub struct Grant {
129    pub agent: AgentId,
130    pub models: Granted,
131}
132
133#[derive(Clone, Debug, Eq, PartialEq)]
134pub enum Granted {
135    Listed(BTreeSet<String>),
136    Inherited(String),
137}
138
139impl Granted {
140    fn names(&self) -> Vec<&str> {
141        match self {
142            Self::Listed(models) => models.iter().map(String::as_str).collect(),
143            Self::Inherited(model) => vec![model.as_str()],
144        }
145    }
146}
147
148pub struct ModelProxy {
149    models: HashMap<String, ProxyModel>,
150    guests: HashMap<String, Grant>,
151    metering: Arc<Metering>,
152    http: reqwest::Client,
153    timing: Timing,
154}
155
156#[derive(Debug, thiserror::Error)]
157#[error("model proxy HTTP client could not be built")]
158pub struct ClientError(#[source] reqwest::Error);
159
160impl ModelProxy {
161    pub fn new(
162        models: Vec<ProxyModel>,
163        guests: HashMap<String, Grant>,
164        metering: Arc<Metering>,
165    ) -> Result<Self, ClientError> {
166        let http = reqwest::Client::builder()
167            .redirect(reqwest::redirect::Policy::none())
168            .no_proxy()
169            .read_timeout(UPSTREAM_IDLE_TIMEOUT)
170            .build()
171            .map_err(ClientError)?;
172        Ok(Self {
173            models: models
174                .into_iter()
175                .map(|model| (model.name.clone(), model))
176                .collect(),
177            guests,
178            metering,
179            http,
180            timing: Timing {
181                ping: PING_INTERVAL,
182                idle: UPSTREAM_IDLE_TIMEOUT,
183            },
184        })
185    }
186
187    #[cfg(test)]
188    fn with_timing(mut self, ping: Duration, idle: Duration) -> Self {
189        self.timing = Timing { ping, idle };
190        self
191    }
192
193    fn model_rule(&self, grant: &Grant, dialect: Dialect) -> String {
194        let granted = || {
195            grant
196                .models
197                .names()
198                .into_iter()
199                .filter_map(|name| self.models.get(name))
200        };
201        let served = granted()
202            .filter(|model| model.upstream.serves(dialect))
203            .map(|model| format!("`{}`", model.name))
204            .collect::<Vec<_>>();
205        if served.is_empty() {
206            let elsewhere = granted()
207                .map(|model| format!("`{}` on `{}`", model.name, model.upstream.path()))
208                .collect::<Vec<_>>();
209            format!(
210                "this agent may only call its configured models, and none is served on this path: {}. Call one on its own path.",
211                elsewhere.join(", ")
212            )
213        } else {
214            format!(
215                "this agent may only call its configured models: {}. Set the model to one of them.",
216                served.join(", ")
217            )
218        }
219    }
220
221    pub fn router_current<C: CurrentProxy>(current: C) -> Router {
222        Router::new()
223            .route(
224                "/v1/messages",
225                post(|state, headers, body| {
226                    handle_current::<C>(state, Dialect::Messages, headers, body)
227                }),
228            )
229            .route(
230                "/v1/messages/count_tokens",
231                post(|state, headers, body| {
232                    handle_current::<C>(state, Dialect::CountTokens, headers, body)
233                }),
234            )
235            .route(
236                "/v1/responses",
237                post(|state, headers, body| {
238                    handle_current::<C>(state, Dialect::Responses, headers, body)
239                }),
240            )
241            .route(
242                "/v1/chat/completions",
243                post(|state, headers, body| {
244                    handle_current::<C>(state, Dialect::Chat, headers, body)
245                }),
246            )
247            .with_state(current)
248    }
249
250    pub fn router(self: Arc<Self>) -> Router {
251        Self::router_current(self)
252    }
253}
254
255struct CallSpan {
256    inner: Span,
257    session: Option<VmSession>,
258    usage: ModelUsage,
259    settled: bool,
260}
261
262impl CallSpan {
263    fn open(headers: &HeaderMap) -> Self {
264        let session = headers
265            .get(VM_SESSION_HEADER)
266            .and_then(|value| value.to_str().ok())
267            .and_then(VmSession::parse);
268        Self {
269            inner: tracing::info_span!(
270                target: "model",
271                "model.proxy.call",
272                vm.subject = label(headers, SUBJECT_HEADER),
273                vm.session = session.as_ref().map_or("", VmSession::as_str),
274                agent = Empty,
275                model.name = Empty,
276                usage.input_tokens = Empty,
277                usage.output_tokens = Empty,
278                outcome = Empty,
279            ),
280            session,
281            usage: ModelUsage::default(),
282            settled: false,
283        }
284    }
285
286    fn observe(&mut self, usage: ModelUsage) {
287        self.usage = self.usage.merged(usage);
288    }
289
290    fn refused(mut self, response: Response) -> Response {
291        self.end("refused");
292        response
293    }
294
295    fn settle(&mut self, outcome: Outcome) {
296        self.end(outcome.as_str());
297    }
298
299    fn end(&mut self, outcome: &'static str) {
300        self.settled = true;
301        let tokens = |count: Option<u64>| i64::try_from(count.unwrap_or(0)).unwrap_or(i64::MAX);
302        self.inner
303            .record("usage.input_tokens", tokens(self.usage.input_tokens))
304            .record("usage.output_tokens", tokens(self.usage.output_tokens))
305            .record("outcome", outcome);
306    }
307}
308
309impl Drop for CallSpan {
310    fn drop(&mut self) {
311        if !self.settled {
312            self.end(Outcome::Cancelled.as_str());
313        }
314    }
315}
316
317fn label<'a>(headers: &'a HeaderMap, name: &str) -> &'a str {
318    headers
319        .get(name)
320        .and_then(|value| value.to_str().ok())
321        .map(str::trim)
322        .filter(|value| {
323            value.len() <= MAX_LABEL_BYTES && value.bytes().all(|byte| matches!(byte, b' '..=b'~'))
324        })
325        .unwrap_or("")
326}
327
328pub trait CurrentProxy: Clone + Send + Sync + 'static {
329    fn current(&self) -> Option<Arc<ModelProxy>>;
330}
331
332impl CurrentProxy for Arc<ModelProxy> {
333    fn current(&self) -> Option<Arc<ModelProxy>> {
334        Some(Arc::clone(self))
335    }
336}
337
338async fn handle_current<C: CurrentProxy>(
339    State(current): State<C>,
340    dialect: Dialect,
341    headers: HeaderMap,
342    body: Body,
343) -> Response {
344    match current.current() {
345        Some(proxy) => handle(State(proxy), dialect, headers, body).await,
346        None => CallSpan::open(&headers).refused(dialect.sandbox(
347            Problem::Forbidden,
348            "this VM is granted no model. Ask the operator to grant its agent one.",
349        )),
350    }
351}
352
353async fn handle(
354    State(proxy): State<Arc<ModelProxy>>,
355    dialect: Dialect,
356    headers: HeaderMap,
357    body: Body,
358) -> Response {
359    let span = CallSpan::open(&headers);
360    let current = span.inner.clone();
361    call(&proxy, dialect, &headers, body, span)
362        .instrument(current)
363        .await
364}
365
366async fn call(
367    proxy: &ModelProxy,
368    dialect: Dialect,
369    headers: &HeaderMap,
370    body: Body,
371    mut span: CallSpan,
372) -> Response {
373    let Some(grant) = headers
374        .get(SUBJECT_HEADER)
375        .and_then(|subject| subject.to_str().ok())
376        .and_then(|subject| proxy.guests.get(subject))
377    else {
378        return span.refused(dialect.sandbox(
379            Problem::Forbidden,
380            "this VM is granted no model. Ask the operator to grant its agent one.",
381        ));
382    };
383    span.inner.record("agent", grant.agent.as_str());
384    let body = match axum::body::to_bytes(body, MAX_BODY_BYTES).await {
385        Ok(body) => body,
386        Err(error) => {
387            tracing::debug!(target: "model", error = %error, "proxy.request.body.refused");
388            return span.refused(dialect.sandbox(
389                Problem::TooLarge,
390                "a request body may be at most 8 MiB. Send a smaller request.",
391            ));
392        }
393    };
394    let peeked = match &grant.models {
395        Granted::Listed(_) => RequestPeek::of(&body),
396        Granted::Inherited(_) => RequestPeek::with_settings(&body),
397    };
398    let peek = match peeked {
399        Ok(peek) => peek,
400        Err(error) => {
401            return span.refused(dialect.sandbox(Problem::Invalid, &peek_rule(&error, grant)));
402        }
403    };
404    let model = match &grant.models {
405        Granted::Listed(models) => proxy
406            .models
407            .get(&peek.model)
408            .filter(|model| models.contains(&model.name) && model.upstream.serves(dialect)),
409        Granted::Inherited(name) => match proxy.models.get(name) {
410            Some(model) if !model.upstream.serves(dialect) => {
411                return span.refused(dialect.sandbox(
412                    Problem::Forbidden,
413                    &format!(
414                        "this VM runs on its agent's model `{}`, which is served on `{}`, not `{}`. Call it on its own path.",
415                        model.name,
416                        model.upstream.path(),
417                        dialect.path()
418                    ),
419                ));
420            }
421            served => served,
422        },
423    };
424    let Some(model) = model else {
425        return span
426            .refused(dialect.sandbox(Problem::Forbidden, &proxy.model_rule(grant, dialect)));
427    };
428    span.inner.record("model.name", model.name.as_str());
429    let wire_model = serde_json::Value::from(model.wire_model.as_str()).to_string();
430    let rewritten = match (&model.upstream, &grant.models) {
431        (Upstream::Codex { .. }, _) if peek.stream != Some(true) => {
432            return span.refused(dialect.sandbox(
433                Problem::Invalid,
434                "this model is served only as a stream. Set `stream` to true.",
435            ));
436        }
437        (Upstream::Codex { .. }, _) => peek.rewrite(
438            &body,
439            &[(Field::Model, &wire_model), (Field::Store, "false")],
440        ),
441        (Upstream::OpenRouter { .. }, _) if peek.has(Field::Models) || peek.has(Field::Route) => {
442            return span.refused(dialect.sandbox(
443                Problem::Invalid,
444                "OpenRouter fallback routing (`models` / `route`) is not allowed; the sandbox pins each call to one configured model. Remove the field.",
445            ));
446        }
447        (Upstream::OpenRouter { settings, .. }, Granted::Inherited(_)) => {
448            let mut replacements = vec![(Field::Model, wire_model.as_str())];
449            replacements.extend(settings.iter().map(|(field, raw)| (*field, raw.as_str())));
450            peek.rewrite(&body, &replacements)
451        }
452        (Upstream::Anthropic { .. }, _) | (Upstream::OpenRouter { .. }, Granted::Listed(_)) => {
453            peek.rewrite(&body, &[(Field::Model, &wire_model)])
454        }
455    };
456    drop(body);
457    let admission = if dialect == Dialect::CountTokens {
458        None
459    } else {
460        let call = Call {
461            agent: grant.agent.clone(),
462            model: model.name.clone(),
463            backend: model.backend,
464            via: Via::Proxy {
465                session: span.session,
466            },
467        };
468        let estimate = Estimate::from_sizes(
469            Sizes {
470                bytes: rewritten.len(),
471                images: peek.images,
472            },
473            None,
474            model.reserve,
475        );
476        match proxy.metering.admit(call, estimate) {
477            Ok(admission) => Some(admission),
478            Err(refusal) => return span.refused(dialect.refusal(&refusal)),
479        }
480    };
481    let upstream = match send(&proxy.http, model, dialect, headers, Bytes::from(rewritten)).await {
482        Ok(response) => response,
483        Err(error) => {
484            tracing::warn!(target: "model", event = "proxy.upstream_failed", error.kind = error.kind(), error = %error, "proxy.upstream_failed");
485            let outcome = match error {
486                SendError::Credential(_) => Outcome::NotSent,
487                SendError::Request(_) => Outcome::Failed,
488            };
489            if let Some(admission) = admission {
490                admission.settle(outcome);
491            }
492            span.settle(outcome);
493            return dialect.error(Problem::Upstream, "the model upstream could not be reached");
494        }
495    };
496    respond(proxy, dialect, upstream, admission, span).await
497}
498
499#[derive(Debug, thiserror::Error)]
500enum SendError {
501    #[error("Codex credential is unavailable")]
502    Credential(#[source] dekopon_model::chatgpt::ChatGptError),
503    #[error("upstream request failed")]
504    Request(#[source] reqwest::Error),
505}
506
507impl SendError {
508    const fn kind(&self) -> &'static str {
509        match self {
510            Self::Credential(_) => "credential",
511            Self::Request(_) => "request",
512        }
513    }
514}
515
516async fn send(
517    http: &reqwest::Client,
518    model: &ProxyModel,
519    dialect: Dialect,
520    guest: &HeaderMap,
521    body: Bytes,
522) -> Result<reqwest::Response, SendError> {
523    let mut request = http
524        .post(model.upstream.url(dialect))
525        .header(header::CONTENT_TYPE, "application/json")
526        .body(body.clone());
527    for name in PASSED_HEADERS {
528        if let Some(value) = guest.get(name) {
529            request = request.header(name, value);
530        }
531    }
532    match &model.upstream {
533        Upstream::Anthropic { api_key, .. } => request
534            .header("x-api-key", api_key.expose().as_str())
535            .send()
536            .await
537            .map_err(SendError::Request),
538        Upstream::OpenRouter { api_key, .. } => request
539            .bearer_auth(api_key.expose())
540            .send()
541            .await
542            .map_err(SendError::Request),
543        Upstream::Codex {
544            credential,
545            refresh,
546            ..
547        } => {
548            let request = request
549                .header("originator", "dekopon")
550                .header("openai-beta", "responses=experimental")
551                .header(header::ACCEPT, "text/event-stream");
552            let retry = request.try_clone();
553            let current = codex_credential(credential, refresh, None).await?;
554            let response = request
555                .bearer_auth(current.access.expose())
556                .header("chatgpt-account-id", &current.account_id)
557                .send()
558                .await
559                .map_err(SendError::Request)?;
560            match retry {
561                Some(retry) if response.status() == StatusCode::UNAUTHORIZED => {
562                    drop(response);
563                    let refreshed =
564                        codex_credential(credential, refresh, Some(current.access)).await?;
565                    retry
566                        .bearer_auth(refreshed.access.expose())
567                        .header("chatgpt-account-id", &refreshed.account_id)
568                        .send()
569                        .await
570                        .map_err(SendError::Request)
571                }
572                Some(_) | None => Ok(response),
573            }
574        }
575    }
576}
577
578async fn codex_credential(
579    credential: &Arc<CredentialFile>,
580    refresh: &RefreshClient,
581    rejected: Option<Redacted<String>>,
582) -> Result<dekopon_model::chatgpt::ResolvedCredential, SendError> {
583    match rejected {
584        Some(rejected) => credential.force_refresh(refresh, &rejected).await,
585        None => credential.current(refresh).await,
586    }
587    .map_err(SendError::Credential)
588}
589
590async fn respond(
591    proxy: &ModelProxy,
592    dialect: Dialect,
593    upstream: reqwest::Response,
594    admission: Option<dekopon_model_token_governor::Admission>,
595    mut span: CallSpan,
596) -> Response {
597    let status = upstream.status();
598    let mut headers = HeaderMap::new();
599    for name in RETURNED_HEADERS {
600        if let Some(value) = upstream.headers().get(name) {
601            headers.insert(HeaderName::from_static(name), value.clone());
602        }
603    }
604    if !status.is_success() {
605        if let Some(admission) = admission {
606            admission.settle(Outcome::NotSent);
607        }
608        span.settle(Outcome::NotSent);
609        let body = bounded(upstream, MAX_ERROR_BYTES).await;
610        let mut response = Response::new(Body::from(body));
611        *response.status_mut() = status;
612        *response.headers_mut() = headers;
613        return response;
614    }
615    let events = headers
616        .get(header::CONTENT_TYPE)
617        .and_then(|value| value.to_str().ok())
618        .is_some_and(|value| value.starts_with("text/event-stream"));
619    let shape = if events { Shape::Events } else { Shape::Json };
620    let stream = tee::tee(
621        upstream.bytes_stream(),
622        dialect,
623        shape,
624        proxy.timing,
625        admission,
626        span,
627    );
628    let mut response = Response::new(Body::from_stream(stream));
629    *response.status_mut() = status;
630    if events {
631        headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
632    }
633    *response.headers_mut() = headers;
634    response
635}
636
637fn peek_rule(error: &PeekError, grant: &Grant) -> String {
638    match error {
639        PeekError::NotJson | PeekError::NotObject => {
640            "the request body must be a JSON object. Send one.".to_owned()
641        }
642        PeekError::NoModel => format!(
643            "the request must name a model. Set `model` to one of this agent's configured models: {}.",
644            grant
645                .models
646                .names()
647                .into_iter()
648                .map(|name| format!("`{name}`"))
649                .collect::<Vec<_>>()
650                .join(", ")
651        ),
652        PeekError::Duplicate(field) => {
653            format!("the request repeats the top-level key `{field}`; send it once.")
654        }
655    }
656}
657
658async fn bounded(mut upstream: reqwest::Response, limit: usize) -> Vec<u8> {
659    let mut body = Vec::new();
660    loop {
661        match upstream.chunk().await {
662            Ok(Some(chunk)) if body.len() + chunk.len() <= limit => body.extend_from_slice(&chunk),
663            Ok(Some(_)) => break,
664            Ok(None) => break,
665            Err(error) => {
666                tracing::debug!(target: "model", error = %error, "proxy.upstream.error.truncated");
667                break;
668            }
669        }
670    }
671    body
672}