Skip to main content

dekopon_model_proxy/
lib.rs

1#![forbid(unsafe_code)]
2#![cfg_attr(test, allow(clippy::unwrap_used))]
3#![cfg_attr(
4    test,
5    allow(
6        clippy::disallowed_methods,
7        reason = "tests spawn freely; production sites carry their own expectation"
8    )
9)]
10
11mod dialect;
12mod tee;
13pub mod tls;
14
15#[cfg(test)]
16mod tests;
17
18use std::{
19    collections::{BTreeSet, HashMap},
20    sync::Arc,
21    time::Duration,
22};
23
24use axum::{
25    Router,
26    body::Body,
27    extract::State,
28    http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header},
29    response::Response,
30    routing::post,
31};
32use bytes::Bytes;
33use dekopon_core::{AgentId, Redacted, VM_SESSION_HEADER, VmSession};
34use dekopon_model::{
35    chatgpt::{CredentialFile, RefreshClient},
36    wire::{Field, PeekError, RequestPeek},
37};
38use dekopon_model_token_governor::{
39    Call, Estimate, Metering, ModelUsage, Outcome, Sizes, Tokens, Via,
40};
41use tracing::{Instrument as _, Span, field::Empty};
42
43use dialect::Problem;
44pub use dialect::{Dialect, SANDBOX};
45use tee::{Shape, Timing};
46
47pub const MAX_BODY_BYTES: usize = 8 * 1024 * 1024;
48pub const SUBJECT_HEADER: &str = "x-dekopon-vm-subject";
49const MAX_LABEL_BYTES: usize = 128;
50pub const PING_INTERVAL: Duration = Duration::from_secs(20);
51pub const UPSTREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
52const MAX_ERROR_BYTES: usize = 64 * 1024;
53const PASSED_HEADERS: [&str; 3] = ["anthropic-version", "anthropic-beta", "x-request-id"];
54const RETURNED_HEADERS: [&str; 5] = [
55    "content-type",
56    "x-request-id",
57    "request-id",
58    "cache-control",
59    "retry-after",
60];
61
62pub const ANTHROPIC_ENDPOINT: &str = "https://api.anthropic.com";
63pub const OPENROUTER_ENDPOINT: &str = "https://openrouter.ai/api";
64pub const CODEX_ENDPOINT: &str = "https://chatgpt.com/backend-api/codex";
65
66#[derive(Clone)]
67pub enum Upstream {
68    Codex {
69        endpoint: String,
70        credential: Arc<CredentialFile>,
71        refresh: RefreshClient,
72    },
73    OpenRouter {
74        endpoint: String,
75        api_key: Redacted<String>,
76        settings: Vec<(Field, String)>,
77    },
78    Anthropic {
79        endpoint: String,
80        api_key: Redacted<String>,
81    },
82}
83
84impl Upstream {
85    const fn serves(&self, dialect: Dialect) -> bool {
86        match (self, dialect) {
87            (Self::Anthropic { .. }, Dialect::Messages | Dialect::CountTokens)
88            | (Self::Codex { .. }, Dialect::Responses)
89            | (Self::OpenRouter { .. }, Dialect::Chat | Dialect::Messages) => true,
90            (Self::Anthropic { .. }, Dialect::Responses | Dialect::Chat)
91            | (Self::Codex { .. }, Dialect::Messages | Dialect::CountTokens | Dialect::Chat)
92            | (Self::OpenRouter { .. }, Dialect::CountTokens | Dialect::Responses) => false,
93        }
94    }
95
96    const fn path(&self) -> &'static str {
97        match self {
98            Self::Anthropic { .. } => "`/v1/messages` or `/v1/messages/count_tokens`",
99            Self::Codex { .. } => "`/v1/responses`",
100            Self::OpenRouter { .. } => "`/v1/chat/completions` or `/v1/messages`",
101        }
102    }
103
104    fn url(&self, dialect: Dialect) -> String {
105        let (endpoint, path) = match (self, dialect) {
106            (Self::Anthropic { endpoint, .. }, Dialect::CountTokens) => {
107                (endpoint, "/v1/messages/count_tokens")
108            }
109            (Self::Anthropic { endpoint, .. }, _) => (endpoint, "/v1/messages"),
110            (Self::Codex { endpoint, .. }, _) => (endpoint, "/responses"),
111            (Self::OpenRouter { endpoint, .. }, Dialect::Messages) => (endpoint, "/v1/messages"),
112            (Self::OpenRouter { endpoint, .. }, _) => (endpoint, "/v1/chat/completions"),
113        };
114        format!("{}{path}", endpoint.trim_end_matches('/'))
115    }
116}
117
118pub struct ProxyModel {
119    pub name: String,
120    pub wire_model: String,
121    pub backend: &'static str,
122    pub reserve: Tokens,
123    pub upstream: Upstream,
124}
125
126pub struct Grant {
127    pub agent: AgentId,
128    pub models: Granted,
129}
130
131#[derive(Clone, Debug, Eq, PartialEq)]
132pub enum Granted {
133    Listed(BTreeSet<String>),
134    Inherited(String),
135}
136
137impl Granted {
138    fn names(&self) -> Vec<&str> {
139        match self {
140            Self::Listed(models) => models.iter().map(String::as_str).collect(),
141            Self::Inherited(model) => vec![model.as_str()],
142        }
143    }
144}
145
146pub struct ModelProxy {
147    models: HashMap<String, ProxyModel>,
148    guests: HashMap<String, Grant>,
149    metering: Arc<Metering>,
150    http: reqwest::Client,
151    timing: Timing,
152}
153
154#[derive(Debug, thiserror::Error)]
155#[error("model proxy HTTP client could not be built")]
156pub struct ClientError(#[source] reqwest::Error);
157
158impl ModelProxy {
159    pub fn new(
160        models: Vec<ProxyModel>,
161        guests: HashMap<String, Grant>,
162        metering: Arc<Metering>,
163    ) -> Result<Self, ClientError> {
164        let http = reqwest::Client::builder()
165            .redirect(reqwest::redirect::Policy::none())
166            .no_proxy()
167            .read_timeout(UPSTREAM_IDLE_TIMEOUT)
168            .build()
169            .map_err(ClientError)?;
170        Ok(Self {
171            models: models
172                .into_iter()
173                .map(|model| (model.name.clone(), model))
174                .collect(),
175            guests,
176            metering,
177            http,
178            timing: Timing {
179                ping: PING_INTERVAL,
180                idle: UPSTREAM_IDLE_TIMEOUT,
181            },
182        })
183    }
184
185    #[cfg(test)]
186    fn with_timing(mut self, ping: Duration, idle: Duration) -> Self {
187        self.timing = Timing { ping, idle };
188        self
189    }
190
191    fn model_rule(&self, grant: &Grant, dialect: Dialect) -> String {
192        let granted = || {
193            grant
194                .models
195                .names()
196                .into_iter()
197                .filter_map(|name| self.models.get(name))
198        };
199        let served = granted()
200            .filter(|model| model.upstream.serves(dialect))
201            .map(|model| format!("`{}`", model.name))
202            .collect::<Vec<_>>();
203        if served.is_empty() {
204            let elsewhere = granted()
205                .map(|model| format!("`{}` on {}", model.name, model.upstream.path()))
206                .collect::<Vec<_>>();
207            format!(
208                "this agent may only call its configured models, and none is served on this path: {}. Call one on its own path.",
209                elsewhere.join(", ")
210            )
211        } else {
212            format!(
213                "this agent may only call its configured models: {}. Set the model to one of them.",
214                served.join(", ")
215            )
216        }
217    }
218
219    pub fn router_current<C: CurrentProxy>(current: C) -> Router {
220        Router::new()
221            .route(
222                "/v1/messages",
223                post(|state, headers, body| {
224                    handle_current::<C>(state, Dialect::Messages, headers, body)
225                }),
226            )
227            .route(
228                "/v1/messages/count_tokens",
229                post(|state, headers, body| {
230                    handle_current::<C>(state, Dialect::CountTokens, headers, body)
231                }),
232            )
233            .route(
234                "/v1/responses",
235                post(|state, headers, body| {
236                    handle_current::<C>(state, Dialect::Responses, headers, body)
237                }),
238            )
239            .route(
240                "/v1/chat/completions",
241                post(|state, headers, body| {
242                    handle_current::<C>(state, Dialect::Chat, headers, body)
243                }),
244            )
245            .with_state(current)
246    }
247
248    pub fn router(self: Arc<Self>) -> Router {
249        Self::router_current(self)
250    }
251}
252
253struct CallSpan {
254    inner: Span,
255    session: Option<VmSession>,
256    usage: ModelUsage,
257    settled: bool,
258}
259
260impl CallSpan {
261    fn open(headers: &HeaderMap) -> Self {
262        let session = headers
263            .get(VM_SESSION_HEADER)
264            .and_then(|value| value.to_str().ok())
265            .and_then(VmSession::parse);
266        Self {
267            inner: tracing::info_span!(
268                target: "model",
269                "model.proxy.call",
270                vm.subject = label(headers, SUBJECT_HEADER),
271                vm.session = session.as_ref().map_or("", VmSession::as_str),
272                agent = Empty,
273                model.name = Empty,
274                usage.input_tokens = Empty,
275                usage.output_tokens = Empty,
276                outcome = Empty,
277            ),
278            session,
279            usage: ModelUsage::default(),
280            settled: false,
281        }
282    }
283
284    fn observe(&mut self, usage: ModelUsage) {
285        self.usage = self.usage.merged(usage);
286    }
287
288    fn refused(mut self, response: Response) -> Response {
289        self.end("refused");
290        response
291    }
292
293    fn settle(&mut self, outcome: Outcome) {
294        self.end(outcome.as_str());
295    }
296
297    fn end(&mut self, outcome: &'static str) {
298        self.settled = true;
299        let tokens = |count: Option<u64>| i64::try_from(count.unwrap_or(0)).unwrap_or(i64::MAX);
300        self.inner
301            .record("usage.input_tokens", tokens(self.usage.input_tokens))
302            .record("usage.output_tokens", tokens(self.usage.output_tokens))
303            .record("outcome", outcome);
304    }
305}
306
307impl Drop for CallSpan {
308    fn drop(&mut self) {
309        if !self.settled {
310            self.end(Outcome::Cancelled.as_str());
311        }
312    }
313}
314
315fn label<'a>(headers: &'a HeaderMap, name: &str) -> &'a str {
316    headers
317        .get(name)
318        .and_then(|value| value.to_str().ok())
319        .map(str::trim)
320        .filter(|value| {
321            value.len() <= MAX_LABEL_BYTES && value.bytes().all(|byte| matches!(byte, b' '..=b'~'))
322        })
323        .unwrap_or("")
324}
325
326pub trait CurrentProxy: Clone + Send + Sync + 'static {
327    fn current(&self) -> Option<Arc<ModelProxy>>;
328}
329
330impl CurrentProxy for Arc<ModelProxy> {
331    fn current(&self) -> Option<Arc<ModelProxy>> {
332        Some(Arc::clone(self))
333    }
334}
335
336async fn handle_current<C: CurrentProxy>(
337    State(current): State<C>,
338    dialect: Dialect,
339    headers: HeaderMap,
340    body: Body,
341) -> Response {
342    match current.current() {
343        Some(proxy) => handle(State(proxy), dialect, headers, body).await,
344        None => CallSpan::open(&headers).refused(dialect.sandbox(
345            Problem::Forbidden,
346            "this VM is granted no model. Ask the operator to grant its agent one.",
347        )),
348    }
349}
350
351async fn handle(
352    State(proxy): State<Arc<ModelProxy>>,
353    dialect: Dialect,
354    headers: HeaderMap,
355    body: Body,
356) -> Response {
357    let span = CallSpan::open(&headers);
358    let current = span.inner.clone();
359    call(&proxy, dialect, &headers, body, span)
360        .instrument(current)
361        .await
362}
363
364async fn call(
365    proxy: &ModelProxy,
366    dialect: Dialect,
367    headers: &HeaderMap,
368    body: Body,
369    mut span: CallSpan,
370) -> Response {
371    let Some(grant) = headers
372        .get(SUBJECT_HEADER)
373        .and_then(|subject| subject.to_str().ok())
374        .and_then(|subject| proxy.guests.get(subject))
375    else {
376        return span.refused(dialect.sandbox(
377            Problem::Forbidden,
378            "this VM is granted no model. Ask the operator to grant its agent one.",
379        ));
380    };
381    span.inner.record("agent", grant.agent.as_str());
382    let body = match axum::body::to_bytes(body, MAX_BODY_BYTES).await {
383        Ok(body) => body,
384        Err(error) => {
385            tracing::debug!(target: "model", error = %error, "proxy.request.body.refused");
386            return span.refused(dialect.sandbox(
387                Problem::TooLarge,
388                "a request body may be at most 8 MiB. Send a smaller request.",
389            ));
390        }
391    };
392    let peeked = match (&grant.models, dialect) {
393        (Granted::Listed(_), _) | (Granted::Inherited(_), Dialect::Messages) => {
394            RequestPeek::of(&body)
395        }
396        (Granted::Inherited(_), Dialect::CountTokens | Dialect::Responses | Dialect::Chat) => {
397            RequestPeek::with_settings(&body)
398        }
399    };
400    let peek = match peeked {
401        Ok(peek) => peek,
402        Err(error) => {
403            return span.refused(dialect.sandbox(Problem::Invalid, &peek_rule(&error, grant)));
404        }
405    };
406    let model = match &grant.models {
407        Granted::Listed(models) => proxy
408            .models
409            .get(&peek.model)
410            .filter(|model| models.contains(&model.name) && model.upstream.serves(dialect)),
411        Granted::Inherited(name) => match proxy.models.get(name) {
412            Some(model) if !model.upstream.serves(dialect) => {
413                return span.refused(dialect.sandbox(
414                    Problem::Forbidden,
415                    &format!(
416                        "this VM runs on its agent's model `{}`, which is served on {}, not `{}`. Call it on its own path.",
417                        model.name,
418                        model.upstream.path(),
419                        dialect.path()
420                    ),
421                ));
422            }
423            served => served,
424        },
425    };
426    let Some(model) = model else {
427        return span
428            .refused(dialect.sandbox(Problem::Forbidden, &proxy.model_rule(grant, dialect)));
429    };
430    span.inner.record("model.name", model.name.as_str());
431    let wire_model = serde_json::Value::from(model.wire_model.as_str()).to_string();
432    let rewritten = match (&model.upstream, &grant.models) {
433        (Upstream::Codex { .. }, _) if peek.stream != Some(true) => {
434            return span.refused(dialect.sandbox(
435                Problem::Invalid,
436                "this model is served only as a stream. Set `stream` to true.",
437            ));
438        }
439        (Upstream::Codex { .. }, _) => peek.rewrite(
440            &body,
441            &[(Field::Model, &wire_model), (Field::Store, "false")],
442        ),
443        (Upstream::OpenRouter { .. }, _) if peek.has(Field::Models) || peek.has(Field::Route) => {
444            return span.refused(dialect.sandbox(
445                Problem::Invalid,
446                "OpenRouter fallback routing (`models` / `route`) is not allowed; the sandbox pins each call to one configured model. Remove the field.",
447            ));
448        }
449        (Upstream::OpenRouter { .. }, _) if dialect == Dialect::Messages => {
450            peek.rewrite(&body, &[(Field::Model, &wire_model)])
451        }
452        (Upstream::OpenRouter { settings, .. }, Granted::Inherited(_)) => {
453            let mut replacements = vec![(Field::Model, wire_model.as_str())];
454            replacements.extend(settings.iter().map(|(field, raw)| (*field, raw.as_str())));
455            peek.rewrite(&body, &replacements)
456        }
457        (Upstream::Anthropic { .. }, _) | (Upstream::OpenRouter { .. }, Granted::Listed(_)) => {
458            peek.rewrite(&body, &[(Field::Model, &wire_model)])
459        }
460    };
461    drop(body);
462    let admission = if dialect == Dialect::CountTokens {
463        None
464    } else {
465        let call = Call {
466            agent: grant.agent.clone(),
467            model: model.name.clone(),
468            backend: model.backend,
469            via: Via::Proxy {
470                session: span.session,
471            },
472        };
473        let estimate = Estimate::from_sizes(
474            Sizes {
475                bytes: rewritten.len(),
476                images: peek.images,
477            },
478            None,
479            model.reserve,
480        );
481        match proxy.metering.admit(call, estimate) {
482            Ok(admission) => Some(admission),
483            Err(refusal) => return span.refused(dialect.refusal(&refusal)),
484        }
485    };
486    let upstream = match send(&proxy.http, model, dialect, headers, Bytes::from(rewritten)).await {
487        Ok(response) => response,
488        Err(error) => {
489            tracing::warn!(target: "model", event = "proxy.upstream_failed", error.kind = error.kind(), error = %error, "proxy.upstream_failed");
490            let outcome = match error {
491                SendError::Credential(_) => Outcome::NotSent,
492                SendError::Request(_) => Outcome::Failed,
493            };
494            if let Some(admission) = admission {
495                admission.settle(outcome);
496            }
497            span.settle(outcome);
498            return dialect.error(Problem::Upstream, "the model upstream could not be reached");
499        }
500    };
501    respond(proxy, dialect, upstream, admission, span).await
502}
503
504#[derive(Debug, thiserror::Error)]
505enum SendError {
506    #[error("Codex credential is unavailable")]
507    Credential(#[source] dekopon_model::chatgpt::ChatGptError),
508    #[error("upstream request failed")]
509    Request(#[source] reqwest::Error),
510}
511
512impl SendError {
513    const fn kind(&self) -> &'static str {
514        match self {
515            Self::Credential(_) => "credential",
516            Self::Request(_) => "request",
517        }
518    }
519}
520
521async fn send(
522    http: &reqwest::Client,
523    model: &ProxyModel,
524    dialect: Dialect,
525    guest: &HeaderMap,
526    body: Bytes,
527) -> Result<reqwest::Response, SendError> {
528    let mut request = http
529        .post(model.upstream.url(dialect))
530        .header(header::CONTENT_TYPE, "application/json")
531        .body(body.clone());
532    for name in PASSED_HEADERS {
533        if let Some(value) = guest.get(name) {
534            request = request.header(name, value);
535        }
536    }
537    match &model.upstream {
538        Upstream::Anthropic { api_key, .. } => request
539            .header("x-api-key", api_key.expose().as_str())
540            .send()
541            .await
542            .map_err(SendError::Request),
543        Upstream::OpenRouter { api_key, .. } => request
544            .bearer_auth(api_key.expose())
545            .send()
546            .await
547            .map_err(SendError::Request),
548        Upstream::Codex {
549            credential,
550            refresh,
551            ..
552        } => {
553            let request = request
554                .header("originator", "dekopon")
555                .header("openai-beta", "responses=experimental")
556                .header(header::ACCEPT, "text/event-stream");
557            let retry = request.try_clone();
558            let current = codex_credential(credential, refresh, None).await?;
559            let response = request
560                .bearer_auth(current.access.expose())
561                .header("chatgpt-account-id", &current.account_id)
562                .send()
563                .await
564                .map_err(SendError::Request)?;
565            match retry {
566                Some(retry) if response.status() == StatusCode::UNAUTHORIZED => {
567                    drop(response);
568                    let refreshed =
569                        codex_credential(credential, refresh, Some(current.access)).await?;
570                    retry
571                        .bearer_auth(refreshed.access.expose())
572                        .header("chatgpt-account-id", &refreshed.account_id)
573                        .send()
574                        .await
575                        .map_err(SendError::Request)
576                }
577                Some(_) | None => Ok(response),
578            }
579        }
580    }
581}
582
583async fn codex_credential(
584    credential: &Arc<CredentialFile>,
585    refresh: &RefreshClient,
586    rejected: Option<Redacted<String>>,
587) -> Result<dekopon_model::chatgpt::ResolvedCredential, SendError> {
588    match rejected {
589        Some(rejected) => credential.force_refresh(refresh, &rejected).await,
590        None => credential.current(refresh).await,
591    }
592    .map_err(SendError::Credential)
593}
594
595async fn respond(
596    proxy: &ModelProxy,
597    dialect: Dialect,
598    upstream: reqwest::Response,
599    admission: Option<dekopon_model_token_governor::Admission>,
600    mut span: CallSpan,
601) -> Response {
602    let status = upstream.status();
603    let mut headers = HeaderMap::new();
604    for name in RETURNED_HEADERS {
605        if let Some(value) = upstream.headers().get(name) {
606            headers.insert(HeaderName::from_static(name), value.clone());
607        }
608    }
609    if !status.is_success() {
610        if let Some(admission) = admission {
611            admission.settle(Outcome::NotSent);
612        }
613        span.settle(Outcome::NotSent);
614        let body = bounded(upstream, MAX_ERROR_BYTES).await;
615        let mut response = Response::new(Body::from(body));
616        *response.status_mut() = status;
617        *response.headers_mut() = headers;
618        return response;
619    }
620    let events = headers
621        .get(header::CONTENT_TYPE)
622        .and_then(|value| value.to_str().ok())
623        .is_some_and(|value| value.starts_with("text/event-stream"));
624    let shape = if events { Shape::Events } else { Shape::Json };
625    let stream = tee::tee(
626        upstream.bytes_stream(),
627        dialect,
628        shape,
629        proxy.timing,
630        admission,
631        span,
632    );
633    let mut response = Response::new(Body::from_stream(stream));
634    *response.status_mut() = status;
635    if events {
636        headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
637    }
638    *response.headers_mut() = headers;
639    response
640}
641
642fn peek_rule(error: &PeekError, grant: &Grant) -> String {
643    match error {
644        PeekError::NotJson | PeekError::NotObject => {
645            "the request body must be a JSON object. Send one.".to_owned()
646        }
647        PeekError::NoModel => format!(
648            "the request must name a model. Set `model` to one of this agent's configured models: {}.",
649            grant
650                .models
651                .names()
652                .into_iter()
653                .map(|name| format!("`{name}`"))
654                .collect::<Vec<_>>()
655                .join(", ")
656        ),
657        PeekError::Duplicate(field) => {
658            format!("the request repeats the top-level key `{field}`; send it once.")
659        }
660    }
661}
662
663async fn bounded(mut upstream: reqwest::Response, limit: usize) -> Vec<u8> {
664    let mut body = Vec::new();
665    loop {
666        match upstream.chunk().await {
667            Ok(Some(chunk)) if body.len() + chunk.len() <= limit => body.extend_from_slice(&chunk),
668            Ok(Some(_)) => break,
669            Ok(None) => break,
670            Err(error) => {
671                tracing::debug!(target: "model", error = %error, "proxy.upstream.error.truncated");
672                break;
673            }
674        }
675    }
676    body
677}