1#![forbid(unsafe_code)]
2#![cfg_attr(test, allow(clippy::unwrap_used))]
3#![cfg_attr(
4 test,
5 allow(
6 clippy::disallowed_methods,
7 reason = "tests spawn freely; production sites carry their own expectation"
8 )
9)]
10
11mod dialect;
12mod tee;
13pub mod tls;
14
15#[cfg(test)]
16mod tests;
17
18use std::{
19 collections::{BTreeSet, HashMap},
20 sync::Arc,
21 time::Duration,
22};
23
24use axum::{
25 Router,
26 body::Body,
27 extract::State,
28 http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header},
29 response::Response,
30 routing::post,
31};
32use bytes::Bytes;
33use dekopon_core::{AgentId, Redacted, VM_SESSION_HEADER, VmSession};
34use dekopon_model::{
35 chatgpt::{CredentialFile, RefreshClient},
36 wire::{Field, PeekError, RequestPeek},
37};
38use dekopon_model_token_governor::{
39 Call, Estimate, Metering, ModelUsage, Outcome, Sizes, Tokens, Via,
40};
41use tracing::{Instrument as _, Span, field::Empty};
42
43use dialect::Problem;
44pub use dialect::{Dialect, SANDBOX};
45use tee::{Shape, Timing};
46
47pub const MAX_BODY_BYTES: usize = 8 * 1024 * 1024;
48pub const SUBJECT_HEADER: &str = "x-dekopon-vm-subject";
49const MAX_LABEL_BYTES: usize = 128;
50pub const PING_INTERVAL: Duration = Duration::from_secs(20);
51pub const UPSTREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
52const MAX_ERROR_BYTES: usize = 64 * 1024;
53const PASSED_HEADERS: [&str; 3] = ["anthropic-version", "anthropic-beta", "x-request-id"];
54const RETURNED_HEADERS: [&str; 5] = [
55 "content-type",
56 "x-request-id",
57 "request-id",
58 "cache-control",
59 "retry-after",
60];
61
62pub const ANTHROPIC_ENDPOINT: &str = "https://api.anthropic.com";
63pub const OPENROUTER_ENDPOINT: &str = "https://openrouter.ai/api";
64pub const CODEX_ENDPOINT: &str = "https://chatgpt.com/backend-api/codex";
65
66#[derive(Clone)]
67pub enum Upstream {
68 Codex {
69 endpoint: String,
70 credential: Arc<CredentialFile>,
71 refresh: RefreshClient,
72 },
73 OpenRouter {
74 endpoint: String,
75 api_key: Redacted<String>,
76 settings: Vec<(Field, String)>,
77 },
78 Anthropic {
79 endpoint: String,
80 api_key: Redacted<String>,
81 },
82}
83
84impl Upstream {
85 const fn serves(&self, dialect: Dialect) -> bool {
86 match (self, dialect) {
87 (Self::Anthropic { .. }, Dialect::Messages | Dialect::CountTokens)
88 | (Self::Codex { .. }, Dialect::Responses)
89 | (Self::OpenRouter { .. }, Dialect::Chat | Dialect::Messages) => true,
90 (Self::Anthropic { .. }, Dialect::Responses | Dialect::Chat)
91 | (Self::Codex { .. }, Dialect::Messages | Dialect::CountTokens | Dialect::Chat)
92 | (Self::OpenRouter { .. }, Dialect::CountTokens | Dialect::Responses) => false,
93 }
94 }
95
96 const fn path(&self) -> &'static str {
97 match self {
98 Self::Anthropic { .. } => "`/v1/messages` or `/v1/messages/count_tokens`",
99 Self::Codex { .. } => "`/v1/responses`",
100 Self::OpenRouter { .. } => "`/v1/chat/completions` or `/v1/messages`",
101 }
102 }
103
104 fn url(&self, dialect: Dialect) -> String {
105 let (endpoint, path) = match (self, dialect) {
106 (Self::Anthropic { endpoint, .. }, Dialect::CountTokens) => {
107 (endpoint, "/v1/messages/count_tokens")
108 }
109 (Self::Anthropic { endpoint, .. }, _) => (endpoint, "/v1/messages"),
110 (Self::Codex { endpoint, .. }, _) => (endpoint, "/responses"),
111 (Self::OpenRouter { endpoint, .. }, Dialect::Messages) => (endpoint, "/v1/messages"),
112 (Self::OpenRouter { endpoint, .. }, _) => (endpoint, "/v1/chat/completions"),
113 };
114 format!("{}{path}", endpoint.trim_end_matches('/'))
115 }
116}
117
118pub struct ProxyModel {
119 pub name: String,
120 pub wire_model: String,
121 pub backend: &'static str,
122 pub reserve: Tokens,
123 pub upstream: Upstream,
124}
125
126pub struct Grant {
127 pub agent: AgentId,
128 pub models: Granted,
129}
130
131#[derive(Clone, Debug, Eq, PartialEq)]
132pub enum Granted {
133 Listed(BTreeSet<String>),
134 Inherited(String),
135}
136
137impl Granted {
138 fn names(&self) -> Vec<&str> {
139 match self {
140 Self::Listed(models) => models.iter().map(String::as_str).collect(),
141 Self::Inherited(model) => vec![model.as_str()],
142 }
143 }
144}
145
146pub struct ModelProxy {
147 models: HashMap<String, ProxyModel>,
148 guests: HashMap<String, Grant>,
149 metering: Arc<Metering>,
150 http: reqwest::Client,
151 timing: Timing,
152}
153
154#[derive(Debug, thiserror::Error)]
155#[error("model proxy HTTP client could not be built")]
156pub struct ClientError(#[source] reqwest::Error);
157
158impl ModelProxy {
159 pub fn new(
160 models: Vec<ProxyModel>,
161 guests: HashMap<String, Grant>,
162 metering: Arc<Metering>,
163 ) -> Result<Self, ClientError> {
164 let http = reqwest::Client::builder()
165 .redirect(reqwest::redirect::Policy::none())
166 .no_proxy()
167 .read_timeout(UPSTREAM_IDLE_TIMEOUT)
168 .build()
169 .map_err(ClientError)?;
170 Ok(Self {
171 models: models
172 .into_iter()
173 .map(|model| (model.name.clone(), model))
174 .collect(),
175 guests,
176 metering,
177 http,
178 timing: Timing {
179 ping: PING_INTERVAL,
180 idle: UPSTREAM_IDLE_TIMEOUT,
181 },
182 })
183 }
184
185 #[cfg(test)]
186 fn with_timing(mut self, ping: Duration, idle: Duration) -> Self {
187 self.timing = Timing { ping, idle };
188 self
189 }
190
191 fn model_rule(&self, grant: &Grant, dialect: Dialect) -> String {
192 let granted = || {
193 grant
194 .models
195 .names()
196 .into_iter()
197 .filter_map(|name| self.models.get(name))
198 };
199 let served = granted()
200 .filter(|model| model.upstream.serves(dialect))
201 .map(|model| format!("`{}`", model.name))
202 .collect::<Vec<_>>();
203 if served.is_empty() {
204 let elsewhere = granted()
205 .map(|model| format!("`{}` on {}", model.name, model.upstream.path()))
206 .collect::<Vec<_>>();
207 format!(
208 "this agent may only call its configured models, and none is served on this path: {}. Call one on its own path.",
209 elsewhere.join(", ")
210 )
211 } else {
212 format!(
213 "this agent may only call its configured models: {}. Set the model to one of them.",
214 served.join(", ")
215 )
216 }
217 }
218
219 pub fn router_current<C: CurrentProxy>(current: C) -> Router {
220 Router::new()
221 .route(
222 "/v1/messages",
223 post(|state, headers, body| {
224 handle_current::<C>(state, Dialect::Messages, headers, body)
225 }),
226 )
227 .route(
228 "/v1/messages/count_tokens",
229 post(|state, headers, body| {
230 handle_current::<C>(state, Dialect::CountTokens, headers, body)
231 }),
232 )
233 .route(
234 "/v1/responses",
235 post(|state, headers, body| {
236 handle_current::<C>(state, Dialect::Responses, headers, body)
237 }),
238 )
239 .route(
240 "/v1/chat/completions",
241 post(|state, headers, body| {
242 handle_current::<C>(state, Dialect::Chat, headers, body)
243 }),
244 )
245 .with_state(current)
246 }
247
248 pub fn router(self: Arc<Self>) -> Router {
249 Self::router_current(self)
250 }
251}
252
253struct CallSpan {
254 inner: Span,
255 session: Option<VmSession>,
256 usage: ModelUsage,
257 settled: bool,
258}
259
260impl CallSpan {
261 fn open(headers: &HeaderMap) -> Self {
262 let session = headers
263 .get(VM_SESSION_HEADER)
264 .and_then(|value| value.to_str().ok())
265 .and_then(VmSession::parse);
266 Self {
267 inner: tracing::info_span!(
268 target: "model",
269 "model.proxy.call",
270 vm.subject = label(headers, SUBJECT_HEADER),
271 vm.session = session.as_ref().map_or("", VmSession::as_str),
272 agent = Empty,
273 model.name = Empty,
274 usage.input_tokens = Empty,
275 usage.output_tokens = Empty,
276 outcome = Empty,
277 ),
278 session,
279 usage: ModelUsage::default(),
280 settled: false,
281 }
282 }
283
284 fn observe(&mut self, usage: ModelUsage) {
285 self.usage = self.usage.merged(usage);
286 }
287
288 fn refused(mut self, response: Response) -> Response {
289 self.end("refused");
290 response
291 }
292
293 fn settle(&mut self, outcome: Outcome) {
294 self.end(outcome.as_str());
295 }
296
297 fn end(&mut self, outcome: &'static str) {
298 self.settled = true;
299 let tokens = |count: Option<u64>| i64::try_from(count.unwrap_or(0)).unwrap_or(i64::MAX);
300 self.inner
301 .record("usage.input_tokens", tokens(self.usage.input_tokens))
302 .record("usage.output_tokens", tokens(self.usage.output_tokens))
303 .record("outcome", outcome);
304 }
305}
306
307impl Drop for CallSpan {
308 fn drop(&mut self) {
309 if !self.settled {
310 self.end(Outcome::Cancelled.as_str());
311 }
312 }
313}
314
315fn label<'a>(headers: &'a HeaderMap, name: &str) -> &'a str {
316 headers
317 .get(name)
318 .and_then(|value| value.to_str().ok())
319 .map(str::trim)
320 .filter(|value| {
321 value.len() <= MAX_LABEL_BYTES && value.bytes().all(|byte| matches!(byte, b' '..=b'~'))
322 })
323 .unwrap_or("")
324}
325
326pub trait CurrentProxy: Clone + Send + Sync + 'static {
327 fn current(&self) -> Option<Arc<ModelProxy>>;
328}
329
330impl CurrentProxy for Arc<ModelProxy> {
331 fn current(&self) -> Option<Arc<ModelProxy>> {
332 Some(Arc::clone(self))
333 }
334}
335
336async fn handle_current<C: CurrentProxy>(
337 State(current): State<C>,
338 dialect: Dialect,
339 headers: HeaderMap,
340 body: Body,
341) -> Response {
342 match current.current() {
343 Some(proxy) => handle(State(proxy), dialect, headers, body).await,
344 None => CallSpan::open(&headers).refused(dialect.sandbox(
345 Problem::Forbidden,
346 "this VM is granted no model. Ask the operator to grant its agent one.",
347 )),
348 }
349}
350
351async fn handle(
352 State(proxy): State<Arc<ModelProxy>>,
353 dialect: Dialect,
354 headers: HeaderMap,
355 body: Body,
356) -> Response {
357 let span = CallSpan::open(&headers);
358 let current = span.inner.clone();
359 call(&proxy, dialect, &headers, body, span)
360 .instrument(current)
361 .await
362}
363
364async fn call(
365 proxy: &ModelProxy,
366 dialect: Dialect,
367 headers: &HeaderMap,
368 body: Body,
369 mut span: CallSpan,
370) -> Response {
371 let Some(grant) = headers
372 .get(SUBJECT_HEADER)
373 .and_then(|subject| subject.to_str().ok())
374 .and_then(|subject| proxy.guests.get(subject))
375 else {
376 return span.refused(dialect.sandbox(
377 Problem::Forbidden,
378 "this VM is granted no model. Ask the operator to grant its agent one.",
379 ));
380 };
381 span.inner.record("agent", grant.agent.as_str());
382 let body = match axum::body::to_bytes(body, MAX_BODY_BYTES).await {
383 Ok(body) => body,
384 Err(error) => {
385 tracing::debug!(target: "model", error = %error, "proxy.request.body.refused");
386 return span.refused(dialect.sandbox(
387 Problem::TooLarge,
388 "a request body may be at most 8 MiB. Send a smaller request.",
389 ));
390 }
391 };
392 let peeked = match (&grant.models, dialect) {
393 (Granted::Listed(_), _) | (Granted::Inherited(_), Dialect::Messages) => {
394 RequestPeek::of(&body)
395 }
396 (Granted::Inherited(_), Dialect::CountTokens | Dialect::Responses | Dialect::Chat) => {
397 RequestPeek::with_settings(&body)
398 }
399 };
400 let peek = match peeked {
401 Ok(peek) => peek,
402 Err(error) => {
403 return span.refused(dialect.sandbox(Problem::Invalid, &peek_rule(&error, grant)));
404 }
405 };
406 let model = match &grant.models {
407 Granted::Listed(models) => proxy
408 .models
409 .get(&peek.model)
410 .filter(|model| models.contains(&model.name) && model.upstream.serves(dialect)),
411 Granted::Inherited(name) => match proxy.models.get(name) {
412 Some(model) if !model.upstream.serves(dialect) => {
413 return span.refused(dialect.sandbox(
414 Problem::Forbidden,
415 &format!(
416 "this VM runs on its agent's model `{}`, which is served on {}, not `{}`. Call it on its own path.",
417 model.name,
418 model.upstream.path(),
419 dialect.path()
420 ),
421 ));
422 }
423 served => served,
424 },
425 };
426 let Some(model) = model else {
427 return span
428 .refused(dialect.sandbox(Problem::Forbidden, &proxy.model_rule(grant, dialect)));
429 };
430 span.inner.record("model.name", model.name.as_str());
431 let wire_model = serde_json::Value::from(model.wire_model.as_str()).to_string();
432 let rewritten = match (&model.upstream, &grant.models) {
433 (Upstream::Codex { .. }, _) if peek.stream != Some(true) => {
434 return span.refused(dialect.sandbox(
435 Problem::Invalid,
436 "this model is served only as a stream. Set `stream` to true.",
437 ));
438 }
439 (Upstream::Codex { .. }, _) => peek.rewrite(
440 &body,
441 &[(Field::Model, &wire_model), (Field::Store, "false")],
442 ),
443 (Upstream::OpenRouter { .. }, _) if peek.has(Field::Models) || peek.has(Field::Route) => {
444 return span.refused(dialect.sandbox(
445 Problem::Invalid,
446 "OpenRouter fallback routing (`models` / `route`) is not allowed; the sandbox pins each call to one configured model. Remove the field.",
447 ));
448 }
449 (Upstream::OpenRouter { .. }, _) if dialect == Dialect::Messages => {
450 peek.rewrite(&body, &[(Field::Model, &wire_model)])
451 }
452 (Upstream::OpenRouter { settings, .. }, Granted::Inherited(_)) => {
453 let mut replacements = vec![(Field::Model, wire_model.as_str())];
454 replacements.extend(settings.iter().map(|(field, raw)| (*field, raw.as_str())));
455 peek.rewrite(&body, &replacements)
456 }
457 (Upstream::Anthropic { .. }, _) | (Upstream::OpenRouter { .. }, Granted::Listed(_)) => {
458 peek.rewrite(&body, &[(Field::Model, &wire_model)])
459 }
460 };
461 drop(body);
462 let admission = if dialect == Dialect::CountTokens {
463 None
464 } else {
465 let call = Call {
466 agent: grant.agent.clone(),
467 model: model.name.clone(),
468 backend: model.backend,
469 via: Via::Proxy {
470 session: span.session,
471 },
472 };
473 let estimate = Estimate::from_sizes(
474 Sizes {
475 bytes: rewritten.len(),
476 images: peek.images,
477 },
478 None,
479 model.reserve,
480 );
481 match proxy.metering.admit(call, estimate) {
482 Ok(admission) => Some(admission),
483 Err(refusal) => return span.refused(dialect.refusal(&refusal)),
484 }
485 };
486 let upstream = match send(&proxy.http, model, dialect, headers, Bytes::from(rewritten)).await {
487 Ok(response) => response,
488 Err(error) => {
489 tracing::warn!(target: "model", event = "proxy.upstream_failed", error.kind = error.kind(), error = %error, "proxy.upstream_failed");
490 let outcome = match error {
491 SendError::Credential(_) => Outcome::NotSent,
492 SendError::Request(_) => Outcome::Failed,
493 };
494 if let Some(admission) = admission {
495 admission.settle(outcome);
496 }
497 span.settle(outcome);
498 return dialect.error(Problem::Upstream, "the model upstream could not be reached");
499 }
500 };
501 respond(proxy, dialect, upstream, admission, span).await
502}
503
504#[derive(Debug, thiserror::Error)]
505enum SendError {
506 #[error("Codex credential is unavailable")]
507 Credential(#[source] dekopon_model::chatgpt::ChatGptError),
508 #[error("upstream request failed")]
509 Request(#[source] reqwest::Error),
510}
511
512impl SendError {
513 const fn kind(&self) -> &'static str {
514 match self {
515 Self::Credential(_) => "credential",
516 Self::Request(_) => "request",
517 }
518 }
519}
520
521async fn send(
522 http: &reqwest::Client,
523 model: &ProxyModel,
524 dialect: Dialect,
525 guest: &HeaderMap,
526 body: Bytes,
527) -> Result<reqwest::Response, SendError> {
528 let mut request = http
529 .post(model.upstream.url(dialect))
530 .header(header::CONTENT_TYPE, "application/json")
531 .body(body.clone());
532 for name in PASSED_HEADERS {
533 if let Some(value) = guest.get(name) {
534 request = request.header(name, value);
535 }
536 }
537 match &model.upstream {
538 Upstream::Anthropic { api_key, .. } => request
539 .header("x-api-key", api_key.expose().as_str())
540 .send()
541 .await
542 .map_err(SendError::Request),
543 Upstream::OpenRouter { api_key, .. } => request
544 .bearer_auth(api_key.expose())
545 .send()
546 .await
547 .map_err(SendError::Request),
548 Upstream::Codex {
549 credential,
550 refresh,
551 ..
552 } => {
553 let request = request
554 .header("originator", "dekopon")
555 .header("openai-beta", "responses=experimental")
556 .header(header::ACCEPT, "text/event-stream");
557 let retry = request.try_clone();
558 let current = codex_credential(credential, refresh, None).await?;
559 let response = request
560 .bearer_auth(current.access.expose())
561 .header("chatgpt-account-id", ¤t.account_id)
562 .send()
563 .await
564 .map_err(SendError::Request)?;
565 match retry {
566 Some(retry) if response.status() == StatusCode::UNAUTHORIZED => {
567 drop(response);
568 let refreshed =
569 codex_credential(credential, refresh, Some(current.access)).await?;
570 retry
571 .bearer_auth(refreshed.access.expose())
572 .header("chatgpt-account-id", &refreshed.account_id)
573 .send()
574 .await
575 .map_err(SendError::Request)
576 }
577 Some(_) | None => Ok(response),
578 }
579 }
580 }
581}
582
583async fn codex_credential(
584 credential: &Arc<CredentialFile>,
585 refresh: &RefreshClient,
586 rejected: Option<Redacted<String>>,
587) -> Result<dekopon_model::chatgpt::ResolvedCredential, SendError> {
588 match rejected {
589 Some(rejected) => credential.force_refresh(refresh, &rejected).await,
590 None => credential.current(refresh).await,
591 }
592 .map_err(SendError::Credential)
593}
594
595async fn respond(
596 proxy: &ModelProxy,
597 dialect: Dialect,
598 upstream: reqwest::Response,
599 admission: Option<dekopon_model_token_governor::Admission>,
600 mut span: CallSpan,
601) -> Response {
602 let status = upstream.status();
603 let mut headers = HeaderMap::new();
604 for name in RETURNED_HEADERS {
605 if let Some(value) = upstream.headers().get(name) {
606 headers.insert(HeaderName::from_static(name), value.clone());
607 }
608 }
609 if !status.is_success() {
610 if let Some(admission) = admission {
611 admission.settle(Outcome::NotSent);
612 }
613 span.settle(Outcome::NotSent);
614 let body = bounded(upstream, MAX_ERROR_BYTES).await;
615 let mut response = Response::new(Body::from(body));
616 *response.status_mut() = status;
617 *response.headers_mut() = headers;
618 return response;
619 }
620 let events = headers
621 .get(header::CONTENT_TYPE)
622 .and_then(|value| value.to_str().ok())
623 .is_some_and(|value| value.starts_with("text/event-stream"));
624 let shape = if events { Shape::Events } else { Shape::Json };
625 let stream = tee::tee(
626 upstream.bytes_stream(),
627 dialect,
628 shape,
629 proxy.timing,
630 admission,
631 span,
632 );
633 let mut response = Response::new(Body::from_stream(stream));
634 *response.status_mut() = status;
635 if events {
636 headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
637 }
638 *response.headers_mut() = headers;
639 response
640}
641
642fn peek_rule(error: &PeekError, grant: &Grant) -> String {
643 match error {
644 PeekError::NotJson | PeekError::NotObject => {
645 "the request body must be a JSON object. Send one.".to_owned()
646 }
647 PeekError::NoModel => format!(
648 "the request must name a model. Set `model` to one of this agent's configured models: {}.",
649 grant
650 .models
651 .names()
652 .into_iter()
653 .map(|name| format!("`{name}`"))
654 .collect::<Vec<_>>()
655 .join(", ")
656 ),
657 PeekError::Duplicate(field) => {
658 format!("the request repeats the top-level key `{field}`; send it once.")
659 }
660 }
661}
662
663async fn bounded(mut upstream: reqwest::Response, limit: usize) -> Vec<u8> {
664 let mut body = Vec::new();
665 loop {
666 match upstream.chunk().await {
667 Ok(Some(chunk)) if body.len() + chunk.len() <= limit => body.extend_from_slice(&chunk),
668 Ok(Some(_)) => break,
669 Ok(None) => break,
670 Err(error) => {
671 tracing::debug!(target: "model", error = %error, "proxy.upstream.error.truncated");
672 break;
673 }
674 }
675 }
676 body
677}