1#![forbid(unsafe_code)]
2#![cfg_attr(test, allow(clippy::unwrap_used))]
3#![cfg_attr(
4 test,
5 allow(
6 clippy::disallowed_methods,
7 reason = "tests spawn freely; production sites carry their own expectation"
8 )
9)]
10
11mod dialect;
12mod tee;
13pub mod tls;
14
15#[cfg(test)]
16mod tests;
17
18use std::{
19 collections::{BTreeSet, HashMap},
20 sync::Arc,
21 time::Duration,
22};
23
24use axum::{
25 Router,
26 body::Body,
27 extract::State,
28 http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header},
29 response::Response,
30 routing::post,
31};
32use bytes::Bytes;
33use dekopon_core::{AgentId, Redacted, VM_SESSION_HEADER, VmSession};
34use dekopon_model::{
35 chatgpt::{CredentialFile, RefreshClient},
36 wire::{Field, PeekError, RequestPeek},
37};
38use dekopon_model_token_governor::{
39 Call, Estimate, Metering, ModelUsage, Outcome, Sizes, Tokens, Via,
40};
41use tracing::{Instrument as _, Span, field::Empty};
42
43use dialect::Problem;
44pub use dialect::{Dialect, SANDBOX};
45use tee::{Shape, Timing};
46
47pub const MAX_BODY_BYTES: usize = 8 * 1024 * 1024;
48pub const SUBJECT_HEADER: &str = "x-dekopon-vm-subject";
49const MAX_LABEL_BYTES: usize = 128;
50pub const PING_INTERVAL: Duration = Duration::from_secs(20);
51pub const UPSTREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
52const MAX_ERROR_BYTES: usize = 64 * 1024;
53const PASSED_HEADERS: [&str; 3] = ["anthropic-version", "anthropic-beta", "x-request-id"];
54const RETURNED_HEADERS: [&str; 5] = [
55 "content-type",
56 "x-request-id",
57 "request-id",
58 "cache-control",
59 "retry-after",
60];
61
62pub const ANTHROPIC_ENDPOINT: &str = "https://api.anthropic.com";
63pub const OPENROUTER_ENDPOINT: &str = "https://openrouter.ai/api";
64pub const CODEX_ENDPOINT: &str = "https://chatgpt.com/backend-api/codex";
65
66#[derive(Clone)]
67pub enum Upstream {
68 Codex {
69 endpoint: String,
70 credential: Arc<CredentialFile>,
71 refresh: RefreshClient,
72 },
73 OpenRouter {
74 endpoint: String,
75 api_key: Redacted<String>,
76 settings: Vec<(Field, String)>,
77 },
78 Anthropic {
79 endpoint: String,
80 api_key: Redacted<String>,
81 },
82}
83
84impl Upstream {
85 const fn serves(&self, dialect: Dialect) -> bool {
86 match (self, dialect) {
87 (Self::Anthropic { .. }, Dialect::Messages | Dialect::CountTokens)
88 | (Self::Codex { .. }, Dialect::Responses)
89 | (Self::OpenRouter { .. }, Dialect::Chat) => true,
90 (Self::Anthropic { .. }, Dialect::Responses | Dialect::Chat)
91 | (Self::Codex { .. }, Dialect::Messages | Dialect::CountTokens | Dialect::Chat)
92 | (
93 Self::OpenRouter { .. },
94 Dialect::Messages | Dialect::CountTokens | Dialect::Responses,
95 ) => false,
96 }
97 }
98
99 const fn path(&self) -> &'static str {
100 match self {
101 Self::Anthropic { .. } => "/v1/messages",
102 Self::Codex { .. } => "/v1/responses",
103 Self::OpenRouter { .. } => "/v1/chat/completions",
104 }
105 }
106
107 fn url(&self, dialect: Dialect) -> String {
108 let (endpoint, path) = match (self, dialect) {
109 (Self::Anthropic { endpoint, .. }, Dialect::CountTokens) => {
110 (endpoint, "/v1/messages/count_tokens")
111 }
112 (Self::Anthropic { endpoint, .. }, _) => (endpoint, "/v1/messages"),
113 (Self::Codex { endpoint, .. }, _) => (endpoint, "/responses"),
114 (Self::OpenRouter { endpoint, .. }, _) => (endpoint, "/v1/chat/completions"),
115 };
116 format!("{}{path}", endpoint.trim_end_matches('/'))
117 }
118}
119
120pub struct ProxyModel {
121 pub name: String,
122 pub wire_model: String,
123 pub backend: &'static str,
124 pub reserve: Tokens,
125 pub upstream: Upstream,
126}
127
128pub struct Grant {
129 pub agent: AgentId,
130 pub models: Granted,
131}
132
133#[derive(Clone, Debug, Eq, PartialEq)]
134pub enum Granted {
135 Listed(BTreeSet<String>),
136 Inherited(String),
137}
138
139impl Granted {
140 fn names(&self) -> Vec<&str> {
141 match self {
142 Self::Listed(models) => models.iter().map(String::as_str).collect(),
143 Self::Inherited(model) => vec![model.as_str()],
144 }
145 }
146}
147
148pub struct ModelProxy {
149 models: HashMap<String, ProxyModel>,
150 guests: HashMap<String, Grant>,
151 metering: Arc<Metering>,
152 http: reqwest::Client,
153 timing: Timing,
154}
155
156#[derive(Debug, thiserror::Error)]
157#[error("model proxy HTTP client could not be built")]
158pub struct ClientError(#[source] reqwest::Error);
159
160impl ModelProxy {
161 pub fn new(
162 models: Vec<ProxyModel>,
163 guests: HashMap<String, Grant>,
164 metering: Arc<Metering>,
165 ) -> Result<Self, ClientError> {
166 let http = reqwest::Client::builder()
167 .redirect(reqwest::redirect::Policy::none())
168 .no_proxy()
169 .read_timeout(UPSTREAM_IDLE_TIMEOUT)
170 .build()
171 .map_err(ClientError)?;
172 Ok(Self {
173 models: models
174 .into_iter()
175 .map(|model| (model.name.clone(), model))
176 .collect(),
177 guests,
178 metering,
179 http,
180 timing: Timing {
181 ping: PING_INTERVAL,
182 idle: UPSTREAM_IDLE_TIMEOUT,
183 },
184 })
185 }
186
187 #[cfg(test)]
188 fn with_timing(mut self, ping: Duration, idle: Duration) -> Self {
189 self.timing = Timing { ping, idle };
190 self
191 }
192
193 fn model_rule(&self, grant: &Grant, dialect: Dialect) -> String {
194 let granted = || {
195 grant
196 .models
197 .names()
198 .into_iter()
199 .filter_map(|name| self.models.get(name))
200 };
201 let served = granted()
202 .filter(|model| model.upstream.serves(dialect))
203 .map(|model| format!("`{}`", model.name))
204 .collect::<Vec<_>>();
205 if served.is_empty() {
206 let elsewhere = granted()
207 .map(|model| format!("`{}` on `{}`", model.name, model.upstream.path()))
208 .collect::<Vec<_>>();
209 format!(
210 "this agent may only call its configured models, and none is served on this path: {}. Call one on its own path.",
211 elsewhere.join(", ")
212 )
213 } else {
214 format!(
215 "this agent may only call its configured models: {}. Set the model to one of them.",
216 served.join(", ")
217 )
218 }
219 }
220
221 pub fn router_current<C: CurrentProxy>(current: C) -> Router {
222 Router::new()
223 .route(
224 "/v1/messages",
225 post(|state, headers, body| {
226 handle_current::<C>(state, Dialect::Messages, headers, body)
227 }),
228 )
229 .route(
230 "/v1/messages/count_tokens",
231 post(|state, headers, body| {
232 handle_current::<C>(state, Dialect::CountTokens, headers, body)
233 }),
234 )
235 .route(
236 "/v1/responses",
237 post(|state, headers, body| {
238 handle_current::<C>(state, Dialect::Responses, headers, body)
239 }),
240 )
241 .route(
242 "/v1/chat/completions",
243 post(|state, headers, body| {
244 handle_current::<C>(state, Dialect::Chat, headers, body)
245 }),
246 )
247 .with_state(current)
248 }
249
250 pub fn router(self: Arc<Self>) -> Router {
251 Self::router_current(self)
252 }
253}
254
255struct CallSpan {
256 inner: Span,
257 session: Option<VmSession>,
258 usage: ModelUsage,
259 settled: bool,
260}
261
262impl CallSpan {
263 fn open(headers: &HeaderMap) -> Self {
264 let session = headers
265 .get(VM_SESSION_HEADER)
266 .and_then(|value| value.to_str().ok())
267 .and_then(VmSession::parse);
268 Self {
269 inner: tracing::info_span!(
270 target: "model",
271 "model.proxy.call",
272 vm.subject = label(headers, SUBJECT_HEADER),
273 vm.session = session.as_ref().map_or("", VmSession::as_str),
274 agent = Empty,
275 model.name = Empty,
276 usage.input_tokens = Empty,
277 usage.output_tokens = Empty,
278 outcome = Empty,
279 ),
280 session,
281 usage: ModelUsage::default(),
282 settled: false,
283 }
284 }
285
286 fn observe(&mut self, usage: ModelUsage) {
287 self.usage = self.usage.merged(usage);
288 }
289
290 fn refused(mut self, response: Response) -> Response {
291 self.end("refused");
292 response
293 }
294
295 fn settle(&mut self, outcome: Outcome) {
296 self.end(outcome.as_str());
297 }
298
299 fn end(&mut self, outcome: &'static str) {
300 self.settled = true;
301 let tokens = |count: Option<u64>| i64::try_from(count.unwrap_or(0)).unwrap_or(i64::MAX);
302 self.inner
303 .record("usage.input_tokens", tokens(self.usage.input_tokens))
304 .record("usage.output_tokens", tokens(self.usage.output_tokens))
305 .record("outcome", outcome);
306 }
307}
308
309impl Drop for CallSpan {
310 fn drop(&mut self) {
311 if !self.settled {
312 self.end(Outcome::Cancelled.as_str());
313 }
314 }
315}
316
317fn label<'a>(headers: &'a HeaderMap, name: &str) -> &'a str {
318 headers
319 .get(name)
320 .and_then(|value| value.to_str().ok())
321 .map(str::trim)
322 .filter(|value| {
323 value.len() <= MAX_LABEL_BYTES && value.bytes().all(|byte| matches!(byte, b' '..=b'~'))
324 })
325 .unwrap_or("")
326}
327
328pub trait CurrentProxy: Clone + Send + Sync + 'static {
329 fn current(&self) -> Option<Arc<ModelProxy>>;
330}
331
332impl CurrentProxy for Arc<ModelProxy> {
333 fn current(&self) -> Option<Arc<ModelProxy>> {
334 Some(Arc::clone(self))
335 }
336}
337
338async fn handle_current<C: CurrentProxy>(
339 State(current): State<C>,
340 dialect: Dialect,
341 headers: HeaderMap,
342 body: Body,
343) -> Response {
344 match current.current() {
345 Some(proxy) => handle(State(proxy), dialect, headers, body).await,
346 None => CallSpan::open(&headers).refused(dialect.sandbox(
347 Problem::Forbidden,
348 "this VM is granted no model. Ask the operator to grant its agent one.",
349 )),
350 }
351}
352
353async fn handle(
354 State(proxy): State<Arc<ModelProxy>>,
355 dialect: Dialect,
356 headers: HeaderMap,
357 body: Body,
358) -> Response {
359 let span = CallSpan::open(&headers);
360 let current = span.inner.clone();
361 call(&proxy, dialect, &headers, body, span)
362 .instrument(current)
363 .await
364}
365
366async fn call(
367 proxy: &ModelProxy,
368 dialect: Dialect,
369 headers: &HeaderMap,
370 body: Body,
371 mut span: CallSpan,
372) -> Response {
373 let Some(grant) = headers
374 .get(SUBJECT_HEADER)
375 .and_then(|subject| subject.to_str().ok())
376 .and_then(|subject| proxy.guests.get(subject))
377 else {
378 return span.refused(dialect.sandbox(
379 Problem::Forbidden,
380 "this VM is granted no model. Ask the operator to grant its agent one.",
381 ));
382 };
383 span.inner.record("agent", grant.agent.as_str());
384 let body = match axum::body::to_bytes(body, MAX_BODY_BYTES).await {
385 Ok(body) => body,
386 Err(error) => {
387 tracing::debug!(target: "model", error = %error, "proxy.request.body.refused");
388 return span.refused(dialect.sandbox(
389 Problem::TooLarge,
390 "a request body may be at most 8 MiB. Send a smaller request.",
391 ));
392 }
393 };
394 let peeked = match &grant.models {
395 Granted::Listed(_) => RequestPeek::of(&body),
396 Granted::Inherited(_) => RequestPeek::with_settings(&body),
397 };
398 let peek = match peeked {
399 Ok(peek) => peek,
400 Err(error) => {
401 return span.refused(dialect.sandbox(Problem::Invalid, &peek_rule(&error, grant)));
402 }
403 };
404 let model = match &grant.models {
405 Granted::Listed(models) => proxy
406 .models
407 .get(&peek.model)
408 .filter(|model| models.contains(&model.name) && model.upstream.serves(dialect)),
409 Granted::Inherited(name) => match proxy.models.get(name) {
410 Some(model) if !model.upstream.serves(dialect) => {
411 return span.refused(dialect.sandbox(
412 Problem::Forbidden,
413 &format!(
414 "this VM runs on its agent's model `{}`, which is served on `{}`, not `{}`. Call it on its own path.",
415 model.name,
416 model.upstream.path(),
417 dialect.path()
418 ),
419 ));
420 }
421 served => served,
422 },
423 };
424 let Some(model) = model else {
425 return span
426 .refused(dialect.sandbox(Problem::Forbidden, &proxy.model_rule(grant, dialect)));
427 };
428 span.inner.record("model.name", model.name.as_str());
429 let wire_model = serde_json::Value::from(model.wire_model.as_str()).to_string();
430 let rewritten = match (&model.upstream, &grant.models) {
431 (Upstream::Codex { .. }, _) if peek.stream != Some(true) => {
432 return span.refused(dialect.sandbox(
433 Problem::Invalid,
434 "this model is served only as a stream. Set `stream` to true.",
435 ));
436 }
437 (Upstream::Codex { .. }, _) => peek.rewrite(
438 &body,
439 &[(Field::Model, &wire_model), (Field::Store, "false")],
440 ),
441 (Upstream::OpenRouter { .. }, _) if peek.has(Field::Models) || peek.has(Field::Route) => {
442 return span.refused(dialect.sandbox(
443 Problem::Invalid,
444 "OpenRouter fallback routing (`models` / `route`) is not allowed; the sandbox pins each call to one configured model. Remove the field.",
445 ));
446 }
447 (Upstream::OpenRouter { settings, .. }, Granted::Inherited(_)) => {
448 let mut replacements = vec![(Field::Model, wire_model.as_str())];
449 replacements.extend(settings.iter().map(|(field, raw)| (*field, raw.as_str())));
450 peek.rewrite(&body, &replacements)
451 }
452 (Upstream::Anthropic { .. }, _) | (Upstream::OpenRouter { .. }, Granted::Listed(_)) => {
453 peek.rewrite(&body, &[(Field::Model, &wire_model)])
454 }
455 };
456 drop(body);
457 let admission = if dialect == Dialect::CountTokens {
458 None
459 } else {
460 let call = Call {
461 agent: grant.agent.clone(),
462 model: model.name.clone(),
463 backend: model.backend,
464 via: Via::Proxy {
465 session: span.session,
466 },
467 };
468 let estimate = Estimate::from_sizes(
469 Sizes {
470 bytes: rewritten.len(),
471 images: peek.images,
472 },
473 None,
474 model.reserve,
475 );
476 match proxy.metering.admit(call, estimate) {
477 Ok(admission) => Some(admission),
478 Err(refusal) => return span.refused(dialect.refusal(&refusal)),
479 }
480 };
481 let upstream = match send(&proxy.http, model, dialect, headers, Bytes::from(rewritten)).await {
482 Ok(response) => response,
483 Err(error) => {
484 tracing::warn!(target: "model", event = "proxy.upstream_failed", error.kind = error.kind(), error = %error, "proxy.upstream_failed");
485 let outcome = match error {
486 SendError::Credential(_) => Outcome::NotSent,
487 SendError::Request(_) => Outcome::Failed,
488 };
489 if let Some(admission) = admission {
490 admission.settle(outcome);
491 }
492 span.settle(outcome);
493 return dialect.error(Problem::Upstream, "the model upstream could not be reached");
494 }
495 };
496 respond(proxy, dialect, upstream, admission, span).await
497}
498
499#[derive(Debug, thiserror::Error)]
500enum SendError {
501 #[error("Codex credential is unavailable")]
502 Credential(#[source] dekopon_model::chatgpt::ChatGptError),
503 #[error("upstream request failed")]
504 Request(#[source] reqwest::Error),
505}
506
507impl SendError {
508 const fn kind(&self) -> &'static str {
509 match self {
510 Self::Credential(_) => "credential",
511 Self::Request(_) => "request",
512 }
513 }
514}
515
516async fn send(
517 http: &reqwest::Client,
518 model: &ProxyModel,
519 dialect: Dialect,
520 guest: &HeaderMap,
521 body: Bytes,
522) -> Result<reqwest::Response, SendError> {
523 let mut request = http
524 .post(model.upstream.url(dialect))
525 .header(header::CONTENT_TYPE, "application/json")
526 .body(body.clone());
527 for name in PASSED_HEADERS {
528 if let Some(value) = guest.get(name) {
529 request = request.header(name, value);
530 }
531 }
532 match &model.upstream {
533 Upstream::Anthropic { api_key, .. } => request
534 .header("x-api-key", api_key.expose().as_str())
535 .send()
536 .await
537 .map_err(SendError::Request),
538 Upstream::OpenRouter { api_key, .. } => request
539 .bearer_auth(api_key.expose())
540 .send()
541 .await
542 .map_err(SendError::Request),
543 Upstream::Codex {
544 credential,
545 refresh,
546 ..
547 } => {
548 let request = request
549 .header("originator", "dekopon")
550 .header("openai-beta", "responses=experimental")
551 .header(header::ACCEPT, "text/event-stream");
552 let retry = request.try_clone();
553 let current = codex_credential(credential, refresh, None).await?;
554 let response = request
555 .bearer_auth(current.access.expose())
556 .header("chatgpt-account-id", ¤t.account_id)
557 .send()
558 .await
559 .map_err(SendError::Request)?;
560 match retry {
561 Some(retry) if response.status() == StatusCode::UNAUTHORIZED => {
562 drop(response);
563 let refreshed =
564 codex_credential(credential, refresh, Some(current.access)).await?;
565 retry
566 .bearer_auth(refreshed.access.expose())
567 .header("chatgpt-account-id", &refreshed.account_id)
568 .send()
569 .await
570 .map_err(SendError::Request)
571 }
572 Some(_) | None => Ok(response),
573 }
574 }
575 }
576}
577
578async fn codex_credential(
579 credential: &Arc<CredentialFile>,
580 refresh: &RefreshClient,
581 rejected: Option<Redacted<String>>,
582) -> Result<dekopon_model::chatgpt::ResolvedCredential, SendError> {
583 match rejected {
584 Some(rejected) => credential.force_refresh(refresh, &rejected).await,
585 None => credential.current(refresh).await,
586 }
587 .map_err(SendError::Credential)
588}
589
590async fn respond(
591 proxy: &ModelProxy,
592 dialect: Dialect,
593 upstream: reqwest::Response,
594 admission: Option<dekopon_model_token_governor::Admission>,
595 mut span: CallSpan,
596) -> Response {
597 let status = upstream.status();
598 let mut headers = HeaderMap::new();
599 for name in RETURNED_HEADERS {
600 if let Some(value) = upstream.headers().get(name) {
601 headers.insert(HeaderName::from_static(name), value.clone());
602 }
603 }
604 if !status.is_success() {
605 if let Some(admission) = admission {
606 admission.settle(Outcome::NotSent);
607 }
608 span.settle(Outcome::NotSent);
609 let body = bounded(upstream, MAX_ERROR_BYTES).await;
610 let mut response = Response::new(Body::from(body));
611 *response.status_mut() = status;
612 *response.headers_mut() = headers;
613 return response;
614 }
615 let events = headers
616 .get(header::CONTENT_TYPE)
617 .and_then(|value| value.to_str().ok())
618 .is_some_and(|value| value.starts_with("text/event-stream"));
619 let shape = if events { Shape::Events } else { Shape::Json };
620 let stream = tee::tee(
621 upstream.bytes_stream(),
622 dialect,
623 shape,
624 proxy.timing,
625 admission,
626 span,
627 );
628 let mut response = Response::new(Body::from_stream(stream));
629 *response.status_mut() = status;
630 if events {
631 headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
632 }
633 *response.headers_mut() = headers;
634 response
635}
636
637fn peek_rule(error: &PeekError, grant: &Grant) -> String {
638 match error {
639 PeekError::NotJson | PeekError::NotObject => {
640 "the request body must be a JSON object. Send one.".to_owned()
641 }
642 PeekError::NoModel => format!(
643 "the request must name a model. Set `model` to one of this agent's configured models: {}.",
644 grant
645 .models
646 .names()
647 .into_iter()
648 .map(|name| format!("`{name}`"))
649 .collect::<Vec<_>>()
650 .join(", ")
651 ),
652 PeekError::Duplicate(field) => {
653 format!("the request repeats the top-level key `{field}`; send it once.")
654 }
655 }
656}
657
658async fn bounded(mut upstream: reqwest::Response, limit: usize) -> Vec<u8> {
659 let mut body = Vec::new();
660 loop {
661 match upstream.chunk().await {
662 Ok(Some(chunk)) if body.len() + chunk.len() <= limit => body.extend_from_slice(&chunk),
663 Ok(Some(_)) => break,
664 Ok(None) => break,
665 Err(error) => {
666 tracing::debug!(target: "model", error = %error, "proxy.upstream.error.truncated");
667 break;
668 }
669 }
670 }
671 body
672}