1use std::net::SocketAddr;
4use std::pin::Pin;
5use std::sync::Arc;
6use std::task::{Context, Poll};
7use std::time::{Duration, Instant};
8
9use axum::body::{Body, HttpBody};
10use axum::extract::{ConnectInfo, Request, State};
11use axum::middleware::Next;
12use axum::response::Response;
13use cpa_core::config::TrustedProxies;
14
15use crate::Runtime;
16
17#[derive(Clone)]
19pub struct RequestId(pub String);
20
21tokio::task_local! {
22 static REQUEST_ID: String;
23 static CLOCK: Arc<Clock>;
24}
25
26pub(crate) fn current_request_id() -> Option<String> {
29 REQUEST_ID.try_with(Clone::clone).ok().filter(|id| !id.is_empty())
30}
31
32#[derive(Debug)]
37pub struct Clock {
38 accepted: Instant,
39 body: std::sync::OnceLock<(Instant, u64)>,
40 response: std::sync::OnceLock<Instant>,
41}
42
43impl Default for Clock {
44 fn default() -> Self {
47 Self {
48 accepted: Instant::now(),
49 body: std::sync::OnceLock::new(),
50 response: std::sync::OnceLock::new(),
51 }
52 }
53}
54
55impl Clock {
56 pub fn accepted(&self) -> Instant {
58 self.accepted
59 }
60
61 pub fn body_received(&self) -> Option<(Instant, u64)> {
63 self.body.get().copied()
64 }
65
66 pub fn response_started(&self) -> Option<Instant> {
69 self.response.get().copied()
70 }
71}
72
73pub(crate) fn current_clock() -> Arc<Clock> {
75 CLOCK.try_with(Arc::clone).unwrap_or_default()
76}
77
78struct Received {
81 body: Body,
82 bytes: u64,
83 clock: Option<Arc<Clock>>,
84}
85
86impl Received {
87 fn finish(&mut self) {
88 if let Some(clock) = self.clock.take() {
89 let _ = clock.body.set((Instant::now(), self.bytes));
90 }
91 }
92}
93
94impl HttpBody for Received {
95 type Data = bytes::Bytes;
96 type Error = axum::Error;
97
98 fn poll_frame(
99 mut self: Pin<&mut Self>,
100 cx: &mut Context<'_>,
101 ) -> Poll<Option<Result<hyper::body::Frame<Self::Data>, Self::Error>>> {
102 let result = Pin::new(&mut self.body).poll_frame(cx);
103 match &result {
104 Poll::Ready(Some(Ok(frame))) => {
105 self.bytes += frame.data_ref().map_or(0, |data| data.len() as u64);
106 if self.body.is_end_stream() {
107 self.finish();
108 }
109 }
110 Poll::Ready(None) => self.finish(),
111 _ => {}
112 }
113 result
114 }
115
116 fn is_end_stream(&self) -> bool {
117 self.body.is_end_stream()
118 }
119
120 fn size_hint(&self) -> hyper::body::SizeHint {
121 self.body.size_hint()
122 }
123}
124
125#[derive(Clone)]
126struct Access {
127 proxies: TrustedProxies,
128}
129
130pub fn router(rt: &Arc<Runtime>, app: axum::Router) -> axum::Router {
133 app.layer(axum::middleware::from_fn_with_state(
134 Access {
135 proxies: TrustedProxies::new(&rt.config().trusted_proxies),
136 },
137 access_log,
138 ))
139}
140
141pub(crate) fn is_ai_path(path: &str) -> bool {
142 ["/v1", "/v1beta", "/openai/v1", "/backend-api/codex"]
143 .iter()
144 .any(|prefix| path == *prefix || path.strip_prefix(prefix).is_some_and(|tail| tail.starts_with('/')))
145}
146
147async fn access_log(State(state): State<Access>, mut request: Request, next: Next) -> Response {
148 let start = Instant::now();
149 let path = crate::management::percent_decode(request.uri().path());
150 let query = logged_query(&path, request.uri().query().unwrap_or_default());
151 let method = request.method().to_string();
152 let peer = request.extensions().get::<ConnectInfo<SocketAddr>>().map(|p| p.0);
153 let client = state
154 .proxies
155 .client_ip(peer, |name| request.headers().get(name).map(|v| v.as_bytes()));
156 let inference = is_ai_path(&path);
157 let id = if inference {
158 let id = crate::dispatch::request_id();
159 request.extensions_mut().insert(RequestId(id.clone()));
160 id
161 } else {
162 "--------".to_owned()
163 };
164 let context_id = if id == "--------" { String::new() } else { id.clone() };
165 let health = path == "/healthz" && matches!(method.as_str(), "GET" | "HEAD");
166 let path = if query.is_empty() {
167 path
168 } else {
169 format!("{path}?{query}")
170 };
171 let mut log = AccessGuard(Some(AccessLine {
172 start,
173 status: 200,
174 client,
175 method,
176 path,
177 id,
178 health,
179 }));
180 let clock = inference.then(|| {
181 Arc::new(Clock {
182 accepted: start,
183 ..Clock::default()
184 })
185 });
186 let response = match &clock {
187 Some(clock) => {
188 let request = request.map(|body| {
189 Body::new(Received {
190 body,
191 bytes: 0,
192 clock: Some(clock.clone()),
193 })
194 });
195 REQUEST_ID
196 .scope(context_id, CLOCK.scope(clock.clone(), next.run(request)))
197 .await
198 }
199 None => REQUEST_ID.scope(context_id, next.run(request)).await,
200 };
201 let status = response.status().as_u16();
202 if health && (200..300).contains(&status) {
203 log.0 = None;
204 return response;
205 }
206 if let Some(line) = &mut log.0 {
207 line.status = status;
208 }
209 let (parts, body) = response.into_parts();
210 Response::from_parts(parts, Body::new(AccessBody { body, log, clock }))
211}
212
213struct AccessLine {
214 start: Instant,
215 status: u16,
216 client: String,
217 method: String,
218 path: String,
219 id: String,
220 health: bool,
221}
222
223impl AccessLine {
224 fn emit(self) {
225 if self.health && (200..300).contains(&self.status) {
226 return;
227 }
228 let line = access_line(
229 self.status,
230 self.start.elapsed(),
231 &self.client,
232 &self.method,
233 &self.path,
234 );
235 match self.status {
236 500.. => tracing::error!(request_id = self.id, "{line}"),
237 400.. => tracing::warn!(request_id = self.id, "{line}"),
238 _ => tracing::info!(request_id = self.id, "{line}"),
239 }
240 }
241}
242
243fn access_line(status: u16, latency: Duration, client: &str, method: &str, path: &str) -> String {
245 let latency = if latency >= Duration::from_secs(60) {
246 let seconds = latency.as_secs();
247 let (hours, minutes, seconds) = (seconds / 3600, seconds / 60 % 60, seconds % 60);
248 if hours > 0 {
249 format!("{hours}h{minutes}m{seconds}s")
250 } else {
251 format!("{minutes}m{seconds}s")
252 }
253 } else {
254 let millis = latency.as_millis();
255 if millis == 0 {
256 "0s".to_owned()
257 } else if millis < 1000 {
258 format!("{millis}ms")
259 } else {
260 let fraction = format!("{:03}", millis % 1000);
261 if millis.is_multiple_of(1000) {
262 format!("{}s", millis / 1000)
263 } else {
264 format!("{}.{}s", millis / 1000, fraction.trim_end_matches('0'))
265 }
266 }
267 };
268 let path = cpa_common::gostr::quote(path);
271 format!("{status:3} | {latency:>13} | {client:>15} | {method:<7} {path}")
272}
273
274struct AccessBody {
275 body: Body,
276 log: AccessGuard,
277 clock: Option<Arc<Clock>>,
279}
280
281struct AccessGuard(Option<AccessLine>);
282
283impl AccessGuard {
284 fn finish(&mut self) {
285 if let Some(log) = self.0.take() {
286 log.emit();
287 }
288 }
289}
290
291impl Drop for AccessGuard {
292 fn drop(&mut self) {
293 self.finish();
294 }
295}
296
297impl HttpBody for AccessBody {
298 type Data = bytes::Bytes;
299 type Error = axum::Error;
300
301 fn poll_frame(
302 mut self: Pin<&mut Self>,
303 cx: &mut Context<'_>,
304 ) -> Poll<Option<Result<hyper::body::Frame<Self::Data>, Self::Error>>> {
305 let result = Pin::new(&mut self.body).poll_frame(cx);
306 if let Poll::Ready(Some(Ok(frame))) = &result
307 && frame.data_ref().is_some_and(|data| !data.is_empty())
308 && let Some(clock) = self.clock.take()
309 {
310 let _ = clock.response.set(Instant::now());
311 }
312 if matches!(result, Poll::Ready(None | Some(Err(_)))) {
313 self.log.finish();
314 }
315 result
316 }
317
318 fn is_end_stream(&self) -> bool {
319 self.body.is_end_stream()
320 }
321
322 fn size_hint(&self) -> hyper::body::SizeHint {
323 self.body.size_hint()
324 }
325}
326
327pub(crate) fn hide_key(value: &[u8]) -> Vec<u8> {
332 let keep = match value.len() {
333 9.. => 4,
334 5.. => 2,
335 3.. => 1,
336 0 => return Vec::new(),
337 _ => return b"...".to_vec(),
338 };
339 [&value[..keep], b"...", &value[value.len() - keep..]].concat()
340}
341
342fn logged_query(path: &str, raw: &str) -> String {
347 if path.starts_with("/v8/management/oauth/") {
348 return String::new();
349 }
350 mask_query(raw)
351}
352
353pub(crate) fn mask_query(raw: &str) -> String {
354 mask_query_by(raw, go_sensitive_name)
355}
356
357fn go_sensitive_name(lower: &[u8]) -> bool {
360 lower == b"key"
361 || lower == b"value"
362 || [&b"api-key"[..], b"apikey", b"api_key", b"token", b"secret"]
363 .iter()
364 .any(|word| lower.windows(word.len()).any(|w| w == *word))
365}
366
367pub(crate) fn mask_upstream_query(raw: &str) -> String {
372 mask_query_by(raw, |lower| {
373 go_sensitive_name(lower)
374 || [&b"key"[..], b"token", b"secret", b"auth", b"password", b"sig"]
375 .iter()
376 .any(|word| lower.windows(word.len()).any(|w| w == *word))
377 })
378}
379
380fn mask_query_by(raw: &str, sensitive: impl Fn(&[u8]) -> bool) -> String {
381 raw.split('&')
382 .map(|part| {
383 let (key, value) = part.split_once('=').unwrap_or((part, ""));
384 let decoded = crate::access::unescape(key).unwrap_or_else(|| key.as_bytes().to_owned());
385 let lower = cpa_common::gostr::lower_bytes(cpa_core::config::go_trim_space(&decoded));
386 let lower = lower.strip_suffix("[]").unwrap_or(&lower).as_bytes();
387 if !sensitive(lower) {
388 return part.to_owned();
389 }
390 let value = crate::access::unescape(value).unwrap_or_else(|| value.as_bytes().to_owned());
391 let value = hide_key(cpa_core::config::go_trim_space(&value));
392 let escaped: String = value
393 .iter()
394 .map(|b| match b {
395 b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (*b as char).to_string(),
396 b' ' => "+".to_owned(),
397 _ => format!("%{b:02X}"),
398 })
399 .collect();
400 format!("{key}={escaped}")
401 })
402 .collect::<Vec<_>>()
403 .join("&")
404}
405
406#[cfg(test)]
407mod tests {
408 use super::*;
409 use axum::http::StatusCode;
410 use axum::response::IntoResponse;
411 use tower_service::Service;
412 use tracing::instrument::WithSubscriber;
413
414 #[derive(Clone, Default)]
415 struct Capture(Arc<std::sync::Mutex<Vec<u8>>>);
416
417 impl std::io::Write for Capture {
418 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
419 self.0.lock().unwrap().extend_from_slice(buf);
420 Ok(buf.len())
421 }
422 fn flush(&mut self) -> std::io::Result<()> {
423 Ok(())
424 }
425 }
426
427 fn subscriber(capture: &Capture) -> impl tracing::Subscriber + Send + Sync + use<> {
428 let capture = capture.clone();
429 tracing_subscriber::fmt()
430 .without_time()
431 .with_ansi(false)
432 .with_writer(move || capture.clone())
433 .finish()
434 }
435
436 fn runtime() -> Arc<Runtime> {
437 Arc::new(crate::testing::runtime(
438 cpa_core::config::Config::parse("server: {trusted-proxies: [127.0.0.1]}\n").unwrap(),
439 Vec::new(),
440 cpa_exec::Executors {
441 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:1").unwrap(),
442 codex: Default::default(),
443 devices: Default::default(),
444 openai: Default::default(),
445 google: Default::default(),
446 },
447 ))
448 }
449
450 #[tokio::test]
451 async fn health_status_levels_and_stream_completion_follow_go() {
452 let fixture: serde_json::Value =
453 serde_json::from_str(include_str!("../tests/fixtures/observe_go.json")).unwrap();
454 let rt = runtime();
455 for case in fixture["health"].as_array().unwrap() {
456 let status = StatusCode::from_u16(case["status"].as_u64().unwrap() as u16).unwrap();
457 let app = axum::Router::new().fallback(move || async move { status });
458 let mut app = router(&rt, app);
459 let request = Request::builder()
460 .method(case["method"].as_str().unwrap())
461 .uri("/healthz")
462 .body(Body::empty())
463 .unwrap();
464 let capture = Capture::default();
465 async {
466 let response = app.call(request).await.unwrap();
467 assert_eq!(response.status(), status);
468 drop(response);
469 }
470 .with_subscriber(subscriber(&capture))
471 .await;
472 let text = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap();
473 let level = case["level"].as_str().unwrap();
474 if level.is_empty() {
475 assert!(text.is_empty(), "{text}");
476 } else {
477 assert!(
478 text.contains(match level {
479 "warning" => "WARN",
480 "error" => "ERROR",
481 _ => "INFO",
482 }),
483 "{text}"
484 );
485 }
486 }
487 let capture = Capture::default();
488 async {
489 let app = axum::Router::new().fallback(|axum::Extension(id): axum::Extension<RequestId>| async move {
490 let trace = crate::dispatch::Trace::with_request_id(current_request_id());
491 assert_eq!(id.0, trace.request_id());
492 Body::from_stream(futures_util::stream::pending::<Result<bytes::Bytes, std::io::Error>>())
494 .into_response()
495 });
496 let mut app = router(&rt, app);
497 let mut request = Request::builder()
498 .uri("/backend-api/codex/responses?key=abcdefghijk")
499 .header("x-forwarded-for", "203.0.113.8")
500 .body(Body::empty())
501 .unwrap();
502 request
503 .extensions_mut()
504 .insert(ConnectInfo("127.0.0.1:1".parse::<SocketAddr>().unwrap()));
505 let response = app.call(request).await.unwrap();
506 assert!(capture.0.lock().unwrap().is_empty(), "stream has not completed");
507 drop(response);
508 }
509 .with_subscriber(subscriber(&capture))
510 .await;
511 let text = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap();
512 assert_eq!(text.lines().count(), 1, "{text}");
513 assert!(text.contains("203.0.113.8"), "{text}");
514 assert!(
515 text.contains("key=abcd...hijk") && !text.contains("abcdefghijk"),
516 "{text}"
517 );
518 }
519
520 #[tokio::test]
521 async fn cancellation_before_headers_and_body_end_log_exactly_once() {
522 let rt = runtime();
523 let health_capture = Capture::default();
524 async {
525 let app = axum::Router::new().fallback(std::future::pending::<StatusCode>);
526 let mut app = router(&rt, app);
527 let request = Request::builder().uri("/healthz").body(Body::empty()).unwrap();
528 let mut pending = Box::pin(app.call(request));
529 assert!(futures_util::poll!(&mut pending).is_pending());
530 drop(pending);
531 }
532 .with_subscriber(subscriber(&health_capture))
533 .await;
534 assert!(health_capture.0.lock().unwrap().is_empty());
535 let capture = Capture::default();
536 async {
537 let app = axum::Router::new().fallback(std::future::pending::<StatusCode>);
538 let mut app = router(&rt, app);
539 let request = Request::builder().uri("/v1/responses").body(Body::empty()).unwrap();
540 let mut pending = Box::pin(app.call(request));
541 assert!(futures_util::poll!(&mut pending).is_pending());
542 assert!(capture.0.lock().unwrap().is_empty());
543 drop(pending);
544 }
545 .with_subscriber(subscriber(&capture))
546 .await;
547 let text = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap();
548 assert_eq!(text.lines().count(), 1, "{text}");
549 assert!(text.contains("200 |") && text.contains("request_id="), "{text}");
550
551 for fail in [false, true] {
552 let capture = Capture::default();
553 async {
554 let app = axum::Router::new().fallback(move || async move {
555 let result = if fail {
556 Err(std::io::Error::other("fixture"))
557 } else {
558 Ok(bytes::Bytes::from_static(b"ok"))
559 };
560 Body::from_stream(futures_util::stream::iter([result]))
561 });
562 let mut app = router(&rt, app);
563 let response = app
564 .call(Request::builder().uri("/v1/responses").body(Body::empty()).unwrap())
565 .await
566 .unwrap();
567 assert!(capture.0.lock().unwrap().is_empty());
568 let result = axum::body::to_bytes(response.into_body(), usize::MAX).await;
569 assert_eq!(result.is_err(), fail);
570 }
571 .with_subscriber(subscriber(&capture))
572 .await;
573 assert_eq!(
574 String::from_utf8(capture.0.lock().unwrap().clone())
575 .unwrap()
576 .lines()
577 .count(),
578 1
579 );
580 }
581 }
582
583 #[tokio::test]
584 async fn real_dispatch_header_access_and_usage_share_identity() {
585 let config = cpa_core::config::Config::parse("claude-api-key:\n - api-key: fake-observe-key\n base-url: http://127.0.0.1:9\n models: [{name: claude-sonnet-4-6}]\nobservability: {usage: {usage-statistics-enabled: true}}\n").unwrap();
586 let credentials = cpa_core::config::credentials::from_config(&config);
587 let executors = cpa_exec::Executors {
588 claude: cpa_exec::claude::ClaudeExecutor::new("http://127.0.0.1:9").unwrap(),
589 codex: Default::default(),
590 devices: Default::default(),
591 openai: Default::default(),
592 google: Default::default(),
593 };
594 let rt = Arc::new(crate::testing::runtime(config, credentials, executors));
595 rt.usage_queue().configure(true, &rt.config());
596 let capture = Capture::default();
597 let id = async {
598 let mut app = router(&rt, crate::router(rt.clone()));
599 let request = Request::builder()
600 .method("POST")
601 .uri("/v1/messages")
602 .header("content-type", "application/json")
603 .body(Body::from(
604 r#"{"model":"claude-sonnet-4-6","max_tokens":1,"messages":[]}"#,
605 ))
606 .unwrap();
607 let response = app.call(request).await.unwrap();
608 let trace = response
609 .headers()
610 .get("x-cpa-trace-id")
611 .expect("selected credential trace")
612 .to_str()
613 .unwrap()
614 .to_owned();
615 let id = trace.splitn(3, '-').nth(2).unwrap().to_owned();
616 assert_eq!(id.as_bytes()[14], b'7');
617 let _ = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
618 id
619 }
620 .with_subscriber(subscriber(&capture))
621 .await;
622 let text = String::from_utf8(capture.0.lock().unwrap().clone()).unwrap();
623 assert!(text.contains(&id), "{text}");
624 let records = rt.usage_queue().pop_oldest(10);
625 assert!(!records.is_empty());
626 for record in records {
627 let record: serde_json::Value = serde_json::from_slice(&record).unwrap();
628 assert_eq!(record["request_id"], id);
629 assert_eq!(record["trace_id"], id);
630 }
631 }
632
633 #[test]
634 fn go_access_and_redaction_goldens() {
635 let fixture: serde_json::Value =
636 serde_json::from_str(include_str!("../tests/fixtures/observe_go.json")).unwrap();
637 for case in fixture["query"].as_array().unwrap() {
638 let want = case["out"].as_str().unwrap().replacen("key=ab&", "key=...&", 1);
640 assert_eq!(mask_query(case["in"].as_str().unwrap()), want);
641 }
642 assert_eq!(
644 mask_query("value=sk-secret-client-key&index=3"),
645 "value=sk-s...-key&index=3"
646 );
647 let oauth = "state=secret-state&code=secret-code&client_id=issued-secret&error=secret-error";
650 for path in [
651 "/v8/management/oauth/callback",
652 "/v8/management/oauth/status",
653 "/v8/management/oauth/session",
654 ] {
655 assert_eq!(logged_query(path, oauth), "");
656 }
657 assert_eq!(logged_query("/v0/management/oauth-callback", "state=s"), "state=s");
658 assert_eq!(logged_query("/v8/management/oauthx", "state=s"), "state=s");
659 for case in fixture["lines"].as_array().unwrap() {
660 assert_eq!(
661 access_line(
662 case["status"].as_u64().unwrap() as u16,
663 Duration::from_nanos(case["nanos"].as_u64().unwrap()),
664 case["client"].as_str().unwrap(),
665 case["method"].as_str().unwrap(),
666 case["path"].as_str().unwrap()
667 ),
668 case["out"]
669 );
670 }
671 assert!(
673 access_line(404, Duration::ZERO, "127.0.0.1", "GET", "/x\n200 forged")
674 .ends_with(r#"GET "/x\n200 forged""#)
675 );
676 let query = logged_query("/v1/models", "key=sk-0123456789cdef&limit=5");
679 assert!(
680 access_line(200, Duration::ZERO, "127.0.0.1", "GET", &format!("/v1/models?{query}"))
681 .ends_with(r#"GET "/v1/models?key=sk-0...cdef&limit=5""#)
682 );
683 for path in [
684 "/v1",
685 "/v1/images/edits",
686 "/openai/v1/videos",
687 "/backend-api/codex/responses",
688 ] {
689 assert!(is_ai_path(path));
690 }
691 for path in ["/v10", "/openai/v10", "/backend-api/codex-status", "/healthz"] {
692 assert!(!is_ai_path(path));
693 }
694 }
695}